diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 16595d1..2e582aa 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -89,7 +89,14 @@ "Bash(wait:*)", "WebSearch", "Bash(SKILL_CREATOR_PATH=\"C:\\\\Users\\\\A00963355\\\\.claude\\\\plugins\\\\cache\\\\claude-plugins-official\\\\skill-creator\\\\d5c15b861cd2\\\\skills\\\\skill-creator\" && WORKSPACE=\"C:\\\\Users\\\\A00963355\\\\OneDrive - Amaris Zorggroep\\\\Documents\\\\VScodeProjects\\\\DragonsStash\\\\.claude\\\\skills\\\\tdlib-telegram-workspace\\\\iteration-1\" && python \"$SKILL_CREATOR_PATH/eval-viewer/generate_review.py\" \"$WORKSPACE\" --skill-name \"tdlib-telegram\" --benchmark \"$WORKSPACE/benchmark.json\" --static \"$WORKSPACE/review.html\" 2>&1)", - "Bash(start:*)" + "Bash(start:*)", + "Bash(npm run:*)", + "Bash(DATABASE_URL=\"postgresql://dragons:stash@localhost:5432/dragonsstash\" npx prisma migrate dev --name add-skipped-packages)", + "Bash(git checkout:*)", + "Bash(DATABASE_URL=\"postgresql://dragons:stash@localhost:5432/dragonsstash?schema=public\" npx prisma migrate dev --name add_package_groups 2>&1)", + "Bash(psql:*)", + "Bash(git log:*)", + "Bash(git merge:*)" ] } } diff --git a/.drone.yml b/.drone.yml index 7392bba..8299a23 100644 --- a/.drone.yml +++ b/.drone.yml @@ -54,9 +54,24 @@ steps: password: from_secret: gitea_password + - name: build-backup + image: plugins/docker + depends_on: [clone] + settings: + repo: git.samagsteribbe.nl/admin/dragonsstash-backup + registry: git.samagsteribbe.nl + dockerfile: backup/Dockerfile + tags: + - latest + - "${DRONE_COMMIT_SHA:0:8}" + username: + from_secret: gitea_username + password: + from_secret: gitea_password + - name: deploy image: alpine - depends_on: [build-app, build-worker, build-bot] + depends_on: [build-app, build-worker, build-bot, build-backup] environment: SSH_KEY: from_secret: ssh_key diff --git a/.env.example b/.env.example index 223add6..97e509a 100644 --- a/.env.example +++ b/.env.example @@ -37,10 +37,11 @@ WORKER_MAX_ZIP_SIZE_MB=4096 MULTIPART_TIMEOUT_HOURS=0 LOG_LEVEL="info" -# Disaster recovery backups -BACKUP_MOUNT_PATH="/mnt/dragonsstash-backups" -BACKUP_STAGING_PATH="/var/lib/dragons-stash/backup-staging" -BACKUP_REPOSITORY="/backup/restic" -BACKUP_RESTIC_PASSWORD_FILE="/etc/dragons-stash/restic-password" -BACKUP_RETENTION_DAYS=30 -BACKUP_APP_VERSION="unknown" +# Backup (NAS via SMB/CIFS + restic) +NAS_HOST="" # Synology NAS IP or hostname reachable from this host +NAS_SHARE="" # SMB share name, e.g. dragonsstash_backups +NAS_USERNAME="" # SMB user with read/write on the share +NAS_PASSWORD="" # SMB user password (avoid commas — they delimit cifs mount opts) +RESTIC_PASSWORD="" # generate with: openssl rand -base64 32 +KUMA_PUSH_URL="" # optional: Uptime Kuma Push monitor URL; leave empty to disable alerting +TZ="Etc/UTC" diff --git a/backup/Dockerfile b/backup/Dockerfile index 9edf3ac..104ac6e 100644 --- a/backup/Dockerfile +++ b/backup/Dockerfile @@ -1,8 +1,13 @@ -FROM postgres:16-alpine +FROM alpine:3.20 -RUN apk add --no-cache bash restic coreutils +# Note: use busybox's built-in crond (Alpine base), NOT the dcron package — +# dcron's crond fails with "setpgid: Operation not permitted" in this runtime. +RUN apk add --no-cache restic postgresql16-client curl tzdata tar bash -COPY scripts/backup/container-entrypoint.sh /usr/local/bin/dragons-stash-backup -RUN chmod 0755 /usr/local/bin/dragons-stash-backup +COPY backup/backup.sh /backup.sh +COPY backup/entrypoint.sh /entrypoint.sh +COPY backup/crontab /etc/crontabs/root -ENTRYPOINT ["/usr/local/bin/dragons-stash-backup"] +RUN chmod +x /backup.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/backup/backup.sh b/backup/backup.sh new file mode 100644 index 0000000..cc415f7 --- /dev/null +++ b/backup/backup.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -euo pipefail + +report_failure() { + [ -n "${KUMA_PUSH_URL:-}" ] || return 0 + curl -fsS "$KUMA_PUSH_URL" --get \ + --data-urlencode "status=down" \ + --data-urlencode "msg=$BASH_COMMAND failed" || true +} +trap report_failure ERR + +DUMP_FILE=/tmp/dragonsstash.dump +TAR_FILE=/tmp/tdlib.tar.gz + +trap 'rm -f "$DUMP_FILE" "$TAR_FILE"' EXIT + +pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE" + +# TDLib volumes are tarred live (best-effort, per design). A file changing +# mid-read makes GNU tar exit 1 (warning) — that is expected here and must not +# abort the backup. Only a genuine error (exit >= 2) is fatal. +tar --warning=no-file-changed -czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot \ + || { rc=$?; [ "$rc" -le 1 ] || exit "$rc"; } + +restic backup "$DUMP_FILE" "$TAR_FILE" +restic forget --keep-daily 14 --prune + +if [ -n "${KUMA_PUSH_URL:-}" ]; then + curl -fsS "$KUMA_PUSH_URL" --get \ + --data-urlencode "status=up" \ + --data-urlencode "msg=OK" +fi diff --git a/backup/crontab b/backup/crontab new file mode 100644 index 0000000..3686087 --- /dev/null +++ b/backup/crontab @@ -0,0 +1,2 @@ +0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1 +0 4 * * 0 restic check >> /proc/1/fd/1 2>&1 diff --git a/backup/entrypoint.sh b/backup/entrypoint.sh new file mode 100644 index 0000000..238abae --- /dev/null +++ b/backup/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -uo pipefail + +# Ensure the repo exists, but never crash-loop on it: a transient error reading +# the repo (CIFS hiccup, stale lock) must not kill PID 1. `restic init` failing +# because the repo already exists is expected and harmless here. +if ! restic cat config >/dev/null 2>&1; then + restic init || echo "restic init skipped (repo already exists or temporarily unreachable)" +fi + +exec crond -f -l 2 diff --git a/docker-compose.yml b/docker-compose.yml index 2e7363a..cc447eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,26 +98,31 @@ services: - backend backup: - profiles: ["backup"] build: context: . dockerfile: backup/Dockerfile + pull_policy: never environment: - DATABASE_URL: postgresql://${POSTGRES_USER:-dragons}:${POSTGRES_PASSWORD:-stash}@db:5432/${POSTGRES_DB:-dragonsstash} - RESTIC_REPOSITORY: ${BACKUP_REPOSITORY:-/backup/restic} - RESTIC_PASSWORD_FILE: /run/secrets/restic-password - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30} - BACKUP_APP_VERSION: ${BACKUP_APP_VERSION:-unknown} - user: "0:0" + - POSTGRES_USER=${POSTGRES_USER:-dragons} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash} + - PGPASSWORD=${POSTGRES_PASSWORD:-stash} + - POSTGRES_DB=${POSTGRES_DB:-dragonsstash} + - RESTIC_REPOSITORY=/backups/restic-repo + - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env} + - KUMA_PUSH_URL=${KUMA_PUSH_URL:-} + - TZ=${TZ:-Etc/UTC} volumes: - tdlib_state:/data/tdlib-worker:ro - tdlib_bot_state:/data/tdlib-bot:ro - - ${BACKUP_MOUNT_PATH:?Set BACKUP_MOUNT_PATH to the mounted Synology share}:/backup:rw - - ${BACKUP_STAGING_PATH:?Set BACKUP_STAGING_PATH to a local staging directory}:/staging:rw - - ${BACKUP_RESTIC_PASSWORD_FILE:?Set BACKUP_RESTIC_PASSWORD_FILE to a root-readable secret file}:/run/secrets/restic-password:ro + - nas_backups:/backups depends_on: db: condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + memory: 1G networks: - backend @@ -140,8 +145,10 @@ services: limits: memory: 1G networks: - - frontend - - backend + frontend: {} + backend: + aliases: + - dragonsstash-db volumes: postgres_data: @@ -149,6 +156,11 @@ volumes: tdlib_bot_state: tmp_zips: manual_uploads: + nas_backups: + driver_opts: + type: cifs + o: "username=${NAS_USERNAME},password=${NAS_PASSWORD},vers=3.0,uid=0,gid=0,file_mode=0660,dir_mode=0770" + device: "//${NAS_HOST}/${NAS_SHARE}" networks: frontend: diff --git a/docs/superpowers/plans/2026-07-23-nas-backup.md b/docs/superpowers/plans/2026-07-23-nas-backup.md new file mode 100644 index 0000000..3dfe398 --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-nas-backup.md @@ -0,0 +1,555 @@ +# NAS Backup for Postgres + TDLib State Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `backup` container to the DragonsStash stack that takes daily, encrypted, deduplicated backups of the Postgres database and both TDLib state volumes, and ships them to a Synology NAS over NFS. + +**Architecture:** A small Alpine-based image (restic + postgresql16-client + curl + dcron) runs as its own compose service. A crontab fires `backup.sh` daily at 03:00, which dumps Postgres, tars the TDLib volumes, hands both to `restic backup` against an NFS-backed Docker volume, prunes with `restic forget --keep-daily 14`, and reports success/failure to an Uptime Kuma push monitor. Matches the existing `worker`/`bot` pattern: build context in the repo's `docker-compose.yml`, prebuilt image in `/opt/stacks/DragonsStash/docker-compose.yml`, built and pushed by `.drone.yml`. + +**Tech Stack:** Alpine 3.20, restic 0.16, postgresql16-client, dcron, bash, Docker Compose NFS volume driver. + +## Global Constraints + +- Retention: `restic forget --keep-daily 14 --prune` (14-day window, per approved spec). +- Schedule: daily backup at 03:00, weekly `restic check` at 04:00 Sunday. +- No Docker socket mount, no `privileged: true` — the backup container must not be able to control sibling containers. +- No host-level mount — NFS access only via Docker's native `driver_opts: type: nfs` volume, never `/etc/fstab`. +- Encryption and retention are restic's job — no hand-rolled `age`/`gpg`/`find -mtime` logic. +- TDLib volumes are tarred live (best-effort) — never pause `worker`/`bot` for the backup. +- Restore is a manual, documented procedure only — never scripted/automated. + +**Required user input before Task 5 can run:** `NAS_HOST` and `NAS_EXPORT_PATH` (the Synology NFS share details) and a Kuma Push-monitor URL (`KUMA_PUSH_URL`, created manually in the existing Uptime Kuma instance, ~26h expected heartbeat interval). Tasks 1–4 need none of these and can proceed immediately; do not substitute placeholder values for them in Task 5 — stop and ask the user instead. + +--- + +### Task 1: Backup image (Dockerfile + entrypoint) + +**Files:** +- Create: `backup/Dockerfile` +- Create: `backup/entrypoint.sh` +- Create: `backup/crontab` + +**Interfaces:** +- Produces: a buildable image tagged `dragonsstash-backup:test` locally, with `/entrypoint.sh` as `ENTRYPOINT`, `/backup.sh` present at the image root (written in Task 2 — this task only needs the `COPY` line and a placeholder-free stub isn't acceptable, so create an empty‑body-but-real `backup/backup.sh` here containing just `#!/bin/bash` + `exit 0`, and Task 2 replaces its contents), `restic`, `pg_dump`/`pg_restore`, `curl`, `tar`, `bash`, `dcron` all on `PATH`. +- Consumes: nothing from earlier tasks. + +- [ ] **Step 1: Write `backup/backup.sh` stub** + +```bash +#!/bin/bash +set -euo pipefail +exit 0 +``` + +- [ ] **Step 2: Write `backup/entrypoint.sh`** + +```bash +#!/bin/bash +set -euo pipefail + +if ! restic snapshots >/dev/null 2>&1; then + restic init +fi + +exec crond -f -l 2 +``` + +- [ ] **Step 3: Write `backup/crontab`** + +``` +0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1 +0 4 * * 0 restic check >> /proc/1/fd/1 2>&1 +``` + +- [ ] **Step 4: Write `backup/Dockerfile`** + +```dockerfile +FROM alpine:3.20 + +RUN apk add --no-cache restic postgresql16-client curl tzdata dcron tar bash + +COPY backup/backup.sh /backup.sh +COPY backup/entrypoint.sh /entrypoint.sh +COPY backup/crontab /etc/crontabs/root + +RUN chmod +x /backup.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +``` + +- [ ] **Step 5: Build the image** + +Run: `cd /home/sam/Documents/DragonsStash && docker build -t dragonsstash-backup:test -f backup/Dockerfile .` +Expected: build completes with `Successfully tagged dragonsstash-backup:test` (or Buildkit's equivalent final `naming to docker.io/library/dragonsstash-backup:test done`), no errors. + +- [ ] **Step 6: Verify the tools are present** + +Run: `docker run --rm dragonsstash-backup:test restic version && docker run --rm dragonsstash-backup:test pg_dump --version` +Expected: `restic 0.16.x ...` and `pg_dump (PostgreSQL) 16.x` printed, both commands exit 0. + +- [ ] **Step 7: Verify the entrypoint initializes an empty repo and starts cron** + +```bash +mkdir -p /tmp/backup-repo-smoke +docker run -d --name backup-smoke \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=smoketest \ + -v /tmp/backup-repo-smoke:/backups \ + dragonsstash-backup:test +sleep 2 +docker logs backup-smoke +docker exec backup-smoke restic snapshots +docker rm -f backup-smoke +rm -rf /tmp/backup-repo-smoke +``` + +Expected: `docker logs` shows no errors (restic init ran silently); `restic snapshots` prints an empty snapshot list (repo exists, header row only, no error). + +- [ ] **Step 8: Commit** + +```bash +git add backup/Dockerfile backup/entrypoint.sh backup/backup.sh backup/crontab +git commit -m "Add backup service image (Dockerfile, entrypoint, crontab)" +``` + +--- + +### Task 2: `backup.sh` script + +**Files:** +- Modify: `backup/backup.sh` (replace Task 1's stub with the real script) + +**Interfaces:** +- Consumes: the image built in Task 1 (`dragonsstash-backup:test`), rebuilt after this change. +- Produces: `/backup.sh`, invoked by cron in Task 1's `crontab` and manually in Task 6's verification. Reads env vars `POSTGRES_USER`, `PGPASSWORD`, `POSTGRES_DB`, `RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, `KUMA_PUSH_URL`. Assumes network hostname `dragonsstash-db:5432` for Postgres and mounts `/data/tdlib-worker`, `/data/tdlib-bot` (read-only) for TDLib state. + +- [ ] **Step 1: Replace `backup/backup.sh` with the real script** + +```bash +#!/bin/bash +set -euo pipefail + +report_failure() { + curl -fsS "$KUMA_PUSH_URL" --get \ + --data-urlencode "status=down" \ + --data-urlencode "msg=$BASH_COMMAND failed" || true +} +trap report_failure ERR + +DUMP_FILE=/tmp/dragonsstash.dump +TAR_FILE=/tmp/tdlib.tar.gz + +pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE" + +tar czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot + +restic backup "$DUMP_FILE" "$TAR_FILE" +restic forget --keep-daily 14 --prune + +rm -f "$DUMP_FILE" "$TAR_FILE" + +curl -fsS "$KUMA_PUSH_URL" --get \ + --data-urlencode "status=up" \ + --data-urlencode "msg=OK" +``` + +- [ ] **Step 2: Rebuild the image** + +Run: `docker build -t dragonsstash-backup:test -f backup/Dockerfile .` +Expected: build succeeds. + +- [ ] **Step 3: Stand up a scratch Postgres aliased as `dragonsstash-db`** + +```bash +docker network create backup-test-net 2>/dev/null || true +docker run -d --name test-pg --network backup-test-net --network-alias dragonsstash-db \ + -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e POSTGRES_DB=dragonsstash \ + postgres:16-alpine +sleep 5 +docker exec test-pg pg_isready -U dragons -d dragonsstash +``` + +Expected: `accepting connections`. + +- [ ] **Step 4: Stand up a mock Kuma push endpoint** + +```bash +mkdir -p /tmp/mock-kuma-root && touch /tmp/mock-kuma-root/push +docker run -d --name mock-kuma --network backup-test-net \ + -v /tmp/mock-kuma-root:/srv -w /srv python:3-alpine \ + python3 -m http.server 8000 +sleep 1 +``` + +- [ ] **Step 5: Prepare fake TDLib state and a local restic repo dir** + +```bash +mkdir -p /tmp/backup-test/tdlib-worker /tmp/backup-test/tdlib-bot /tmp/backup-test/repo +echo "fake-session" > /tmp/backup-test/tdlib-worker/state.bin +echo "fake-session" > /tmp/backup-test/tdlib-bot/state.bin +``` + +- [ ] **Step 6: Initialize the test restic repo and run `backup.sh` (success path)** + +```bash +docker run --rm --network backup-test-net \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \ + -v /tmp/backup-test/repo:/backups \ + --entrypoint restic dragonsstash-backup:test init + +docker run --rm --network backup-test-net \ + -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e PGPASSWORD=stash -e POSTGRES_DB=dragonsstash \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \ + -e KUMA_PUSH_URL=http://mock-kuma:8000/push \ + -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \ + -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \ + -v /tmp/backup-test/repo:/backups \ + --entrypoint /backup.sh dragonsstash-backup:test +echo "exit code: $?" +``` + +Expected: exit code `0`, restic prints a line like `snapshot xxxxxxxx saved`, no error output. + +- [ ] **Step 7: Verify the snapshot landed and contains both files** + +```bash +docker run --rm -v /tmp/backup-test/repo:/backups \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \ + --entrypoint restic dragonsstash-backup:test snapshots + +docker run --rm -v /tmp/backup-test/repo:/backups \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \ + --entrypoint restic dragonsstash-backup:test ls latest +``` + +Expected: `snapshots` shows exactly one entry; `ls latest` lists `/tmp/dragonsstash.dump` and `/tmp/tdlib.tar.gz`. + +- [ ] **Step 8: Verify the failure path reports to Kuma** + +```bash +docker run --rm --network backup-test-net \ + -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=wrongpass -e PGPASSWORD=wrongpass -e POSTGRES_DB=dragonsstash \ + -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \ + -e KUMA_PUSH_URL=http://mock-kuma:8000/push \ + -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \ + -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \ + -v /tmp/backup-test/repo:/backups \ + --entrypoint /backup.sh dragonsstash-backup:test +echo "exit code: $?" +docker logs mock-kuma | tail -5 +``` + +Expected: exit code nonzero (pg_dump auth failure trips `set -e`); `docker logs mock-kuma` shows a GET request line containing `status=down`. + +- [ ] **Step 9: Clean up test resources** + +```bash +docker rm -f test-pg mock-kuma +docker network rm backup-test-net +rm -rf /tmp/backup-test /tmp/mock-kuma-root +``` + +- [ ] **Step 10: Commit** + +```bash +git add backup/backup.sh +git commit -m "Implement backup.sh: pg_dump + tdlib tar + restic backup/forget + Kuma reporting" +``` + +--- + +### Task 3: Wire the `backup` service into the repo's `docker-compose.yml` + +**Files:** +- Modify: `docker-compose.yml` +- Modify: `.env.example` + +**Interfaces:** +- Consumes: `backup/Dockerfile` (Task 1), `backup/backup.sh` (Task 2). +- Produces: a `backup` compose service buildable via `docker compose build backup`, and a `nas_backups` named volume other tasks (5) will mirror into the production compose file. + +- [ ] **Step 1: Add the `backup` service to `docker-compose.yml`** + +Insert after the existing `bot` service (before `db`): + +```yaml + backup: + build: + context: . + dockerfile: backup/Dockerfile + pull_policy: never + environment: + - POSTGRES_USER=${POSTGRES_USER:-dragons} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash} + - PGPASSWORD=${POSTGRES_PASSWORD:-stash} + - POSTGRES_DB=${POSTGRES_DB:-dragonsstash} + - RESTIC_REPOSITORY=/backups/restic-repo + - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env} + - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env} + - TZ=${TZ:-Etc/UTC} + volumes: + - tdlib_state:/data/tdlib-worker:ro + - tdlib_bot_state:/data/tdlib-bot:ro + - nas_backups:/backups + depends_on: + db: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + memory: 256M + networks: + - backend +``` + +- [ ] **Step 2: Add the `nas_backups` volume to the `volumes:` block** + +```yaml + nas_backups: + driver_opts: + type: nfs + o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100" + device: ":${NAS_EXPORT_PATH}" +``` + +- [ ] **Step 3: Document the new env vars in `.env.example`** + +Append: + +``` +# Backup (NAS via NFS + restic) +NAS_HOST="" # Synology NAS IP or hostname reachable from this host +NAS_EXPORT_PATH="" # NFS export path, e.g. /volume1/dragonsstash-backups +RESTIC_PASSWORD="" # generate with: openssl rand -base64 32 +KUMA_PUSH_URL="" # Uptime Kuma Push monitor URL (create the monitor first) +TZ="Etc/UTC" +``` + +- [ ] **Step 4: Validate the compose file parses** + +Run (with dummy values so the `:?` guards don't fail parsing — `AUTH_SECRET` is required by the existing `app` service, not by this change, but `config` validates the whole file): +```bash +RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \ + docker compose -f docker-compose.yml config --quiet +``` +Expected: no output, exit code 0 (a syntax/interpolation error would print to stderr and exit nonzero). + +- [ ] **Step 5: Validate the service actually builds through Compose** + +Run: +```bash +RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \ + docker compose -f docker-compose.yml build backup +``` +Expected: build succeeds (reuses Task 1's image layers). + +- [ ] **Step 6: Commit** + +```bash +git add docker-compose.yml .env.example +git commit -m "Add backup service and nas_backups volume to docker-compose.yml" +``` + +--- + +### Task 4: CI — build and push the backup image + +**Files:** +- Modify: `.drone.yml` + +**Interfaces:** +- Consumes: `backup/Dockerfile` (Task 1). +- Produces: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (and `:`), pushed on every push to `main`. Task 5's production compose file references this image tag. + +- [ ] **Step 1: Add a `build-backup` step, mirroring `build-worker`/`build-bot`** + +Insert after the existing `build-bot` step in `.drone.yml`: + +```yaml + - name: build-backup + image: plugins/docker + depends_on: [clone] + settings: + repo: git.samagsteribbe.nl/admin/dragonsstash-backup + registry: git.samagsteribbe.nl + dockerfile: backup/Dockerfile + tags: + - latest + - "${DRONE_COMMIT_SHA:0:8}" + username: + from_secret: gitea_username + password: + from_secret: gitea_password +``` + +- [ ] **Step 2: Add `build-backup` to the `deploy` step's `depends_on`** + +Change: +```yaml + - name: deploy + image: alpine + depends_on: [build-app, build-worker, build-bot] +``` +to: +```yaml + - name: deploy + image: alpine + depends_on: [build-app, build-worker, build-bot, build-backup] +``` + +- [ ] **Step 3: Validate YAML syntax** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.drone.yml')); print('OK')"` +Expected: `OK` printed, no exception. + +- [ ] **Step 4: Commit** + +```bash +git add .drone.yml +git commit -m "Add build-backup CI step, include it in deploy dependencies" +``` + +--- + +### Task 5: Wire production (`/opt/stacks/DragonsStash`) — requires NAS + Kuma details from the user + +**Do not start this task until the user has supplied `NAS_HOST` and `NAS_EXPORT_PATH` for the Synology NFS share, and has created an Uptime Kuma Push monitor (name it `dragonsstash-backup`, ~26h expected heartbeat interval) and shared its push URL. If any of these are missing, stop and ask — do not substitute placeholder values here, since this file drives the real deployment.** + +**Files:** +- Modify: `/opt/stacks/DragonsStash/docker-compose.yml` +- Modify: `/opt/stacks/DragonsStash/.env` + +**Interfaces:** +- Consumes: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (published by Task 4's CI step once merged/pushed), the real `NAS_HOST`/`NAS_EXPORT_PATH`/`KUMA_PUSH_URL` values gathered above. +- Produces: a running `dragonsstash-backup` container on the production host, verified in Task 6. + +- [ ] **Step 1: Generate `RESTIC_PASSWORD` and add all four new vars to `/opt/stacks/DragonsStash/.env`** + +```bash +cd /opt/stacks/DragonsStash +printf '\n# Backup (NAS via NFS + restic)\nNAS_HOST=""\nNAS_EXPORT_PATH=""\nRESTIC_PASSWORD="%s"\nKUMA_PUSH_URL=""\nTZ="Etc/UTC"\n' "$(openssl rand -base64 32)" >> .env +``` + +Replace the two `` placeholders with the real NAS details and Kuma push URL before saving — this step cannot be completed with the literal placeholder text left in place. + +- [ ] **Step 2: Add the `backup` service to `/opt/stacks/DragonsStash/docker-compose.yml`** + +Insert after the existing `bot` service (before `db`): + +```yaml + backup: + image: git.samagsteribbe.nl/admin/dragonsstash-backup:latest + container_name: dragonsstash-backup + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER:-dragons} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash} + - PGPASSWORD=${POSTGRES_PASSWORD:-stash} + - POSTGRES_DB=${POSTGRES_DB:-dragonsstash} + - RESTIC_REPOSITORY=/backups/restic-repo + - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env} + - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env} + - TZ=${TZ:-Etc/UTC} + volumes: + - tdlib_state:/data/tdlib-worker:ro + - tdlib_bot_state:/data/tdlib-bot:ro + - nas_backups:/backups + depends_on: + db: + condition: service_healthy + deploy: + resources: + limits: + memory: 256M + networks: + - internal +``` + +- [ ] **Step 3: Add the `nas_backups` volume** + +```yaml + nas_backups: + driver_opts: + type: nfs + o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100" + device: ":${NAS_EXPORT_PATH}" +``` + +- [ ] **Step 4: Validate the production compose file parses with the real `.env`** + +Run: `cd /opt/stacks/DragonsStash && docker compose config --quiet` +Expected: no output, exit code 0. + +- [ ] **Step 5: Commit is not applicable here** — `/opt/stacks/DragonsStash` is a deployed copy, not the git repo (confirm with `git -C /opt/stacks/DragonsStash status` — expect "not a git repository"). Skip committing; Task 6 deploys these file changes directly. + +--- + +### Task 6: Deploy and verify + +**Files:** none (operational task) + +**Interfaces:** +- Consumes: everything from Tasks 1–5. +- Produces: a running, verified backup on the real NAS, and one completed restore drill. + +- [ ] **Step 1: Confirm with the user before pushing/deploying** + +Pushing to `main` triggers Drone CI to build all four images and deploy to the production host via SSH (`docker compose pull && docker compose up -d`). Confirm the user wants this to happen now before proceeding — this affects the live stack. + +- [ ] **Step 2: Push to `main`** + +```bash +cd /home/sam/Documents/DragonsStash +git push origin main +``` + +Expected: Drone pipeline runs `build-app`, `build-worker`, `build-bot`, `build-backup`, then `deploy`, all green. Check via the Drone UI or `drone build info admin/DragonsStash ` if the `drone` CLI is configured. + +- [ ] **Step 3: Confirm the container is up on the production host** + +```bash +ssh sam@192.168.68.68 "docker ps --filter name=dragonsstash-backup --format '{{.Names}}\t{{.Status}}'" +``` + +Expected: `dragonsstash-backup Up ...`. + +- [ ] **Step 4: Trigger one manual backup run and confirm a snapshot lands on the NAS** + +```bash +ssh sam@192.168.68.68 "docker exec dragonsstash-backup /backup.sh" +ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic snapshots" +``` + +Expected: `backup.sh` exits 0; `restic snapshots` lists exactly one entry. + +- [ ] **Step 5: Confirm the Uptime Kuma push monitor shows green** + +Open the Uptime Kuma dashboard and check the `dragonsstash-backup` monitor's status is up with a recent heartbeat. + +- [ ] **Step 6: Restore drill — prove the backup is actually restorable** + +```bash +ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic restore latest --target /tmp/restore-drill" +ssh sam@192.168.68.68 "docker exec dragonsstash-backup ls -la /tmp/restore-drill/tmp" +``` + +Expected: `/tmp/restore-drill/tmp/dragonsstash.dump` and `/tmp/restore-drill/tmp/tdlib.tar.gz` both present with nonzero size. Then, on a scratch Postgres (not the live `dragonsstash-db`), confirm the dump restores cleanly: + +```bash +ssh sam@192.168.68.68 "docker run -d --rm --name restore-drill-pg --network dragonsstash_internal \ + -e POSTGRES_USER=drill -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill postgres:16-alpine" +ssh sam@192.168.68.68 "docker cp dragonsstash-backup:/tmp/restore-drill/tmp/dragonsstash.dump /tmp/dragonsstash.dump" +ssh sam@192.168.68.68 "docker cp /tmp/dragonsstash.dump restore-drill-pg:/tmp/dragonsstash.dump" +ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg pg_restore -U drill -d drill --clean --if-exists /tmp/dragonsstash.dump" +ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg psql -U drill -d drill -c '\\dt' | head -20" +ssh sam@192.168.68.68 "docker rm -f restore-drill-pg" +``` + +Expected: `pg_restore` completes without fatal errors; `\dt` lists the app's tables (e.g. `Package`, `User`, `TelegramLink`). + +- [ ] **Step 7: Clean up drill artifacts** + +```bash +ssh sam@192.168.68.68 "docker exec dragonsstash-backup rm -rf /tmp/restore-drill" +ssh sam@192.168.68.68 "rm -f /tmp/dragonsstash.dump" +``` diff --git a/docs/superpowers/plans/2026-07-23-provenance-backfill.md b/docs/superpowers/plans/2026-07-23-provenance-backfill.md new file mode 100644 index 0000000..a05cf60 --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-provenance-backfill.md @@ -0,0 +1,1091 @@ +# Provenance Backfill Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** During normal source-channel re-indexing, attribute the true origin (source channel/message/topic/caption/creator) to placeholder-provenance packages, confirmed by a CRC32 fingerprint read from a ranged central-directory download — without downloading whole archives. + +**Architecture:** A new step in `processOneArchiveSet` (between the same-channel repost check and the full download) looks up a placeholder candidate by `fileName`+`fileSize`, confirms it by comparing the archive's internal CRC32 multiset (candidate side already in `PackageFile.crc32`; scanned side read via a ranged tail download of the ZIP/7z central directory), and on a match transactionally overwrites the placeholder fields and skips the download. RAR and listing-less candidates fall back to name+size only. Pure logic (fingerprint compare, central-directory parse) is unit-tested; TDLib range behavior is verified with a spike before it's built on. + +**Tech Stack:** TypeScript (ESM, NodeNext), `tdl`/TDLib, Prisma v7 (`@prisma/adapter-pg`), Vitest (new, worker only). + +## Global Constraints + +- Worker code is ESM: relative imports MUST use the `.js` extension (e.g. `./archive/central-directory.js`). +- ESLint does NOT cover `worker/` — do not rely on lint; rely on `tsc` and Vitest. +- All new `client.invoke(...)` calls go through `invokeWithTimeout`/`withFloodWait` (see `worker/src/tdlib/download.ts`, `worker/src/util/retry.ts`) — never bare invokes (per the tdlib-telegram skill). +- Never overwrite a package that already has real provenance. A candidate is defined ONLY as `sourceChannelId === destChannelId`. +- `crc32` values in `FileEntry`/`PackageFile` are lowercase hex, zero-padded to 8 chars, or `null` (see `worker/src/archive/zip-reader.ts:57`). +- Prisma PKs are `cuid()` strings; message IDs and file sizes are `BigInt`. +- Commit after every task. Branch is `nas-backup` (already a feature branch) — commit there. + +## File Structure + +- Create `worker/vitest.config.ts` — Vitest config (node env). +- Create `worker/src/archive/fingerprint.ts` — pure CRC32 multiset fingerprint + compare. +- Create `worker/src/archive/fingerprint.test.ts` — unit tests. +- Create `worker/src/archive/central-directory.ts` — pure ZIP EOCD/central-directory parser over a tail `Buffer`; pure 7z end-header locator. +- Create `worker/src/archive/central-directory.test.ts` — unit tests using generated fixtures. +- Create `worker/src/tdlib/range-download.ts` — `downloadFileRange()` (ranged TDLib download → `Buffer`). +- Create `worker/src/provenance-backfill.ts` — orchestrator `tryProvenanceBackfill()` (candidate lookup → ranged listing → fingerprint confirm → transactional update). +- Modify `worker/src/db/queries.ts` — add `findPlaceholderCandidate`, `getPackageFileCrcs`, `backfillProvenance`. +- Modify `worker/src/worker.ts` — call `tryProvenanceBackfill` in `processOneArchiveSet`; add `zipsBackfilled` counter plumbing. +- Modify `prisma/schema.prisma` + new migration — `IngestionRun.zipsBackfilled Int @default(0)`. +- Modify `worker/package.json` — add `vitest` devDependency + `test` script. + +--- + +### Task 1: Add the Vitest test harness to the worker + +**Files:** +- Modify: `worker/package.json` +- Create: `worker/vitest.config.ts` +- Create: `worker/src/archive/fingerprint.test.ts` (temporary smoke test, replaced in Task 2) + +**Interfaces:** +- Produces: `npm test` (in `worker/`) runs Vitest over `src/**/*.test.ts`. + +- [ ] **Step 1: Add vitest devDependency and test script** + +In `worker/package.json`, add to `devDependencies`: `"vitest": "^3.2.4"`, and to `scripts`: `"test": "vitest run"`, `"test:watch": "vitest"`. + +- [ ] **Step 2: Install** + +Run: `cd worker && npm install` +Expected: vitest added, no errors. (If the environment blocks writes to `node_modules`, run in the container/owner context.) + +- [ ] **Step 3: Create `worker/vitest.config.ts`** + +```ts +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + }, +}); +``` + +- [ ] **Step 4: Create a smoke test `worker/src/archive/fingerprint.test.ts`** + +```ts +import { describe, it, expect } from "vitest"; + +describe("harness", () => { + it("runs", () => { + expect(1 + 1).toBe(2); + }); +}); +``` + +- [ ] **Step 5: Run the harness** + +Run: `cd worker && npm test` +Expected: 1 passing test. + +- [ ] **Step 6: Commit** + +```bash +git add worker/package.json worker/package-lock.json worker/vitest.config.ts worker/src/archive/fingerprint.test.ts +git commit -m "test(worker): add vitest harness" +``` + +--- + +### Task 2: CRC32 fingerprint compare (pure) + +**Files:** +- Create: `worker/src/archive/fingerprint.ts` +- Test: `worker/src/archive/fingerprint.test.ts` (replace smoke test) + +**Interfaces:** +- Consumes: `FileEntry` from `./zip-reader.js` (`{ crc32: string | null, ... }`). +- Produces: + - `crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean }` — sorted lowercase crc list; `complete=false` if any entry has `crc32 === null` OR `entries` is empty. + - `fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean` — true iff both fingerprints are `complete`, equal length, and equal sorted crc arrays. + +- [ ] **Step 1: Write the failing test** (`worker/src/archive/fingerprint.test.ts`) + +```ts +import { describe, it, expect } from "vitest"; +import { crcFingerprint, fingerprintsMatch } from "./fingerprint.js"; +import type { FileEntry } from "./zip-reader.js"; + +const fe = (crc: string | null): FileEntry => ({ + path: "a", fileName: "a", extension: null, + compressedSize: 0n, uncompressedSize: 0n, crc32: crc, +}); + +describe("crcFingerprint", () => { + it("sorts crcs and marks complete", () => { + expect(crcFingerprint([fe("00ff"), fe("00aa")])).toEqual({ crcs: ["00aa", "00ff"], complete: true }); + }); + it("is incomplete when any crc is null", () => { + expect(crcFingerprint([fe("00aa"), fe(null)]).complete).toBe(false); + }); + it("is incomplete when empty", () => { + expect(crcFingerprint([]).complete).toBe(false); + }); +}); + +describe("fingerprintsMatch", () => { + it("matches identical crc multisets regardless of order", () => { + expect(fingerprintsMatch([fe("01"), fe("02")], [fe("02"), fe("01")])).toBe(true); + }); + it("rejects different counts", () => { + expect(fingerprintsMatch([fe("01")], [fe("01"), fe("02")])).toBe(false); + }); + it("rejects disjoint sets", () => { + expect(fingerprintsMatch([fe("01")], [fe("09")])).toBe(false); + }); + it("rejects when either side is incomplete", () => { + expect(fingerprintsMatch([fe("01"), fe(null)], [fe("01"), fe("02")])).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd worker && npm test -- fingerprint` +Expected: FAIL (`crcFingerprint` not found). + +- [ ] **Step 3: Implement `worker/src/archive/fingerprint.ts`** + +```ts +import type { FileEntry } from "./zip-reader.js"; + +export function crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean } { + if (entries.length === 0) return { crcs: [], complete: false }; + const crcs: string[] = []; + let complete = true; + for (const e of entries) { + if (e.crc32 == null) { complete = false; continue; } + crcs.push(e.crc32.toLowerCase()); + } + crcs.sort(); + return { crcs, complete }; +} + +export function fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean { + const fa = crcFingerprint(a); + const fb = crcFingerprint(b); + if (!fa.complete || !fb.complete) return false; + if (fa.crcs.length !== fb.crcs.length) return false; + return fa.crcs.every((c, i) => c === fb.crcs[i]); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd worker && npm test -- fingerprint` +Expected: PASS (all cases). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/fingerprint.ts worker/src/archive/fingerprint.test.ts +git commit -m "feat(worker): CRC32 archive fingerprint compare" +``` + +--- + +### Task 3: Parse a ZIP central directory from a tail buffer (pure) + +**Files:** +- Create: `worker/src/archive/central-directory.ts` +- Test: `worker/src/archive/central-directory.test.ts` + +**Interfaces:** +- Consumes: `FileEntry` from `./zip-reader.js`. +- Produces: + - `parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[]` — `tail` is the last bytes of the archive, `tailStart` is the absolute offset in the whole file where `tail` begins. Locates the End-Of-Central-Directory record (signature `0x06054b50`) by scanning backward, then walks central-directory file headers (signature `0x02014b50`). Throws `RangeError("EOCD not found in tail")` if the EOCD (or a referenced central-directory byte) falls before `tailStart` (caller must fetch a larger tail). + - `MIN_ZIP_TAIL_BYTES = 65_557` (max EOCD comment 65_535 + 22-byte EOCD). + +**Reference (ZIP format, little-endian):** +- EOCD (22 bytes + comment): sig `50 4b 05 06`; offset 12 = CD size (u32); offset 16 = CD start offset in file (u32); offset 20 = comment length (u16). ZIP64: if CD offset == `0xFFFFFFFF`, locate ZIP64 EOCD locator (sig `0x07064b50`) preceding EOCD and read 8-byte fields. +- Central directory header (46 bytes + names): sig `50 4b 01 02`; off 16 = crc32 (u32); off 20 = compressed size (u32); off 24 = uncompressed size (u32); off 28 = name len (u16); off 30 = extra len (u16); off 32 = comment len (u16); name follows at off 46. ZIP64 extra (id `0x0001`) overrides sizes when they are `0xFFFFFFFF`. + +- [ ] **Step 1: Write the failing test** (`worker/src/archive/central-directory.test.ts`) + +Generate a real ZIP with Node's `zlib`-free approach via a fixture builder helper in the test (store-only entries so we control CRCs), then assert parsing the whole buffer as its own tail returns the entries. + +```ts +import { describe, it, expect } from "vitest"; +import { parseZipCentralDirectoryFromTail } from "./central-directory.js"; +import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32 + +// Build a minimal STORE (no compression) ZIP in-memory with the given files. +function buildStoreZip(files: { name: string; data: Buffer }[]): Buffer { + const chunks: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const f of files) { + const crc = crc32(f.data) >>> 0; + const nameBuf = Buffer.from(f.name, "utf8"); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); // version needed + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // method = store + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(f.data.length, 18); // compressed + local.writeUInt32LE(f.data.length, 22); // uncompressed + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); // extra len + const localHeader = Buffer.concat([local, nameBuf, f.data]); + chunks.push(localHeader); + + const cd = Buffer.alloc(46); + cd.writeUInt32LE(0x02014b50, 0); + cd.writeUInt16LE(20, 4); cd.writeUInt16LE(20, 6); + cd.writeUInt16LE(0, 8); cd.writeUInt16LE(0, 10); + cd.writeUInt32LE(crc, 16); + cd.writeUInt32LE(f.data.length, 20); + cd.writeUInt32LE(f.data.length, 24); + cd.writeUInt16LE(nameBuf.length, 28); + cd.writeUInt32LE(offset, 42); // local header offset + central.push(Buffer.concat([cd, nameBuf])); + offset += localHeader.length; + } + const cdBuf = Buffer.concat(central); + const cdOffset = offset; + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(files.length, 8); + eocd.writeUInt16LE(files.length, 10); + eocd.writeUInt32LE(cdBuf.length, 12); + eocd.writeUInt32LE(cdOffset, 16); + return Buffer.concat([...chunks, cdBuf, eocd]); +} + +describe("parseZipCentralDirectoryFromTail", () => { + it("lists entries with correct names, sizes, and crc32", () => { + const zip = buildStoreZip([ + { name: "models/dragon.stl", data: Buffer.from("DRAGON") }, + { name: "readme.txt", data: Buffer.from("hello world") }, + ]); + const entries = parseZipCentralDirectoryFromTail(zip, 0); + expect(entries.map((e) => e.fileName).sort()).toEqual(["dragon.stl", "readme.txt"]); + const dragon = entries.find((e) => e.fileName === "dragon.stl")!; + expect(dragon.path).toBe("models/dragon.stl"); + expect(dragon.uncompressedSize).toBe(6n); + expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/); + }); + + it("throws when the central directory begins before the tail window", () => { + const zip = buildStoreZip([{ name: "a.txt", data: Buffer.alloc(100) }]); + // Provide only the last 30 bytes but claim they start at offset (len-30): + const tail = zip.subarray(zip.length - 30); + expect(() => parseZipCentralDirectoryFromTail(tail, zip.length - 30)).toThrow(RangeError); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd worker && npm test -- central-directory` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `worker/src/archive/central-directory.ts`** + +```ts +import path from "path"; +import type { FileEntry } from "./zip-reader.js"; + +export const MIN_ZIP_TAIL_BYTES = 65_557; + +const EOCD_SIG = 0x06054b50; +const CD_SIG = 0x02014b50; + +function extOf(name: string): string | null { + const e = path.extname(name).replace(/^\./, "").toLowerCase(); + return e === "" ? null : e; +} + +/** Parse a ZIP central directory from the tail of an archive. */ +export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] { + // 1. Find EOCD by scanning backward for its signature. + let eocd = -1; + for (let i = tail.length - 22; i >= 0; i--) { + if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; } + } + if (eocd < 0) throw new RangeError("EOCD not found in tail"); + + let cdSize = tail.readUInt32LE(eocd + 12); + let cdOffset = tail.readUInt32LE(eocd + 16); + + // ZIP64: sizes/offsets of 0xFFFFFFFF mean "see ZIP64 EOCD". + if (cdOffset === 0xffffffff || cdSize === 0xffffffff) { + const locSig = 0x07064b50; + let loc = -1; + for (let i = eocd - 20; i >= 0; i--) { + if (tail.readUInt32LE(i) === locSig) { loc = i; break; } + } + if (loc < 0) throw new RangeError("ZIP64 EOCD locator not in tail"); + const z64Abs = Number(tail.readBigUInt64LE(loc + 8)); // absolute offset of ZIP64 EOCD + const z64 = z64Abs - tailStart; + if (z64 < 0) throw new RangeError("ZIP64 EOCD before tail window"); + cdSize = Number(tail.readBigUInt64LE(z64 + 40)); + cdOffset = Number(tail.readBigUInt64LE(z64 + 48)); + } + + // 2. Map the absolute central-directory offset into the tail buffer. + const cdLocal = cdOffset - tailStart; + if (cdLocal < 0 || cdLocal + cdSize > tail.length) { + throw new RangeError("Central directory begins before tail window"); + } + + // 3. Walk central-directory headers. + const entries: FileEntry[] = []; + let p = cdLocal; + const end = cdLocal + cdSize; + while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) { + let crc = tail.readUInt32LE(p + 16) >>> 0; + let comp = BigInt(tail.readUInt32LE(p + 20)); + let uncomp = BigInt(tail.readUInt32LE(p + 24)); + const nameLen = tail.readUInt16LE(p + 28); + const extraLen = tail.readUInt16LE(p + 30); + const commentLen = tail.readUInt16LE(p + 32); + const name = tail.toString("utf8", p + 46, p + 46 + nameLen); + + // ZIP64 extra field overrides 0xFFFFFFFF sizes. + if (comp === 0xffffffffn || uncomp === 0xffffffffn) { + let ep = p + 46 + nameLen; + const extraEnd = ep + extraLen; + while (ep + 4 <= extraEnd) { + const id = tail.readUInt16LE(ep); + const sz = tail.readUInt16LE(ep + 2); + if (id === 0x0001) { + let fp = ep + 4; + if (uncomp === 0xffffffffn) { uncomp = tail.readBigUInt64LE(fp); fp += 8; } + if (comp === 0xffffffffn) { comp = tail.readBigUInt64LE(fp); fp += 8; } + } + ep += 4 + sz; + } + } + + const isDir = name.endsWith("/"); + if (!isDir) { + entries.push({ + path: name, + fileName: path.basename(name), + extension: extOf(name), + compressedSize: comp, + uncompressedSize: uncomp, + crc32: crc !== 0 ? crc.toString(16).padStart(8, "0") : null, + }); + } + p += 46 + nameLen + extraLen + commentLen; + } + return entries; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd worker && npm test -- central-directory` +Expected: PASS (both cases). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/central-directory.ts worker/src/archive/central-directory.test.ts +git commit -m "feat(worker): parse ZIP central directory from a tail buffer" +``` + +--- + +### Task 4: SPIKE — verify TDLib ranged download, then implement `downloadFileRange` + +**Files:** +- Create: `worker/src/tdlib/range-download.ts` + +**Interfaces:** +- Consumes: `Client` from `tdl`; `invokeWithTimeout` from `./download.js` (add export if needed); `withFloodWait` from `../util/retry.js`. +- Produces: `downloadFileRange(client, fileId: string, offset: number, limit: number, expectedSize: bigint): Promise` — returns exactly the requested byte range from the remote file. + +**Why a spike:** TDLib's `downloadFile` with `offset`/`limit`/`synchronous:true` downloads a region into its file cache; reading the exact range back from the on-disk file (which may be a sparse "prefix" file) must be verified empirically before other tasks depend on it. Do NOT skip the spike. + +- [ ] **Step 1: Spike — confirm behavior against a real file** + +Write a throwaway script `worker/src/tdlib/_spike-range.ts` that: creates a client (reuse `createTdlibClient`), picks a known large document message in the destination channel, calls `downloadFile` with `{ offset: , limit: 65557, synchronous: true, priority: 1 }`, inspects the returned `file.local` (`path`, `downloaded_prefix_size`, `download_offset`), then reads bytes `[offset, offset+limit)` from `file.local.path` and prints their length + last 4 bytes as hex. +Run it and record: (a) does `synchronous:true` block until the region is present? (b) is the region at absolute file offset on disk, or at offset 0 of a prefix file? Note the answer in a comment in `range-download.ts`. +Delete `_spike-range.ts` after. + +- [ ] **Step 2: Implement `worker/src/tdlib/range-download.ts` using the verified behavior** + +```ts +import { open } from "fs/promises"; +import type { Client } from "tdl"; +import { childLogger } from "../util/logger.js"; +import { withFloodWait } from "../util/retry.js"; + +const log = childLogger("range-download"); +const RANGE_TIMEOUT_MS = 120_000; + +// NOTE (from Task 4 spike): TDLib writes the requested region into +// file.local.path at its ABSOLUTE file offset; file.local.downloaded_prefix_size +// counts contiguous bytes from download_offset. We request a 1KB-aligned offset +// so downloaded_prefix_size covers our whole [offset, offset+limit) window. +export async function downloadFileRange( + client: Client, + fileId: string, + offset: number, + limit: number, + expectedSize: bigint, +): Promise { + const numericId = parseInt(fileId, 10); + const alignedOffset = Math.max(0, offset - (offset % 1024)); + const alignedLimit = limit + (offset - alignedOffset); + + const file = await withFloodWait( + () => + new Promise<{ local: { path: string; download_offset: number; downloaded_prefix_size: number } }>( + (resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Range download timed out for ${fileId}`)), RANGE_TIMEOUT_MS); + client + .invoke({ + _: "downloadFile", + file_id: numericId, + priority: 1, + offset: alignedOffset, + limit: alignedLimit, + synchronous: true, + } as never) + .then((f) => { clearTimeout(timer); resolve(f as never); }) + .catch((e) => { clearTimeout(timer); reject(e); }); + }, + ), + `downloadFileRange:${fileId}`, + ); + + const start = offset; + const fh = await open(file.local.path, "r"); + try { + const buf = Buffer.alloc(limit); + const { bytesRead } = await fh.read(buf, 0, limit, start); + log.debug({ fileId, offset, limit, bytesRead }, "range read"); + return bytesRead < limit ? buf.subarray(0, bytesRead) : buf; + } finally { + await fh.close(); + } +} +``` + +- [ ] **Step 3: Manual verification** + +Run the worker against a real destination archive via Task 8's integration path (deferred), OR re-run a trimmed spike confirming `downloadFileRange` returns a buffer whose last 22+ bytes contain the EOCD signature `50 4b 05 06` for a known ZIP. +Expected: buffer length == limit (or less near EOF); EOCD signature present for ZIPs. + +- [ ] **Step 4: Commit** + +```bash +git add worker/src/tdlib/range-download.ts +git commit -m "feat(worker): ranged TDLib file download (downloadFileRange)" +``` + +--- + +### Task 5: DB helpers — candidate lookup, candidate CRCs, transactional backfill + +**Files:** +- Modify: `worker/src/db/queries.ts` + +**Interfaces:** +- Produces: + - `findPlaceholderCandidate(destChannelId: string, fileName: string, fileSize: bigint): Promise<{ id: string; archiveType: string; fileCount: number } | null>` — a **placeholder** package (`sourceChannelId === destChannelId` OR `sourceMessageId === 0` — see spec §1) AND `fileName` AND `fileSize` match, with a real destination (`destMessageId != null`). + - `getPackageFileCrcs(packageId: string): Promise<(string | null)[]>` — `PackageFile.crc32` values for the candidate. + - `backfillProvenance(input: BackfillProvenanceInput): Promise` — transactionally overwrite placeholder fields; returns `false` (no-op) if the row is no longer a placeholder. Type: + ```ts + export interface BackfillProvenanceInput { + packageId: string; + destChannelId: string; // to re-check placeholder status in-txn + sourceChannelId: string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + sourceCaption: string | null; + remoteUniqueId: string | null; + creator: string | null; // always set (re-derived by caller) + entries?: FileEntry[]; // set only if candidate had fileCount === 0 + previewData?: Buffer | null; // set only if provided and candidate lacks one + previewMsgId?: bigint | null; + } + ``` + +- [ ] **Step 1: Implement the three helpers in `worker/src/db/queries.ts`** + +```ts +export async function findPlaceholderCandidate( + destChannelId: string, + fileName: string, + fileSize: bigint, +): Promise<{ id: string; archiveType: string; fileCount: number } | null> { + return db.package.findFirst({ + where: { + fileName, + fileSize, + destMessageId: { not: null }, + // Placeholder provenance (spec §1): manual-upload (source == destination) + // OR rebuild record (sourceMessageId == 0 "unknown" sentinel). + OR: [ + { sourceChannelId: destChannelId }, + { sourceMessageId: 0n }, + ], + }, + select: { id: true, archiveType: true, fileCount: true }, + orderBy: { indexedAt: "asc" }, + }); +} + +export async function getPackageFileCrcs(packageId: string): Promise<(string | null)[]> { + const rows = await db.packageFile.findMany({ + where: { packageId }, + select: { crc32: true }, + }); + return rows.map((r) => r.crc32); +} + +export async function backfillProvenance(input: BackfillProvenanceInput): Promise { + return db.$transaction(async (tx) => { + const current = await tx.package.findUnique({ + where: { id: input.packageId }, + select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true }, + }); + // Re-check placeholder status inside the txn (another worker may have won). + // Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0). + const stillPlaceholder = + !!current && + (current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n); + if (!stillPlaceholder) return false; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data: any = { + sourceChannelId: input.sourceChannelId, + sourceMessageId: input.sourceMessageId, + sourceTopicId: input.sourceTopicId, + sourceCaption: input.sourceCaption, + remoteUniqueId: input.remoteUniqueId, + creator: input.creator, + }; + if (input.entries && current.fileCount === 0) { + await tx.packageFile.deleteMany({ where: { packageId: input.packageId } }); + await tx.packageFile.createMany({ + data: input.entries.map((e) => ({ + packageId: input.packageId, + path: e.path, fileName: e.fileName, extension: e.extension, + compressedSize: e.compressedSize, uncompressedSize: e.uncompressedSize, crc32: e.crc32, + })), + }); + data.fileCount = input.entries.length; + } + if (input.previewData && !current.previewData) { + data.previewData = input.previewData; + data.previewMsgId = input.previewMsgId ?? null; + } + await tx.package.update({ where: { id: input.packageId }, data }); + return true; + }); +} +``` + +Add `import type { FileEntry } from "../archive/zip-reader.js";` at the top if not present, and export `BackfillProvenanceInput`. + +- [ ] **Step 2: Typecheck** + +Run: `cd worker && npx tsc --noEmit` +Expected: no NEW errors in `db/queries.ts` (pre-existing stale-client errors, if the client isn't regenerated in this env, are unrelated — regenerate with `npm run db:generate` in the owner context first). + +- [ ] **Step 3: Commit** + +```bash +git add worker/src/db/queries.ts +git commit -m "feat(worker): DB helpers for provenance backfill" +``` + +--- + +### Task 6: Ranged listing orchestrator + backfill decision (`provenance-backfill.ts`) + +**Files:** +- Create: `worker/src/provenance-backfill.ts` + +**Interfaces:** +- Consumes: `findPlaceholderCandidate`, `getPackageFileCrcs`, `backfillProvenance` (Task 5); `downloadFileRange` (Task 4); `parseZipCentralDirectoryFromTail`, `MIN_ZIP_TAIL_BYTES` (Task 3); `fingerprintsMatch`, `crcFingerprint` (Task 2); `FileEntry` (`zip-reader.js`). +- Produces: `tryProvenanceBackfill(args: BackfillArgs): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }>`. + ```ts + export interface BackfillArgs { + client: import("tdl").Client; + destChannelId: string; // DB id of the destination channel + scannedSourceChannelId: string; // DB id of the channel being scanned + fileName: string; + fileSize: bigint; // total across parts + archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT" | string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + sourceCaption: string | null; + remoteUniqueId: string | null; + creator: string | null; // caller-derived (topic > filename > channel) + scannedFileId: string; // TDLib file id of the scanned archive (last part for multipart ZIP) + previewData?: Buffer | null; + previewMsgId?: bigint | null; + } + ``` + +**Behavior:** +1. `findPlaceholderCandidate(destChannelId, fileName, fileSize)`. If none → return `{ backfilled: false }`. +2. If `archiveType === "ZIP"` or `"SEVEN_Z"`: read the scanned archive's listing via `readScannedListing()` (below). Compare to the candidate's CRCs via `fingerprintsMatch`. On match → `backfillProvenance(..., entries: )` → confidence `"fingerprint"`. +3. Else (RAR, or ranged read failed, or candidate has no stored CRCs): confirm by **name+size only** (already true from step 1) → `backfillProvenance(...)` → confidence `"name-size"`. (Do NOT pass entries unless a listing was actually read.) +4. If `backfillProvenance` returns `false` (lost the race) → `{ backfilled: false }`. + +`readScannedListing(client, scannedFileId, fileSize, archiveType)`: for ZIP, `downloadFileRange(client, scannedFileId, max(0, fileSize - MIN_ZIP_TAIL_BYTES), MIN_ZIP_TAIL_BYTES, fileSize)` then `parseZipCentralDirectoryFromTail(tail, tailStart)`; on `RangeError`, retry once with a 4× larger tail; on any error return `null`. For 7z, return `null` in v1 (falls back to name+size) — a follow-up can add 7z end-header parsing. Wrap in try/catch; return `null` on failure. + +- [ ] **Step 1: Write the module** (full code) + +```ts +import { childLogger } from "./util/logger.js"; +import { downloadFileRange } from "./tdlib/range-download.js"; +import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js"; +import { fingerprintsMatch } from "./archive/fingerprint.js"; +import { + findPlaceholderCandidate, + getPackageFileCrcs, + backfillProvenance, +} from "./db/queries.js"; +import type { FileEntry } from "./archive/zip-reader.js"; +import type { Client } from "tdl"; + +const log = childLogger("provenance-backfill"); + +export interface BackfillArgs { + client: Client; + destChannelId: string; + scannedSourceChannelId: string; + fileName: string; + fileSize: bigint; + archiveType: string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + sourceCaption: string | null; + remoteUniqueId: string | null; + creator: string | null; + scannedFileId: string; + previewData?: Buffer | null; + previewMsgId?: bigint | null; +} + +async function readScannedZipListing( + client: Client, + fileId: string, + fileSize: bigint, +): Promise { + const total = Number(fileSize); + for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) { + const start = Math.max(0, total - tailBytes); + try { + const tail = await downloadFileRange(client, fileId, start, Math.min(tailBytes, total), fileSize); + return parseZipCentralDirectoryFromTail(tail, start); + } catch (err) { + if (err instanceof RangeError) continue; // try a larger tail + log.warn({ err, fileId }, "ranged ZIP listing failed"); + return null; + } + } + return null; +} + +export async function tryProvenanceBackfill( + args: BackfillArgs, +): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }> { + const candidate = await findPlaceholderCandidate(args.destChannelId, args.fileName, args.fileSize); + if (!candidate) return { backfilled: false }; + + let entries: FileEntry[] | null = null; + let confidence: "fingerprint" | "name-size" = "name-size"; + + if (args.archiveType === "ZIP") { + entries = await readScannedZipListing(args.client, args.scannedFileId, args.fileSize); + if (entries) { + const candidateCrcs = await getPackageFileCrcs(candidate.id); + const candidateEntries: FileEntry[] = candidateCrcs.map((crc) => ({ + path: "", fileName: "", extension: null, compressedSize: 0n, uncompressedSize: 0n, crc32: crc, + })); + if (fingerprintsMatch(entries, candidateEntries)) { + confidence = "fingerprint"; + } else { + // Fingerprint mismatch: NOT the same content despite name+size. Do not backfill. + log.info({ candidateId: candidate.id, fileName: args.fileName }, "fingerprint mismatch — not backfilling"); + return { backfilled: false }; + } + } + } + + const ok = await backfillProvenance({ + packageId: candidate.id, + destChannelId: args.destChannelId, + sourceChannelId: args.scannedSourceChannelId, + sourceMessageId: args.sourceMessageId, + sourceTopicId: args.sourceTopicId, + sourceCaption: args.sourceCaption, + remoteUniqueId: args.remoteUniqueId, + creator: args.creator, + entries: candidate.fileCount === 0 && entries ? entries : undefined, + previewData: args.previewData ?? undefined, + previewMsgId: args.previewMsgId ?? undefined, + }); + + if (!ok) return { backfilled: false }; + log.info( + { candidateId: candidate.id, fileName: args.fileName, confidence, source: args.scannedSourceChannelId }, + "provenance backfilled", + ); + return { backfilled: true, confidence }; +} +``` + +- [ ] **Step 2: Typecheck** + +Run: `cd worker && npx tsc --noEmit` +Expected: no new errors in `provenance-backfill.ts`. + +- [ ] **Step 3: Commit** + +```bash +git add worker/src/provenance-backfill.ts +git commit -m "feat(worker): provenance-backfill orchestrator" +``` + +--- + +### Task 7: Add `zipsBackfilled` counter to IngestionRun + +**Files:** +- Modify: `prisma/schema.prisma` (model `IngestionRun`, ~line 562) +- Create: migration via `npx prisma migrate dev --name ingestion_run_zips_backfilled` + +**Interfaces:** +- Produces: `IngestionRun.zipsBackfilled Int @default(0)`. + +- [ ] **Step 1: Add the field** in `prisma/schema.prisma` after `zipsIngested Int @default(0)`: + +```prisma + zipsBackfilled Int @default(0) +``` + +- [ ] **Step 2: Create the migration** + +Run (owner/container context with DB access): `npx prisma migrate dev --name ingestion_run_zips_backfilled` +Expected: migration created + applied; Prisma client regenerated. + +- [ ] **Step 3: Commit** + +```bash +git add prisma/schema.prisma prisma/migrations +git commit -m "feat(db): IngestionRun.zipsBackfilled counter" +``` + +--- + +### Task 8: Wire the hook into `processOneArchiveSet` + +**Files:** +- Modify: `worker/src/worker.ts` (`processOneArchiveSet`, after the `findRepostedPackage` block ~line 1613–1660, BEFORE the download/size-guard section; and counters plumbing) + +**Interfaces:** +- Consumes: `tryProvenanceBackfill` (Task 6); existing `ctx` (has `client`, `channel`, `destChannelId`, `sourceTopicId`, `topicCreator`, `counters`, `accountLog`, `runId`), `archiveSet.parts[0]` (`{ id, fileName, remoteUniqueId, ... }`), `previewMatches`. +- Produces: a new early-return path that skips the download when a backfill occurs; `counters.zipsBackfilled` increments. + +- [ ] **Step 1: Import and counter type** + +Add near the other imports in `worker.ts`: +```ts +import { tryProvenanceBackfill } from "./provenance-backfill.js"; +``` +Find the `counters` object/type (search `zipsDuplicate`) and add `zipsBackfilled: number` to the type and initialize `zipsBackfilled: 0` where the run counters are created (near `zipsDuplicate: 0`, ~line 423 and the local `counters` type ~line 315). + +- [ ] **Step 2: Insert the backfill step** immediately AFTER the `findRepostedPackage` `if (reposted) { ... return null; }` block and BEFORE the download begins: + +```ts + // ── Cross-channel provenance backfill ── + // The same-channel checks above missed. Before downloading, see if this + // archive is the true origin of a placeholder-source package (manual upload + // / rebuild record whose sourceChannelId == destChannelId). If so, backfill + // its real provenance and skip the download entirely. + if (destChannelId && (archType === "ZIP" || archType === "RAR" || archType === "SEVEN_Z")) { + try { + const derivedCreator = + topicCreator && topicCreator !== "General" + ? topicCreator + : (extractCreatorFromFileName(archiveName) ?? topicCreator ?? null); + const preview = previewMatches.get(archiveSet.parts[0].id.toString()); + const result = await tryProvenanceBackfill({ + client, + destChannelId, + scannedSourceChannelId: channel.id, + fileName: archiveName, + fileSize: totalArchiveSize, + archiveType: archType, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId, + sourceCaption: archiveSet.parts[0].caption ?? null, + remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null, + creator: derivedCreator, + scannedFileId: archiveSet.parts[archiveSet.parts.length - 1].fileId, + previewData: null, + previewMsgId: preview?.id ?? null, + }); + if (result.backfilled) { + counters.zipsBackfilled++; + accountLog.info( + { fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id), confidence: result.confidence }, + "Backfilled provenance for placeholder package — skipping download", + ); + await updateRunActivity(runId, { + currentActivity: `Backfilled provenance for ${archiveName}`, + currentStep: "backfilling", + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + return null; + } + } catch (err) { + accountLog.warn({ err, fileName: archiveName }, "Provenance backfill attempt failed (non-fatal), continuing to normal ingestion"); + } + } +``` + +Notes for the implementer: +- `archType` is the detected archive type variable already in scope in this function (search for where `archiveType`/`archType` is computed for the set). If the variable has a different name, use it. +- `extractCreatorFromFileName` is already imported in `worker.ts` (used elsewhere). If not, add `import { extractCreatorFromFileName } from "./archive/creator.js";`. +- `archiveSet.parts[i].fileId` / `.caption` / `.remoteUniqueId`: confirm these fields exist on the part type (see `worker/src/archive/multipart.ts` `TelegramMessage`/part type). If `caption` isn't on the part, pass `null` (the source caption enrichment is best-effort). +- `previewMatches` maps `baseName`/`firstMessageId` → `{ id, fileId }` (see `matchPreviewToArchive`); adjust the key to match its actual keying. + +- [ ] **Step 3: Surface the counter** — where the run summary/`updateRunActivity` writes `zipsDuplicate`, also write `zipsBackfilled: counters.zipsBackfilled`. Update the final `IngestionRun` update to persist it. + +- [ ] **Step 4: Typecheck** + +Run: `cd worker && npx tsc --noEmit` +Expected: no new errors in `worker.ts`. + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/worker.ts +git commit -m "feat(worker): opportunistic provenance backfill during scan" +``` + +--- + +### Task 9: Destination-copy second tail read for listing-less ZIP candidates + +**Files:** +- Modify: `worker/src/db/queries.ts` (extend `findPlaceholderCandidate`) +- Modify: `worker/src/provenance-backfill.ts` + +**Rationale:** Rebuild-created candidates have `fileCount === 0` and no `PackageFile.crc32`, so name-side CRCs are empty and `fingerprintsMatch` can't confirm. For ZIP candidates we can still get a fingerprint by doing a *second* ranged tail read of the candidate's own copy in the destination channel, then compare scanned-vs-destination CRC sets. + +**Interfaces:** +- `findPlaceholderCandidate` now also selects: `destMessageId: bigint | null`, `destMessageIds: bigint[]`, and `destChannel: { telegramId: bigint }` (destination chat id). Return type extended accordingly. +- Produces (in `provenance-backfill.ts`): `readZipListingFromDestination(client, destChatTelegramId: bigint, destMessageId: bigint, fileSize: bigint): Promise`. + +- [ ] **Step 1: Extend `findPlaceholderCandidate`** select + return type in `db/queries.ts`: + +```ts +select: { + id: true, archiveType: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, + destChannel: { select: { telegramId: true } }, +}, +``` +Return type: `{ id: string; archiveType: string; fileCount: number; fileSize: bigint; destMessageId: bigint | null; destMessageIds: bigint[]; destChannel: { telegramId: bigint } | null } | null`. + +- [ ] **Step 2: Add `readZipListingFromDestination` to `provenance-backfill.ts`** + +```ts +import { invokeWithTimeout } from "./tdlib/download.js"; + +async function readZipListingFromDestination( + client: Client, + destChatTelegramId: bigint, + destMessageId: bigint, + fileSize: bigint, +): Promise { + try { + // Resolve the destination message's document file id. + const msg = (await invokeWithTimeout(client, { + _: "getMessage", + chat_id: Number(destChatTelegramId), + message_id: Number(destMessageId), + })) as { content?: { document?: { document?: { id: number } } } }; + const fid = msg?.content?.document?.document?.id; + if (!fid) return null; + return await readScannedZipListing(client, String(fid), fileSize); + } catch (err) { + log.warn({ err, destMessageId: Number(destMessageId) }, "destination ZIP listing read failed"); + return null; + } +} +``` + +- [ ] **Step 3: Use it in `tryProvenanceBackfill`** — when `args.archiveType === "ZIP"`, after computing `candidateEntries` from `getPackageFileCrcs`, if `crcFingerprint(candidateEntries).complete === false` and the candidate has a destination copy, replace `candidateEntries` with the result of `readZipListingFromDestination(...)` (use the last of `destMessageIds` if present, else `destMessageId`). Then run `fingerprintsMatch(entries, candidateEntries)` as before. If the destination read returns null, fall through to name+size. + +- [ ] **Step 4: Typecheck** + +Run: `cd worker && npx tsc --noEmit` +Expected: no new errors. + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/db/queries.ts worker/src/provenance-backfill.ts +git commit -m "feat(worker): fingerprint listing-less ZIP candidates via destination copy" +``` + +--- + +### Task 10: Multi-candidate ambiguity notification + +**Files:** +- Modify: `worker/src/db/queries.ts` (add `findPlaceholderCandidates` returning all matches; add `createGroupingConflictNotification`-style helper or reuse `db.systemNotification.create`) +- Modify: `worker/src/provenance-backfill.ts` + +**Interfaces:** +- `findPlaceholderCandidates(destChannelId, fileName, fileSize): Promise` — all placeholder matches (same select as Task 9), oldest first. +- Orchestrator resolves ambiguity via fingerprint; if it can't, emits a `SystemNotification` and backfills nothing. + +- [ ] **Step 1: Add `findPlaceholderCandidates`** (plural) in `db/queries.ts` — identical to `findPlaceholderCandidate` but `findMany`. Keep the singular as `return (await findPlaceholderCandidates(...))[0] ?? null` to avoid duplication. + +- [ ] **Step 2: Update `tryProvenanceBackfill`** + +```ts +const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize); +if (candidates.length === 0) return { backfilled: false }; + +let chosen = candidates[0]; +if (candidates.length > 1) { + // Try to disambiguate by fingerprint (ZIP only). If exactly one matches, pick it. + if (args.archiveType === "ZIP" && scannedEntries) { + const matches = []; + for (const c of candidates) { + const crcs = await getPackageFileCrcs(c.id); + const ce: FileEntry[] = crcs.map((crc) => ({ path:"", fileName:"", extension:null, compressedSize:0n, uncompressedSize:0n, crc32: crc })); + if (fingerprintsMatch(scannedEntries, ce)) matches.push(c); + } + if (matches.length === 1) { chosen = matches[0]; } + else { + await db.systemNotification.create({ data: { + type: "INTEGRITY_AUDIT", severity: "WARNING", + title: `Ambiguous provenance match: ${args.fileName}`, + message: `${candidates.length} placeholder packages share this name+size and the fingerprint did not uniquely disambiguate. No provenance was backfilled.`, + context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) }, + }}); + return { backfilled: false }; + } + } else { + // Can't disambiguate without a fingerprint — notify, don't guess. + await db.systemNotification.create({ data: { + type: "INTEGRITY_AUDIT", severity: "WARNING", + title: `Ambiguous provenance match: ${args.fileName}`, + message: `${candidates.length} placeholder packages share this name+size (archive type ${args.archiveType} — no cheap fingerprint). No provenance was backfilled.`, + context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) }, + }}); + return { backfilled: false }; + } +} +``` + +(Refactor the single-candidate ZIP fingerprint logic from Task 6 to compute `scannedEntries` once up front and reuse `chosen` in the `backfillProvenance` call. Import `db` from `./db/client.js`. Confirm `SkipReason`/`NotificationType` enum has `INTEGRITY_AUDIT`; if not, use an existing value like `UPLOAD_FAILED` or the correct `NotificationType`.) + +- [ ] **Step 3: Typecheck + Commit** + +```bash +cd worker && npx tsc --noEmit +git add worker/src/db/queries.ts worker/src/provenance-backfill.ts +git commit -m "feat(worker): notify on ambiguous provenance candidates instead of guessing" +``` + +--- + +### Task 11: 7z ranged listing (SPIKE-gated) + +**Files:** +- Modify: `worker/src/archive/central-directory.ts` (add 7z end-header locator) +- Modify: `worker/src/provenance-backfill.ts` (add 7z branch) + +**⚠️ Honesty note:** Full 7z listing-from-tail is materially harder than ZIP. 7z stores its metadata header at the end (locatable from the 32-byte start header: sig `37 7A BC AF 27 1C`, then at offset 12 an 8-byte `NextHeaderOffset` relative to byte 32, and at offset 20 an 8-byte `NextHeaderSize`). BUT the header is frequently an *encoded (LZMA-compressed) header*, which requires decoding the packed header stream (also near EOF) to recover file CRCs — non-trivial to implement correctly from scratch. This task is therefore SPIKE-GATED: implement only the plain-header common case; fall back to name+size otherwise. Do not ship a half-correct LZMA decoder. + +**Interfaces:** +- `locate7zHeader(startHeader: Buffer): { nextHeaderOffset: number; nextHeaderSize: number } | null` (pure) — parses the 32-byte start header. +- `parse7zPlainHeaderCrcs(header: Buffer): string[] | null` (pure) — returns file CRC32s if the header is a plain (kHeader) structure with a `kCRC` section; returns `null` if the header is encoded (`kEncodedHeader`, id `0x17`) or otherwise unparseable. + +- [ ] **Step 1: Spike** — download the last ~4 MB of a known 7z from the destination channel via `downloadFileRange`, plus the first 32 bytes. Compute the header region from the start header, extract it from the tail buffer, and inspect its first byte: `0x01` = plain header (`kHeader`), `0x17` = encoded header (`kEncodedHeader`). Record the observed distribution across a few real archives. If the vast majority are encoded, STOP and leave 7z as name+size (document the finding in a comment) — do not implement steps 2–4. + +- [ ] **Step 2: Implement `locate7zHeader` (pure)** with a unit test (signature check + offset/size read). Add to `central-directory.test.ts`. + +```ts +export function locate7zHeader(startHeader: Buffer): { nextHeaderOffset: number; nextHeaderSize: number } | null { + const SIG = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + if (startHeader.length < 32 || !startHeader.subarray(0, 6).equals(SIG)) return null; + return { + nextHeaderOffset: Number(startHeader.readBigUInt64LE(12)), + nextHeaderSize: Number(startHeader.readBigUInt64LE(20)), + }; +} +``` + +- [ ] **Step 3: Implement `parse7zPlainHeaderCrcs` (pure)** for the kHeader→kMainStreamsInfo→kSubStreamsInfo→kCRC path, returning CRCs; return `null` on `kEncodedHeader` (0x17) or any unrecognized property id. Unit-test with a fixture 7z created by the system `7z a -mhc=off` (header-compression off → plain header) in the test setup, skipped if `7z` is unavailable. + +- [ ] **Step 4: Wire the 7z branch** in `tryProvenanceBackfill`: for `archiveType === "SEVEN_Z"`, download the start header (first 32 bytes) + a generous tail, `locate7zHeader`, extract the header bytes, `parse7zPlainHeaderCrcs`. If non-null, build `FileEntry[]` with those CRCs (path/name empty is fine for fingerprinting) and fingerprint-compare; else fall back to name+size. + +- [ ] **Step 5: Typecheck + Commit** + +```bash +cd worker && npx tsc --noEmit && npm test -- central-directory +git add worker/src/archive/central-directory.ts worker/src/archive/central-directory.test.ts worker/src/provenance-backfill.ts +git commit -m "feat(worker): 7z plain-header ranged fingerprint (name+size fallback for encoded headers)" +``` + +--- + +### Task 12: Manual integration verification + +**Files:** none (verification only). + +- [ ] **Step 1: Pick a known placeholder package** — a manually-uploaded ZIP whose content also exists in a real source channel. Confirm in DB: `SELECT id, "sourceChannelId", "destChannelId", "fileCount" FROM packages WHERE ...` shows `sourceChannelId == destChannelId`. + +- [ ] **Step 2: Re-index the real source channel** (trigger a fetch/scan of the channel containing that archive). + +- [ ] **Step 3: Verify backfill** — the package now has the real `sourceChannelId`, `sourceMessageId`, `sourceCaption`, `creator`; `fileCount > 0` if it was 0 before; the run's `zipsBackfilled` incremented; worker logs show "Backfilled provenance … confidence=fingerprint"; no full download occurred (no large download in logs for that archive). + +- [ ] **Step 4: Idempotency** — re-run the same scan; verify the package is now skipped via the existing dedup (remoteUniqueId/repost) and NOT mutated again; `zipsBackfilled` does not increment. + +- [ ] **Step 5: Collision safety** — (if available) a different archive sharing name+size with a placeholder but different contents ingests as a NEW package (fingerprint mismatch path), not mis-attributed. + +- [ ] **Step 6: RAR fallback** — a RAR placeholder match backfills with `confidence=name-size` and logs the lower-confidence notice. + +--- + +## Self-Review + +**Spec coverage:** +- Candidate = `sourceChannelId == destChannelId` → Task 5 `findPlaceholderCandidate`, re-checked in `backfillProvenance`. ✓ +- Hook between repost check and download → Task 8. ✓ +- Fingerprint via ranged tail (ZIP) → Tasks 3, 4, 6. ✓ +- 7z end-header → Task 11 (SPIKE-gated: plain-header case implemented, encoded-header falls back to name+size). Task 6 returns null for 7z until Task 11 wires the branch. +- Candidate side CRCs from `PackageFile` → Task 5 `getPackageFileCrcs`, Task 6. ✓ +- Rebuild candidates (no CRCs) → Task 9 adds the destination-copy second tail read for ZIP so they can still be fingerprint-confirmed; non-ZIP listing-less candidates fall to name+size. +- Fields written (source*, remoteUniqueId, creator always, fileCount/PackageFile if empty, preview if empty) → Task 5. ✓ +- RAR name+size, logged → Task 6/8. ✓ +- Fingerprint mismatch → fall through to normal ingestion → Task 6 returns `{backfilled:false}`, Task 8 continues. ✓ +- Ambiguity notification → Task 10 (`findPlaceholderCandidates` + fingerprint disambiguation + `SystemNotification` when it can't uniquely resolve). +- Idempotency → Task 8 relies on existing checks after `remoteUniqueId` is set; verified in Task 9. ✓ +- Tail-download failure → return null / leave untouched → Task 6. ✓ +- Transaction re-check → Task 5. ✓ +- `zipsBackfilled` counter → Tasks 7, 8. ✓ +- Lightweight harness + pure-fn tests → Tasks 1–3. ✓ + +**Remaining bounded risk:** Task 11's 7z support is spike-gated — if real archives use encoded headers, 7z stays at name+size confidence (documented, safe). No other spec items are deferred. + +**Placeholder scan:** No TBD/TODO left; all code steps contain full code. Spike (Task 4) is a genuine verification step, not a placeholder. + +**Type consistency:** `FileEntry` shape consistent across Tasks 2/3/5/6; `BackfillProvenanceInput`/`BackfillArgs` names match between Tasks 5 and 6; `tryProvenanceBackfill` return type consistent between Tasks 6 and 8. diff --git a/docs/superpowers/plans/2026-07-27-ranged-archive-listing.md b/docs/superpowers/plans/2026-07-27-ranged-archive-listing.md new file mode 100644 index 0000000..762c270 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-ranged-archive-listing.md @@ -0,0 +1,1073 @@ +# Ranged inner-file listing (RAR & 7z) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Index the inner files of RAR & 7z placeholder packages via small ranged reads (no full download), so the reindex/provenance-backfill path fills `fileCount`/`package_files` for all archive types. + +**Architecture:** For each archive, do the minimum binary parsing to locate its header bytes, fetch only those via `downloadFileRange`, write them into a sparse temp file at their true offsets (data regions left as zero holes), then run the existing `7z l` / `unrar lt` (via `read7zContents` / `readRarContents`) and reuse their parsers. A ranged reader returning `null` triggers a size-capped full-download fallback. + +**Tech Stack:** TypeScript (strict, ESM, `.js` import specifiers), TDLib via `tdl`, vitest, `unrar`/`7z` CLIs (already installed in the worker image), Prisma/Postgres. + +## Global Constraints + +- TypeScript strict; ESM import specifiers end in `.js`. Copy the surrounding files' style. +- ESLint does NOT cover `worker/` — but keep types clean; no `any` unless mirroring existing patterns. +- Tests: vitest, files match `src/**/*.test.ts`; run from `worker/` with `npx vitest run`. +- All TDLib calls must go through FLOOD_WAIT-safe wrappers. `downloadFileRange` already wraps `client.invoke` in `withFloodWait` — do not add a second wrapper. Keep ranged reads **sequential** (never `Promise.all` a walk). +- Full-download fallback is gated by `config.maxZipSizeMB` (env `WORKER_MAX_ZIP_SIZE_MB`, default 204800). Over the cap → do NOT download; write a `SystemNotification` and return `null`. +- No new DB migration. Reuse the existing `zipsBackfilled` counter. +- Shared return type is `FileEntry` from `worker/src/archive/zip-reader.ts`: + `{ path: string; fileName: string; extension: string | null; compressedSize: bigint; uncompressedSize: bigint; crc32: string | null }`. +- Deploy is local (no GitHub push): build `worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image WITHOUT `pull` (see Task 4 / Task 8 deploy steps). + +--- + +## File Structure + +- Create `worker/src/archive/ranged/sparse-list.ts` — sparse temp-file reconstruction + CLI lister. (+ `sparse-list.test.ts`) +- Create `worker/src/archive/ranged/sevenz-ranged.ts` — 7z signature parse + ranged listing. (+ `sevenz-ranged.test.ts`) +- Create `worker/src/archive/ranged/rar-ranged.ts` — RAR signature/vint/block-extent parse + ranged walk. (+ `rar-ranged.test.ts`) +- Create `worker/src/archive/ranged/range-reader.ts` — `RangeReader` type + default TDLib impl. +- Create `worker/src/archive/ranged/fallback.ts` — size-capped full-download fallback + notification. +- Modify `worker/src/provenance-backfill.ts` — dispatch scanned/destination listing by archive type; call fallback. +- Modify `worker/src/worker.ts` — include `fileName` in the `scannedParts` passed to `tryProvenanceBackfill`. + +--- + +## Task 1: Sparse reconstruction helper (`sparse-list.ts`) + +**Files:** +- Create: `worker/src/archive/ranged/sparse-list.ts` +- Test: `worker/src/archive/ranged/sparse-list.test.ts` + +**Interfaces:** +- Consumes: `FileEntry` from `../zip-reader.js`; `config.tempDir` from `../../util/config.js`. +- Produces: + - `interface SparsePart { fileName: string; size: number; regions: { offset: number; bytes: Buffer }[] }` + - `type SparseLister = (firstPartPath: string) => Promise` + - `async function listFromSparse(parts: SparsePart[], lister: SparseLister): Promise` + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/archive/ranged/sparse-list.test.ts +import { describe, it, expect } from "vitest"; +import { open } from "fs/promises"; +import { listFromSparse } from "./sparse-list.js"; + +describe("listFromSparse", () => { + it("writes each region at its offset into a sparse file and passes the path to the lister", async () => { + const size = 1_000_000; + const regions = [ + { offset: 0, bytes: Buffer.from("HEAD") }, + { offset: size - 4, bytes: Buffer.from("TAIL") }, + ]; + let seenPath = ""; + const entries = await listFromSparse( + [{ fileName: "sample.7z", size, regions }], + async (firstPartPath) => { + seenPath = firstPartPath; + const fh = await open(firstPartPath, "r"); + try { + const head = Buffer.alloc(4); await fh.read(head, 0, 4, 0); + const tail = Buffer.alloc(4); await fh.read(tail, 0, 4, size - 4); + const hole = Buffer.alloc(4); await fh.read(hole, 0, 4, 500_000); + expect(head.toString()).toBe("HEAD"); + expect(tail.toString()).toBe("TAIL"); + expect(hole.equals(Buffer.alloc(4))).toBe(true); // gap is zero + } finally { await fh.close(); } + return [{ path: "a/b.stl", fileName: "b.stl", extension: "stl", compressedSize: 1n, uncompressedSize: 1n, crc32: null }]; + }, + ); + expect(seenPath.endsWith("sample.7z")).toBe(true); + expect(entries).not.toBeNull(); + expect(entries!).toHaveLength(1); + }); + + it("returns null when the lister yields no entries", async () => { + const res = await listFromSparse( + [{ fileName: "x.7z", size: 100, regions: [{ offset: 0, bytes: Buffer.from("A") }] }], + async () => [], + ); + expect(res).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/sparse-list.test.ts` +Expected: FAIL — `Cannot find module './sparse-list.js'`. + +- [ ] **Step 3: Write minimal implementation** + +```typescript +// worker/src/archive/ranged/sparse-list.ts +import { mkdtemp, open, rm } from "fs/promises"; +import path from "path"; +import { config } from "../../util/config.js"; +import { childLogger } from "../../util/logger.js"; +import type { FileEntry } from "../zip-reader.js"; + +const log = childLogger("sparse-list"); + +export interface SparsePart { + fileName: string; + size: number; + regions: { offset: number; bytes: Buffer }[]; +} + +export type SparseLister = (firstPartPath: string) => Promise; + +/** + * Reconstruct archive header bytes into sparse temp files (data areas left as + * zero holes), run `lister` on the first part, return its entries. + * Returns null on any error or when the lister finds nothing. + */ +export async function listFromSparse( + parts: SparsePart[], + lister: SparseLister, +): Promise { + if (parts.length === 0) return null; + const dir = await mkdtemp(path.join(config.tempDir, "ranged-")); + try { + let firstPath = ""; + for (let i = 0; i < parts.length; i++) { + const p = parts[i]; + const filePath = path.join(dir, p.fileName); + if (i === 0) firstPath = filePath; + const fh = await open(filePath, "w"); + try { + await fh.truncate(p.size); // create the sparse hole + for (const r of p.regions) { + await fh.write(r.bytes, 0, r.bytes.length, r.offset); + } + } finally { + await fh.close(); + } + } + const entries = await lister(firstPath); + return entries.length > 0 ? entries : null; + } catch (err) { + log.warn({ err }, "sparse listing failed"); + return null; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/sparse-list.test.ts` +Expected: PASS (2 passed). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/ranged/sparse-list.ts worker/src/archive/ranged/sparse-list.test.ts +git commit -m "feat(worker): sparse-file reconstruction helper for ranged archive listing" +``` + +--- + +## Task 2: 7z signature-header parser (`sevenz-ranged.ts`, pure part) + +**Files:** +- Create: `worker/src/archive/ranged/sevenz-ranged.ts` +- Test: `worker/src/archive/ranged/sevenz-ranged.test.ts` + +**Interfaces:** +- Produces: `function parseSevenZSignatureHeader(buf: Buffer): { nextHeaderOffset: number; nextHeaderSize: number } | null` + - Returns null unless `buf` starts with the 6-byte 7z magic and is ≥ 32 bytes. + - `nextHeaderOffset` is relative to the end of the 32-byte signature header (absolute end-header start = `32 + nextHeaderOffset`). + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/archive/ranged/sevenz-ranged.test.ts +import { describe, it, expect } from "vitest"; +import { parseSevenZSignatureHeader } from "./sevenz-ranged.js"; + +const MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + +function buildSignatureHeader(nextOffset: bigint, nextSize: bigint): Buffer { + const buf = Buffer.alloc(32); + MAGIC.copy(buf, 0); + buf.writeUInt8(0, 6); buf.writeUInt8(4, 7); // version 0.4 + buf.writeUInt32LE(0, 8); // StartHeaderCRC (unused here) + buf.writeBigUInt64LE(nextOffset, 12); + buf.writeBigUInt64LE(nextSize, 20); + buf.writeUInt32LE(0, 28); // NextHeaderCRC (unused here) + return buf; +} + +describe("parseSevenZSignatureHeader", () => { + it("reads NextHeaderOffset and NextHeaderSize", () => { + const buf = buildSignatureHeader(1_000_000n, 4096n); + expect(parseSevenZSignatureHeader(buf)).toEqual({ nextHeaderOffset: 1_000_000, nextHeaderSize: 4096 }); + }); + + it("returns null on bad magic", () => { + expect(parseSevenZSignatureHeader(Buffer.alloc(32))).toBeNull(); + }); + + it("returns null when shorter than 32 bytes", () => { + expect(parseSevenZSignatureHeader(MAGIC)).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/sevenz-ranged.test.ts` +Expected: FAIL — `Cannot find module './sevenz-ranged.js'`. + +- [ ] **Step 3: Write minimal implementation** (signature parser only — orchestrator added in Task 3) + +```typescript +// worker/src/archive/ranged/sevenz-ranged.ts +const SEVENZ_MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + +export function parseSevenZSignatureHeader( + buf: Buffer, +): { nextHeaderOffset: number; nextHeaderSize: number } | null { + if (buf.length < 32) return null; + if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null; + return { + nextHeaderOffset: Number(buf.readBigUInt64LE(12)), + nextHeaderSize: Number(buf.readBigUInt64LE(20)), + }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/sevenz-ranged.test.ts` +Expected: PASS (3 passed). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/ranged/sevenz-ranged.ts worker/src/archive/ranged/sevenz-ranged.test.ts +git commit -m "feat(worker): 7z signature-header parser" +``` + +--- + +## Task 3: `RangeReader` + 7z ranged listing orchestrator + +**Files:** +- Create: `worker/src/archive/ranged/range-reader.ts` +- Modify: `worker/src/archive/ranged/sevenz-ranged.ts` (add orchestrator) +- Test: `worker/src/archive/ranged/sevenz-ranged.test.ts` (add case) + +**Interfaces:** +- Consumes: `parseSevenZSignatureHeader` (Task 2); `listFromSparse`, `SparsePart` (Task 1); `read7zContents` from `../sevenz-reader.js`; `downloadFileRange` from `../../tdlib/range-download.js`. +- Produces: + - `worker/src/archive/ranged/range-reader.ts`: + - `type RangeReader = (fileId: string, offset: number, length: number, partSize: bigint) => Promise` + - `function tdlibRangeReader(client: import("tdl").Client): RangeReader` + - `sevenz-ranged.ts`: + - `interface RangedPart { fileId: string; fileSize: bigint; fileName: string }` + - `async function readSevenZListingRanged(parts: RangedPart[], read: RangeReader): Promise` + (7z placeholders are single-part; uses `parts[0]`.) + +- [ ] **Step 1: Write the failing test** (append to `sevenz-ranged.test.ts`) + +```typescript +import { readSevenZListingRanged } from "./sevenz-ranged.js"; +import type { RangeReader } from "./range-reader.js"; + +describe("readSevenZListingRanged", () => { + it("reads the signature + end-header regions and reconstructs for 7z l", async () => { + const size = 5_000_000; + const endHeaderOffset = 4_900_000; // absolute + const nextHeaderOffset = endHeaderOffset - 32; + const sig = Buffer.alloc(32); + Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0); + sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12); + sig.writeBigUInt64LE(100n, 20); + + const reads: { offset: number; length: number }[] = []; + const read: RangeReader = async (_id, offset, length) => { + reads.push({ offset, length }); + if (offset === 0) return sig.subarray(0, length); + return Buffer.alloc(length, 0xAB); // stand-in end-header bytes + }; + + // Inject a fake lister via the module boundary: readSevenZListingRanged + // calls listFromSparse(parts, read7zContents). We assert the ranged reads + // it issued; the sparse file + real 7z is covered by live verification. + const entries = await readSevenZListingRanged( + [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }], + read, + ); + // entries may be null here because the stand-in bytes aren't a real 7z; + // the contract under test is the ranged-read offsets: + expect(reads[0]).toEqual({ offset: 0, length: 32 }); + expect(reads[1]).toEqual({ offset: endHeaderOffset, length: 100 }); + expect(entries === null || Array.isArray(entries)).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/sevenz-ranged.test.ts` +Expected: FAIL — `readSevenZListingRanged`/`range-reader.js` not found. + +- [ ] **Step 3: Write minimal implementation** + +```typescript +// worker/src/archive/ranged/range-reader.ts +import type { Client } from "tdl"; +import { downloadFileRange } from "../../tdlib/range-download.js"; + +export type RangeReader = ( + fileId: string, + offset: number, + length: number, + partSize: bigint, +) => Promise; + +export function tdlibRangeReader(client: Client): RangeReader { + return (fileId, offset, length, partSize) => + downloadFileRange(client, fileId, offset, length, partSize); +} +``` + +```typescript +// append to worker/src/archive/ranged/sevenz-ranged.ts +import type { FileEntry } from "../zip-reader.js"; +import { read7zContents } from "../sevenz-reader.js"; +import { listFromSparse } from "./sparse-list.js"; +import type { RangeReader } from "./range-reader.js"; +import { childLogger } from "../../util/logger.js"; + +const log = childLogger("sevenz-ranged"); + +export interface RangedPart { fileId: string; fileSize: bigint; fileName: string } + +export async function readSevenZListingRanged( + parts: RangedPart[], + read: RangeReader, +): Promise { + const part = parts[0]; + if (!part) return null; + const size = Number(part.fileSize); + try { + const sig = await read(part.fileId, 0, 32, part.fileSize); + const parsed = parseSevenZSignatureHeader(sig); + if (!parsed) return null; + const endStart = 32 + parsed.nextHeaderOffset; + if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null; + const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize); + return listFromSparse( + [{ + fileName: part.fileName, + size, + regions: [ + { offset: 0, bytes: sig }, + { offset: endStart, bytes: endHeader }, + ], + }], + read7zContents, + ); + } catch (err) { + log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed"); + return null; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/sevenz-ranged.test.ts` +Expected: PASS (all cases). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/ranged/range-reader.ts worker/src/archive/ranged/sevenz-ranged.ts worker/src/archive/ranged/sevenz-ranged.test.ts +git commit -m "feat(worker): ranged 7z listing orchestrator + RangeReader" +``` + +--- + +## Task 4: Wire 7z into provenance-backfill + fallback stub; deploy & live-verify (the productive spike) + +**Files:** +- Create: `worker/src/archive/ranged/fallback.ts` +- Modify: `worker/src/provenance-backfill.ts` (dispatch scanned/dest listing; call fallback) +- Modify: `worker/src/worker.ts` (add `fileName` to `scannedParts`) +- Test: `worker/src/archive/ranged/fallback.test.ts` + +**Interfaces:** +- Consumes: `readSevenZListingRanged`, `RangedPart` (Task 3); `tdlibRangeReader` (Task 3); `read7zContents`/`readRarContents`; `downloadFile` from `../../tdlib/download.js`; `config.maxZipSizeMB`, `config.tempDir`; `db` from `../../db/client.js`. +- Produces: + - `fallback.ts`: `async function fullDownloadListing(args: { client: import("tdl").Client; parts: RangedPart[]; archiveType: string; totalSize: bigint; fileName: string; }): Promise` — downloads all parts if `totalSize <= maxZipSizeMB`, runs the CLI reader, returns entries; over cap → writes a `SystemNotification` and returns null. + - In `provenance-backfill.ts`: `async function readScannedListingRanged(archiveType: string, client, parts: RangedPart[]): Promise` used in place of the ZIP-only branch. (RAR routes to Task 6/7's function — see Task 8; for now RAR returns null → fallback.) + +- [ ] **Step 1: Write the failing test** (fallback size-cap behavior — pure, no real download) + +```typescript +// worker/src/archive/ranged/fallback.test.ts +import { describe, it, expect, vi } from "vitest"; + +vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp" } })); +const created: unknown[] = []; +vi.mock("../../db/client.js", () => ({ + db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } }, +})); + +import { fullDownloadListing } from "./fallback.js"; + +describe("fullDownloadListing", () => { + it("refuses to download over the size cap and records a notification", async () => { + const res = await fullDownloadListing({ + client: {} as never, + parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }], + archiveType: "RAR", + totalSize: 2n * 1024n * 1024n * 1024n, + fileName: "big.rar", + }); + expect(res).toBeNull(); + expect(created).toHaveLength(1); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/fallback.test.ts` +Expected: FAIL — `Cannot find module './fallback.js'`. + +- [ ] **Step 3: Write minimal implementation** + +```typescript +// worker/src/archive/ranged/fallback.ts +import { mkdtemp, rm } from "fs/promises"; +import path from "path"; +import type { Client } from "tdl"; +import { config } from "../../util/config.js"; +import { db } from "../../db/client.js"; +import { childLogger } from "../../util/logger.js"; +import { downloadFile } from "../../tdlib/download.js"; +import { read7zContents } from "../sevenz-reader.js"; +import { readRarContents } from "../rar-reader.js"; +import type { FileEntry } from "../zip-reader.js"; +import type { RangedPart } from "./sevenz-ranged.js"; + +const log = childLogger("ranged-fallback"); + +export async function fullDownloadListing(args: { + client: Client; + parts: RangedPart[]; + archiveType: string; + totalSize: bigint; + fileName: string; +}): Promise { + const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n; + if (args.totalSize > capBytes) { + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "WARNING", + title: `Listing skipped (over size cap): ${args.fileName}`, + message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`, + context: { fileName: args.fileName, archiveType: args.archiveType }, + }, + }); + log.warn({ fileName: args.fileName }, "fallback skipped — over size cap"); + return null; + } + const dir = await mkdtemp(path.join(config.tempDir, "fallback-")); + const paths: string[] = []; + try { + for (const p of args.parts) { + const dest = path.join(dir, p.fileName); + await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {}); + paths.push(dest); + } + const entries = + args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0]) + : args.archiveType === "RAR" ? await readRarContents(paths[0]) + : []; + return entries.length > 0 ? entries : null; + } catch (err) { + log.warn({ err, fileName: args.fileName }, "full-download fallback failed"); + return null; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/fallback.test.ts` +Expected: PASS (1 passed). + +- [ ] **Step 5: Wire the dispatcher into `provenance-backfill.ts`** + +Replace the ZIP-only scanned-listing branch. Find (around line 146-149): + +```typescript + let scannedEntries: FileEntry[] | null = null; + if (args.archiveType === "ZIP") { + scannedEntries = await readScannedZipListing(args.client, args.scannedParts); + } +``` + +Replace with: + +```typescript + let scannedEntries: FileEntry[] | null = await readScannedListingRanged( + args.archiveType, args.client, args.scannedParts, + ); + // Cheap ranged read failed — fall back to a size-capped full download so the + // listing still gets indexed. Only worth it when the candidate lacks a listing. + if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) { + const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n); + scannedEntries = await fullDownloadListing({ + client: args.client, parts: args.scannedParts, archiveType: args.archiveType, + totalSize, fileName: args.fileName, + }); + } +``` + +Add this dispatcher function near the other read helpers in `provenance-backfill.ts`: + +```typescript +async function readScannedListingRanged( + archiveType: string, + client: Client, + parts: RangedPart[], +): Promise { + const read = tdlibRangeReader(client); + if (archiveType === "ZIP") return readScannedZipListing(client, parts); + if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read); + // RAR enabled in Task 8. + return null; +} +``` + +Add imports at the top of `provenance-backfill.ts`: + +```typescript +import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js"; +import { tdlibRangeReader } from "./archive/ranged/range-reader.js"; +import { fullDownloadListing } from "./archive/ranged/fallback.js"; +``` + +Change `args.scannedParts` type in `BackfillArgs` from `{ fileId: string; fileSize: bigint }[]` to `RangedPart[]` (adds `fileName`). + +- [ ] **Step 6: Add `fileName` to `scannedParts` in `worker.ts`** + +Find (around line 1674): + +```typescript + scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize })), +``` + +Replace with: + +```typescript + scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize, fileName: p.fileName })), +``` + +- [ ] **Step 7: Typecheck + full test run + build** + +```bash +cd worker && npx tsc --noEmit && npx vitest run +``` +Expected: no TS errors; all tests pass. + +- [ ] **Step 8: Commit** + +```bash +git add worker/src/archive/ranged/fallback.ts worker/src/archive/ranged/fallback.test.ts worker/src/provenance-backfill.ts worker/src/worker.ts +git commit -m "feat(worker): dispatch 7z ranged listing + size-capped full-download fallback" +``` + +- [ ] **Step 9: Deploy locally (no push) and live-verify 7z** + +```bash +cd /home/sam/Documents/DragonsStash +docker build -f worker/Dockerfile -t git.samagsteribbe.nl/admin/dragonsstash-worker:latest . +docker compose --project-name dragonsstash --project-directory /opt/stacks/DragonsStash \ + -f /opt/stacks/DragonsStash/docker-compose.yml up -d --no-deps --force-recreate worker +``` + +Then verify the 7z ranged path is populating listings (not full-downloading): + +```bash +# Watch for ranged 7z work + confirm no multi-GB downloads for 7z +docker logs -f --since 30s dragonsstash-worker 2>&1 | grep -iE "sevenz-ranged|sparse-list|Backfilled provenance|Downloading archive part" +``` + +DB check — 7z placeholders gaining a listing: +```sql +SELECT count(*) FROM packages WHERE "contentHash" LIKE 'rebuild:%' AND "archiveType"='SEVEN_Z' AND "fileCount">0; +``` +Expected: climbs over successive cycles. Spot-check one against truth: pick a backfilled 7z package's `fileName`, and compare its `package_files` count to `7z l` on a real copy. + +**Gate:** if 7z listings populate correctly via the ranged path with no full downloads, the two spike risks (arbitrary-offset ranged read + `7z l` on a sparse file) are proven. Proceed to RAR. If not, stop and debug before building RAR. + +--- + +## Task 5: RAR parsers — vint, signature, block-extent (pure) + +**Files:** +- Create: `worker/src/archive/ranged/rar-ranged.ts` (pure parsers only in this task) +- Test: `worker/src/archive/ranged/rar-ranged.test.ts` + +**Interfaces:** +- Produces: + - `function readVint(buf: Buffer, pos: number): { value: number; bytes: number }` (RAR5 base-128 LE varint) + - `function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null` + - `interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean }` + - `function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent` + - `function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent` + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/archive/ranged/rar-ranged.test.ts +import { describe, it, expect } from "vitest"; +import { readVint, detectRarSignature, parseRar5BlockExtent, parseRar4BlockExtent } from "./rar-ranged.js"; + +describe("readVint", () => { + it("reads single-byte and multi-byte values (base-128 LE)", () => { + expect(readVint(Buffer.from([0x08]), 0)).toEqual({ value: 8, bytes: 1 }); + // 0x80,0x01 => 0 | (1<<7) = 128 + expect(readVint(Buffer.from([0x80, 0x01]), 0)).toEqual({ value: 128, bytes: 2 }); + }); +}); + +describe("detectRarSignature", () => { + it("detects RAR5 and RAR4", () => { + expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]))).toEqual({ version: 5, sigLen: 8 }); + expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x00]))).toEqual({ version: 4, sigLen: 7 }); + expect(detectRarSignature(Buffer.alloc(8))).toBeNull(); + }); +}); + +describe("parseRar5BlockExtent", () => { + it("computes header+data extent and flags end-of-archive", () => { + // CRC32(4) | HeaderSize vint=5 | Type vint=2 (file) | Flags vint=2 (data present) | DataSize vint=100 | (pad to headerSize) + const b = Buffer.concat([ + Buffer.from([0,0,0,0]), // CRC + Buffer.from([0x05]), // HeaderSize = 5 (bytes after this vint) + Buffer.from([0x02]), // Type = 2 (file) + Buffer.from([0x02]), // Flags = 0x02 -> data present + Buffer.from([0x64]), // DataSize = 100 + Buffer.from([0x00, 0x00]), // padding to fill HeaderSize(5): Type+Flags+DataSize=3, +2 pad =5 + ]); + const ext = parseRar5BlockExtent(b, 0); + // headerBytes = 4 (CRC) + 1 (HeaderSize vint) + 5 (HeaderSize) = 10 + expect(ext.headerBytes).toBe(10); + expect(ext.dataSize).toBe(100); + expect(ext.isEnd).toBe(false); + + const endBlk = Buffer.from([0,0,0,0, 0x02, 0x05, 0x00]); // HeaderSize=2, Type=5(end), Flags=0 + const e2 = parseRar5BlockExtent(endBlk, 0); + expect(e2.isEnd).toBe(true); + }); +}); + +describe("parseRar4BlockExtent", () => { + it("computes extent with ADD_SIZE when flag 0x8000 is set", () => { + // CRC(2) TYPE(1)=0x74 FLAGS(2)=0x8000 HEAD_SIZE(2)=11 ADD_SIZE(4)=200 + const b = Buffer.alloc(11); + b.writeUInt8(0x74, 2); + b.writeUInt16LE(0x8000, 3); + b.writeUInt16LE(11, 5); + b.writeUInt32LE(200, 7); + const ext = parseRar4BlockExtent(b, 0); + expect(ext.headerBytes).toBe(11); + expect(ext.dataSize).toBe(200); + expect(ext.isEnd).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/rar-ranged.test.ts` +Expected: FAIL — module/exports not found. + +- [ ] **Step 3: Write minimal implementation** (parsers only) + +```typescript +// worker/src/archive/ranged/rar-ranged.ts +const RAR4_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]); +const RAR5_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); + +export function readVint(buf: Buffer, pos: number): { value: number; bytes: number } { + let value = 0, shift = 0, bytes = 0; + while (pos + bytes < buf.length) { + const b = buf[pos + bytes]; + value += (b & 0x7f) * Math.pow(2, shift); // Math.pow keeps >32-bit sizes exact up to 2^53 + bytes++; + if ((b & 0x80) === 0) return { value, bytes }; + shift += 7; + if (shift > 63) break; + } + throw new RangeError("incomplete RAR vint"); +} + +export function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null { + if (buf.length >= 8 && buf.subarray(0, 8).equals(RAR5_SIG)) return { version: 5, sigLen: 8 }; + if (buf.length >= 7 && buf.subarray(0, 7).equals(RAR4_SIG)) return { version: 4, sigLen: 7 }; + return null; +} + +export interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean } + +// RAR5: CRC32(4) | HeaderSize(vint) | HeaderType(vint) | HeaderFlags(vint) +// [ExtraAreaSize(vint) if flags&0x0001] [DataSize(vint) if flags&0x0002] ... +export function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent { + let p = pos + 4; // skip CRC32 + const hs = readVint(buf, p); p += hs.bytes; + const headerBytes = 4 + hs.bytes + hs.value; // CRC + HeaderSize-vint + HeaderSize + const type = readVint(buf, p); p += type.bytes; + const flags = readVint(buf, p); p += flags.bytes; + if (flags.value & 0x0001) { const ea = readVint(buf, p); p += ea.bytes; } // extra area size (skip) + let dataSize = 0; + if (flags.value & 0x0002) { const ds = readVint(buf, p); p += ds.bytes; dataSize = ds.value; } + return { headerBytes, dataSize, isEnd: type.value === 5 }; +} + +// RAR4: HEAD_CRC(2) | HEAD_TYPE(1) | HEAD_FLAGS(2) | HEAD_SIZE(2) [ADD_SIZE(4) if flags&0x8000] +export function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent { + const type = buf.readUInt8(pos + 2); + const flags = buf.readUInt16LE(pos + 3); + const headSize = buf.readUInt16LE(pos + 5); + const dataSize = (flags & 0x8000) ? buf.readUInt32LE(pos + 7) : 0; + return { headerBytes: headSize, dataSize, isEnd: type === 0x7b }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/rar-ranged.test.ts` +Expected: PASS (all). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/ranged/rar-ranged.ts worker/src/archive/ranged/rar-ranged.test.ts +git commit -m "feat(worker): RAR vint/signature/block-extent parsers" +``` + +--- + +## Task 6: RAR ranged walk — single volume + +**Files:** +- Modify: `worker/src/archive/ranged/rar-ranged.ts` (add walk + orchestrator) +- Test: `worker/src/archive/ranged/rar-ranged.test.ts` (add case) + +**Interfaces:** +- Consumes: parsers (Task 5); `RangeReader`, `RangedPart` (Task 3); `listFromSparse`, `SparsePart` (Task 1); `readRarContents` from `../rar-reader.js`. +- Produces: + - `async function walkRarVolume(read: RangeReader, part: RangedPart, version: 4 | 5, sigLen: number): Promise<{ offset: number; bytes: Buffer }[] | null>` — sequential header harvest; returns null if a block is unparseable or the block count exceeds `MAX_RAR_BLOCKS = 50000`. + - `async function readRarListingRanged(parts: RangedPart[], read: RangeReader): Promise` (single-part path in this task; multipart in Task 7). + +- [ ] **Step 1: Write the failing test** — synthetic RAR5 volume driven by an in-memory `RangeReader` + +```typescript +import { walkRarVolume, readRarListingRanged } from "./rar-ranged.js"; +import type { RangeReader } from "./range-reader.js"; + +// Build a synthetic RAR5 volume: signature + main header + 2 file blocks (each +// with data) + end block. We only need extents to be walkable. +function buildRar5Volume(): Buffer { + const sig = Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]); + const block = (type: number, flags: number, dataSize: number, pad = 0) => { + const body = [Buffer.from([type]), Buffer.from([flags])]; + if (flags & 0x0002) body.push(Buffer.from([dataSize])); // DataSize (<=127 for test) + if (pad) body.push(Buffer.alloc(pad)); + const bodyBuf = Buffer.concat(body); + const hs = Buffer.from([bodyBuf.length]); // HeaderSize vint (<=127) + const header = Buffer.concat([Buffer.alloc(4), hs, bodyBuf]); // CRC(4)+HeaderSize+body + const data = Buffer.alloc(flags & 0x0002 ? dataSize : 0, 0xEE); + return Buffer.concat([header, data]); + }; + const main = block(1, 0, 0); // main archive header, no data + const f1 = block(2, 0x02, 20); // file header + 20 bytes data + const f2 = block(2, 0x02, 30); // file header + 30 bytes data + const end = block(5, 0, 0); // end of archive + return Buffer.concat([sig, main, f1, f2, end]); +} + +describe("walkRarVolume", () => { + it("harvests every block header and stops at end-of-archive", async () => { + const vol = buildRar5Volume(); + const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length); + const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }, 5, 8); + expect(regions).not.toBeNull(); + // main + 2 files + end = 4 header regions + expect(regions!).toHaveLength(4); + // First region starts right after the 8-byte signature + expect(regions![0].offset).toBe(8); + }); +}); + +describe("readRarListingRanged (single part)", () => { + it("returns null cleanly when the reconstructed file isn't a real RAR", async () => { + const vol = buildRar5Volume(); + const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length); + const res = await readRarListingRanged([{ fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }], read); + expect(res === null || Array.isArray(res)).toBe(true); // real unrar parse covered live + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/rar-ranged.test.ts` +Expected: FAIL — `walkRarVolume`/`readRarListingRanged` not exported. + +- [ ] **Step 3: Write minimal implementation** (append to `rar-ranged.ts`) + +```typescript +import type { FileEntry } from "../zip-reader.js"; +import type { RangeReader } from "./range-reader.js"; +import type { RangedPart } from "./sevenz-ranged.js"; +import { listFromSparse, type SparsePart } from "./sparse-list.js"; +import { readRarContents } from "../rar-reader.js"; +import { childLogger } from "../../util/logger.js"; + +const rlog = childLogger("rar-ranged"); +const MAX_RAR_BLOCKS = 50000; +const HEADER_CHUNK = 8192; + +export async function walkRarVolume( + read: RangeReader, + part: RangedPart, + version: 4 | 5, + sigLen: number, +): Promise<{ offset: number; bytes: Buffer }[] | null> { + const size = Number(part.fileSize); + const regions: { offset: number; bytes: Buffer }[] = []; + let pos = sigLen; + let blocks = 0; + try { + while (pos < size) { + if (++blocks > MAX_RAR_BLOCKS) return null; + const chunkLen = Math.min(HEADER_CHUNK, size - pos); + let chunk = await read(part.fileId, pos, chunkLen, part.fileSize); + const ext = version === 5 ? parseRar5BlockExtent(chunk, 0) : parseRar4BlockExtent(chunk, 0); + // Ensure we have the full header bytes to harvest (long filenames). + let headerBuf = chunk; + if (ext.headerBytes > chunk.length) { + headerBuf = await read(part.fileId, pos, Math.min(ext.headerBytes, size - pos), part.fileSize); + } + regions.push({ offset: pos, bytes: headerBuf.subarray(0, Math.min(ext.headerBytes, size - pos)) }); + if (ext.isEnd) break; + const advance = ext.headerBytes + ext.dataSize; + if (advance <= 0) return null; + if (pos + advance > size) break; // data clamped at the volume boundary (multipart continuation) + pos += advance; + } + return regions; + } catch (err) { + rlog.warn({ err, fileId: part.fileId }, "RAR volume walk failed"); + return null; + } +} + +export async function readRarListingRanged( + parts: RangedPart[], + read: RangeReader, +): Promise { + const sparseParts: SparsePart[] = []; + for (const part of parts) { + const head = await read(part.fileId, 0, 16, part.fileSize); + const sig = detectRarSignature(head); + if (!sig) return null; + const regions = await walkRarVolume(read, part, sig.version, sig.sigLen); + if (!regions) return null; + sparseParts.push({ fileName: part.fileName, size: Number(part.fileSize), regions }); + } + return listFromSparse(sparseParts, readRarContents); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/rar-ranged.test.ts` +Expected: PASS (all). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/ranged/rar-ranged.ts worker/src/archive/ranged/rar-ranged.test.ts +git commit -m "feat(worker): RAR ranged header-walk + single-part listing" +``` + +--- + +## Task 7: RAR multipart walk + +**Files:** +- Test: `worker/src/archive/ranged/rar-ranged.test.ts` (add multipart case) + +**Interfaces:** +- Consumes: `readRarListingRanged` (Task 6) — already loops over `parts`, walking each volume from its own signature and co-locating sparse files under their real names for `unrar` sibling discovery. This task adds a test to lock that behavior in; no new production code unless the test reveals a gap. + +- [ ] **Step 1: Write the failing/【regression】 test** — two volumes, each with its own signature + +```typescript +describe("readRarListingRanged (multipart)", () => { + it("walks each volume from its own signature and reconstructs all parts", async () => { + const vol = buildRar5Volume(); // reuse from Task 6 test + // Two volumes with identical structure; each RangeReader read is scoped by fileId. + const byId: Record = { p1: vol, p2: vol }; + const reads: Record = { p1: 0, p2: 0 }; + const read: RangeReader = async (fileId, offset, length) => { + reads[fileId]++; + return byId[fileId].subarray(offset, offset + length); + }; + const res = await readRarListingRanged( + [ + { fileId: "p1", fileSize: BigInt(vol.length), fileName: "x.part1.rar" }, + { fileId: "p2", fileSize: BigInt(vol.length), fileName: "x.part2.rar" }, + ], + read, + ); + // Both volumes were walked (each read at least its signature + blocks). + expect(reads.p1).toBeGreaterThan(0); + expect(reads.p2).toBeGreaterThan(0); + expect(res === null || Array.isArray(res)).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run test** + +Run: `cd worker && npx vitest run src/archive/ranged/rar-ranged.test.ts` +Expected: PASS — `readRarListingRanged` already loops parts. If it does NOT pass (e.g. it read only `p1`), fix `readRarListingRanged` to iterate all `parts` (it should already), then re-run. + +- [ ] **Step 3: Commit** + +```bash +git add worker/src/archive/ranged/rar-ranged.test.ts +git commit -m "test(worker): lock in RAR multipart per-volume walk" +``` + +--- + +## Task 8: Enable RAR in the dispatcher; deploy & live-verify + +**Files:** +- Modify: `worker/src/provenance-backfill.ts` (route RAR + destination reads through the ranged readers) + +**Interfaces:** +- Consumes: `readRarListingRanged` (Task 6), `readSevenZListingRanged` (Task 3). + +- [ ] **Step 1: Route RAR in the scanned dispatcher** + +In `readScannedListingRanged` (added in Task 4), replace the `// RAR enabled in Task 8.` line so RAR routes to the walker: + +```typescript + if (archiveType === "RAR") return readRarListingRanged(parts, read); +``` + +Add import to `provenance-backfill.ts`: + +```typescript +import { readRarListingRanged } from "./archive/ranged/rar-ranged.js"; +``` + +- [ ] **Step 2: Route 7z/RAR for the destination-copy read** + +Find `readZipListingFromDestination` usage inside `resolveCandidateFingerprintEntries` (around line 113) and generalize it. Change the call: + +```typescript + const destEntries = await readZipListingFromDestination( + client, candidate.destChannel.telegramId, candidate.destMessageIds, candidate.destMessageId, + ); +``` + +to resolve the doc parts once and dispatch by the candidate's archive type: + +```typescript + const destParts = await resolveDestParts( + client, candidate.destChannel.telegramId, candidate.destMessageIds, candidate.destMessageId, + ); + let destEntries: FileEntry[] | null = null; + if (destParts) { + const read = tdlibRangeReader(client); + destEntries = + candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts) + : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read) + : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read) + : null; + } +``` + +Refactor `readZipListingFromDestination` into `resolveDestParts` returning `RangedPart[] | null` (it already resolves each message's document `id` + `size`; also capture the document `file_name` for `fileName`, falling back to the candidate’s stored `fileName` when TDLib omits it). `PlaceholderCandidate` must expose `archiveType` and `fileName` — extend the select in `findPlaceholderCandidates` (`worker/src/db/queries.ts`) and the `PlaceholderCandidate` type to include `archiveType` and `fileName` (both already columns on `packages`). + +- [ ] **Step 3: Typecheck + tests + build** + +```bash +cd worker && npx tsc --noEmit && npx vitest run +``` +Expected: clean. + +- [ ] **Step 4: Commit** + +```bash +git add worker/src/provenance-backfill.ts worker/src/db/queries.ts +git commit -m "feat(worker): enable RAR ranged listing + format-aware destination reads" +``` + +- [ ] **Step 5: Deploy locally (no push)** + +```bash +cd /home/sam/Documents/DragonsStash +docker build -f worker/Dockerfile -t git.samagsteribbe.nl/admin/dragonsstash-worker:latest . +docker compose --project-name dragonsstash --project-directory /opt/stacks/DragonsStash \ + -f /opt/stacks/DragonsStash/docker-compose.yml up -d --no-deps --force-recreate worker +``` + +- [ ] **Step 6: Live-verify RAR (incl. multipart) and the fallback** + +```bash +docker logs -f --since 30s dragonsstash-worker 2>&1 | grep -iE "rar-ranged|Backfilled provenance|full-download fallback|over size cap" +``` + +DB checks: +```sql +-- RAR listings populating without full downloads: +SELECT count(*) FROM packages WHERE "contentHash" LIKE 'rebuild:%' AND "archiveType"='RAR' AND "fileCount">0; +-- Fallback/flag events (should be rare): +SELECT count(*) FROM system_notifications WHERE title LIKE 'Listing skipped%'; +``` +Spot-check: pick one backfilled multipart RAR, compare its `package_files` count to `unrar lt` run on a real download of the same archive. Confirm the 116 GB RAR (if it surfaces) is flagged, not downloaded. + +**Done when:** RAR + 7z placeholders gain accurate `fileCount`/`package_files` via the ranged path, no multi-GB downloads occur except deliberate fallbacks under the size cap, and oversized stragglers are flagged. + +--- + +## Task 4b: 7z encoded-header support (added 2026-07-27 after the Task 4 live spike) + +The Task 4 deploy revealed the two-region (start+end) 7z reconstruction is rejected by `7z l` +for archives with an **encoded/LZMA-compressed header** — the packed header stream lives mid-file, +not at EOF. Fix: parse the encoded header's `PackInfo` and fetch that packed region as a third +sparse region. Full brief with exact code + tests: `.superpowers/sdd/task-4b-brief.md`. Adds +`read7zNumber` + `locate7zEncodedHeaderPack` to `sevenz-ranged.ts` and branches +`readSevenZListingRanged` on the next-header type (`0x01` plain → 2 regions; `0x17` encoded → 3 +regions; else → null/fallback). Sequenced between Task 5 and Task 6; re-verified live at Task 8. + +--- + +## Self-Review + +- **Spec coverage:** 7z ranged read (Tasks 2-4) ✓; RAR walk incl. multipart (Tasks 5-7) ✓; sparse+CLI reconstruction (Task 1) ✓; dispatcher + downstream unchanged (Tasks 4, 8) ✓; size-capped full-download fallback + notification (Task 4) ✓; scanned + destination reads format-aware (Tasks 4, 8) ✓; spike via 7z-first-behind-fallback (Task 4 gate) ✓; unit tests + live verification (throughout) ✓; local deploy recipe (Tasks 4, 8) ✓. +- **Placeholder scan:** no TBD/TODO; every code step has full code; commands have expected output. ✓ +- **Type consistency:** `FileEntry`, `RangedPart` (`{fileId,fileSize,fileName}`), `RangeReader` signature, `SparsePart`, `BlockExtent`, `listFromSparse`/`readSevenZListingRanged`/`readRarListingRanged`/`walkRarVolume`/`fullDownloadListing` names/signatures are consistent across tasks. ✓ diff --git a/docs/superpowers/plans/2026-07-30-forward-priority-ingestion.md b/docs/superpowers/plans/2026-07-30-forward-priority-ingestion.md new file mode 100644 index 0000000..ba77da7 --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-forward-priority-ingestion.md @@ -0,0 +1,1475 @@ +# Forward-Priority Ingestion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** For Telegram source channels that allow forwarding, skip the download+reupload round-trip for new archives — use Telegram-native forwarding to move the file to the destination channel, and the existing ranged (no-download) inner-file listing readers to keep every archive fully indexed. Channels that block forwarding, and any specific archive where the cheap listing fails, keep using today's unchanged download+reupload pipeline. + +**Architecture:** Add one fork point inside `processOneArchiveSet` (`worker/src/worker.ts`), right after the existing pre-download dedup checks and size guard. If the source channel's cached `allowsForwarding` flag is true, attempt a new forward path: read the inner-file listing via the already-built ranged readers (`worker/src/archive/ranged/*`), derive a dedup identity from the listing (no downloaded bytes available), forward the message(s) natively via TDLib, and write the same `Package`/`PackageFile` records the download path would have produced. Any failure in that path (ranged listing miss, ambiguous/blocked forward) falls straight through into the existing, unchanged download/hash/split/upload code below for that one archive. + +**Tech Stack:** TypeScript (strict, ESM, `.js` import specifiers), TDLib via `tdl`, vitest, Prisma/Postgres (`@prisma/client` v7). + +**Design doc:** `docs/superpowers/specs/2026-07-30-forward-priority-ingestion-design.md` — read it before starting if anything below is unclear about *why*, not just *what*. + +## Global Constraints + +- TypeScript strict; ESM import specifiers end in `.js`. Copy the surrounding files' style exactly. +- ESLint does NOT cover `worker/` — but keep types clean; no `any` unless mirroring an existing pattern in the same file. +- Tests: vitest, files match `worker/src/**/*.test.ts`; run from `worker/` with `npx vitest run`. +- All TDLib calls must go through the existing FLOOD_WAIT-safe wrappers (`withFloodWait` from `worker/src/util/retry.js`, or `invokeWithTimeout` from `worker/src/tdlib/download.js`). Do not add a second wrapper on top of an already-wrapped call. +- DB migrations run from the **repo root** (the Prisma schema lives at `prisma/schema.prisma`, shared by all three services), not from `worker/`. Command: `npx prisma migrate dev --name ` (ensure `DATABASE_URL` is set — check `.env` at repo root first before passing it inline). +- No new test framework — this repo uses vitest only for pure/isolable logic (archive parsers, fingerprint math, request-shape builders). The large `processOneArchiveSet`/`runWorkerForAccount` orchestration in `worker.ts` has no automated test coverage today and is verified live post-deploy; this plan follows that same convention rather than inventing new orchestration-level tests. +- Deploy is local (no GitHub push): build `worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image WITHOUT `pull` — same recipe used by the `ranged-archive-listing` work. +- **Task 1 touches the shared `master` branch (merge + push). Do not run its merge/push commands unattended — stop and get explicit human confirmation before executing them, even when running under an autonomous execution skill.** + +--- + +## File Structure + +- Create `worker/src/archive/ranged/dispatch.ts` — the shared (no-download) archive-listing dispatcher, promoted out of `provenance-backfill.ts` so both backfill and fresh ingestion can call it. (+ `dispatch.test.ts`) +- Create `worker/src/archive/forward-identity.ts` — derives a `Package.contentHash`-compatible identity string when there are no downloaded bytes to hash. (+ `forward-identity.test.ts`) +- Create `worker/src/archive/forward-repost-check.ts` — cross-channel CRC-fingerprint duplicate check for the forward path. (+ `forward-repost-check.test.ts`) +- Create `worker/src/upload/forward.ts` — native TDLib forward from source chat to destination chat, mirroring `upload/channel.ts`'s shape. (+ `forward.test.ts`) +- Modify `prisma/schema.prisma` — `TelegramChannel.allowsForwarding`, `IngestionRun.zipsForwarded`. +- Modify `worker/src/db/queries.ts` — `setChannelAllowsForwarding`, `findFingerprintDedupCandidates`, counter plumbing (`ActivityUpdate`, `updateRunActivity`, `completeIngestionRun`). +- Modify `worker/src/provenance-backfill.ts` — import the dispatcher from its new shared location instead of defining it locally; export `resolveCandidateFingerprintEntries` and `compareFingerprints` for reuse. +- Modify `worker/src/worker.ts` — persist the per-channel forwarding flag; add the fork point + `tryForwardArchiveSet` helper; wire the new counter. + +--- + +## Task 1: Merge `feat/ranged-archive-listing` to master, branch for this feature + +**This task requires a human to confirm before the merge/push commands run.** `feat/ranged-archive-listing` is already complete and serves a different purpose (backfilling listings onto already-deduped placeholder packages) — it merges independently of this feature. + +- [ ] **Step 1: Confirm the branch is clean and up to date** + +```bash +cd /path/to/DragonsStash +git status +git log --oneline -5 +``` +Expected: working tree clean (or only expected local files), branch `feat/ranged-archive-listing` up to date with its remote. + +- [ ] **Step 2: STOP — get explicit human confirmation** + +Show the human the commit list that will land on `master` (`git log master..feat/ranged-archive-listing --oneline`) and ask them to confirm before proceeding. Do not continue to Step 3 without an explicit yes. + +- [ ] **Step 3: Merge to master and push** + +```bash +git checkout master +git pull +git merge --no-ff feat/ranged-archive-listing +git push +``` +Expected: fast-forward or clean merge commit, push succeeds. + +- [ ] **Step 4: Branch for this feature** + +```bash +git checkout -b feat/forward-priority-ingestion +``` +Expected: new branch created off the just-updated `master`. + +--- + +## Task 2: Schema — `TelegramChannel.allowsForwarding` + `IngestionRun.zipsForwarded` + +**Files:** +- Modify: `prisma/schema.prisma` + +**Interfaces:** +- Produces: `TelegramChannel.allowsForwarding: boolean | null` (Prisma-generated type, consumed by Task 3 and Task 8), `IngestionRun.zipsForwarded: number` (consumed by Task 8). + +- [ ] **Step 1: Edit `TelegramChannel`** + +In `prisma/schema.prisma`, find: + +```prisma +model TelegramChannel { + id String @id @default(cuid()) + telegramId BigInt @unique + title String + type ChannelType + isForum Boolean @default(false) + isActive Boolean @default(false) + category String? @db.VarChar(64) +``` + +Replace with: + +```prisma +model TelegramChannel { + id String @id @default(cuid()) + telegramId BigInt @unique + title String + type ChannelType + isForum Boolean @default(false) + isActive Boolean @default(false) + category String? @db.VarChar(64) + /// Whether this chat currently allows forwarding/saving (the inverse of + /// TDLib's chat.has_protected_content). Null = not yet checked; treated the + /// same as false everywhere in the worker (safe default: use the + /// download+reupload path until this is confirmed true). + allowsForwarding Boolean? +``` + +- [ ] **Step 2: Edit `IngestionRun`** + +Find: + +```prisma + zipsIngested Int @default(0) + zipsBackfilled Int @default(0) + errorMessage String? +``` + +Replace with: + +```prisma + zipsIngested Int @default(0) + zipsBackfilled Int @default(0) + zipsForwarded Int @default(0) + errorMessage String? +``` + +- [ ] **Step 3: Create and apply the migration** + +```bash +cd /path/to/DragonsStash +npx prisma migrate dev --name add_forward_priority_ingestion +``` +Expected: a new folder under `prisma/migrations/`, migration applied to the local dev DB, Prisma client regenerated (no errors). + +- [ ] **Step 4: Commit** + +```bash +git add prisma/schema.prisma prisma/migrations +git commit -m "feat(db): add TelegramChannel.allowsForwarding + IngestionRun.zipsForwarded" +``` + +--- + +## Task 3: Detect + persist per-channel forwarding permission + +**Files:** +- Modify: `worker/src/db/queries.ts` +- Modify: `worker/src/worker.ts:497-518` (the existing per-channel `getChat` + `isForum` check block, inside `runWorkerForAccount`) + +**Interfaces:** +- Produces: `setChannelAllowsForwarding(channelId: string, allowsForwarding: boolean): Promise` (mirrors the existing `setChannelForum`). +- Consumes: nothing new — reuses the `getChat` call already made per channel per cycle. + +- [ ] **Step 1: Add the query function** + +In `worker/src/db/queries.ts`, find: + +```typescript +export async function setChannelForum(channelId: string, isForum: boolean) { + return db.telegramChannel.update({ + where: { id: channelId }, + data: { isForum }, + }); +} +``` + +Add immediately after it: + +```typescript +export async function setChannelAllowsForwarding(channelId: string, allowsForwarding: boolean) { + return db.telegramChannel.update({ + where: { id: channelId }, + data: { allowsForwarding }, + }); +} +``` + +- [ ] **Step 2: Capture the `getChat` response and persist the flag** + +In `worker/src/worker.ts`, find (inside the per-channel loop in `runWorkerForAccount`): + +```typescript + // ── Ensure TDLib knows about this chat ── + // getChats may not have loaded all channels (pagination, archive folder, etc.) + // so we explicitly load each channel before scanning. + try { + await client.invoke({ + _: "getChat", + chat_id: Number(channel.telegramId), + }); + } catch (chatErr) { + accountLog.warn( + { err: chatErr, channelId: channel.id, title: channel.title, telegramId: channel.telegramId.toString() }, + "TDLib does not know about this chat — it may not be accessible to this account. Skipping." + ); + continue; + } + + // ── Check if channel is a forum ── + const forum = await isChatForum(client, channel.telegramId); + if (forum !== channel.isForum) { + await setChannelForum(channel.id, forum); + accountLog.info( + { channelId: channel.id, title: channel.title, isForum: forum }, + "Updated channel forum status" + ); + } +``` + +Replace with: + +```typescript + // ── Ensure TDLib knows about this chat ── + // getChats may not have loaded all channels (pagination, archive folder, etc.) + // so we explicitly load each channel before scanning. The response is + // also where we read has_protected_content (below) to decide whether + // this channel is eligible for the forward-priority ingestion path. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let chatInfo: any; + try { + chatInfo = await client.invoke({ + _: "getChat", + chat_id: Number(channel.telegramId), + }); + } catch (chatErr) { + accountLog.warn( + { err: chatErr, channelId: channel.id, title: channel.title, telegramId: channel.telegramId.toString() }, + "TDLib does not know about this chat — it may not be accessible to this account. Skipping." + ); + continue; + } + + // ── Check if channel is a forum ── + const forum = await isChatForum(client, channel.telegramId); + if (forum !== channel.isForum) { + await setChannelForum(channel.id, forum); + accountLog.info( + { channelId: channel.id, title: channel.title, isForum: forum }, + "Updated channel forum status" + ); + } + + // ── Check if channel allows forwarding ── + // TDLib's chat.has_protected_content is documented on the general + // Chat object (core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1chat.html), + // but PENDING LIVE VERIFICATION here: confirm on first deploy that a + // real chatTypeSupergroup/channel response actually populates this + // field (some TDLib doc pages describe it in the context of basic + // groups only). If it's ever `undefined` in practice, this block is a + // no-op and allowsForwarding stays at its last-known/null value — + // which safely keeps the channel on the download path. + const hasProtectedContent: boolean | undefined = chatInfo?.has_protected_content; + if (typeof hasProtectedContent === "boolean") { + const allowsForwarding = !hasProtectedContent; + if (allowsForwarding !== channel.allowsForwarding) { + await setChannelAllowsForwarding(channel.id, allowsForwarding); + accountLog.info( + { channelId: channel.id, title: channel.title, allowsForwarding }, + "Updated channel forwarding permission" + ); + } + channel.allowsForwarding = allowsForwarding; + } +``` + +- [ ] **Step 3: Typecheck** + +```bash +cd worker && npx tsc --noEmit +``` +Expected: no errors. (`channel.allowsForwarding` is assignable because `channel` comes from the Prisma-generated `TelegramChannel` type, which now includes the field from Task 2's migration.) + +- [ ] **Step 4: Commit** + +```bash +git add worker/src/db/queries.ts worker/src/worker.ts +git commit -m "feat(worker): detect + persist per-channel forwarding permission" +``` + +--- + +## Task 4: Promote the ranged-listing dispatcher to a shared module + +**Files:** +- Create: `worker/src/archive/ranged/dispatch.ts` +- Test: `worker/src/archive/ranged/dispatch.test.ts` +- Modify: `worker/src/provenance-backfill.ts` + +**Interfaces:** +- Consumes: `parseZipCentralDirectoryFromTail`, `MIN_ZIP_TAIL_BYTES` from `../central-directory.js`; `downloadFileRange` from `../../tdlib/range-download.js`; `readSevenZListingRanged` from `./sevenz-ranged.js`; `readRarListingRanged` from `./rar-ranged.js`; `tdlibRangeReader`, `RangeReader` from `./range-reader.js`; `RangedPart` from `./sevenz-ranged.js`; `FileEntry` from `../zip-reader.js`. +- Produces: `readScannedZipListing(client: Client, parts: { fileId: string; fileSize: bigint }[]): Promise`, `readScannedListingRanged(archiveType: string, client: Client, parts: RangedPart[]): Promise` — both consumed by Task 8 (worker.ts) and already-existing callers in `provenance-backfill.ts`. + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/archive/ranged/dispatch.test.ts +import { describe, it, expect } from "vitest"; +import { readScannedListingRanged } from "./dispatch.js"; + +describe("readScannedListingRanged", () => { + it("returns null for an unknown archive type without calling the reader", async () => { + const read = async () => Buffer.alloc(0); + const result = await readScannedListingRanged( + "DOCUMENT", + { invoke: async () => ({}) } as never, + [{ fileId: "1", fileSize: 100n, fileName: "a.pdf" }], + ); + expect(result).toBeNull(); + void read; // unused placeholder kept out of the dispatch call — DOCUMENT never reaches a reader + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/ranged/dispatch.test.ts` +Expected: FAIL — `Cannot find module './dispatch.js'`. + +- [ ] **Step 3: Create `dispatch.ts` — move `readScannedZipListing` and `readScannedListingRanged` out of `provenance-backfill.ts`** + +```typescript +// worker/src/archive/ranged/dispatch.ts +import type { Client } from "tdl"; +import { downloadFileRange } from "../../tdlib/range-download.js"; +import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "../central-directory.js"; +import { childLogger } from "../../util/logger.js"; +import type { FileEntry } from "../zip-reader.js"; +import { readSevenZListingRanged, type RangedPart } from "./sevenz-ranged.js"; +import { readRarListingRanged } from "./rar-ranged.js"; +import { tdlibRangeReader } from "./range-reader.js"; + +const log = childLogger("ranged-dispatch"); + +/** + * Read a ZIP central directory from the tail of a (possibly multipart) + * archive. `parts` is ordered; only the LAST part carries the EOCD record. + * `fileSize` on each part is that part's own size (NOT the whole-archive + * total) so the download offset stays within that part's bounds, while + * `tailStart` passed to the parser is the logical whole-archive offset + * (preceding parts' sizes + the offset within the last part). + */ +export async function readScannedZipListing( + client: Client, + parts: { fileId: string; fileSize: bigint }[], +): Promise { + if (parts.length === 0) return null; + const lastPart = parts[parts.length - 1]; + const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0); + const lastSize = Number(lastPart.fileSize); + for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) { + const partOffset = Math.max(0, lastSize - tailBytes); + const downloadLen = Math.min(tailBytes, lastSize); + try { + const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize); + const tailStart = precedingSize + partOffset; + return parseZipCentralDirectoryFromTail(buf, tailStart); + } catch (err) { + if (err instanceof RangeError) continue; // try a larger tail + log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed"); + return null; + } + } + return null; +} + +/** + * Dispatch a (no-download) inner-file listing read by archive type. Used both + * by the provenance-backfill path (reading an already-uploaded copy) and the + * forward-priority ingestion path (reading the source channel's copy before + * any download/forward decision is made) — the read itself only needs + * {fileId, fileSize, fileName}, so it doesn't matter which channel the file + * currently lives in. + */ +export async function readScannedListingRanged( + archiveType: string, + client: Client, + parts: RangedPart[], +): Promise { + const read = tdlibRangeReader(client); + if (archiveType === "ZIP") return readScannedZipListing(client, parts); + if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read); + if (archiveType === "RAR") return readRarListingRanged(parts, read); + return null; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/ranged/dispatch.test.ts` +Expected: PASS (1 passed). + +- [ ] **Step 5: Update `provenance-backfill.ts` to import instead of define** + +In `worker/src/provenance-backfill.ts`, delete the `readScannedZipListing` function body (lines ~47-69) and the `readScannedListingRanged` function body (lines ~109-119). + +Change the top-of-file imports from: + +```typescript +import { downloadFileRange } from "./tdlib/range-download.js"; +import { invokeWithTimeout } from "./tdlib/download.js"; +import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js"; +import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js"; +``` + +to: + +```typescript +import { invokeWithTimeout } from "./tdlib/download.js"; +import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js"; +import { readScannedZipListing, readScannedListingRanged } from "./archive/ranged/dispatch.js"; +``` + +(`downloadFileRange`, `parseZipCentralDirectoryFromTail`, and `MIN_ZIP_TAIL_BYTES` are no longer used directly in this file — they're now only used inside `dispatch.ts`. `readSevenZListingRanged`, `readRarListingRanged`, and `tdlibRangeReader` stay imported in `provenance-backfill.ts` as-is; `resolveCandidateFingerprintEntries` still calls them directly for the destination-copy read.) + +- [ ] **Step 6: Typecheck + run the full suite** + +```bash +cd worker && npx tsc --noEmit && npx vitest run +``` +Expected: no TS errors; all existing tests still pass (this is a pure move — behavior is unchanged). + +- [ ] **Step 7: Commit** + +```bash +git add worker/src/archive/ranged/dispatch.ts worker/src/archive/ranged/dispatch.test.ts worker/src/provenance-backfill.ts +git commit -m "refactor(worker): promote ranged-listing dispatcher to a shared module" +``` + +--- + +## Task 5: Dedup-identity derivation for forward-path packages + +**Files:** +- Create: `worker/src/archive/forward-identity.ts` +- Test: `worker/src/archive/forward-identity.test.ts` + +**Interfaces:** +- Consumes: `FileEntry` from `./zip-reader.js`; `crcFingerprint` from `./fingerprint.js`. +- Produces: `deriveForwardContentHash(entries: FileEntry[], remoteUniqueId: string | null, sourceChannelId: string, sourceMessageId: bigint): string` — consumed by Task 8. + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/archive/forward-identity.test.ts +import { describe, it, expect } from "vitest"; +import { createHash } from "crypto"; +import { deriveForwardContentHash } from "./forward-identity.js"; +import type { FileEntry } from "./zip-reader.js"; + +function entry(crc32: string | null): FileEntry { + return { path: "a", fileName: "a", extension: null, compressedSize: 1n, uncompressedSize: 1n, crc32 }; +} + +describe("deriveForwardContentHash", () => { + it("hashes the sorted CRC list when all entries have a CRC32 (ZIP/RAR)", () => { + const entries = [entry("BBBB"), entry("AAAA")]; + const expectedHash = createHash("sha256").update(["aaaa", "bbbb"].join(",")).digest("hex"); + expect(deriveForwardContentHash(entries, "unique-1", "chan-1", 42n)).toBe(`fingerprint:${expectedHash}`); + }); + + it("falls back to remoteUniqueId when CRCs are incomplete (7z today)", () => { + const entries = [entry(null), entry("AAAA")]; + expect(deriveForwardContentHash(entries, "unique-42", "chan-1", 42n)).toBe("forward:unique-42"); + }); + + it("falls back to sourceChannelId+sourceMessageId when there's no CRC and no remoteUniqueId", () => { + const entries = [entry(null)]; + expect(deriveForwardContentHash(entries, null, "chan-1", 42n)).toBe("forward:chan-1:42"); + }); + + it("falls back past an empty entries list the same way", () => { + expect(deriveForwardContentHash([], null, "chan-1", 7n)).toBe("forward:chan-1:7"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/forward-identity.test.ts` +Expected: FAIL — `Cannot find module './forward-identity.js'`. + +- [ ] **Step 3: Write minimal implementation** + +```typescript +// worker/src/archive/forward-identity.ts +import { createHash } from "crypto"; +import { crcFingerprint } from "./fingerprint.js"; +import type { FileEntry } from "./zip-reader.js"; + +/** + * Derive a Package.contentHash-compatible identity string for a forward-path + * package (no downloaded bytes exist to hash directly). Priority order: + * 1. A CRC32-fingerprint hash, when the ranged listing's CRCs are complete + * (ZIP/RAR today) — the strongest available signal, since it lets + * forward-path and download-path copies of the same archive still + * collide/dedupe on identical content. + * 2. TDLib's remote.unique_id, when CRCs are incomplete (7z today has none). + * 3. sourceChannelId+sourceMessageId, as a last-resort unique value so the + * required-unique Package.contentHash column is always satisfiable. + * Follows the same `:` synthetic-hash convention already used + * by `rebuild.ts`'s `rebuild:${destChannelId}:${destMessageId}` placeholder. + */ +export function deriveForwardContentHash( + entries: FileEntry[], + remoteUniqueId: string | null, + sourceChannelId: string, + sourceMessageId: bigint, +): string { + const fp = crcFingerprint(entries); + if (fp.complete && fp.crcs.length > 0) { + const hash = createHash("sha256").update(fp.crcs.join(",")).digest("hex"); + return `fingerprint:${hash}`; + } + if (remoteUniqueId) { + return `forward:${remoteUniqueId}`; + } + return `forward:${sourceChannelId}:${sourceMessageId}`; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/forward-identity.test.ts` +Expected: PASS (4 passed). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/archive/forward-identity.ts worker/src/archive/forward-identity.test.ts +git commit -m "feat(worker): derive a dedup identity for forward-path packages without bytes" +``` + +--- + +## Task 6: Cross-channel CRC-fingerprint repost check + +**Files:** +- Modify: `worker/src/db/queries.ts` +- Modify: `worker/src/provenance-backfill.ts` +- Create: `worker/src/archive/forward-repost-check.ts` +- Test: `worker/src/archive/forward-repost-check.test.ts` + +**Interfaces:** +- Consumes: `PlaceholderCandidate` type from `../db/queries.js`. +- Produces: + - `db/queries.ts`: `findFingerprintDedupCandidates(fileName: string, fileSize: bigint): Promise`. + - `provenance-backfill.ts`: `resolveCandidateFingerprintEntries` and `compareFingerprints` become exported (unchanged behavior, just no longer private). + - `forward-repost-check.ts`: `checkFingerprintRepost(client: Client, entries: FileEntry[], fileName: string, fileSize: bigint): Promise<{ isDuplicate: boolean; matchedPackageId: string | null }>` — consumed by Task 8. + +- [ ] **Step 1: Extract the shared row-enrichment helper and add the new query** + +In `worker/src/db/queries.ts`, find `findPlaceholderCandidates` (around line 1032): + +```typescript +export async function findPlaceholderCandidates( + destChannelId: string, + fileName: string, + fileSize: bigint, +): Promise { + const rows = await db.package.findMany({ + where: { + fileName, + fileSize, + destMessageId: { not: null }, + // Placeholder provenance (spec §1): manual-upload (source == destination) + // OR rebuild record (sourceMessageId == 0 "unknown" sentinel). + OR: [ + { sourceChannelId: destChannelId }, + { sourceMessageId: 0n }, + ], + }, + select: { + id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, destChannelId: true, + }, + orderBy: { indexedAt: "asc" }, + }); + if (rows.length === 0) return []; + + const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))]; + const channels = destChannelIds.length + ? await db.telegramChannel.findMany({ + where: { id: { in: destChannelIds } }, + select: { id: true, telegramId: true }, + }) + : []; + const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId])); + + return rows.map((row) => ({ +``` + +(the function continues with the `.map` return — leave that as-is). Extract the row shape + enrichment into a shared helper by adding this function right ABOVE `findPlaceholderCandidates`: + +```typescript +type PlaceholderRow = { + id: string; archiveType: string; fileName: string; fileCount: number; fileSize: bigint; + destMessageId: bigint | null; destMessageIds: bigint[]; destChannelId: string | null; +}; + +async function enrichWithDestChannel(rows: PlaceholderRow[]): Promise { + if (rows.length === 0) return []; + const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))]; + const channels = destChannelIds.length + ? await db.telegramChannel.findMany({ + where: { id: { in: destChannelIds } }, + select: { id: true, telegramId: true }, + }) + : []; + const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId])); + return rows.map((row) => ({ + id: row.id, archiveType: row.archiveType, fileName: row.fileName, fileCount: row.fileCount, fileSize: row.fileSize, + destMessageId: row.destMessageId, destMessageIds: row.destMessageIds, + destChannel: row.destChannelId && telegramIdById.has(row.destChannelId) + ? { telegramId: telegramIdById.get(row.destChannelId)! } + : null, + })); +} +``` + +Then simplify `findPlaceholderCandidates` to use it — replace the whole function body with: + +```typescript +export async function findPlaceholderCandidates( + destChannelId: string, + fileName: string, + fileSize: bigint, +): Promise { + const rows = await db.package.findMany({ + where: { + fileName, + fileSize, + destMessageId: { not: null }, + // Placeholder provenance (spec §1): manual-upload (source == destination) + // OR rebuild record (sourceMessageId == 0 "unknown" sentinel). + OR: [ + { sourceChannelId: destChannelId }, + { sourceMessageId: 0n }, + ], + }, + select: { + id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, destChannelId: true, + }, + orderBy: { indexedAt: "asc" }, + }); + return enrichWithDestChannel(rows); +} +``` + +Add the new, broader query right after it (no placeholder-only restriction — matches ANY package, any channel, so a forward-path candidate can dedupe against a normal fully-downloaded package elsewhere): + +```typescript +/** + * Find every uploaded Package (any provenance, any channel) matching + * name+size, for the forward-priority path's cross-channel CRC-fingerprint + * dedup check. Unlike findPlaceholderCandidates, this is NOT restricted to + * placeholder rows — it exists to catch the case where the exact same + * archive was independently uploaded (not reposted/forwarded) to two + * different source channels. + */ +export async function findFingerprintDedupCandidates( + fileName: string, + fileSize: bigint, +): Promise { + const rows = await db.package.findMany({ + where: { fileName, fileSize, destMessageId: { not: null } }, + select: { + id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, destChannelId: true, + }, + orderBy: { indexedAt: "asc" }, + }); + return enrichWithDestChannel(rows); +} +``` + +- [ ] **Step 2: Export the two helpers already defined in `provenance-backfill.ts`** + +In `worker/src/provenance-backfill.ts`, find: + +```typescript +async function resolveCandidateFingerprintEntries( +``` + +Change to: + +```typescript +export async function resolveCandidateFingerprintEntries( +``` + +Find: + +```typescript +function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" { +``` + +Change to: + +```typescript +export function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" { +``` + +No other changes in this file — both functions' bodies and all existing call sites are untouched. + +- [ ] **Step 3: Write the failing test for the new orchestrator** + +```typescript +// worker/src/archive/forward-repost-check.test.ts +import { describe, it, expect, vi } from "vitest"; + +const candidate = { + id: "pkg-1", archiveType: "ZIP", fileName: "a.zip", fileCount: 3, fileSize: 100n, + destMessageId: 1n, destMessageIds: [1n], destChannel: { telegramId: 999n }, +}; + +vi.mock("../db/queries.js", () => ({ + findFingerprintDedupCandidates: vi.fn(async () => [candidate]), +})); +const resolveMock = vi.fn(async () => [{ path: "x", fileName: "x", extension: null, compressedSize: 1n, uncompressedSize: 1n, crc32: "AAAA" }]); +const compareMock = vi.fn(); +vi.mock("../provenance-backfill.js", () => ({ + resolveCandidateFingerprintEntries: (...args: unknown[]) => resolveMock(...args), + compareFingerprints: (...args: unknown[]) => compareMock(...args), +})); + +import { checkFingerprintRepost } from "./forward-repost-check.js"; +import type { FileEntry } from "./zip-reader.js"; + +const newEntries: FileEntry[] = [{ path: "x", fileName: "x", extension: null, compressedSize: 1n, uncompressedSize: 1n, crc32: "AAAA" }]; + +describe("checkFingerprintRepost", () => { + it("reports a duplicate when a candidate's fingerprint matches", async () => { + compareMock.mockReturnValueOnce("match"); + const result = await checkFingerprintRepost({} as never, newEntries, "a.zip", 100n); + expect(result).toEqual({ isDuplicate: true, matchedPackageId: "pkg-1" }); + }); + + it("reports no duplicate when no candidate matches", async () => { + compareMock.mockReturnValueOnce("mismatch"); + const result = await checkFingerprintRepost({} as never, newEntries, "a.zip", 100n); + expect(result).toEqual({ isDuplicate: false, matchedPackageId: null }); + }); +}); +``` + +- [ ] **Step 4: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/archive/forward-repost-check.test.ts` +Expected: FAIL — `Cannot find module './forward-repost-check.js'`. + +- [ ] **Step 5: Write minimal implementation** + +```typescript +// worker/src/archive/forward-repost-check.ts +import type { Client } from "tdl"; +import type { FileEntry } from "./zip-reader.js"; +import { compareFingerprints, resolveCandidateFingerprintEntries } from "../provenance-backfill.js"; +import { findFingerprintDedupCandidates } from "../db/queries.js"; + +export interface FingerprintRepostResult { + isDuplicate: boolean; + matchedPackageId: string | null; +} + +/** + * Cross-channel duplicate check for the forward-priority path: compare the + * new archive's CRC fingerprint against every existing Package sharing its + * name+size, regardless of which channel or ingestion path produced them. + * This is what lets a forwarded copy dedupe against a previously + * fully-downloaded copy of the same archive, despite never sharing a + * byte-hash-derived contentHash. + */ +export async function checkFingerprintRepost( + client: Client, + entries: FileEntry[], + fileName: string, + fileSize: bigint, +): Promise { + const candidates = await findFingerprintDedupCandidates(fileName, fileSize); + for (const candidate of candidates) { + const candidateEntries = await resolveCandidateFingerprintEntries(client, candidate); + if (compareFingerprints(entries, candidateEntries) === "match") { + return { isDuplicate: true, matchedPackageId: candidate.id }; + } + } + return { isDuplicate: false, matchedPackageId: null }; +} +``` + +- [ ] **Step 6: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/archive/forward-repost-check.test.ts` +Expected: PASS (2 passed). + +- [ ] **Step 7: Typecheck the whole worker** + +```bash +cd worker && npx tsc --noEmit +``` +Expected: no errors. + +- [ ] **Step 8: Commit** + +```bash +git add worker/src/db/queries.ts worker/src/provenance-backfill.ts worker/src/archive/forward-repost-check.ts worker/src/archive/forward-repost-check.test.ts +git commit -m "feat(worker): cross-channel CRC-fingerprint repost check for the forward path" +``` + +--- + +## Task 7: `forwardArchiveToChannel` + +**Files:** +- Create: `worker/src/upload/forward.ts` +- Test: `worker/src/upload/forward.test.ts` + +**Interfaces:** +- Consumes: `withFloodWait` from `../util/retry.js`. +- Produces: `interface ForwardResult { messageId: bigint; messageIds: bigint[] }`, `forwardArchiveToChannel(client: Client, fromChatId: bigint, toChatId: bigint, sourceMessageIds: bigint[]): Promise` — consumed by Task 8. + +TDLib reference (confirmed via docs, `forwardMessages`): `chat_id` (destination), `topic_id` (pass `null`), `from_chat_id` (source), `message_ids` (int53 array, **must be in strictly increasing order**, max 100 per call), `options` (pass `null` for defaults), `send_copy` (`false` = plain forward, keeps "Forwarded from" attribution; the destination archive channel is not user-facing so this plan keeps it simple and forwards plainly), `remove_caption` (`false`, only relevant when `send_copy` is true). Response is `{ messages: (Message | null)[] }` in the same order as the request — Telegram returns `null` for any message that couldn't be forwarded (e.g. if `has_protected_content` blocks it), which this function treats as a failure. + +- [ ] **Step 1: Write the failing test** + +```typescript +// worker/src/upload/forward.test.ts +import { describe, it, expect, vi } from "vitest"; +import { forwardArchiveToChannel } from "./forward.js"; + +function fakeClient(response: unknown) { + return { invoke: vi.fn(async () => response) } as never; +} + +describe("forwardArchiveToChannel", () => { + it("sorts message ids ascending and sends them via forwardMessages", async () => { + const invoke = vi.fn(async (req: { message_ids: number[] }) => ({ + messages: req.message_ids.map((id) => ({ id: id + 1000 })), + })); + const client = { invoke } as never; + + const result = await forwardArchiveToChannel(client, 111n, 222n, [30n, 10n, 20n]); + + expect(invoke).toHaveBeenCalledWith( + expect.objectContaining({ + _: "forwardMessages", + chat_id: 222, + from_chat_id: 111, + message_ids: [10, 20, 30], + send_copy: false, + }), + ); + expect(result.messageId).toBe(1010n); + expect(result.messageIds).toEqual([1010n, 1020n, 1030n]); + }); + + it("throws when Telegram returns null for a message (can't be forwarded)", async () => { + const client = fakeClient({ messages: [{ id: 1001 }, null] }); + await expect(forwardArchiveToChannel(client, 111n, 222n, [10n, 20n])).rejects.toThrow(/could not forward/); + }); + + it("throws when the response has the wrong number of messages", async () => { + const client = fakeClient({ messages: [{ id: 1001 }] }); + await expect(forwardArchiveToChannel(client, 111n, 222n, [10n, 20n])).rejects.toThrow(/expected 2/); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd worker && npx vitest run src/upload/forward.test.ts` +Expected: FAIL — `Cannot find module './forward.js'`. + +- [ ] **Step 3: Write minimal implementation** + +```typescript +// worker/src/upload/forward.ts +import type { Client } from "tdl"; +import { childLogger } from "../util/logger.js"; +import { withFloodWait } from "../util/retry.js"; + +const log = childLogger("forward"); + +export interface ForwardResult { + messageId: bigint; + messageIds: bigint[]; +} + +/** + * Forward all parts of an archive set from the source chat directly to the + * destination chat via TDLib's forwardMessages — no download, no re-upload. + * Only usable when the source channel allows forwarding + * (TelegramChannel.allowsForwarding); the caller is responsible for that + * check. message_ids must be in strictly increasing order per the TDLib API, + * so this always sorts them regardless of the order they're passed in. + */ +export async function forwardArchiveToChannel( + client: Client, + fromChatId: bigint, + toChatId: bigint, + sourceMessageIds: bigint[], +): Promise { + const sortedIds = [...sourceMessageIds].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + const numericIds = sortedIds.map((id) => Number(id)); + + log.info( + { fromChatId: Number(fromChatId), toChatId: Number(toChatId), count: numericIds.length }, + "Forwarding archive to destination channel" + ); + + const result = (await withFloodWait( + () => + client.invoke({ + _: "forwardMessages", + chat_id: Number(toChatId), + topic_id: null, + from_chat_id: Number(fromChatId), + message_ids: numericIds, + options: null, + send_copy: false, + remove_caption: false, + } as never), + "forwardMessages" + )) as { messages: ({ id: number } | null)[] }; + + const forwarded = result.messages; + if (!forwarded || forwarded.length !== numericIds.length) { + throw new Error( + `forwardMessages returned ${forwarded?.length ?? 0} messages, expected ${numericIds.length}` + ); + } + + const messageIds: bigint[] = []; + for (let i = 0; i < forwarded.length; i++) { + const msg = forwarded[i]; + if (!msg) { + throw new Error( + `forwardMessages could not forward source message ${sortedIds[i]} (Telegram returned null — message may not be forwardable)` + ); + } + messageIds.push(BigInt(msg.id)); + } + + log.info( + { fromChatId: Number(fromChatId), toChatId: Number(toChatId), messageIds: messageIds.map(Number) }, + "Forward confirmed by Telegram" + ); + + return { messageId: messageIds[0], messageIds }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd worker && npx vitest run src/upload/forward.test.ts` +Expected: PASS (3 passed). + +- [ ] **Step 5: Commit** + +```bash +git add worker/src/upload/forward.ts worker/src/upload/forward.test.ts +git commit -m "feat(worker): native TDLib forward from source to destination channel" +``` + +--- + +## Task 8: Wire the fork point into `processOneArchiveSet` + +**Files:** +- Modify: `worker/src/db/queries.ts` (counter plumbing) +- Modify: `worker/src/worker.ts` (fork point, new helper function, counter wiring, `inferSkipReason`) + +**Interfaces:** +- Consumes: `readScannedListingRanged` (Task 4), `deriveForwardContentHash` (Task 5), `checkFingerprintRepost` (Task 6), `forwardArchiveToChannel` (Task 7). +- Produces: no new exports — this is the integration task. No isolated unit test (matches this file's existing convention of live-only verification for orchestration code); verified in Tasks 9-11. + +- [ ] **Step 1: Add `zipsForwarded` to the counter plumbing in `db/queries.ts`** + +Find the `ActivityUpdate` interface's `zipsBackfilled` field (around line 379): + +```typescript + zipsBackfilled?: number; +} +``` + +Add a sibling field: + +```typescript + zipsBackfilled?: number; + zipsForwarded?: number; +} +``` + +Find, in `updateRunActivity` (around line 403): + +```typescript + ...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }), +``` + +Add immediately after: + +```typescript + ...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }), + ...(activity.zipsForwarded !== undefined && { zipsForwarded: activity.zipsForwarded }), +``` + +Find `completeIngestionRun`'s counters parameter type (around line 429): + +```typescript + counters: { + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + zipsBackfilled: number; + } +``` + +Add a sibling field: + +```typescript + counters: { + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + zipsBackfilled: number; + zipsForwarded: number; + } +``` + +- [ ] **Step 2: Add `zipsForwarded` to `PipelineContext.counters` and its initializer in `worker.ts`** + +Find (around line 313-319): + +```typescript + counters: { + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + zipsBackfilled: number; + }; +``` + +Replace with: + +```typescript + counters: { + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + zipsBackfilled: number; + zipsForwarded: number; + }; +``` + +Find (around line 422-428): + +```typescript + const counters = { + messagesScanned: 0, + zipsFound: 0, + zipsDuplicate: 0, + zipsIngested: 0, + zipsBackfilled: 0, + }; +``` + +Replace with: + +```typescript + const counters = { + messagesScanned: 0, + zipsFound: 0, + zipsDuplicate: 0, + zipsIngested: 0, + zipsBackfilled: 0, + zipsForwarded: 0, + }; +``` + +- [ ] **Step 3: Add the new imports** + +Find the import block at the top of `worker.ts` (near the other `archive/*` imports): + +```typescript +import { readZipCentralDirectory } from "./archive/zip-reader.js"; +import { readRarContents } from "./archive/rar-reader.js"; +import { read7zContents } from "./archive/sevenz-reader.js"; +``` + +Add immediately after: + +```typescript +import { readScannedListingRanged } from "./archive/ranged/dispatch.js"; +import { deriveForwardContentHash } from "./archive/forward-identity.js"; +import { checkFingerprintRepost } from "./archive/forward-repost-check.js"; +import { forwardArchiveToChannel } from "./upload/forward.js"; +``` + +- [ ] **Step 4: Extend `inferSkipReason` to recognize forward failures** + +Find: + +```typescript +function inferSkipReason(errMsg: string): "DOWNLOAD_FAILED" | "UPLOAD_FAILED" | "EXTRACT_FAILED" { + const lower = errMsg.toLowerCase(); + if (lower.includes("upload") || lower.includes("too many requests") || lower.includes("retry after") || lower.includes("send")) { + return "UPLOAD_FAILED"; + } +``` + +Replace with: + +```typescript +function inferSkipReason(errMsg: string): "DOWNLOAD_FAILED" | "UPLOAD_FAILED" | "EXTRACT_FAILED" { + const lower = errMsg.toLowerCase(); + if (lower.includes("upload") || lower.includes("forward") || lower.includes("too many requests") || lower.includes("retry after") || lower.includes("send")) { + return "UPLOAD_FAILED"; + } +``` + +- [ ] **Step 5: Add the `tryForwardArchiveSet` helper** + +Find the closing brace of `processOneArchiveSet` (search for `async function deleteFiles(paths: string[])` — the helper goes right BEFORE that function, i.e. right after `processOneArchiveSet` ends). Insert: + +```typescript +/** + * Attempt the forward-priority path for one archive set: ranged listing (no + * download) + native Telegram forward to the destination channel. + * + * Returns `undefined` when the forward path isn't usable for this specific + * archive (ranged listing failed, or the forward itself failed) — the caller + * falls through to the existing download+reupload pipeline in that case, so + * indexing completeness never regresses. + * + * Returns `null` when the archive is a confirmed duplicate (skip, same + * contract as the pre-download dedup checks earlier in the caller). + * + * Returns the new Package id on success. + */ +async function tryForwardArchiveSet( + ctx: PipelineContext, + archiveSet: ArchiveSet, + setIdx: number, + totalSets: number, + previewMatches: Map, + ingestionRunId: string, +): Promise { + const { + client, channelTitle, channel, + destChannelTelegramId, destChannelId, + counters, topicCreator, sourceTopicId, accountLog, + } = ctx; + void setIdx; + void totalSets; + + const archiveName = archiveSet.parts[0].fileName; + const archType = archiveSet.type === "7Z" ? ("SEVEN_Z" as const) : archiveSet.type; + if (archType !== "ZIP" && archType !== "RAR" && archType !== "SEVEN_Z") { + // The ranged listing readers only cover archive formats. Standalone + // DOCUMENT attachments always go through the existing download path, + // which for DOCUMENT is already cheap (no extraction, single entry). + return undefined; + } + + const scannedParts = archiveSet.parts.map((p) => ({ + fileId: p.fileId, + fileSize: p.fileSize, + fileName: p.fileName, + })); + + const entries = await readScannedListingRanged(archType, client, scannedParts); + if (!entries) return undefined; + + const totalArchiveSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n); + const firstRemoteUniqueId = archiveSet.parts[0].remoteUniqueId ?? null; + const contentHash = deriveForwardContentHash( + entries, + firstRemoteUniqueId, + channel.id, + archiveSet.parts[0].id, + ); + + if (await packageExistsByHash(contentHash)) { + counters.zipsDuplicate++; + accountLog.debug({ fileName: archiveName, contentHash }, "Forward-path duplicate (hash), skipping"); + return null; + } + + const repost = await checkFingerprintRepost(client, entries, archiveName, totalArchiveSize); + if (repost.isDuplicate) { + counters.zipsDuplicate++; + accountLog.info( + { fileName: archiveName, matchedPackageId: repost.matchedPackageId }, + "Forward-path duplicate (CRC fingerprint match against another channel's copy), skipping" + ); + return null; + } + + const hashLockAcquired = await tryAcquireHashLock(contentHash); + if (!hashLockAcquired) { + counters.zipsDuplicate++; + accountLog.info( + { fileName: archiveName, contentHash }, + "Hash lock held by another worker — skipping concurrent duplicate" + ); + return null; + } + + try { + if (await packageExistsByHash(contentHash)) { + counters.zipsDuplicate++; + return null; + } + + let destResult: { messageId: bigint; messageIds: bigint[] }; + try { + destResult = await forwardArchiveToChannel( + client, + channel.telegramId, + destChannelTelegramId, + archiveSet.parts.map((p) => p.id), + ); + } catch (forwardErr) { + accountLog.warn( + { err: forwardErr, fileName: archiveName }, + "Forward failed — falling back to download+reupload for this archive" + ); + return undefined; + } + + await deleteOrphanedPackageByHash(contentHash); + + const creator = + topicCreator ?? + extractCreatorFromFileName(archiveName) ?? + extractCreatorFromChannelTitle(channelTitle) ?? + null; + + const tags: string[] = []; + if (channel.category) tags.push(channel.category); + for (const tag of extractSlicerTags(entries)) { + if (!tags.includes(tag)) tags.push(tag); + } + + const stub = await createPackageStub({ + contentHash, + fileName: archiveName, + fileSize: totalArchiveSize, + archiveType: archType, + sourceChannelId: channel.id, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId, + remoteUniqueId: firstRemoteUniqueId, + destChannelId, + destMessageId: destResult.messageId, + destMessageIds: destResult.messageIds, + isMultipart: archiveSet.parts.length > 1, + partCount: archiveSet.parts.length, + ingestionRunId, + creator, + tags, + }); + + counters.zipsForwarded++; + await deleteSkippedPackage(channel.id, archiveSet.parts[0].id); + + let previewData: Buffer | null = null; + let previewMsgId: bigint | null = null; + const matchedPhoto = previewMatches.get(archiveSet.baseName); + if (matchedPhoto) { + previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId); + if (previewData) previewMsgId = matchedPhoto.id; + } + + await updatePackageWithMetadata(stub.id, { files: entries, previewData, previewMsgId }); + + accountLog.info( + { fileName: archiveName, contentHash, fileCount: entries.length, creator }, + "Archive forwarded (no download)" + ); + + return stub.id; + } finally { + await releaseHashLock(contentHash); + } +} +``` + +- [ ] **Step 6: Insert the fork point in `processOneArchiveSet`** + +Find the end of the size-guard block: + +```typescript + await upsertSkippedPackage({ + fileName: archiveName, + fileSize: totalArchiveSize, + reason: "SIZE_LIMIT", + sourceChannelId: channel.id, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId: ctx.sourceTopicId, + isMultipart: archiveSet.isMultipart, + partCount: archiveSet.parts.length, + accountId: ctx.accountId, + }); + return null; + } + + const tempPaths: string[] = []; +``` + +Replace with: + +```typescript + await upsertSkippedPackage({ + fileName: archiveName, + fileSize: totalArchiveSize, + reason: "SIZE_LIMIT", + sourceChannelId: channel.id, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId: ctx.sourceTopicId, + isMultipart: archiveSet.isMultipart, + partCount: archiveSet.parts.length, + accountId: ctx.accountId, + }); + return null; + } + + // ── Forward-priority path ── + // If the source channel allows forwarding, try to index + forward without a + // local download. Any failure (ranged listing miss, blocked/failed forward) + // falls through into the existing download pipeline below so indexing + // completeness never regresses. + if (channel.allowsForwarding === true) { + const forwardResult = await tryForwardArchiveSet( + ctx, archiveSet, setIdx, totalSets, previewMatches, ingestionRunId + ); + if (forwardResult !== undefined) { + return forwardResult; + } + accountLog.info( + { fileName: archiveName }, + "Forward path unavailable for this archive — falling back to download+reupload" + ); + } + + const tempPaths: string[] = []; +``` + +- [ ] **Step 7: Thread `zipsForwarded` through the final run summary** + +Find, near the end of `runWorkerForAccount`: + +```typescript + await throttled.flush(); + await completeIngestionRun(activeRunId, counters); + accountLog.info({ counters }, "Ingestion run completed"); +``` + +This already passes the whole `counters` object, which now includes `zipsForwarded` from Step 2 — no change needed here, just confirming it flows through. (No edit — verification only.) + +- [ ] **Step 8: Typecheck** + +```bash +cd worker && npx tsc --noEmit +``` +Expected: no errors. `archiveSet.parts[0].id` is `bigint` (from `TelegramMessage.id`); `channel.telegramId` is `bigint` (from Prisma) — both line up with `forwardArchiveToChannel`'s signature. + +- [ ] **Step 9: Commit** + +```bash +git add worker/src/db/queries.ts worker/src/worker.ts +git commit -m "feat(worker): fork to the forward-priority path in processOneArchiveSet" +``` + +--- + +## Task 9: Full typecheck + test suite + +- [ ] **Step 1: Run the full worker test suite** + +```bash +cd worker && npx tsc --noEmit && npx vitest run +``` +Expected: no TS errors; all tests pass (existing archive/ranged/*.test.ts tests plus all new tests added in Tasks 4-7). + +- [ ] **Step 2: Fix anything that fails, then re-run until clean.** + +--- + +## Task 10: Local build + deploy (no push) + +Same recipe as the `ranged-archive-listing` work — this repo does not push worker images to a registry; the container is rebuilt and recreated locally. + +- [ ] **Step 1: Build** + +```bash +cd /path/to/DragonsStash +docker build -f worker/Dockerfile -t git.samagsteribbe.nl/admin/dragonsstash-worker:latest . +``` +Expected: image builds successfully. + +- [ ] **Step 2: Recreate the running container** + +```bash +docker compose --project-name dragonsstash --project-directory /opt/stacks/DragonsStash \ + -f /opt/stacks/DragonsStash/docker-compose.yml up -d --no-deps --force-recreate worker +``` +Expected: `dragonsstash-worker` container recreated from the local image, starts cleanly. + +- [ ] **Step 3: Watch startup logs** + +```bash +docker logs -f --since 30s dragonsstash-worker +``` +Expected: no errors on startup; first ingestion cycle begins on schedule. + +--- + +## Task 11: Live verification + +Requires two test source channels already linked to a worker account: one with forwarding allowed, one with "restrict saving content" enabled (create/toggle via `toggleChatHasProtectedContent` or the Telegram client UI if you don't already have one). + +- [ ] **Step 1: Watch for the forwarding-permission detection** + +```bash +docker logs -f --since 30s dragonsstash-worker 2>&1 | grep -iE "Updated channel forwarding permission" +``` +Expected: both test channels get their `allowsForwarding` flag set correctly on the next scan cycle (confirm against the DB: `SELECT title, "allowsForwarding" FROM telegram_channels WHERE title IN ('', '');`). + +- [ ] **Step 2: Post a fresh archive into the forwarding-enabled test channel** + +```bash +docker logs -f --since 30s dragonsstash-worker 2>&1 | grep -iE "Archive forwarded|Forward path unavailable|Forward failed" +``` +Expected: `Archive forwarded (no download)` appears, with no matching `Downloading archive part` log line for that file. + +DB check: +```sql +SELECT "fileName", "contentHash", "fileCount", "destMessageId", "archiveType" +FROM packages ORDER BY "indexedAt" DESC LIMIT 5; +``` +Expected: the new row has `fileCount > 0`, a `contentHash` prefixed `fingerprint:` or `forward:`, and a non-null `destMessageId`. Spot-check `fileCount` against a real `unzip -l` / `unrar lt` / `7z l` on a manually-downloaded copy of the same file. + +- [ ] **Step 3: Post a fresh archive into the protected-content test channel** + +Expected: the existing download+reupload path runs unchanged (`Downloading archive part` appears in logs), and the resulting Package has a normal sha256-style `contentHash` (no `fingerprint:`/`forward:` prefix). + +- [ ] **Step 4: Force a ranged-listing failure in the forwarding-enabled channel** + +Post a deliberately-corrupted or unsupported-format archive (e.g. a password-protected-header 7z) into the forwarding-enabled test channel. + +Expected: `Forward path unavailable for this archive — falling back to download+reupload` appears, followed by the normal download pipeline completing successfully — the package still ends up with `fileCount > 0`. + +- [ ] **Step 5: Confirm bot delivery still works for a forwarded package** + +Use the bot to request the forwarded package from Step 2. Expected: delivery succeeds via the existing `copyMessageToUser` path (unaffected by this feature — it already reads `destMessageId` the same way regardless of how the package was ingested). + +--- + +## Self-Review + +- **Spec coverage:** sequencing/merge-first (Task 1) ✓; `allowsForwarding` schema + detection (Tasks 2-3) ✓; shared ranged-listing dispatcher (Task 4) ✓; dedup identity incl. rebuild:-style fallback chain (Task 5) ✓; cross-channel fingerprint repost check (Task 6) ✓; native forward (Task 7) ✓; fork point + fallback-to-download semantics + zipsForwarded observability (Task 8) ✓; typecheck/tests (Task 9) ✓; local no-push deploy (Task 10) ✓; live verification of both channel types plus the ranged-listing-failure fallback (Task 11) ✓. Non-goals from the spec (no ranged single-entry preview extraction, no reprocessing existing packages, no bot-delivery changes, no size-guard/split changes) require no tasks — confirmed nothing in this plan touches them. +- **Placeholder scan:** no TBD/TODO; every code step has full code; every command has an expected outcome. The one open item (`has_protected_content` field availability on supergroup/channel chats) is explicitly flagged as PENDING LIVE VERIFICATION in Task 3, Step 2's comment and covered by Task 11, Step 1 — not a placeholder, a spike already scheduled for live verification, matching this repo's own established convention for the same kind of TDLib-behavior uncertainty (`range-download.ts`'s absolute-offset note). +- **Type consistency:** `RangedPart` (`{fileId, fileSize, fileName}`), `FileEntry`, `PlaceholderCandidate`, `ForwardResult` (`{messageId, messageIds}`), and `deriveForwardContentHash`/`checkFingerprintRepost`/`forwardArchiveToChannel`/`tryForwardArchiveSet` signatures are consistent everywhere they're referenced across tasks. ✓ diff --git a/docs/superpowers/specs/2026-07-23-nas-backup-design.md b/docs/superpowers/specs/2026-07-23-nas-backup-design.md new file mode 100644 index 0000000..9cdd86f --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-nas-backup-design.md @@ -0,0 +1,205 @@ +# NAS Backup for Postgres + TDLib State — Design + +**Date:** 2026-07-23 +**Status:** Approved for planning + +## Summary + +Add a dedicated `backup` container to the DragonsStash stack that takes daily, +encrypted, deduplicated backups of the two things that can't be regenerated — +the Postgres database (inventory/STL metadata, users, everything the app +manages) and the two TDLib state volumes (Telegram session/auth state for the +worker and bot) — and ships them to a Synology NAS over NFS. STL archive +contents themselves are explicitly out of scope: they only live on this host +temporarily and are not backed up. + +Backups are stored via [restic](https://restic.net/), which provides +encryption-at-rest, block-level dedup, and retention pruning natively, so no +custom encryption or pruning scripts need to be written or maintained. + +## Context + +Current state (as of this design): + +- Production stack runs from `/opt/stacks/DragonsStash/docker-compose.yml` on + this Dockge-managed host, pulling prebuilt images from + `git.samagsteribbe.nl`. The `docker-compose.yml` in this repo is the + build/dev reference and should be kept in sync. +- Named volumes in use: `postgres_data` (Postgres 16 data directory), + `tdlib_state` (worker's TDLib session), `tdlib_bot_state` (bot's TDLib + session), `tmp_zips` and `manual_uploads` (both transient, explicitly out of + scope here). +- No backup mechanism, NFS mount, or host cron currently exists anywhere in + this deployment. +- The host already runs Uptime Kuma (used here for backup alerting) and Loki + (container logs are presumably already collected there). + +## Requirements + +1. Daily backup of the Postgres database and both TDLib state volumes. +2. Backups stored on a Synology NAS via NFS, not on local disk. +3. 14-day retention, oldest snapshots pruned automatically. +4. Backups encrypted at rest (Postgres dumps and TDLib session files both + contain sensitive material — password hashes, Telegram API secrets, live + session state). +5. Postgres backups must be transactionally consistent regardless of live app + traffic. TDLib state backups are best-effort (see Decisions below) — this + is an accepted trade-off, not a defect. +6. Alert (via existing Uptime Kuma) if a backup run fails or doesn't happen. +7. No new host-level state (no `/etc/fstab` entries, no host crontab) — the + backup mechanism should be a container, consistent with how everything + else on this host is deployed and versioned. +8. No new privileged access — specifically, the backup container must not + have Docker socket access or any ability to control sibling containers. + +## Decisions + +- **NFS mounted via Docker's native NFS volume driver** (`driver_opts: type: + nfs`), not a host-level mount. Keeps all backup-related state inside the + compose file instead of split across host config. +- **Restic, not hand-rolled tar+age+find.** Restic already solves encryption, + dedup, and retention correctly; hand-rolled scripts would be reinventing + that logic with more room for bugs. +- **TDLib state is tarred live (best-effort), not paused.** Pausing the + worker/bot for a clean snapshot would require mounting the Docker socket + into the backup container so it could stop/start sibling containers — a + real privilege escalation (a compromised backup container could then + control any container on the host). The downside of a best-effort tar is + bounded: worst case, a bad TDLib restore means redoing the Telegram SMS + auth flow, which is the same outcome as having no backup at all. That + bounded, low-severity downside doesn't justify the privilege escalation. +- **Fixed-time cron (`crond`), not a sleep-loop.** A `sleep 86400` loop drifts + on every container restart; a real crontab entry fires at a fixed time of + day regardless of restarts, for negligible extra complexity. +- **Restore is manual, not automated.** A script capable of restoring can + overwrite live state; that should always require a human deliberately + running it, not run unattended. + +## Design + +### New service: `backup` + +Added to both `/opt/stacks/DragonsStash/docker-compose.yml` (production) and +this repo's `docker-compose.yml` (dev/build reference). + +- **Image**: custom, `FROM alpine:3.20`, `apk add --no-cache restic + postgresql16-client curl tzdata dcron tar bash`. No dependency on app + source — independent Dockerfile, e.g. `backup/Dockerfile`. +- **Scheduling**: `crond -f` in the foreground as the container's entrypoint, + with a crontab installed at build time: + ``` + 0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1 + 0 4 * * 0 restic check >> /proc/1/fd/1 2>&1 + ``` + (daily dump/backup at 03:00, weekly repo integrity check at 04:00 Sunday). + `restic init` runs once at container startup (entrypoint, before `crond` + starts), swallowing the "already initialized" error on subsequent + container (re)starts. +- **Network**: `internal` only — reaches `dragonsstash-db:5432` for + `pg_dump`. No ports exposed. +- **Volumes**: + - `tdlib_state:/data/tdlib-worker:ro` + - `tdlib_bot_state:/data/tdlib-bot:ro` + - `nas_backups:/backups`, a named volume defined with: + ```yaml + nas_backups: + driver_opts: + type: nfs + o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100" + device: ":${NAS_EXPORT_PATH}" + ``` +- **New `.env` entries**: `NAS_HOST`, `NAS_EXPORT_PATH` (NFS share details), + `RESTIC_PASSWORD` (repo encryption key), `KUMA_PUSH_URL` (Uptime Kuma push + monitor URL). All four are inputs to gather during implementation, not + hardcoded. +- `restart: unless-stopped`, no `privileged`, no Docker socket mount. + +### `backup.sh` + +``` +set -euo pipefail +trap 'curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=down" \ + --data-urlencode "msg=$BASH_COMMAND failed"' ERR + +pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -Fc -f /tmp/dragonsstash.dump + +tar czf /tmp/tdlib.tar.gz -C /data tdlib-worker tdlib-bot + +restic backup /tmp/dragonsstash.dump /tmp/tdlib.tar.gz +restic forget --keep-daily 14 --prune + +rm -f /tmp/dragonsstash.dump /tmp/tdlib.tar.gz + +curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=up" \ + --data-urlencode "msg=OK" +``` + +`PGPASSWORD` and `RESTIC_REPOSITORY=/backups/restic-repo` are set as +container environment variables (from `.env`), not inline in the script. + +### Data flow + +``` +crond (daily 03:00) + → pg_dump (consistent snapshot via Postgres MVCC) → /tmp/dragonsstash.dump + → tar tdlib_state + tdlib_bot_state (best-effort, live) → /tmp/tdlib.tar.gz + → restic backup (encrypt + dedup) → NFS-mounted repo on Synology NAS + → restic forget --keep-daily 14 --prune + → curl Uptime Kuma push monitor (up on success, down + reason on any failure) +``` + +### Restore (manual, documented procedure — not scripted/automated) + +``` +restic -r /backups/restic-repo restore latest --target /tmp/restore +pg_restore -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + --clean --if-exists /tmp/restore/tmp/dragonsstash.dump +# untar /tmp/restore/tmp/tdlib.tar.gz back into the tdlib_state / +# tdlib_bot_state volumes (via a throwaway container mounting both) +``` + +## Alerting + +- One Uptime Kuma **Push** monitor, created manually in the existing Kuma + instance, with an expected heartbeat interval of ~26 hours (slack past the + 24h schedule so one slow run doesn't false-positive). Whatever notification + channels are already configured on that monitor fire automatically — no new + alerting integration. +- `backup.sh` pushes `status=up` on success and `status=down` (with the + failing command in `msg`) on any failure, via the `ERR` trap. +- Container logs go to stdout, collected the same way every other container's + logs already are on this host. + +## Testing + +This repo has no automated test framework (documented convention: manual +testing). For this infra change: + +- After deploy: manually run `docker exec dragonsstash-backup /backup.sh` + once, confirm a snapshot appears (`restic snapshots`), confirm the Kuma + monitor goes green. +- **Restore drill** (once, during setup): actually restore the dump into a + scratch Postgres and untar the TDLib archive into scratch volumes, to prove + the backup is really restorable. Not automated or recurring for now. + +## Out of scope / non-goals + +- Backing up `tmp_zips` or `manual_uploads` — both transient by design. +- Automated/scheduled restore testing. +- Backing up any other stack on this host (this design is DragonsStash-only, + though the pattern — Docker-native NFS volume + restic — could be reused + for other stacks later). +- Pausing worker/bot for a guaranteed-consistent TDLib snapshot (see + Decisions). + +## Files touched + +- `docker-compose.yml` (this repo) and + `/opt/stacks/DragonsStash/docker-compose.yml` (production) — add `backup` + service, `nas_backups` volume. +- `backup/Dockerfile` — new. +- `backup/backup.sh` — new. +- `backup/crontab` — new. +- `.env.example` / `.env` — add `NAS_HOST`, `NAS_EXPORT_PATH`, + `RESTIC_PASSWORD`, `KUMA_PUSH_URL`. diff --git a/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md b/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md new file mode 100644 index 0000000..60aedd4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md @@ -0,0 +1,271 @@ +# Provenance Backfill on Re-Index — Design + +**Date:** 2026-07-23 +**Status:** Approved for planning + +## Summary + +Recover the true origin of packages whose recorded "source" is a placeholder — +specifically the manual-upload and `rebuild.ts`-created records whose +`sourceChannelId` points at the destination (archive) channel itself. When a +real source channel is re-indexed and the worker encounters an archive that +matches such a placeholder package, it backfills the real +`sourceChannelId` / `sourceMessageId` / `sourceTopicId` / `sourceCaption` / +`creator` (and, for records that never had one, the file listing and preview) +onto the existing package — **without downloading the full archive**. + +Matching is a two-stage process: a zero-download candidate lookup by +`fileName` + total `fileSize`, confirmed by a CRC32 fingerprint read from just +the archive's central directory via a **ranged (tail) download** of a few +KB–MB, instead of the multi-GB whole file. + +This is the first of four linked sub-projects. The others (creator +normalization, provenance display, missing-files) are out of scope here and get +their own spec → plan cycles. "Missing files" is explicitly deferred until this +lands. + +## Context + +Current state (as of this design): + +- `Package` records their origin via `sourceChannelId`, `sourceMessageId`, + `sourceTopicId`, `sourceCaption`, and (for content dedup) `contentHash` and + `remoteUniqueId`. `creator` is a free-text string extracted at ingestion. +- Two ingestion paths create records with **placeholder provenance**, where the + "source" is the destination channel, not a real origin: + - **Manual uploads** (`worker/src/manual-upload.ts`) set + `sourceChannelId = destChannel.id` and `sourceMessageId = destResult.messageId`. + They *do* populate `PackageFile` (with `crc32`) from a local central-directory + read at upload time. + - **`rebuild.ts`** scans the destination channel and creates minimal records + with `fileCount == 0` and **no** `PackageFile` rows. +- The main worker upload path (`worker/src/worker.ts` → `uploadToChannel`) sends + archives to the destination channel with **no caption**, so provenance cannot + be recovered by re-reading destination messages — it has to come from matching + against real source channels. +- The scan/dedup ladder in `processOneArchiveSet` (`worker/src/worker.ts:1521`) + is entirely **source-channel-scoped**: + 1. `findPackageByRemoteUniqueId(channel.id, …)` — same channel only + 2. `packageExistsBySourceMessage(channel.id, …)` — same channel only + 3. `findRepostedPackage(channel.id, fileName, size)` — same channel only + 4. …then full download → `packageExistsByHash(contentHash)` — global, but only + *after* downloading the whole archive. +- Because a placeholder package's `sourceChannelId` is the destination, checks + 1–3 never match it when a real source channel is scanned. Today the worker + therefore downloads the entire archive, hits check 4, finds it is a duplicate, + and skips — **wasting the download and leaving the wrong provenance in place.** +- There is already an in-production precedent for "match an already-known + archive during scan and enrich its metadata": when `findRepostedPackage` + matches, the worker backfills richer *topic* context onto the existing package + via `updatePackageTopicContext` (`worker/src/worker.ts:1608`+). Provenance + backfill is the same move, widened from same-channel to cross-channel. +- `remote.unique_id` is **not** a reliable cross-channel key: it identifies a + stored file object on Telegram's servers, not the content. The same archive + independently uploaded to two channels gets different `unique_id`s; it only + matches for forwarded messages (same underlying file object). This is why the + existing dedup scopes it to a single channel. + +## Goals / Non-Goals + +**Goals** + +- Attribute true origin to placeholder-provenance packages during normal + re-index scans, opportunistically (no separate pass). +- Do it without downloading whole archives (ranged central-directory read only). +- Be non-destructive: only ever touch packages that currently have placeholder + provenance; never overwrite a real, non-placeholder source. +- Be idempotent: re-running a re-index does not re-mutate or duplicate. + +**Non-Goals (separate sub-projects / deferred)** + +- Creator name normalization / canonical creator entity. +- UI changes to display provenance. +- Recovering "missing files." +- Choosing a *preferred* origin when an archive genuinely exists in several + source channels (first confirmed source wins). + +## Design + +### 1. Candidate definition + +> A package is a backfill candidate iff **`sourceChannelId == destChannelId` +> OR `sourceMessageId == 0`**. + +Two placeholder shapes exist (verified against live data 2026-07-23): +- **Manual uploads** (`manual-upload.ts`): `sourceChannelId == destChannelId`, + real `contentHash`, real `sourceMessageId`, has a `PackageFile` listing. +- **Rebuild records** (`rebuild.ts`): `sourceMessageId == 0n` (deliberate + "unknown" sentinel), synthetic `contentHash = "rebuild::"`, + `fileCount == 0`, and `sourceChannelId` set to an **arbitrary fallback source + channel** (`sourceChannels[0]`) — NOT the destination. (This is the common + case: e.g. 59,893 records after a destination rebuild.) + +Normal ingestion always sets a real `sourceMessageId` (> 0) and a real source +channel, so neither marker matches a genuinely-sourced package. Both markers are +overwritten on backfill (source channel + message become real), so a record +stops being a candidate once fixed — this is what makes re-scans idempotent. + +**Known limitation:** backfill does NOT rewrite a rebuild record's synthetic +`"rebuild:"` `contentHash` (the true content hash would require a full download, +which this feature avoids). That is acceptable — dedup after backfill relies on +`remoteUniqueId` + name/size within the source channel, not on `contentHash`. + +### 2. Where it hooks + +A new step is inserted into `processOneArchiveSet` **between check #3 +(`findRepostedPackage`) and the full download**. It runs only after the +same-channel checks have missed (so genuine same-channel reposts keep their +existing fast paths). + +Flow for the scanned archive set: + +1. Stage A — **candidate lookup (zero download).** Query for a package where + `(sourceChannelId == destChannelId OR sourceMessageId == 0)` AND + `fileName == archiveName` AND `fileSize == totalArchiveSize`. + (`Package` has `@@index([fileName])`.) + - No candidate → fall through to normal ingestion unchanged. +2. Stage B — **fingerprint confirmation (tiny download).** See §3. +3. On confirmation → **backfill** (see §4) and return `null` (treated as a + duplicate; no full download, no new package). Increment a `zipsBackfilled` + counter. +4. On mismatch / failure / ambiguity → fall through to normal ingestion (see §6). + +### 3. Fingerprint confirmation + +The confirmation signal is the **multiset of internal CRC32s** of the archive's +entries (CRC32 of each entry's *uncompressed* data). This value is a property of +the archive contents and is identical regardless of which channel hosts the +file. + +- **Candidate side:** for manual uploads, `PackageFile.crc32` is already + populated from the local central-directory read — zero cost. For **rebuild + records** (`fileCount == 0`, no `crc32`), there is nothing stored to compare + against; obtain the candidate's fingerprint with a **second ranged tail read + of its destination copy** (ZIP/7z — still no full download). RAR candidates + cannot be tail-read on either side, so they fall back to name+size (see §5). +- **Scanned-source side:** read via a **ranged (tail) download** of the central + directory: + - **ZIP** (incl. multipart): the End-of-Central-Directory record + central + directory live at the tail of the last part. Download only that tail and + parse entries. (Reuses the lightweight-listing mechanism that is + sub-project 4's core.) + - **7z**: a start header at byte 0 points to an end header at the tail; fetch + both small pieces. + - **RAR**: headers are scattered through the file — no cheap tail read. See §5. +- **Match rule:** confirmed iff the sorted CRC32 multisets are equal **and** file + counts are equal. + +The comparison logic (CRC32 multiset equality) and the candidate-match predicate +are implemented as **pure functions** with no TDLib dependency, so they are unit +testable (see §7). + +### 4. What gets written + +On a confirmed match, update the candidate package in a single transaction, +overwriting only placeholder/empty fields: + +| Field | New value | Condition | +|---|---|---| +| `sourceChannelId` | scanned `channel.id` | always | +| `sourceMessageId` | scanned `parts[0].id` | always | +| `sourceTopicId` | `ctx.sourceTopicId` | always | +| `sourceCaption` | scanned message caption | always | +| `remoteUniqueId` | scanned `firstRemoteUniqueId` | always (enables future same-channel dedup via check #1) | +| `creator` | re-derived from source (topic > filename > channel) | **always** — un-normalized; the later creator-normalization sub-project cleans it up | +| `fileCount` + `PackageFile[]` | from the central-directory read | only if candidate had none (`fileCount == 0`) — folds in the sub-project 4 outcome for rebuild records | +| `previewData` / `previewMsgId` | matched preview from scan | only if candidate has none | + +**Left untouched:** `contentHash`, `destChannelId`, `destMessageId`, +`destMessageIds` — the bytes physically live in the destination channel; that is +correct and must not change. + +**Transaction safety:** re-check `sourceChannelId == destChannelId` *inside* the +transaction before writing, so a concurrent worker that already backfilled the +record causes this one to no-op (mirrors the existing `backfill.ts` guard). + +### 5. RAR handling + +RAR sources cannot be tail-read, so no cheap CRC fingerprint is available. +Decision: **backfill RAR matches on `fileName` + `fileSize` alone**, flagged as +lower-confidence in logs and via a `SystemNotification`, so they can be audited. +No full download. + +This name+size-only fallback applies to any candidate that cannot produce a +CRC32 fingerprint cheaply: a RAR **source**, a RAR **candidate**, or a rebuild +candidate whose destination copy is RAR. ZIP/7z rebuild candidates are still +confirmed by fingerprint via the second tail read described in §3. + +### 6. Conflict, mismatch, ambiguity + +- **Fingerprint mismatch** → the scanned archive is genuinely different content + that merely shares name+size. Fall through to **normal ingestion** (download + + index) — it is a new package for this source. +- **Ambiguous candidates** (2+ match name+size and the fingerprint cannot + disambiguate) → log a `SystemNotification`, backfill nothing, and fall through + to normal ingestion; the post-download `packageExistsByHash` check still + dedups it safely. +- **Same archive in multiple source channels** → the **first re-indexed source + that confirms wins.** After backfill the package has a real source, so it is no + longer a candidate; later scans treat it as an ordinary duplicate via checks #1 + (the `remoteUniqueId` we set) or #3. + +### 7. Idempotency + +Falls out of the candidate definition. Once backfilled: +- `sourceChannelId` is the real channel → no longer a candidate. +- A re-scan of that source hits check #1 (`remoteUniqueId`, now set) or check #3 + (`findRepostedPackage`) → normal dedup skip. No re-mutation, no duplicate. + +### 8. Error handling + +- **Tail-download failure** (network / `FLOOD_WAIT`) → cannot confirm this round. + Do **not** fall back to a full download and do **not** guess: leave the + candidate untouched and let the next re-index retry. Wrap the ranged read in + `withFloodWait` (per the TDLib skill). +- All new TDLib calls follow the skill's patterns: `withFloodWait`, listener + attached before the async op, client closed in `finally`. + +### 9. Visibility + +- Add a `zipsBackfilled` counter to `IngestionRun` activity so a re-index run + reports "N provenance backfills" instead of silently mutating records. +- Info log per backfill (candidate id, old vs new source, confidence: + fingerprint | name+size-RAR). +- `SystemNotification` for ambiguous-candidate cases. + +## Testing + +The repo currently has no test framework (`CLAUDE.md`: "testing is manual"). This +work introduces a **lightweight test harness** for the worker (e.g. `vitest` or +node's built-in `node:test`) and unit-tests the correctness-critical pure logic: + +- **Unit (automated):** + - CRC32 multiset fingerprint equality (equal sets, different order, differing + counts, disjoint sets). + - Candidate-match predicate (name+size+placeholder true/false cases). + - Field-merge rules (which fields overwrite, which only fill-if-empty). +- **Manual integration checklist:** + 1. Re-index a real source channel containing a known manually-uploaded pack → + verify `sourceChannelId`/`sourceMessageId`/`sourceCaption`/`creator` are + backfilled and no full download occurs. + 2. A rebuild-created record (`fileCount == 0`) → verify listing + provenance + are both populated from the single tail read. + 3. A RAR pack → verify name+size backfill with the lower-confidence log/notice. + 4. Re-run the same re-index → verify it is a no-op (idempotent). + 5. A genuine name+size collision (different content) → verify it ingests as a + new package rather than being mis-attributed. + +## Open Questions + +None blocking. Preferred-origin selection among multiple real sources is +deliberately out of scope (first confirmed wins). + +## Affected Code (indicative, for planning) + +- `worker/src/worker.ts` — `processOneArchiveSet`: new Stage A/B step; new counter. +- `worker/src/archive/` — ranged central-directory reader (ZIP/7z tail); shared + with sub-project 4. Pure CRC32-fingerprint compare helper. +- `worker/src/tdlib/download.ts` — ranged/partial download support (`offset`/`limit`). +- `worker/src/db/queries.ts` — candidate lookup + transactional backfill update. +- `prisma/schema.prisma` — `IngestionRun.zipsBackfilled` (and run-counter plumbing). +- Worker test harness + first unit tests. diff --git a/docs/superpowers/specs/2026-07-27-ranged-archive-listing-design.md b/docs/superpowers/specs/2026-07-27-ranged-archive-listing-design.md new file mode 100644 index 0000000..3ca68a8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-ranged-archive-listing-design.md @@ -0,0 +1,180 @@ +# Ranged inner-file listing for RAR & 7z — design + +**Date:** 2026-07-27 +**Status:** Approved (design), pending spec review → implementation plan + +## Problem + +The reindex/provenance-backfill path (`worker/src/provenance-backfill.ts`) can index an +archive's inner files *without* re-downloading it, by reading the file listing from a small +ranged read of the copy already in the source/destination channel. This works **only for +ZIP** today (ZIP keeps its central directory in a tail that `parseZipCentralDirectoryFromTail` +reads). For **RAR and 7z**, `tryProvenanceBackfill` backfills provenance (creator, source +channel, `remoteUniqueId`) so the file is skipped on re-scan and never re-downloaded — but it +leaves the inner listing empty (`fileCount = 0`), because `scannedEntries` is only computed for +ZIP. + +Scope of the gap (rebuild placeholders with `fileCount = 0`, as of 2026-07-27): + +| Type | Count | Total | Avg | Max | Multipart | +|---|---|---|---|---|---| +| 7z | 19,646 | 14 TB | 0.73 GB | 3.9 GB | 0 | +| RAR | 12,780 | 13 TB | 1.04 GB | 116 GB | 1,016 | + +A "just download the whole archive" fallback for all of these means ~27 TB of re-downloads — +the exact cost this path exists to avoid. + +## Goals + +- Index the inner files (names + sizes; CRCs where cheaply available) of RAR and 7z + placeholders **without** downloading the whole archive in the common case. +- Reuse the existing, battle-tested CLI listing parsers (`parse7zOutput`, + `parseUnrarTechnical`) rather than reimplementing filename/size/CRC extraction. +- Keep cost proportional to **file count**, not archive size (so even the 116 GB RAR is cheap). +- Guarantee a listing for the rare archives the cheap path can't handle, via a full-download + fallback that respects the existing max-size guard. + +## Non-goals + +- No change to ingestion of genuinely-new files (those are downloaded in full to be re-uploaded + regardless, so a cheap listing does not help there). This feature only affects the + provenance-backfill / skip path. +- No new ZIP behaviour — the existing ZIP tail reader stays as-is. +- Not attempting to list password-encrypted-header archives from ranged reads (no password); + those take the fallback and, if oversized, are flagged. + +## Approach (chosen: "harvest header regions → sparse file → native CLI") + +Do the *minimum* binary parsing needed to locate an archive's header bytes, fetch only those +via ranged reads, write them into a sparse temp file at their true offsets (data regions left +as unwritten zero holes → ~no disk use), then run the real `7z l` / `unrar lt` and reuse the +existing parsers. The native tools do the hard parsing (7z's LZMA-encoded headers, RAR's two +format versions, Unicode names) — we only compute where the headers are. + +Rejected alternatives: full native TS parsers (most custom binary code, highest risk); +RAR5-quick-open-only (most community RARs lack it → collapses to ~13 TB of RAR downloads). + +## Components + +New directory `worker/src/archive/ranged/`, one focused module per concern, all returning the +existing `FileEntry[]` type from `zip-reader.ts`: + +- `sparse-list.ts` — `listFromSparse(parts, runner, parse) → FileEntry[] | null`, where each + `part` is `{ fileName, size, regions: {offset, bytes}[] }`. For each part it writes a sparse + temp file (`truncate` to `size`, then write only the header `regions` at their offsets), + co-locates all parts in one temp dir under their real names, invokes the supplied CLI runner + (`7z l` / `unrar lt`) on the first part, feeds stdout to the supplied `parse` fn + (`parse7zOutput` / `parseUnrarTechnical`), and cleans up. Single-part archives are just the + one-element case. Returns `null` on CLI error / empty parse. +- `sevenz-ranged.ts` — `readSevenZListingRanged(client, parts) → FileEntry[] | null`. +- `rar-ranged.ts` — `readRarListingRanged(client, parts) → FileEntry[] | null`. +- Dispatcher in `provenance-backfill.ts`: `readScannedListingRanged(archiveType, client, parts)` + replacing the current `if (archiveType === "ZIP")` branch; the destination-copy read in + `resolveCandidateFingerprintEntries` gets the same dispatch. + +`FileEntry` shape (unchanged): `{ path, fileName, extension, compressedSize, uncompressedSize, crc32 }`. + +### 7z ranged listing + +7z layout: 32-byte signature header at offset 0 → packed streams → end header (lists files) at +the end; the signature header stores the end header's location. + +1. Ranged-read `[0, 32)`; validate magic `37 7A BC AF 27 1C`. Read LE `uint64` + `NextHeaderOffset` (byte 12) and `NextHeaderSize` (byte 20). End header is at absolute offset + `32 + NextHeaderOffset`, length `NextHeaderSize`. +2. Ranged-read `[32 + NextHeaderOffset, NextHeaderSize)` — the "next header". +3. Branch on the next header's first byte (a 7z property id): + - **`0x01` (kHeader, plain/uncompressed header):** two regions suffice — + `{0: sigHeader}` and `{32+NextHeaderOffset: endHeader}`. + - **`0x17` (kEncodedHeader, LZMA-compressed header):** the next header is only a *descriptor* + whose `PackInfo` points at a packed header stream stored **in the middle** of the file (not + at EOF). Parse the descriptor's `StreamsInfo → kPackInfo (0x06)` to read `PackPos` and the + `PackSize`s (7z variable-length "numbers"; sum them). Ranged-read the contiguous packed + region `[32 + PackPos, Σ PackSize)` and add it as a **third** sparse region. `7z l` then + decodes the header from that region. + - **anything else:** return `null` (→ fallback). +4. `listFromSparse` with the 2 or 3 regions, runner = `7z l`. `parse7zOutput` yields + names+sizes (`crc32: null`, as today). Return `null` on bad magic / read failure / CLI error. + +**Why the third region is required (spike finding, 2026-07-27):** the original two-region +(start+end) reconstruction was proven insufficient in a live test — `7z l` rejected it with +"Cannot open the file as [7z] archive" because these archives use an *encoded* header whose +compressed bytes live in a packed stream in the file body (a sparse hole), not at EOF. The +`0x17` branch fetches exactly that packed region. `7z l` still never touches the file-data +packed streams (it only lists), so those gaps stay sparse. The `read7zNumber` reader (7z's +base-128-ish variable-length integer with a first-byte length mask) and the minimal +`kPackInfo` walk are the only new 7z binary parsing; `7z l` still does the actual file listing. +All 7z placeholders are single-part. + +### RAR ranged listing + +RAR has no index; walk the block chain, parsing only each block's **size fields** to step +forward and harvest header bytes. + +1. Read first ~16 bytes; detect **RAR4** (`52 61 72 21 1A 07 00`) vs **RAR5** (`…07 01 00`) and + the signature length. +2. From just after the signature, loop: + - Ranged-read a header chunk (start 8 KB; if parsed `HeaderSize` exceeds it — long filenames + — re-read exactly). + - Minimal block-extent parse: + - RAR5: `CRC32(4)` + vint `HeaderSize` + vint `HeaderType` + vint `HeaderFlags`; if the + "extra area" flag (`0x0001`) → vint `ExtraAreaSize`; if the "data present" flag + (`0x0002`) → vint `DataSize`. Next block = `pos + 4 + len(HeaderSize vint) + HeaderSize + + DataSize`. + - RAR4: `HEAD_CRC(2)` + `HEAD_TYPE(1)` + `HEAD_FLAGS(2)` + `HEAD_SIZE(2)`; if flag `0x8000` + → `ADD_SIZE(4)`. Next block = `pos + HEAD_SIZE + ADD_SIZE`. + - Harvest `[blockOffset, blockOffset + HeaderSize)` into the regions list. + - Stop at the end-of-archive block or EOF. +3. `listFromSparse` (headers present, data sparse) → `unrar lt` → `parseUnrarTechnical`. RAR + headers carry CRC32, so RAR contributes CRCs (fingerprint disambiguation keeps working). + +**Multipart RAR** (1,016): each volume starts with its own signature + headers. Walk **each +part from its own signature**, reconstruct one sparse temp file per part with correct names +(`name.part1.rar`, `.part2.rar`, …) co-located in a temp dir, and run `unrar lt` on part 1 — +`unrar` auto-discovers co-located siblings (per the existing reader's note). The global-offset → +`(part, offsetInPart)` mapping reuses the multipart size math the ZIP path already uses. + +## Fallback & integration + +- A ranged reader returning `null` = cheap read failed (bad magic, read error, walk gave up, or + CLI error on the sparse file) → **full-download fallback**: download the whole archive, run the + existing `readRarContents` / `read7zContents`, backfill. +- The fallback is gated by `config.maxZipSizeMB` (the same guard used at ingest). Over the cap → + no download; write a `SystemNotification` (`INTEGRITY_AUDIT`, WARNING) and leave the listing + empty for manual review. This ensures nothing pathological (e.g. the 116 GB RAR) is pulled. +- Downstream is unchanged: `compareFingerprints` already treats null/incomplete CRCs as + "incomplete" (name-size path), and `backfillProvenance` writes entries when the candidate's + `fileCount === 0`. +- Observability: reuse the `zipsBackfilled` counter; add structured logs with + `confidence: "ranged" | "full-download-fallback"` and a WARN on fallback so miss-rate is + visible. + +## Risks & de-risking spike (do before the full build) + +On 3–4 real placeholder archives per format: + +1. Confirm `downloadFileRange` returns correct bytes at **arbitrary (non-tail) offsets** — + currently only tail-verified in production. Underpins everything; if it fails, stop and + rethink. (Note: `range-download.ts` flags absolute-offset behaviour as pending live + verification; tail reads are proven by the 43 ZIP backfills done 2026-07-26.) +2. Confirm `7z l` and `unrar lt` list correctly from a **sparse reconstructed file** — single + part first, then multipart RAR (the highest-risk case). + +If multipart-RAR sparse reconstruction proves unreliable in the spike, multipart RAR uses the +full-download fallback (respecting the size cap → oversized ones flagged, not downloaded). + +## Testing + +- **Unit (vitest, alongside `central-directory.test.ts`):** 7z signature-header parse; RAR4 & + RAR5 block-extent walk against committed small fixtures; `sparse-list` writes the correct + regions. Pure logic, no TDLib. +- **Live post-deploy:** watch `zipsBackfilled` climb for RAR/7z via the ranged path; spot-check + a handful of backfilled packages' `package_files` against a real `unrar lt` / `7z l` on a full + download of the same file; confirm the fallback/flag path fires on a deliberately-broken case. + +## Rollout + +Local build + deploy (no GitHub push required), per the established recipe: build +`worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image +(no `pull`). No new DB migration. The scheduler re-runs hourly and will backfill RAR/7z +placeholders on subsequent cycles. diff --git a/docs/superpowers/specs/2026-07-30-forward-priority-ingestion-design.md b/docs/superpowers/specs/2026-07-30-forward-priority-ingestion-design.md new file mode 100644 index 0000000..f3d1154 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-forward-priority-ingestion-design.md @@ -0,0 +1,211 @@ +# Forward-priority ingestion — design + +**Date:** 2026-07-30 +**Status:** Approved (design), pending spec review → implementation plan + +## Problem + +The worker ingests every archive the same way regardless of whether it needs to: download the +full file from the source channel, then re-upload the full file to the destination (archive) +channel. That download+reupload round-trip was originally necessary because some source channels +have "restrict saving content" (protected content) enabled, which blocks Telegram-native +forwarding — for those channels there is no alternative to moving the bytes through the worker. + +But most source channels do NOT restrict forwarding. For those, the round-trip is pure waste: +Telegram can copy the message from source chat to destination chat server-side, with no bytes +ever passing through the worker. The worker still needs to end up with the same outcome it has +today — a destination-channel copy, a dedup-safe identity, and a full inner-file listing — just +without paying for a download and re-upload to get there. + +Separately, `feat/ranged-archive-listing` (merged to master ahead of this feature) already built +exactly the missing piece: reading a ZIP/RAR/7z archive's inner-file listing via small ranged +reads against the file wherever it currently lives (source channel, destination channel — doesn't +matter), with no full download. It was built for backfilling listings onto already-deduped +placeholder packages. This feature generalizes that same capability to fresh ingestion, and pairs +it with a new native-forward upload path. + +## Goals + +- For channels that allow forwarding: skip download and re-upload entirely for new archives. Use + Telegram-native forwarding from source chat to destination chat, and the existing ranged-listing + readers to index inner files, with no full download in the common case. +- For channels that block forwarding (or when forwarding isn't yet known): keep today's + download+reupload pipeline exactly as-is. +- Every ingested package — regardless of path — ends up with the same outcome as today: a + `Package` row with a valid dedup identity, `destMessageId`/`destMessageIds`, creator, tags, and a + full inner-file listing (`PackageFile` rows). Indexing completeness must not regress. +- If the cheap ranged listing fails for a specific archive (bad/unsupported header, CLI error, + etc.) in an otherwise-forwarding-eligible channel, fall back to today's full download+reupload + pipeline for that one archive — never forward with an empty or partial listing. + +## Non-goals + +- No ranged single-entry preview extraction. Forward-path packages still get a preview when a + channel photo message matches (cheap, unrelated to archive bytes); when there's no matching + photo, forward-path packages simply have no preview, same as any package where preview + extraction fails today. In-archive preview extraction (unzip/unrar/7z against a local file) stays + as a download-path-only feature. May be revisited as a follow-up if it turns out to matter. +- No reprocessing of already-ingested packages. This only changes behavior for newly-scanned + archives going forward. +- No change to the bot's user-delivery leg (`bot/src/tdlib/client.ts` `copyMessageToUser`) — it + already sends via `inputFileRemote` with no download, and is unaffected by this feature. +- No change to `config.maxZipSizeMB` or the multipart byte-level split/repack logic. The existing + size guard runs before either path is chosen, so nothing above the cap reaches the forward path's + fallback-to-download step either. Splitting simply never engages on the forward path — a + forwarded message is already within whatever size Telegram accepted when it was first uploaded. + +## Approaches considered + +**A — Branch inside the existing pipeline (chosen).** Add one fork point in +`processOneArchiveSet`, immediately after the existing pre-download dedup checks: if the channel +allows forwarding, attempt the ranged-listing + forward path; on any failure, fall through into +today's download-based code for that one archive, unchanged. Smallest diff; reuses the existing +dedup/retry/watermark machinery as-is; matches the file's existing forum-vs-non-forum branching +style. + +**B — Separate pipeline per channel.** Decide once per channel and route the whole channel through +either a "forward module" or the existing "download module." Cleaner separation on paper, but +duplicates the SkippedPackage/stall/watermark bookkeeping that currently lives once in +`processArchiveSets`/`processOneArchiveSet` — higher regression risk in a large orchestration file +with no tests at that level. Rejected. + +**C — Strategy-object refactor.** Extract an `IngestStrategy` interface (`download` / `forward`) +and slim `processOneArchiveSet` to delegate to it. The more "proper" abstraction, but it's a +structural refactor of already-battle-tested code that doesn't need it for this feature to work. +Rejected — can revisit later if a third strategy ever appears. + +## Sequencing + +`feat/ranged-archive-listing` merges to master first, as-is (it's complete and serves a different +purpose already). This feature is built on a fresh branch off master afterward. + +## Components + +### 1. `TelegramChannel.allowsForwarding` (new column, new migration) + +`Boolean?` — nullable, `null` means "not yet checked". Refreshed from TDLib's chat +protected-content flag (exact field name to be confirmed against the pinned `tdl`/TDLib version +via docs lookup during implementation — expected to be `chat.has_protected_content`) at the same +point the worker already calls `getChat` per channel per cycle, mirroring the existing +`isForum`/`setChannelForum` read-and-persist pattern precisely. `null` or `false` both route to the +download path — a channel never uses the forward path on unverified permission. + +### 2. Shared ranged-listing dispatcher + +`readScannedListingRanged` (plus `RangedPart`, `tdlibRangeReader`, and the format-specific +ZIP/RAR/7z readers) currently live inside `provenance-backfill.ts`. Promote the dispatcher (and +whatever it depends on) into a shared module (e.g. `worker/src/archive/ranged/dispatch.ts`) so +`worker.ts` can call the same no-download listing logic for fresh ingestion without a circular +import. `provenance-backfill.ts` switches to importing from the new shared location; behavior +unchanged for the existing backfill path. + +### 3. `forwardArchiveToChannel` (new, `worker/src/upload/forward.ts`) + +Mirrors `uploadToChannel`'s shape and return type (`{ messageId, messageIds }`). Uses TDLib +`forwardMessages` to copy all parts of an archive set from the source chat to the destination chat +in one batch call (message IDs in original order), wrapped in the same flood-wait/retry handling +style as `uploadToChannel`. Followed by the same destination read-back verification style as +today's post-upload check (`getMessage` on each new destination message ID, confirm a document is +present). + +### 4. Dedup identity for forward-path packages + +`Package.contentHash` stays a required unique string, but forward-path packages can't hash real +bytes. Derivation order: +1. If the ranged listing's CRC32s are complete (ZIP/RAR today) — hash the sorted CRC32 list into a + synthetic `fingerprint:` value, reusing `archive/fingerprint.ts`'s existing + `crcFingerprint`. +2. Otherwise (7z, or any incomplete-CRC case) — synthesize `forward:`, following + the existing `rebuild:`-prefixed placeholder-hash precedent in `rebuild.ts`. + +Additionally, extend repost detection: before committing to the forward path, compare the new +listing's CRC fingerprint (via the existing `compareFingerprints`/`fingerprintsMatch` logic already +used in `provenance-backfill.ts`'s ambiguous-candidate disambiguation) against recent Packages +sharing the same file name + size. A fingerprint match is treated as a duplicate and skipped, same +as today's `findRepostedPackage` handling — this is what lets a forwarded copy and a previously +fully-downloaded copy of the same archive still dedupe against each other, despite never sharing a +byte-hash-derived `contentHash`. + +### 5. Fork point in `processOneArchiveSet` + +All existing pre-download checks run first, completely unchanged, in the same order: +`remote.unique_id` match → `packageExistsBySourceMessage` → `findRepostedPackage` (name+size) → +cross-channel provenance backfill → size guard (`maxZipSizeMB`). + +Then: + +``` +if channel.allowsForwarding === true: + entries = readScannedListingRanged(archiveType, client, scannedParts) + if entries is not null: + contentHash = deriveForwardContentHash(entries, remoteUniqueId) + if fingerprintRepostCheck(entries, fileName, fileSize) finds a match: + → treat as duplicate, skip (same bookkeeping as today's dup path) + destResult = forwardArchiveToChannel(client, sourceChatId, partMessageIds, destChatId) + creator, tags ← derived from entries/filename/channel/topic, same as today + preview ← channel-photo match only (no in-archive extraction) + createPackageStub(...) + updatePackageWithMetadata(...), same as today + counters.zipsForwarded++ + → done + else: + → fall through into the existing download/hash/split/upload flow below, unchanged + (log the fallback for observability) +else: + → existing download/hash/split/upload flow, completely unchanged +``` + +### 6. Observability + +New `zipsForwarded` counter alongside the existing `zipsFound`/`zipsDuplicate`/`zipsIngested`/ +`zipsBackfilled` counters, surfaced the same way (run activity, ingestion run summary). A WARN-level +log line when a forwarding-eligible archive falls back to download (mirrors the existing +`confidence: "ranged" | "full-download-fallback"` logging convention from the ranged-listing +backfill work), so the fallback rate is visible without digging through debug logs. + +## Data flow + +``` +scan → pre-download dedup + size guard (unchanged) + → channel.allowsForwarding? + true → ranged listing + ok → fingerprint dedup check → forward → stub + entries + tags (no in-archive preview) → done + null → [fall through] existing download pipeline + false/unknown → existing download pipeline (unchanged) +``` + +## Error handling + +- `forwardMessages` failure (permission revoked mid-run, rate limit, transient Telegram error) — + same `SkippedPackage`/`SystemNotification` bookkeeping as today's upload failures. Extend + `inferSkipReason` to recognize forward-specific error text the same way it already recognizes + upload errors. +- Fingerprint-repost check finds multiple ambiguous same-name/size candidates that can't be + uniquely disambiguated — same `INTEGRITY_AUDIT` notification pattern already used in + `provenance-backfill.ts`: don't guess, surface for manual triage. +- `allowsForwarding` unknown (channel just linked, not yet scanned by the refresh point) — treated + as `false`; the download path runs. No channel uses an unverified forwarding permission. +- Ranged listing throwing instead of returning `null` — treated identically to returning `null` + (fall through to download), consistent with how the existing ranged readers already treat + internal errors (they catch and return `null` themselves). + +## Testing + +- Unit tests (vitest, alongside the existing `archive/*.test.ts` and `archive/ranged/*.test.ts` + files): the dedup-identity derivation function (fingerprint-hash vs remoteUniqueId-fallback + branches), the extended fingerprint-based repost check, and `forwardArchiveToChannel`'s + request-building logic against a mocked TDLib client — same style as the existing ranged-reader + tests (pure logic, no live TDLib). +- Live verification (manual — matches this repo's existing convention that the large + `worker.ts`/`worker.py`-equivalent orchestration function has no automated test coverage and is + verified live post-deploy): one forwarding-enabled test channel and one protected-content test + channel. Confirm forward-path packages land with correct entries/tags/dedup identity and + `destMessageIds`; confirm the protected channel still goes through the unchanged full pipeline; + confirm a deliberately-unparseable archive in a forwarding-enabled channel correctly falls back + to download+reupload and still ends up fully indexed. + +## Rollout + +Local build + deploy, following the same recipe as the ranged-archive-listing work: build +`worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image (no +`pull`, no GitHub push required). New DB migration for `TelegramChannel.allowsForwarding`. No +changes required to the bot or app services. diff --git a/prisma/migrations/20260723000000_ingestion_run_zips_backfilled/migration.sql b/prisma/migrations/20260723000000_ingestion_run_zips_backfilled/migration.sql new file mode 100644 index 0000000..f13b9b4 --- /dev/null +++ b/prisma/migrations/20260723000000_ingestion_run_zips_backfilled/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable: count of packages whose provenance was backfilled during a run +-- (opportunistic cross-channel provenance backfill). Additive, non-null with a +-- default of 0 — no data change for existing rows. +ALTER TABLE "ingestion_runs" + ADD COLUMN "zipsBackfilled" INTEGER NOT NULL DEFAULT 0; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 679a74e..c25b8ce 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -569,6 +569,7 @@ model IngestionRun { zipsFound Int @default(0) zipsDuplicate Int @default(0) zipsIngested Int @default(0) + zipsBackfilled Int @default(0) errorMessage String? // Live activity tracking — written by worker in real-time diff --git a/scripts/backup/README.md b/scripts/backup/README.md deleted file mode 100644 index 00f2d95..0000000 --- a/scripts/backup/README.md +++ /dev/null @@ -1,415 +0,0 @@ -# Production backup and recovery - -Dragon's Stash backs up its PostgreSQL database and the worker/bot Telegram -(TDLib) session volumes to an encrypted Restic repository on a Synology NFS -share. Docker volumes are local runtime storage; they are not, by themselves, -backups. - -The backup job runs as `root` because it reads the Restic password file and -mounts host paths into the backup container. Keep the Synology share on the -private network. Do **not** expose NFS to the Internet. - -## Protected data and limits - -Each snapshot contains: - -- a custom-format PostgreSQL dump; -- a backup manifest with dump checksum, migration metadata, version metadata, - and captured source paths; -- the worker TDLib session volume; and -- the bot TDLib session volume. - -The `manual_uploads` and `tmp_zips` volumes are excluded. Completed STL -binaries are not retained as a local recovery set by this backup feature; they -remain in Telegram. Recovery preserves the PostgreSQL archive, message, -package, file, Telegram channel, and Telegram message ID mappings needed for -normal lookup and delivery after restore. - -Future Telegram channel-forwarding behavior and archive/STL-content integrity -auditing are intentionally outside this backup and restore procedure. - -## 1. Configure the Synology NFS share - -On the Synology DSM host: - -1. In **Control Panel > Shared Folder**, create a shared folder named - `dragonsstash-backups`. -2. In **Control Panel > File Services > NFS**, enable NFS. -3. Edit the new shared folder and add an **NFS Permissions** entry for the - Docker host's fixed private-network IP address. Grant read/write access. - Use the least permissive squash and authentication settings that work for - the root-run backup job, and do not use a broad subnet or public address. -4. Record the NFS export path shown by DSM, for example - `/volume1/dragonsstash-backups`. - -On the Docker host, install the NFS client package for its distribution, create -the mountpoint, and mount the export. Replace `NAS_IP` and the export path with -the values for the Synology: - -```bash -sudo mkdir -p /mnt/dragonsstash-backups -sudo mount -t nfs -o nfsvers=4 NAS_IP:/volume1/dragonsstash-backups /mnt/dragonsstash-backups -``` - -For a persistent mount, add an `/etc/fstab` entry using the DSM export path: - -```fstab -NAS_IP:/volume1/dragonsstash-backups /mnt/dragonsstash-backups nfs nfsvers=4,_netdev,nofail,x-systemd.automount 0 0 -``` - -Then verify that the host sees an actual writable mount rather than an empty -local directory: - -```bash -mountpoint /mnt/dragonsstash-backups -touch /mnt/dragonsstash-backups/.write-test -rm /mnt/dragonsstash-backups/.write-test -``` - -The scheduled backup refuses to run if `BACKUP_MOUNT_PATH` is not a mountpoint -or is not writable. - -## 2. Create secrets and host configuration - -Create a root-readable Restic password file. Generate and store a strong, -unique repository password by your approved secret-management process; never -commit it, place it in the Compose file, or paste it into shell history. - -```bash -sudo install -d -m 0700 -o root -g root /etc/dragons-stash -sudo install -m 0600 -o root -g root /dev/null /etc/dragons-stash/restic-password -sudoedit /etc/dragons-stash/restic-password -``` - -Create a local staging directory. It must be on host-local storage, not the NFS -share, and must have enough room for a PostgreSQL dump and restore staging -data. - -```bash -sudo install -d -m 0700 -o root -g root /var/lib/dragons-stash/backup-staging -``` - -Create the systemd environment file at `/etc/dragons-stash/backup.env` and keep -it root-only. It supplies both the host wrapper and the profile-gated backup -container: - -```bash -sudo install -m 0600 -o root -g root /dev/null /etc/dragons-stash/backup.env -sudoedit /etc/dragons-stash/backup.env -``` - -Set these production values, replacing paths and retention as needed: - -```dotenv -BACKUP_MOUNT_PATH=/mnt/dragonsstash-backups -BACKUP_STAGING_PATH=/var/lib/dragons-stash/backup-staging -BACKUP_RESTIC_PASSWORD_FILE=/etc/dragons-stash/restic-password -BACKUP_REPOSITORY=/backup/restic -BACKUP_RETENTION_DAYS=30 -BACKUP_APP_VERSION=unknown -``` - -`BACKUP_REPOSITORY` is evaluated inside the backup container, where the NFS -share is mounted at `/backup`; `/backup/restic` keeps the Restic repository in -the Synology share. `BACKUP_APP_VERSION` is optional metadata. The production -Compose environment must also retain its existing database and application -secrets; do not add secrets to Git. - -## 3. Initialize the Restic repository once - -Before enabling the scheduled job, initialize a new repository explicitly: - -```bash -docker compose --profile backup run --rm backup init -``` - -The backup container preflight refuses to create a repository implicitly. If a -scheduled backup reports that the repository is unavailable or uninitialized, -verify the NFS mount, `BACKUP_REPOSITORY`, and password file before running the -explicit initialization command for an intended new repository. - -## 4. Install the nightly systemd job - -The supplied unit assumes the production Compose checkout is -`/opt/stacks/DragonsStash`. If your deployment lives elsewhere, update the -installed service's `WorkingDirectory` and `ExecStart` to the corresponding -absolute paths before enabling it. - -Install and start the units: - -```bash -sudo install -m 0644 deploy/systemd/dragons-stash-backup.service /etc/systemd/system/ -sudo install -m 0644 deploy/systemd/dragons-stash-backup.timer /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now dragons-stash-backup.timer -sudo systemctl start dragons-stash-backup.service -sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager -``` - -The timer runs nightly at 03:00 with up to 15 minutes of randomized delay and -catches up after downtime. The first run captures PostgreSQL and both TDLib -session volumes. Later Restic snapshots deduplicate unchanged session state and -database dump content. - -## 5. Monitor and maintain backups - -Inspect the next scheduled run and the last service result: - -```bash -systemctl list-timers dragons-stash-backup.timer -systemctl status dragons-stash-backup.timer -systemctl status dragons-stash-backup.service -sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager -sudo systemctl --failed -``` - -List snapshots through the configured backup container: - -```bash -./scripts/backup/restore.sh list -``` - -Check a selected repository snapshot before restoring it: - -```bash -./scripts/backup/restore.sh verify SNAPSHOT_ID -``` - -`verify` runs `restic snapshots SNAPSHOT_ID` and `restic check`. Routine backup -runs apply retention with `restic forget --keep-daily "$BACKUP_RETENTION_DAYS" ---prune`; with the example configuration, this retains 30 daily snapshots. -Choose `BACKUP_RETENTION_DAYS` based on storage capacity and the recovery -window you need. Watch Synology capacity and investigate any failed timer or -service promptly. - -At least monthly, perform a full repository read check: - -```bash -docker compose --profile backup run --rm backup check --read-data -``` - -## 6. Monthly disposable recovery rehearsal - -Perform the following procedure at least monthly. It restores one snapshot into -a unique, disposable Compose project, so the database and Compose volumes are -isolated from production. **Never use the production Compose project name, -production volume names, or `restore-live` for this rehearsal.** Do not run -`docker compose down -v` without the explicit rehearsal `--project-name` shown -below. - -Run these commands from the production Compose checkout as an operator allowed -to read `/etc/dragons-stash/backup.env`. They use a separate application stack; -do not expose its published port beyond the host. The worker and bot are -deliberately replaced with inert processes, so this validates restored images, -database state, and TDLib volume layout without executing Telegram clients or -using production Telegram credentials. - -```bash -set -Eeuo pipefail -set -a -. /etc/dragons-stash/backup.env -set +a - -./scripts/backup/restore.sh list -SNAPSHOT_ID=SNAPSHOT_ID_FROM_LIST -REHEARSAL_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" -REHEARSAL_PROJECT="dragonsstash-rehearsal-$REHEARSAL_ID" -REHEARSAL_DIR="$BACKUP_STAGING_PATH/monthly-rehearsal-$REHEARSAL_ID" -REHEARSAL_ENV="$REHEARSAL_DIR/compose.env" -REHEARSAL_OVERRIDE="$REHEARSAL_DIR/compose.rehearsal.override.yml" -REHEARSAL_APP_PORT=13000 # Choose an unused host-local port. - -./scripts/backup/restore.sh verify "$SNAPSHOT_ID" -./scripts/backup/restore.sh restore-to-staging "$SNAPSHOT_ID" "$REHEARSAL_DIR" -RESTORE_ROOT="$(printf '%s\n' "$REHEARSAL_DIR"/staging/backup-*)" -sha256sum --check "$RESTORE_ROOT/manifest/database.dump.sha256" - -test -s "$RESTORE_ROOT/database.dump" -test -s "$RESTORE_ROOT/manifest/backup-manifest.json" -test -d "$REHEARSAL_DIR/data/tdlib-worker" -test -d "$REHEARSAL_DIR/data/tdlib-bot" - -umask 077 -cp .env "$REHEARSAL_ENV" -printf '\nAPP_PORT=%s\nNEXT_PUBLIC_APP_URL=http://localhost:%s\n' \ - "$REHEARSAL_APP_PORT" "$REHEARSAL_APP_PORT" >> "$REHEARSAL_ENV" - -cat > "$REHEARSAL_OVERRIDE" <<'EOF' -services: - worker: - environment: - TELEGRAM_API_ID: "" - TELEGRAM_API_HASH: "" - entrypoint: ["/bin/sh", "-c"] - command: ["exec sleep infinity"] - bot: - environment: - BOT_TOKEN: "" - TELEGRAM_API_ID: "" - TELEGRAM_API_HASH: "" - entrypoint: ["/bin/sh", "-c"] - command: ["exec sleep infinity"] -EOF -``` - -Confirm that `RESTORE_ROOT` names exactly one `backup-*` directory before -continuing. The staging restore has already verified the custom PostgreSQL dump -and restored `data/tdlib-worker` and `data/tdlib-bot`. - -Create the isolated project and volumes, start only its database, then import -the dump. `compose_rehearsal()` always applies the disposable override created -above: worker and bot retain their restored images and volumes but have their -Telegram credential variables blanked and run only `sleep infinity`, never -Telegram clients. The `create` command makes project-scoped application -volumes without starting app, worker, or bot. - -```bash -compose_rehearsal() { - docker compose --project-name "$REHEARSAL_PROJECT" --env-file "$REHEARSAL_ENV" \ - -f docker-compose.yml -f "$REHEARSAL_OVERRIDE" "$@" -} -rehearsal_volume() { - docker volume ls \ - --filter "label=com.docker.compose.project=$REHEARSAL_PROJECT" \ - --filter "label=com.docker.compose.volume=$1" \ - --format '{{.Name}}' -} - -compose_rehearsal --profile full create app worker bot -compose_rehearsal --profile full up -d db -compose_rehearsal exec -T db dropdb --if-exists --force \ - --username "${POSTGRES_USER:-dragons}" "${POSTGRES_DB:-dragonsstash}" -compose_rehearsal exec -T db createdb --username "${POSTGRES_USER:-dragons}" \ - "${POSTGRES_DB:-dragonsstash}" -compose_rehearsal exec -T db pg_restore --no-owner --exit-on-error \ - --username "${POSTGRES_USER:-dragons}" --dbname "${POSTGRES_DB:-dragonsstash}" \ - < "$RESTORE_ROOT/database.dump" -``` - -Copy each restored TDLib tree into its matching **rehearsal** volume. Each -target is new and empty; the function rejects an ambiguous volume lookup. - -```bash -restore_rehearsal_volume() { - local logical_name="$1" - local source_dir="$2" - local volume_name - volume_name="$(rehearsal_volume "$logical_name")" - test -n "$volume_name" - test "$(printf '%s\n' "$volume_name" | wc -l)" -eq 1 - compose_rehearsal --profile backup run --rm --no-deps \ - --entrypoint bash \ - -v "$source_dir:/restore-source:ro" \ - -v "$volume_name:/restore-target" \ - backup -ceu 'cp -a /restore-source/. /restore-target/' -} - -restore_rehearsal_volume tdlib_state "$REHEARSAL_DIR/data/tdlib-worker" -restore_rehearsal_volume tdlib_bot_state "$REHEARSAL_DIR/data/tdlib-bot" -``` - -Start the disposable app, worker, and bot containers. The app and database -perform their normal health checks; the worker and bot remain inert `sleep -infinity` processes, so this does not execute Telegram clients or send -messages. Check the health endpoint, then retain the `ps` and log output as -rehearsal evidence. - -```bash -compose_rehearsal --profile full up -d app worker bot -curl --fail --silent --show-error \ - "http://localhost:$REHEARSAL_APP_PORT/api/health" -compose_rehearsal --profile full ps -compose_rehearsal --profile full logs --tail=100 app worker bot -``` - -Optionally spot-check the restored database metadata that lets Dragon's Stash -locate Telegram-hosted STL binaries after restore. This checks database -metadata and Telegram IDs only; it does not assert local STL file presence, -STL checksums, Telegram forwarding behavior, or archive/STL binary integrity. - -```bash -compose_rehearsal exec -T db psql --no-psqlrc --tuples-only --no-align --quiet \ - --username "${POSTGRES_USER:-dragons}" \ - --dbname "${POSTGRES_DB:-dragonsstash}" \ - --command 'SELECT count(*) FROM "packages" WHERE "destChannelId" IS NOT NULL AND "destMessageId" IS NOT NULL;' -``` - -After recording the evidence, destroy only the explicitly named disposable -project and its project-labeled volumes. Do not remove `REHEARSAL_DIR` until -the evidence is recorded; then remove only that generated child directory by -your approved host cleanup procedure. Production services, volumes, and backup -snapshots must remain untouched. - -```bash -compose_rehearsal --profile full down --volumes --remove-orphans -docker volume ls --filter "label=com.docker.compose.project=$REHEARSAL_PROJECT" -test -z "$(docker volume ls --quiet --filter "label=com.docker.compose.project=$REHEARSAL_PROJECT")" -``` - -Record this evidence for every rehearsal; do not include secrets, database -dumps, Telegram session contents, or private NAS details. - -```text -Disposable restore rehearsal -Date/time (UTC): -Operator: -Snapshot ID: -Snapshot backup date: -Disposable Compose project: -Full restic check --read-data result: -Database dump manifest checksum result: -TDLib worker tree restored: -TDLib bot tree restored: -Health endpoint result (HTTP/body): -docker compose ps result: -app/worker/bot log review result: -Database Telegram metadata/mapping spot-check result: -Cleanup result (project and rehearsal volumes absent): -Notes/caveats: -``` - -This document describes the procedure only; it has not been run by this -documentation update. - -## 7. Restore modes - -Run restore commands from the production Compose checkout after loading the -same backup environment used by systemd, for example as root with -`/etc/dragons-stash/backup.env` exported. All restore staging directories must -be children of `BACKUP_STAGING_PATH`. - -| Mode | Command | Effect | -| --- | --- | --- | -| List snapshots | `./scripts/backup/restore.sh list` | Lists available Restic snapshots. Does not stop services or change volumes. | -| Verify a snapshot | `./scripts/backup/restore.sh verify SNAPSHOT_ID` | Confirms the snapshot exists and runs a Restic repository check. Does not change live data. | -| Restore to staging | `./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR` | Restores and validates a snapshot in a new or empty child directory of `BACKUP_STAGING_PATH`. Does not stop services or change volumes. | -| Replace live data | `./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data` | Stops application services and replaces the PostgreSQL database plus both TDLib session volumes after validation. | - -`restore-live` is destructive. It requires the exact -`--confirm-replace-live-data` flag and should be used only after a successful -staging restore has been inspected. It creates a safety database dump and -archives of the current TDLib volumes in local staging before replacement. If a -live restore fails, it leaves the application services stopped, retains the -safety artifacts and staging directory, and attempts rollback after replacement -has begun. Review the reported paths and service health before manually -starting services. - -After a successful live restore, confirm the services are running and inspect -their recent logs before declaring recovery complete: - -```bash -docker compose ps -docker compose logs --tail=100 app worker bot -``` - -For a non-destructive recovery rehearsal, choose a snapshot from `list` and -restore it to a fresh staging directory, for example: - -```bash -./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID /var/lib/dragons-stash/backup-staging/rehearsal-SNAPSHOT_ID -``` - -The restored tree must contain a non-empty PostgreSQL dump, its manifest, -worker TDLib state, and bot TDLib state. STL binaries remain in Telegram, and -the restored database mappings and Telegram IDs are what recovery preserves for -normal lookup and delivery. diff --git a/scripts/backup/container-entrypoint.sh b/scripts/backup/container-entrypoint.sh deleted file mode 100644 index f27ccb9..0000000 --- a/scripts/backup/container-entrypoint.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -readonly BACKUP_ROOT="/backup" -readonly STAGING_ROOT="/staging" -readonly TDLIB_WORKER_PATH="/data/tdlib-worker" -readonly TDLIB_BOT_PATH="/data/tdlib-bot" - -RUN_DIR="" - -require_value() { - local name="$1" - - if [[ -z "${!name:-}" ]]; then - printf 'Required environment variable %s is not set.\n' "$name" >&2 - return 1 - fi -} - -require_directory() { - local path="$1" - - if [[ ! -d "$path" ]]; then - printf 'Required mounted directory %s is unavailable.\n' "$path" >&2 - return 1 - fi -} - -validate_restic_repository() { - local canonical_repository - - canonical_repository="$(realpath -m -- "$RESTIC_REPOSITORY")" - if [[ "$canonical_repository" == "$BACKUP_ROOT" || "$canonical_repository" != "$BACKUP_ROOT"/* ]]; then - printf 'RESTIC_REPOSITORY must be strictly below /backup; got %s.\n' "$RESTIC_REPOSITORY" >&2 - return 1 - fi - - RESTIC_REPOSITORY="$canonical_repository" - export RESTIC_REPOSITORY -} - -validate_restic_configuration() { - require_value RESTIC_REPOSITORY - require_value RESTIC_PASSWORD_FILE - - if [[ ! -r "$RESTIC_PASSWORD_FILE" ]]; then - printf 'Restic password file %s is not readable.\n' "$RESTIC_PASSWORD_FILE" >&2 - return 1 - fi - - require_directory "$BACKUP_ROOT" - validate_restic_repository -} - -ensure_repository_initialized() { - if ! restic cat config >/dev/null; then - printf 'Restic repository %s is not initialized or could not be opened. Verify the backup mount, repository path, and password.\n' "$RESTIC_REPOSITORY" >&2 - printf 'For an intended first-time repository, initialize it explicitly with: docker compose --profile backup run --rm backup init\n' >&2 - return 1 - fi -} - -json_escape() { - local input="$1" - local output="" - local character - local code - - while [[ -n "$input" ]]; do - character="${input:0:1}" - input="${input:1}" - - case "$character" in - '"') output+='\"' ;; - '\\') output+='\\' ;; - $'\b') output+='\b' ;; - $'\f') output+='\f' ;; - $'\n') output+='\n' ;; - $'\r') output+='\r' ;; - $'\t') output+='\t' ;; - *) - printf -v code '%d' "'$character" - if (( code < 32 )); then - printf -v character '\u%04x' "$code" - fi - output+="$character" - ;; - esac - done - - printf '%s' "$output" -} - -cleanup() { - local exit_code=$? - - if [[ -n "$RUN_DIR" && "$RUN_DIR" == "$STAGING_ROOT"/* ]]; then - rm -rf -- "$RUN_DIR" - fi - - exit "$exit_code" -} - -run_backup() { - local timestamp - local checksum - local app_version - local applied_migrations - - require_value DATABASE_URL - require_value BACKUP_RETENTION_DAYS - validate_restic_configuration - require_directory "$STAGING_ROOT" - require_directory "$TDLIB_WORKER_PATH" - require_directory "$TDLIB_BOT_PATH" - ensure_repository_initialized - - timestamp="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" - RUN_DIR="$STAGING_ROOT/backup-${timestamp//[:]/}-$$" - mkdir -- "$RUN_DIR" - mkdir -- "$RUN_DIR/manifest" - trap cleanup EXIT - - pg_dump --format=custom --file="$RUN_DIR/database.dump" "$DATABASE_URL" - - if [[ ! -f "$RUN_DIR/database.dump" || ! -s "$RUN_DIR/database.dump" ]]; then - printf 'PostgreSQL dump is missing or empty.\n' >&2 - return 1 - fi - - sha256sum "$RUN_DIR/database.dump" > "$RUN_DIR/manifest/database.dump.sha256" - checksum="$(awk '{print $1}' "$RUN_DIR/manifest/database.dump.sha256")" - app_version="${BACKUP_APP_VERSION:-unknown}" - applied_migrations="$( - psql --dbname "$DATABASE_URL" --no-psqlrc --tuples-only --no-align --quiet \ - --set ON_ERROR_STOP=on <<'SQL' - SELECT COALESCE( - json_agg( - json_build_object( - 'name', "migration_name", - 'finishedAtUtc', to_char( - "finished_at" AT TIME ZONE 'UTC', - 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' - ) - ) - ORDER BY "finished_at", "migration_name" - ), - '[]'::json - )::text - FROM "_prisma_migrations" - WHERE "finished_at" IS NOT NULL AND "rolled_back_at" IS NULL; -SQL - )" - - cat > "$RUN_DIR/manifest/backup-manifest.json" <&2 - exit 1 -} - -setup_fake_environment() { - local fake_bin="$TMP_ROOT/bin" - - mkdir -p -- "$fake_bin" "$TMP_ROOT/mount" "$TMP_ROOT/staging" - printf 'not-secret\n' > "$TMP_ROOT/restic-password" - - cat > "$fake_bin/docker" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail - -printf '%s\t' "$@" >> "$DRAGONS_STASH_DOCKER_CALLS" -printf '\n' >> "$DRAGONS_STASH_DOCKER_CALLS" - -target="" -previous="" -safety_output_host="" -safety_archive="" -for argument in "$@"; do - if [[ "$previous" == "--target" ]]; then - target="$argument" - fi - if [[ "$previous" == "-v" && "$argument" == *":/safety-output" ]]; then - safety_output_host="${argument%:/safety-output}" - fi - if [[ "$argument" == /safety-output/pre-restore-*.tar ]]; then - safety_archive="${argument#/safety-output/}" - fi - previous="$argument" -done - -if [[ "${1:-}" == "compose" && "${2:-}" == "config" && "${3:-}" == "--format" && "${4:-}" == "json" ]]; then - printf '{\n "name": "dragonsstash"\n}\n' - exit 0 -fi - -if [[ "${1:-}" == "compose" && "${2:-}" == "--profile" && "${3:-}" == "full" && "${4:-}" == "ps" && "${5:-}" == "--status" && "${6:-}" == "running" && "${7:-}" == "-q" ]]; then - case " ${RESTORE_ASSERT_RUNNING_SERVICES:-app worker bot} " in - *" ${8:-} "*) printf '%s-container\n' "${8:-}" ;; - esac - exit 0 -fi - -if [[ "${1:-}" == "volume" && "${2:-}" == "ls" ]]; then - if [[ " $* " == *"com.docker.compose.volume=tdlib_state"* ]]; then - printf 'dragonsstash_tdlib_state\n' - elif [[ " $* " == *"com.docker.compose.volume=tdlib_bot_state"* ]]; then - printf 'dragonsstash_tdlib_bot_state\n' - fi - exit 0 -fi - -if [[ " $* " == *" exec -T db pg_dump "* ]]; then - printf 'custom dump\n' - exit 0 -fi - -if [[ " $* " == *" backup restore "* ]]; then - if [[ "$target" != /staging/* ]]; then - printf 'Unexpected restore target: %s\n' "$target" >&2 - exit 1 - fi - - host_target="$BACKUP_STAGING_PATH/${target#/staging/}" - mkdir -p \ - "$host_target/staging/backup-legacy/manifest" \ - "$host_target/data/tdlib-worker" \ - "$host_target/data/tdlib-bot" - printf 'custom dump\n' > "$host_target/staging/backup-legacy/database.dump" - printf '{}\n' > "$host_target/staging/backup-legacy/manifest/backup-manifest.json" - - if [[ "${RESTORE_ASSERT_CREATE_UNEXPECTED:-0}" == "1" ]]; then - mkdir -p "$host_target/data/uploads" "$host_target/data/tmp-zips" "$host_target/data/postgres" - fi -fi - -if [[ -n "$safety_output_host" && -n "$safety_archive" ]]; then - printf 'archive\n' > "$safety_output_host/$safety_archive" -fi - -exit 0 -EOF - chmod +x "$fake_bin/docker" - - cat > "$fake_bin/mountpoint" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail -exit 0 -EOF - chmod +x "$fake_bin/mountpoint" - - cat > "$fake_bin/curl" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail -printf '%s\t' "$@" >> "$DRAGONS_STASH_CURL_CALLS" -printf '\n' >> "$DRAGONS_STASH_CURL_CALLS" -exit 0 -EOF - chmod +x "$fake_bin/curl" - - export PATH="$fake_bin:$PATH" - export BACKUP_MOUNT_PATH="$TMP_ROOT/mount" - export BACKUP_STAGING_PATH="$TMP_ROOT/staging" - export BACKUP_RESTIC_PASSWORD_FILE="$TMP_ROOT/restic-password" - export BACKUP_REPOSITORY="/backup/restic" -} - -run_restore_to_staging() { - local name="$1" - shift - - DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \ - "$@" "$RESTORE_SCRIPT" restore-to-staging snapshot-scope "$TMP_ROOT/staging/$name" -} - -run_restore_live() { - local name="$1" - shift - - DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \ - DRAGONS_STASH_CURL_CALLS="$TMP_ROOT/curl-calls-$name.log" \ - "$@" "$RESTORE_SCRIPT" restore-live snapshot-scope --confirm-replace-live-data -} - -assert_restore_call_is_filtered() { - local calls="$1" - local restore_call - - restore_call="$(grep $'backup\trestore\tsnapshot-scope' "$calls" || true)" - [[ -n "$restore_call" ]] || fail "restore command was not invoked" - [[ "$restore_call" == *$'--include\t/staging/backup-*/database.dump'* ]] || fail "database dump include filter missing" - [[ "$restore_call" == *$'--include\t/staging/backup-*/manifest\t'* ]] || fail "manifest include filter missing" - [[ "$restore_call" == *$'--include\t/staging/backup-*/manifest/**'* ]] || fail "manifest subtree include filter missing" - [[ "$restore_call" == *$'--include\t/data/tdlib-worker\t'* ]] || fail "worker TDLib root include filter missing" - [[ "$restore_call" == *$'--include\t/data/tdlib-worker/**'* ]] || fail "worker TDLib subtree include filter missing" - [[ "$restore_call" == *$'--include\t/data/tdlib-bot\t'* ]] || fail "bot TDLib root include filter missing" - [[ "$restore_call" == *$'--include\t/data/tdlib-bot/**'* ]] || fail "bot TDLib subtree include filter missing" - [[ "$restore_call" != *"/data/uploads"* ]] || fail "restore includes uploads path" -} - -assert_restore_uses_include_filters() { - local calls="$TMP_ROOT/docker-calls-safe.log" - - setup_fake_environment - run_restore_to_staging safe bash >/dev/null - - assert_restore_call_is_filtered "$calls" -} - -assert_live_restore_uses_include_filters() { - local calls="$TMP_ROOT/docker-calls-live.log" - - setup_fake_environment - run_restore_live live bash >/dev/null - - assert_restore_call_is_filtered "$calls" -} - -assert_live_restore_restarts_only_previously_running_services() { - local calls="$TMP_ROOT/docker-calls-live-running-subset.log" - local stop_call - local start_call - - setup_fake_environment - RESTORE_ASSERT_RUNNING_SERVICES="app worker" run_restore_live live-running-subset bash >/dev/null - - stop_call="$(grep $'compose\t--profile\tfull\tstop' "$calls" || true)" - start_call="$(grep $'compose\t--profile\tfull\tup\t-d' "$calls" || true)" - - [[ "$stop_call" == $'compose\t--profile\tfull\tstop\tapp\tworker\t' ]] \ - || fail "restore-live stopped services other than the running app/worker subset: $stop_call" - [[ "$start_call" == $'compose\t--profile\tfull\tup\t-d\tapp\tworker\t' ]] \ - || fail "restore-live started services other than the running app/worker subset: $start_call" -} - -assert_live_restore_skips_health_when_app_was_not_running() { - local calls="$TMP_ROOT/curl-calls-live-without-app.log" - - setup_fake_environment - RESTORE_ASSERT_RUNNING_SERVICES="worker" run_restore_live live-without-app bash >/dev/null - - [[ ! -s "$calls" ]] || fail "restore-live waited for app health even though app was not previously running" -} - -assert_unexpected_volume_content_is_rejected() { - local output="$TMP_ROOT/unexpected-output.log" - - setup_fake_environment - if RESTORE_ASSERT_CREATE_UNEXPECTED=1 run_restore_to_staging unexpected bash >"$output" 2>&1; then - fail "restore accepted unexpected restored data volume content" - fi - grep -Eq 'Unexpected restored data volume content|Refusing restore' "$output" \ - || fail "restore rejected unexpected content without an explicit guard message" -} - -assert_restore_uses_include_filters -assert_live_restore_uses_include_filters -assert_live_restore_restarts_only_previously_running_services -assert_live_restore_skips_health_when_app_was_not_running -assert_unexpected_volume_content_is_rejected - -printf 'restore-path assertions passed\n' diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh deleted file mode 100755 index 5746edd..0000000 --- a/scripts/backup/restore.sh +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -readonly CONFIRM_REPLACE_LIVE_DATA="--confirm-replace-live-data" -readonly STAGING_CONTAINER_ROOT="/staging" -readonly BACKUP_CONTAINER_ROOT="/backup" -readonly -a LIVE_SERVICES=(app worker bot) -readonly -a RESTORE_INCLUDE_FILTERS=( - --include "/staging/backup-*/database.dump" - --include "/staging/backup-*/manifest" - --include "/staging/backup-*/manifest/**" - --include "/data/tdlib-worker" - --include "/data/tdlib-worker/**" - --include "/data/tdlib-bot" - --include "/data/tdlib-bot/**" -) - -RESTORED_DUMP="" -RESTORED_TDLIB_WORKER="" -RESTORED_TDLIB_BOT="" -LIVE_STAGING_DIR="" -SAFETY_DUMP="" -LIVE_RESTORE_ACTIVE=0 -LIVE_REPLACEMENT_STARTED=0 -LIVE_WORKER_VOLUME="" -LIVE_BOT_VOLUME="" -declare -a RUNNING_LIVE_SERVICES=() - -usage() { - cat <<'EOF' -Usage: - ./scripts/backup/restore.sh list - ./scripts/backup/restore.sh verify SNAPSHOT_ID - ./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR - ./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data -EOF -} - -require_value() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - printf 'Required environment variable %s is not set.\n' "$name" >&2 - return 1 - fi -} - -validate_environment() { - require_value BACKUP_MOUNT_PATH - require_value BACKUP_STAGING_PATH - require_value BACKUP_RESTIC_PASSWORD_FILE - require_value BACKUP_REPOSITORY - if [[ ! -r "$BACKUP_RESTIC_PASSWORD_FILE" ]]; then - printf 'Restic password file %s is not readable.\n' "$BACKUP_RESTIC_PASSWORD_FILE" >&2 - return 1 - fi - validate_backup_repository -} - -validate_backup_repository() { - local repository="$BACKUP_REPOSITORY" - local canonical_repository - - canonical_repository="$(realpath -ms -- "$repository")" - if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then - printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2 - return 1 - fi -} - -validate_backup_mount() { - local probe_file - - if ! mountpoint --q "$BACKUP_MOUNT_PATH"; then - printf 'Backup mount %s is not an active mountpoint.\n' "$BACKUP_MOUNT_PATH" >&2 - return 1 - fi - - if ! probe_file="$(mktemp "$BACKUP_MOUNT_PATH/.dragons-stash-backup-write-probe.XXXXXX")"; then - printf 'Backup mount %s is not writable.\n' "$BACKUP_MOUNT_PATH" >&2 - return 1 - fi - - if ! rm -f -- "$probe_file"; then - printf 'Unable to remove writable probe %s.\n' "$probe_file" >&2 - return 1 - fi -} - -backup_restic() { - docker compose --profile backup run --rm --no-deps backup "$@" -} - -restore_snapshot_subset() { - local snapshot_id="$1" - local container_staging_dir="$2" - - backup_restic restore "$snapshot_id" --target "$container_staging_dir" "${RESTORE_INCLUDE_FILTERS[@]}" -} - -canonical_staging_root() { - realpath -m -- "$BACKUP_STAGING_PATH" -} - -validate_staging_directory() { - local staging_dir="$1" - local staging_root - local canonical_staging_dir - - staging_root="$(canonical_staging_root)" - canonical_staging_dir="$(realpath -m -- "$staging_dir")" - if [[ "$canonical_staging_dir" == "$staging_root" || "$canonical_staging_dir" != "$staging_root"/* ]]; then - printf 'Staging directory %s must be inside BACKUP_STAGING_PATH (%s).\n' "$staging_dir" "$staging_root" >&2 - return 1 - fi - printf '%s\n' "$canonical_staging_dir" -} - -prepare_fresh_staging_directory() { - local staging_dir="$1" - - if [[ -e "$staging_dir" ]]; then - if [[ ! -d "$staging_dir" ]]; then - printf 'Staging target %s exists but is not a directory.\n' "$staging_dir" >&2 - return 1 - fi - if [[ -n "$(find "$staging_dir" -mindepth 1 -print -quit)" ]]; then - printf 'Staging target %s must be fresh and empty; remove stale artifacts first.\n' "$staging_dir" >&2 - return 1 - fi - else - mkdir -p -- "$staging_dir" - fi -} - -container_staging_directory() { - local host_staging_dir="$1" - local staging_root - - staging_root="$(canonical_staging_root)" - printf '%s/%s\n' "$STAGING_CONTAINER_ROOT" "${host_staging_dir#"$staging_root"/}" -} - -verify_custom_dump() { - local dump_path="$1" - docker compose --profile backup run --rm --no-deps \ - --entrypoint pg_restore \ - -v "$dump_path:/restore/database.dump:ro" \ - backup --list /restore/database.dump >/dev/null -} - -reject_unexpected_restored_content() { - local staging_dir="$1" - local backup_directory="$2" - local entry - local name - local -a unexpected_entries=() - - while IFS= read -r -d '' entry; do - name="$(basename -- "$entry")" - case "$name" in - data|staging) ;; - *) unexpected_entries+=("$entry") ;; - esac - done < <(find "$staging_dir" -mindepth 1 -maxdepth 1 -type d -print0) - - if [[ -d "$staging_dir/data" ]]; then - while IFS= read -r -d '' entry; do - name="$(basename -- "$entry")" - case "$name" in - tdlib-worker|tdlib-bot) ;; - *) unexpected_entries+=("$entry") ;; - esac - done < <(find "$staging_dir/data" -mindepth 1 -maxdepth 1 -print0) - fi - - while IFS= read -r -d '' entry; do - name="$(basename -- "$entry")" - case "$name" in - database.dump|manifest) ;; - *) unexpected_entries+=("$entry") ;; - esac - done < <(find "$backup_directory" -mindepth 1 -maxdepth 1 -print0) - - if ((${#unexpected_entries[@]})); then - printf 'Refusing restore: unexpected restored data volume content found under %s:\n' "$staging_dir" >&2 - for entry in "${unexpected_entries[@]}"; do - printf ' - %s\n' "${entry#"$staging_dir"/}" >&2 - done - return 1 - fi -} - -validate_restored_tree() { - local staging_dir="$1" - local -a backup_directories=("$staging_dir"/staging/backup-*) - local backup_directory - local manifest - - if ((${#backup_directories[@]} != 1)) || [[ ! -d "${backup_directories[0]:-}" ]]; then - printf 'Restored tree %s must contain exactly one staging/backup-* directory.\n' "$staging_dir" >&2 - return 1 - fi - backup_directory="${backup_directories[0]}" - reject_unexpected_restored_content "$staging_dir" "$backup_directory" - RESTORED_DUMP="$backup_directory/database.dump" - manifest="$backup_directory/manifest/backup-manifest.json" - RESTORED_TDLIB_WORKER="$staging_dir/data/tdlib-worker" - RESTORED_TDLIB_BOT="$staging_dir/data/tdlib-bot" - if [[ ! -s "$RESTORED_DUMP" || ! -s "$manifest" || ! -d "$RESTORED_TDLIB_WORKER" || ! -d "$RESTORED_TDLIB_BOT" ]]; then - printf 'Restored tree %s is incomplete; require staging/backup-*/database.dump, its manifest, data/tdlib-worker, and data/tdlib-bot.\n' "$staging_dir" >&2 - return 1 - fi - verify_custom_dump "$RESTORED_DUMP" -} - -verify_snapshot() { - local snapshot_id="$1" - backup_restic snapshots "$snapshot_id" - backup_restic check -} - -restore_to_staging() { - local snapshot_id="$1" - local requested_staging_dir="$2" - local staging_dir - local container_staging_dir - validate_environment - staging_dir="$(validate_staging_directory "$requested_staging_dir")" - container_staging_dir="$(container_staging_directory "$staging_dir")" - prepare_fresh_staging_directory "$staging_dir" - verify_snapshot "$snapshot_id" - restore_snapshot_subset "$snapshot_id" "$container_staging_dir" - validate_restored_tree "$staging_dir" - printf 'Staging restore verified: %s\n' "$staging_dir" -} - -compose_project_name() { - local project_name - project_name="$(docker compose config --format json | awk -F'"' '/^[[:space:]]*"name"[[:space:]]*:/ { print $4; exit }')" - if [[ -z "$project_name" ]]; then - printf '%s\n' 'Unable to determine the Docker Compose project name.' >&2 - return 1 - fi - printf '%s\n' "$project_name" -} - -compose_volume_name() { - local project_name="$1" - local logical_name="$2" - local -a matches=() - mapfile -t matches < <(docker volume ls \ - --filter "label=com.docker.compose.project=$project_name" \ - --filter "label=com.docker.compose.volume=$logical_name" \ - --format '{{.Name}}') - if ((${#matches[@]} != 1)); then - printf 'Expected exactly one %s volume for Compose project %s; found %s.\n' "$logical_name" "$project_name" "${#matches[@]}" >&2 - return 1 - fi - printf '%s\n' "${matches[0]}" -} - -capture_running_live_services() { - local service - local container_ids - - RUNNING_LIVE_SERVICES=() - for service in "${LIVE_SERVICES[@]}"; do - if ! container_ids="$(docker compose --profile full ps --status running -q "$service")"; then - printf 'Unable to determine whether Compose service %s is running.\n' "$service" >&2 - return 1 - fi - - if [[ -n "$container_ids" ]]; then - RUNNING_LIVE_SERVICES+=("$service") - fi - done -} - -live_service_was_running() { - local requested_service="$1" - local service - - for service in "${RUNNING_LIVE_SERVICES[@]}"; do - [[ "$service" == "$requested_service" ]] && return 0 - done - return 1 -} - -replace_volume() { - local source_dir="$1" - local volume_name="$2" - docker compose --profile backup run --rm --no-deps \ - --entrypoint bash \ - -v "$source_dir:/restore-source:ro" \ - -v "$volume_name:/restore-target" \ - backup -ceu ' - find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + - cp -a /restore-source/. /restore-target/ - ' -} - -create_safety_dump() { - local database_user="${POSTGRES_USER:-dragons}" - local database_name="${POSTGRES_DB:-dragonsstash}" - SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump" - docker compose exec -T db pg_dump --format=custom --no-owner \ - --username "$database_user" --dbname "$database_name" > "$SAFETY_DUMP" - if [[ ! -s "$SAFETY_DUMP" ]]; then - printf 'Safety database dump %s is missing or empty.\n' "$SAFETY_DUMP" >&2 - return 1 - fi -} - -restore_database_dump() { - local dump_path="$1" - local database_user="${POSTGRES_USER:-dragons}" - local database_name="${POSTGRES_DB:-dragonsstash}" - docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$database_name" - docker compose exec -T db createdb --username "$database_user" "$database_name" - docker compose exec -T db pg_restore --no-owner --exit-on-error \ - --username "$database_user" --dbname "$database_name" < "$dump_path" -} - -restore_database() { - restore_database_dump "$RESTORED_DUMP" -} - -archive_live_volume() { - local volume_name="$1" - local logical_name="$2" - local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar" - - docker compose --profile backup run --rm --no-deps \ - --entrypoint bash \ - -v "$volume_name:/safety-source:ro" \ - -v "$LIVE_STAGING_DIR:/safety-output" \ - backup -ceu 'tar -C /safety-source -cf "$1" .' bash \ - "/safety-output/pre-restore-$logical_name.tar" - if [[ ! -s "$archive_path" ]]; then - printf 'Safety archive %s is missing or empty.\n' "$archive_path" >&2 - return 1 - fi -} - -restore_live_volume_archive() { - local volume_name="$1" - local logical_name="$2" - local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar" - - docker compose --profile backup run --rm --no-deps \ - --entrypoint bash \ - -v "$archive_path:/safety-archive:ro" \ - -v "$volume_name:/restore-target" \ - backup -ceu ' - find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + - tar -C /restore-target -xf /safety-archive - ' -} - -wait_for_health() { - local health_url="${RESTORE_HEALTH_URL:-http://localhost:${APP_PORT:-3000}/api/health}" - local attempt - for attempt in {1..30}; do - if curl --fail --silent --show-error "$health_url" >/dev/null; then - return 0 - fi - sleep 2 - done - printf 'Application health endpoint did not become ready: %s\n' "$health_url" >&2 - return 1 -} - -live_restore_failure() { - local exit_code=$? - local rollback_ok=1 - trap - EXIT - if ((LIVE_RESTORE_ACTIVE)); then - if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then - docker compose --profile full stop "${RUNNING_LIVE_SERVICES[@]}" || true - fi - if ((LIVE_REPLACEMENT_STARTED)); then - restore_live_volume_archive "$LIVE_WORKER_VOLUME" tdlib_state || rollback_ok=0 - restore_live_volume_archive "$LIVE_BOT_VOLUME" tdlib_bot_state || rollback_ok=0 - restore_database_dump "$SAFETY_DUMP" || rollback_ok=0 - if ((rollback_ok)); then - printf 'Rollback restored the pre-restore database and both TDLib Docker volumes. Services remain stopped.\n' >&2 - else - printf 'Rollback failed; services remain stopped. Restore the safety archives and database dump manually.\n' >&2 - fi - fi - printf 'Live restore failed (exit %s). Services remain stopped. Staging directory: %s\n' "$exit_code" "$LIVE_STAGING_DIR" >&2 - printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP" >&2 - fi - exit "$exit_code" -} - -restore_live() { - local snapshot_id="$1" - local project_name - local worker_volume - local bot_volume - local timestamp - local container_staging_dir - validate_environment - validate_backup_mount - docker compose --profile backup config --quiet - project_name="$(compose_project_name)" - verify_snapshot "$snapshot_id" - printf 'Live restore target confirmed: Compose project %s, configured Restic repository, snapshot %s.\n' "$project_name" "$snapshot_id" - timestamp="$(date -u +'%Y-%m-%dT%H-%M-%SZ')" - LIVE_STAGING_DIR="$(validate_staging_directory "$BACKUP_STAGING_PATH/live-restore-$timestamp-$$")" - container_staging_dir="$(container_staging_directory "$LIVE_STAGING_DIR")" - prepare_fresh_staging_directory "$LIVE_STAGING_DIR" - SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump" - trap live_restore_failure EXIT - LIVE_RESTORE_ACTIVE=1 - capture_running_live_services - if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then - docker compose --profile full stop "${RUNNING_LIVE_SERVICES[@]}" - fi - create_safety_dump - restore_snapshot_subset "$snapshot_id" "$container_staging_dir" - validate_restored_tree "$LIVE_STAGING_DIR" - worker_volume="$(compose_volume_name "$project_name" tdlib_state)" - bot_volume="$(compose_volume_name "$project_name" tdlib_bot_state)" - LIVE_WORKER_VOLUME="$worker_volume" - LIVE_BOT_VOLUME="$bot_volume" - archive_live_volume "$worker_volume" tdlib_state - archive_live_volume "$bot_volume" tdlib_bot_state - LIVE_REPLACEMENT_STARTED=1 - replace_volume "$RESTORED_TDLIB_WORKER" "$worker_volume" - replace_volume "$RESTORED_TDLIB_BOT" "$bot_volume" - restore_database - if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then - docker compose --profile full up -d "${RUNNING_LIVE_SERVICES[@]}" - fi - if live_service_was_running app; then - wait_for_health - fi - LIVE_RESTORE_ACTIVE=0 - trap - EXIT - printf 'Live restore completed. Staging directory retained at: %s\n' "$LIVE_STAGING_DIR" - printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP" -} - -main() { - local command="${1:-}" - case "$command" in - list) - [[ $# -eq 1 ]] || { usage >&2; return 2; } - validate_environment - backup_restic snapshots - ;; - verify) - [[ $# -eq 2 ]] || { usage >&2; return 2; } - validate_environment - verify_snapshot "$2" - ;; - restore-to-staging) - [[ $# -eq 3 ]] || { usage >&2; return 2; } - restore_to_staging "$2" "$3" - ;; - restore-live) - if [[ $# -ne 3 || "$3" != "$CONFIRM_REPLACE_LIVE_DATA" ]]; then - printf 'restore-live requires the exact confirmation flag: %s\n' "$CONFIRM_REPLACE_LIVE_DATA" >&2 - return 2 - fi - restore_live "$2" - ;; - *) - usage >&2 - return 2 - ;; - esac -} - -main "$@" diff --git a/scripts/backup/run-backup-assertions.sh b/scripts/backup/run-backup-assertions.sh deleted file mode 100644 index a8b6a95..0000000 --- a/scripts/backup/run-backup-assertions.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -PROJECT_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)" -BACKUP_SCRIPT="$PROJECT_ROOT/scripts/backup/run-backup.sh" -TMP_ROOT="$(mktemp -d)" - -cleanup() { - rm -rf -- "$TMP_ROOT" -} -trap cleanup EXIT - -fail() { - printf 'ASSERTION FAILED: %s\n' "$*" >&2 - exit 1 -} - -setup_fake_environment() { - local fake_bin="$TMP_ROOT/bin" - - mkdir -p -- "$fake_bin" "$TMP_ROOT/mount" "$TMP_ROOT/staging" - printf 'not-secret\n' > "$TMP_ROOT/restic-password" - - cat > "$fake_bin/docker" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail - -printf '%s\t' "$@" >> "$DRAGONS_STASH_DOCKER_CALLS" -printf '\n' >> "$DRAGONS_STASH_DOCKER_CALLS" - -if [[ "${1:-}" == "compose" && "${2:-}" == "ps" && "${3:-}" == "--status" && "${4:-}" == "running" && "${5:-}" == "-q" ]]; then - case "${6:-}" in - app) printf 'app-container\n' ;; - esac - exit 0 -fi - -if [[ "${1:-}" == "compose" && "${2:-}" == "stop" ]]; then - exit 0 -fi - -if [[ "${1:-}" == "compose" && "${2:-}" == "start" ]]; then - exit "${BACKUP_ASSERT_START_EXIT:-0}" -fi - -if [[ " $* " == *" backup backup "* ]]; then - exit "${BACKUP_ASSERT_BACKUP_EXIT:-0}" -fi - -exit 0 -EOF - chmod +x "$fake_bin/docker" - - cat > "$fake_bin/mountpoint" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail -exit 0 -EOF - chmod +x "$fake_bin/mountpoint" - - cat > "$fake_bin/flock" <<'EOF' -#!/usr/bin/env bash -set -Eeuo pipefail -exit 0 -EOF - chmod +x "$fake_bin/flock" - - export PATH="$fake_bin:$PATH" - export BACKUP_MOUNT_PATH="$TMP_ROOT/mount" - export BACKUP_STAGING_PATH="$TMP_ROOT/staging" - export BACKUP_RESTIC_PASSWORD_FILE="$TMP_ROOT/restic-password" - export BACKUP_RETENTION_DAYS="30" - export BACKUP_REPOSITORY="/backup/restic" -} - -run_backup() { - local name="$1" - shift - - DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \ - "$@" "$BACKUP_SCRIPT" -} - -assert_restart_failure_makes_successful_backup_fail() { - local output="$TMP_ROOT/restart-fails-output.log" - - setup_fake_environment - if BACKUP_ASSERT_START_EXIT=9 run_backup restart-fails bash >"$output" 2>&1; then - fail "backup reported success when restart failed" - fi - grep -q 'Failed to restart one or more previously running services' "$output" \ - || fail "restart failure did not produce an explicit error" -} - -assert_backup_failure_exit_is_preserved_when_restart_also_fails() { - local output="$TMP_ROOT/backup-and-restart-fail-output.log" - local exit_code - - setup_fake_environment - set +e - BACKUP_ASSERT_BACKUP_EXIT=42 BACKUP_ASSERT_START_EXIT=9 run_backup backup-and-restart-fail bash >"$output" 2>&1 - exit_code=$? - set -e - - [[ "$exit_code" -eq 42 ]] || fail "expected backup failure exit 42 to be preserved; got $exit_code" - grep -q 'Failed to restart one or more previously running services' "$output" \ - || fail "restart failure did not produce an explicit error" -} - -assert_restart_failure_makes_successful_backup_fail -assert_backup_failure_exit_is_preserved_when_restart_also_fails - -printf 'run-backup assertions passed\n' diff --git a/scripts/backup/run-backup.sh b/scripts/backup/run-backup.sh deleted file mode 100755 index 4f46861..0000000 --- a/scripts/backup/run-backup.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -readonly LOCK_FILE="/run/lock/dragons-stash-backup.lock" -readonly BACKUP_CONTAINER_ROOT="/backup" -readonly -a MANAGED_SERVICES=(app worker bot) - -declare -a RUNNING_SERVICES=() - -require_value() { - local name="$1" - - if [[ -z "${!name:-}" ]]; then - printf 'Required environment variable %s is not set.\n' "$name" >&2 - return 1 - fi -} - -validate_environment() { - require_value BACKUP_MOUNT_PATH - require_value BACKUP_STAGING_PATH - require_value BACKUP_RESTIC_PASSWORD_FILE - require_value BACKUP_RETENTION_DAYS -} - -validate_backup_repository() { - local repository="${BACKUP_REPOSITORY:-/backup/restic}" - local canonical_repository - - canonical_repository="$(realpath -ms -- "$repository")" - if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then - printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2 - return 1 - fi -} - -validate_backup_mount() { - local probe_file - - if ! mountpoint --q "$BACKUP_MOUNT_PATH"; then - printf 'Backup mount %s is not an active mountpoint.\n' "$BACKUP_MOUNT_PATH" >&2 - return 1 - fi - - if ! probe_file="$(mktemp "$BACKUP_MOUNT_PATH/.dragons-stash-backup-write-probe.XXXXXX")"; then - printf 'Backup mount %s is not writable.\n' "$BACKUP_MOUNT_PATH" >&2 - return 1 - fi - - if ! rm -f -- "$probe_file"; then - printf 'Unable to remove writable probe %s.\n' "$probe_file" >&2 - return 1 - fi -} - -capture_running_services() { - local service - local container_ids - - for service in "${MANAGED_SERVICES[@]}"; do - if ! container_ids="$(docker compose ps --status running -q "$service")"; then - printf 'Unable to determine whether Compose service %s is running.\n' "$service" >&2 - return 1 - fi - - if [[ -n "$container_ids" ]]; then - RUNNING_SERVICES+=("$service") - fi - done -} - -restart_running_services() { - local exit_code=$? - local restart_exit=0 - - trap - EXIT - - if ((${#RUNNING_SERVICES[@]} > 0)); then - docker compose start "${RUNNING_SERVICES[@]}" || restart_exit=$? - if ((restart_exit != 0)); then - printf 'Failed to restart one or more previously running services: %s\n' "${RUNNING_SERVICES[*]}" >&2 - if ((exit_code == 0)); then - exit_code="$restart_exit" - fi - fi - fi - - exit "$exit_code" -} - -main() { - validate_environment - validate_backup_repository - - exec 9>"$LOCK_FILE" - if ! flock -n 9; then - printf "%s\n" "Another Dragon's Stash backup is already running." >&2 - return 1 - fi - - validate_backup_mount - capture_running_services - trap restart_running_services EXIT - - if ((${#RUNNING_SERVICES[@]} > 0)); then - docker compose stop "${RUNNING_SERVICES[@]}" - fi - - docker compose --profile backup run --rm backup backup -} - -main "$@" diff --git a/src/app/(app)/layout.tsx b/src/app/(app)/layout.tsx index fb98185..5faf03a 100644 --- a/src/app/(app)/layout.tsx +++ b/src/app/(app)/layout.tsx @@ -1,7 +1,23 @@ +import { redirect } from "next/navigation"; +import { auth } from "@/lib/auth"; +import { prisma } from "@/lib/prisma"; import { Sidebar } from "@/components/layout/sidebar"; import { Header } from "@/components/layout/header"; -export default function AppLayout({ children }: { children: React.ReactNode }) { +export default async function AppLayout({ children }: { children: React.ReactNode }) { + // Guard against a stale JWT session whose user no longer exists in the + // database (e.g. after a DB reset). The signed cookie still passes edge + // middleware, but every downstream query keyed on session.user.id would fail. + // Send such sessions to /logout, which clears the cookie and returns to login. + const session = await auth(); + if (session?.user?.id) { + const user = await prisma.user.findUnique({ + where: { id: session.user.id }, + select: { id: true }, + }); + if (!user) redirect("/logout"); + } + return (
diff --git a/src/app/(app)/stls/_components/stl-table.tsx b/src/app/(app)/stls/_components/stl-table.tsx index d914bb0..8814c55 100644 --- a/src/app/(app)/stls/_components/stl-table.tsx +++ b/src/app/(app)/stls/_components/stl-table.tsx @@ -500,14 +500,9 @@ export function StlTable({ )} - - Ungrouped - {ungroupedTotalCount > 0 && ( - - {ungroupedTotalCount} - - )} - + {/* "Ungrouped" tab hidden: the STL list is now flat and grouping is + no longer surfaced here. The tab content below is kept (unreachable) + to avoid churn; remove it and its data fetch if grouping is dropped. */} diff --git a/src/app/logout/route.ts b/src/app/logout/route.ts new file mode 100644 index 0000000..66b4f7d --- /dev/null +++ b/src/app/logout/route.ts @@ -0,0 +1,9 @@ +import { signOut } from "@/lib/auth"; + +// Server-side sign-out that clears the JWT session cookie and redirects to the +// login page. Used to recover from a stale session whose user no longer exists +// in the database (e.g. after a DB reset), which a client-only signOut can't +// reach because the app crashes before rendering the user menu. +export async function GET() { + await signOut({ redirectTo: "/login" }); +} diff --git a/src/data/settings.queries.ts b/src/data/settings.queries.ts index d353462..246fe41 100644 --- a/src/data/settings.queries.ts +++ b/src/data/settings.queries.ts @@ -1,20 +1,37 @@ +import { Prisma } from "@prisma/client"; import { prisma } from "@/lib/prisma"; +const DEFAULT_SETTINGS = { + lowStockThreshold: 20, + currency: "EUR", + theme: "dark", + units: "metric", +} as const; + export async function getUserSettings(userId: string) { let settings = await prisma.userSettings.findUnique({ where: { userId }, }); if (!settings) { - settings = await prisma.userSettings.create({ - data: { - userId, - lowStockThreshold: 20, - currency: "EUR", - theme: "dark", - units: "metric", - }, - }); + try { + settings = await prisma.userSettings.create({ + data: { userId, ...DEFAULT_SETTINGS }, + }); + } catch (err) { + // The session's user may no longer exist (e.g. a stale JWT cookie after a + // database reset). Creating settings then hits a foreign-key violation + // (P2003). Don't crash the Server Component render — return unsaved + // defaults. The (app) layout guard redirects such stale sessions to + // sign-out, so this fallback is only ever momentarily visible. + if ( + err instanceof Prisma.PrismaClientKnownRequestError && + err.code === "P2003" + ) { + return { id: "", userId, ...DEFAULT_SETTINGS }; + } + throw err; + } } return settings; diff --git a/src/lib/telegram/queries.ts b/src/lib/telegram/queries.ts index b667101..a1262bd 100644 --- a/src/lib/telegram/queries.ts +++ b/src/lib/telegram/queries.ts @@ -135,31 +135,31 @@ export async function listDisplayItems(options: { const sortCol = sortBy === "fileName" ? `"fileName"` : sortBy === "fileSize" ? `"fileSize"` : `"indexedAt"`; const sortDir = order === "asc" ? "ASC" : "DESC"; - // Step 1: Count display items + // NOTE: The STL list is intentionally FLAT — every package is its own display + // row regardless of packageGroupId. Grouping is no longer surfaced in this + // view (the creator column/filter organizes the list instead). PackageGroup + // rows and the manual grouping actions still exist in the DB/UI; they just + // don't drive this list's layout anymore. + + // Step 1: Count display items (one per package) const countResult = await prisma.$queryRawUnsafe<[{ count: bigint }]>( - `SELECT COUNT(*) AS count FROM ( - SELECT DISTINCT COALESCE(p."packageGroupId", p."id") AS display_id - FROM packages p - ${whereClause} - ) AS display_items`, + `SELECT COUNT(*) AS count FROM packages p ${whereClause}`, ...params ); const total = Number(countResult[0].count); - // Step 2: Get display item IDs for this page + // Step 2: Get package IDs for this page const limitParam = paramIdx++; const offsetParam = paramIdx++; const displayRows = await prisma.$queryRawUnsafe< { display_id: string; display_type: string }[] >( `SELECT - COALESCE(p."packageGroupId", p."id") AS display_id, - CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END AS display_type, - MAX(p.${sortCol}) AS sort_value + p."id" AS display_id, + 'package' AS display_type, + p.${sortCol} AS sort_value FROM packages p ${whereClause} - GROUP BY COALESCE(p."packageGroupId", p."id"), - CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END ORDER BY sort_value ${sortDir} LIMIT $${limitParam} OFFSET $${offsetParam}`, ...params, limit, (page - 1) * limit diff --git a/worker/package-lock.json b/worker/package-lock.json index 5c71478..3f251ff 100644 --- a/worker/package-lock.json +++ b/worker/package-lock.json @@ -22,7 +22,8 @@ "@types/yauzl": "^2.10.3", "prisma": "^7.4.0", "tsx": "^4.21.0", - "typescript": "^5" + "typescript": "^5", + "vitest": "^3.2.4" } }, "node_modules/@chevrotain/cst-dts-gen": { @@ -547,6 +548,13 @@ "hono": "^4" } }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, "node_modules/@mrleebo/prisma-ast": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/@mrleebo/prisma-ast/-/prisma-ast-0.13.1.tgz", @@ -849,6 +857,356 @@ "react-dom": "^18.0.0 || ^19.0.0" } }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -856,6 +1214,31 @@ "devOptional": true, "license": "MIT" }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "20.19.33", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.33.tgz", @@ -899,6 +1282,131 @@ "@types/node": "*" } }, + "node_modules/@vitest/expect": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", + "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", + "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", + "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.7", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", + "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", + "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", + "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -956,6 +1464,43 @@ } } }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, "node_modules/chevrotain": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-10.5.0.tgz", @@ -1054,6 +1599,16 @@ } } }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/deepmerge-ts": { "version": "7.1.5", "resolved": "https://registry.npmjs.org/deepmerge-ts/-/deepmerge-ts-7.1.5.tgz", @@ -1122,6 +1677,13 @@ "node": ">=14" } }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -1164,6 +1726,26 @@ "@esbuild/win32-x64": "0.27.3" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/exsolve": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.8.tgz", @@ -1194,6 +1776,24 @@ "node": ">=8.0.0" } }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", @@ -1353,6 +1953,13 @@ "jiti": "lib/jiti-cli.mjs" } }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lilconfig": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", @@ -1377,6 +1984,13 @@ "devOptional": true, "license": "Apache-2.0" }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lru.min": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz", @@ -1393,6 +2007,16 @@ "url": "https://github.com/sponsors/wellwelwel" } }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -1433,6 +2057,25 @@ "node": ">=8.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/node-addon-api": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", @@ -1515,6 +2158,16 @@ "devOptional": true, "license": "MIT" }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -1626,6 +2279,26 @@ "split2": "^4.1.0" } }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pino": { "version": "9.14.0", "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", @@ -1675,6 +2348,35 @@ "pathe": "^2.0.3" } }, + "node_modules/postcss": { + "version": "8.5.22", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.22.tgz", + "integrity": "sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, "node_modules/postgres": { "version": "3.4.7", "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.7.tgz", @@ -1932,6 +2634,51 @@ "node": ">= 4" } }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, "node_modules/safe-stable-stringify": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", @@ -1985,6 +2732,13 @@ "node": ">=8" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -2007,6 +2761,16 @@ "atomic-sleep": "^1.0.0" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -2026,6 +2790,13 @@ "node": ">= 0.6" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", @@ -2033,6 +2804,19 @@ "devOptional": true, "license": "MIT" }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/tdl": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/tdl/-/tdl-8.1.0.tgz", @@ -2057,6 +2841,13 @@ "real-require": "^0.2.0" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", @@ -2067,6 +2858,53 @@ "node": ">=18" } }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -2123,6 +2961,184 @@ } } }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", + "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.7", + "@vitest/mocker": "3.2.7", + "@vitest/pretty-format": "^3.2.7", + "@vitest/runner": "3.2.7", + "@vitest/snapshot": "3.2.7", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.7", + "@vitest/ui": "3.2.7", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -2139,6 +3155,23 @@ "node": ">= 8" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/worker/package.json b/worker/package.json index 391fea4..2f82376 100644 --- a/worker/package.json +++ b/worker/package.json @@ -6,7 +6,9 @@ "scripts": { "build": "tsc", "start": "node dist/index.js", - "dev": "tsx watch src/index.ts" + "dev": "tsx watch src/index.ts", + "test": "vitest run", + "test:watch": "vitest" }, "dependencies": { "@prisma/adapter-pg": "^7.4.0", @@ -23,6 +25,7 @@ "@types/yauzl": "^2.10.3", "prisma": "^7.4.0", "tsx": "^4.21.0", - "typescript": "^5" + "typescript": "^5", + "vitest": "^3.2.4" } } diff --git a/worker/src/archive/central-directory.test.ts b/worker/src/archive/central-directory.test.ts new file mode 100644 index 0000000..0cc48b8 --- /dev/null +++ b/worker/src/archive/central-directory.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from "vitest"; +import { parseZipCentralDirectoryFromTail } from "./central-directory.js"; +import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32 + +// Build a minimal STORE (no compression) ZIP in-memory with the given files. +function buildStoreZip(files: { name: string; data: Buffer }[]): Buffer { + const chunks: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const f of files) { + const crc = crc32(f.data) >>> 0; + const nameBuf = Buffer.from(f.name, "utf8"); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); // version needed + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // method = store + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(f.data.length, 18); // compressed + local.writeUInt32LE(f.data.length, 22); // uncompressed + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); // extra len + const localHeader = Buffer.concat([local, nameBuf, f.data]); + chunks.push(localHeader); + + const cd = Buffer.alloc(46); + cd.writeUInt32LE(0x02014b50, 0); + cd.writeUInt16LE(20, 4); cd.writeUInt16LE(20, 6); + cd.writeUInt16LE(0, 8); cd.writeUInt16LE(0, 10); + cd.writeUInt32LE(crc, 16); + cd.writeUInt32LE(f.data.length, 20); + cd.writeUInt32LE(f.data.length, 24); + cd.writeUInt16LE(nameBuf.length, 28); + cd.writeUInt32LE(offset, 42); // local header offset + central.push(Buffer.concat([cd, nameBuf])); + offset += localHeader.length; + } + const cdBuf = Buffer.concat(central); + const cdOffset = offset; + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(files.length, 8); + eocd.writeUInt16LE(files.length, 10); + eocd.writeUInt32LE(cdBuf.length, 12); + eocd.writeUInt32LE(cdOffset, 16); + return Buffer.concat([...chunks, cdBuf, eocd]); +} + +describe("parseZipCentralDirectoryFromTail", () => { + it("lists entries with correct names, sizes, and crc32", () => { + const zip = buildStoreZip([ + { name: "models/dragon.stl", data: Buffer.from("DRAGON") }, + { name: "readme.txt", data: Buffer.from("hello world") }, + ]); + const entries = parseZipCentralDirectoryFromTail(zip, 0); + expect(entries.map((e) => e.fileName).sort()).toEqual(["dragon.stl", "readme.txt"]); + const dragon = entries.find((e) => e.fileName === "dragon.stl")!; + expect(dragon.path).toBe("models/dragon.stl"); + expect(dragon.uncompressedSize).toBe(6n); + expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/); + }); + + it("throws when the central directory begins before the tail window", () => { + const zip = buildStoreZip([{ name: "a.txt", data: Buffer.alloc(100) }]); + // Provide only the last 30 bytes but claim they start at offset (len-30): + const tail = zip.subarray(zip.length - 30); + expect(() => parseZipCentralDirectoryFromTail(tail, zip.length - 30)).toThrow(RangeError); + }); +}); diff --git a/worker/src/archive/central-directory.ts b/worker/src/archive/central-directory.ts new file mode 100644 index 0000000..48f8d46 --- /dev/null +++ b/worker/src/archive/central-directory.ts @@ -0,0 +1,90 @@ +import path from "path"; +import type { FileEntry } from "./zip-reader.js"; + +export const MIN_ZIP_TAIL_BYTES = 65_557; + +const EOCD_SIG = 0x06054b50; +const CD_SIG = 0x02014b50; + +function extOf(name: string): string | null { + const e = path.extname(name).replace(/^\./, "").toLowerCase(); + return e === "" ? null : e; +} + +/** Parse a ZIP central directory from the tail of an archive. */ +export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] { + // 1. Find EOCD by scanning backward for its signature. + let eocd = -1; + for (let i = tail.length - 22; i >= 0; i--) { + if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; } + } + if (eocd < 0) throw new RangeError("EOCD not found in tail"); + + let cdSize = tail.readUInt32LE(eocd + 12); + let cdOffset = tail.readUInt32LE(eocd + 16); + + // ZIP64: sizes/offsets of 0xFFFFFFFF mean "see ZIP64 EOCD". + if (cdOffset === 0xffffffff || cdSize === 0xffffffff) { + const locSig = 0x07064b50; + let loc = -1; + for (let i = eocd - 20; i >= 0; i--) { + if (tail.readUInt32LE(i) === locSig) { loc = i; break; } + } + if (loc < 0) throw new RangeError("ZIP64 EOCD locator not in tail"); + const z64Abs = Number(tail.readBigUInt64LE(loc + 8)); // absolute offset of ZIP64 EOCD + const z64 = z64Abs - tailStart; + if (z64 < 0) throw new RangeError("ZIP64 EOCD before tail window"); + cdSize = Number(tail.readBigUInt64LE(z64 + 40)); + cdOffset = Number(tail.readBigUInt64LE(z64 + 48)); + } + + // 2. Map the absolute central-directory offset into the tail buffer. + const cdLocal = cdOffset - tailStart; + if (cdLocal < 0 || cdLocal + cdSize > tail.length) { + throw new RangeError("Central directory begins before tail window"); + } + + // 3. Walk central-directory headers. + const entries: FileEntry[] = []; + let p = cdLocal; + const end = cdLocal + cdSize; + while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) { + let crc = tail.readUInt32LE(p + 16) >>> 0; + let comp = BigInt(tail.readUInt32LE(p + 20)); + let uncomp = BigInt(tail.readUInt32LE(p + 24)); + const nameLen = tail.readUInt16LE(p + 28); + const extraLen = tail.readUInt16LE(p + 30); + const commentLen = tail.readUInt16LE(p + 32); + const name = tail.toString("utf8", p + 46, p + 46 + nameLen); + + // ZIP64 extra field overrides 0xFFFFFFFF sizes. + if (comp === 0xffffffffn || uncomp === 0xffffffffn) { + let ep = p + 46 + nameLen; + const extraEnd = ep + extraLen; + while (ep + 4 <= extraEnd) { + const id = tail.readUInt16LE(ep); + const sz = tail.readUInt16LE(ep + 2); + if (id === 0x0001) { + let fp = ep + 4; + if (uncomp === 0xffffffffn) { uncomp = tail.readBigUInt64LE(fp); fp += 8; } + if (comp === 0xffffffffn) { comp = tail.readBigUInt64LE(fp); fp += 8; } + } + ep += 4 + sz; + } + } + + const isDir = name.endsWith("/"); + if (!isDir) { + entries.push({ + path: name, + fileName: path.basename(name), + extension: extOf(name), + compressedSize: comp, + uncompressedSize: uncomp, + crc32: crc !== 0 ? crc.toString(16).padStart(8, "0") : null, + }); + } + p += 46 + nameLen + extraLen + commentLen; + } + return entries; +} diff --git a/worker/src/archive/fingerprint.test.ts b/worker/src/archive/fingerprint.test.ts new file mode 100644 index 0000000..e8971cb --- /dev/null +++ b/worker/src/archive/fingerprint.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from "vitest"; +import { crcFingerprint, fingerprintsMatch } from "./fingerprint.js"; +import type { FileEntry } from "./zip-reader.js"; + +const fe = (crc: string | null): FileEntry => ({ + path: "a", fileName: "a", extension: null, + compressedSize: 0n, uncompressedSize: 0n, crc32: crc, +}); + +describe("crcFingerprint", () => { + it("sorts crcs and marks complete", () => { + expect(crcFingerprint([fe("00ff"), fe("00aa")])).toEqual({ crcs: ["00aa", "00ff"], complete: true }); + }); + it("is incomplete when any crc is null", () => { + expect(crcFingerprint([fe("00aa"), fe(null)]).complete).toBe(false); + }); + it("is incomplete when empty", () => { + expect(crcFingerprint([]).complete).toBe(false); + }); +}); + +describe("fingerprintsMatch", () => { + it("matches identical crc multisets regardless of order", () => { + expect(fingerprintsMatch([fe("01"), fe("02")], [fe("02"), fe("01")])).toBe(true); + }); + it("rejects different counts", () => { + expect(fingerprintsMatch([fe("01")], [fe("01"), fe("02")])).toBe(false); + }); + it("rejects disjoint sets", () => { + expect(fingerprintsMatch([fe("01")], [fe("09")])).toBe(false); + }); + it("rejects when either side is incomplete", () => { + expect(fingerprintsMatch([fe("01"), fe(null)], [fe("01"), fe("02")])).toBe(false); + }); +}); diff --git a/worker/src/archive/fingerprint.ts b/worker/src/archive/fingerprint.ts new file mode 100644 index 0000000..a183122 --- /dev/null +++ b/worker/src/archive/fingerprint.ts @@ -0,0 +1,21 @@ +import type { FileEntry } from "./zip-reader.js"; + +export function crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean } { + if (entries.length === 0) return { crcs: [], complete: false }; + const crcs: string[] = []; + let complete = true; + for (const e of entries) { + if (e.crc32 == null) { complete = false; continue; } + crcs.push(e.crc32.toLowerCase()); + } + crcs.sort(); + return { crcs, complete }; +} + +export function fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean { + const fa = crcFingerprint(a); + const fb = crcFingerprint(b); + if (!fa.complete || !fb.complete) return false; + if (fa.crcs.length !== fb.crcs.length) return false; + return fa.crcs.every((c, i) => c === fb.crcs[i]); +} diff --git a/worker/src/archive/ranged/fallback.test.ts b/worker/src/archive/ranged/fallback.test.ts new file mode 100644 index 0000000..ae96164 --- /dev/null +++ b/worker/src/archive/ranged/fallback.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from "vitest"; + +// logLevel is required here too (not just maxZipSizeMB/tempDir) because this +// mock replaces the config module for the whole test-file graph, including +// util/logger.ts's module-level `pino({ level: config.logLevel })` call — +// pino throws at import time if level is undefined. +vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } })); +const created: unknown[] = []; +vi.mock("../../db/client.js", () => ({ + db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } }, +})); + +import { fullDownloadListing } from "./fallback.js"; + +describe("fullDownloadListing", () => { + it("refuses to download over the size cap and records a notification", async () => { + const res = await fullDownloadListing({ + client: {} as never, + parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }], + archiveType: "RAR", + totalSize: 2n * 1024n * 1024n * 1024n, + fileName: "big.rar", + }); + expect(res).toBeNull(); + expect(created).toHaveLength(1); + }); +}); diff --git a/worker/src/archive/ranged/fallback.ts b/worker/src/archive/ranged/fallback.ts new file mode 100644 index 0000000..7c714ef --- /dev/null +++ b/worker/src/archive/ranged/fallback.ts @@ -0,0 +1,55 @@ +import { mkdtemp, rm } from "fs/promises"; +import path from "path"; +import type { Client } from "tdl"; +import { config } from "../../util/config.js"; +import { db } from "../../db/client.js"; +import { childLogger } from "../../util/logger.js"; +import { downloadFile } from "../../tdlib/download.js"; +import { read7zContents } from "../sevenz-reader.js"; +import { readRarContents } from "../rar-reader.js"; +import type { FileEntry } from "../zip-reader.js"; +import type { RangedPart } from "./sevenz-ranged.js"; + +const log = childLogger("ranged-fallback"); + +export async function fullDownloadListing(args: { + client: Client; + parts: RangedPart[]; + archiveType: string; + totalSize: bigint; + fileName: string; +}): Promise { + const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n; + if (args.totalSize > capBytes) { + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "WARNING", + title: `Listing skipped (over size cap): ${args.fileName}`, + message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`, + context: { fileName: args.fileName, archiveType: args.archiveType }, + }, + }); + log.warn({ fileName: args.fileName }, "fallback skipped — over size cap"); + return null; + } + const dir = await mkdtemp(path.join(config.tempDir, "fallback-")); + const paths: string[] = []; + try { + for (const p of args.parts) { + const dest = path.join(dir, p.fileName); + await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {}); + paths.push(dest); + } + const entries = + args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0]) + : args.archiveType === "RAR" ? await readRarContents(paths[0]) + : []; + return entries.length > 0 ? entries : null; + } catch (err) { + log.warn({ err, fileName: args.fileName }, "full-download fallback failed"); + return null; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/worker/src/archive/ranged/range-reader.ts b/worker/src/archive/ranged/range-reader.ts new file mode 100644 index 0000000..3026026 --- /dev/null +++ b/worker/src/archive/ranged/range-reader.ts @@ -0,0 +1,14 @@ +import type { Client } from "tdl"; +import { downloadFileRange } from "../../tdlib/range-download.js"; + +export type RangeReader = ( + fileId: string, + offset: number, + length: number, + partSize: bigint, +) => Promise; + +export function tdlibRangeReader(client: Client): RangeReader { + return (fileId, offset, length, partSize) => + downloadFileRange(client, fileId, offset, length, partSize); +} diff --git a/worker/src/archive/ranged/rar-ranged.test.ts b/worker/src/archive/ranged/rar-ranged.test.ts new file mode 100644 index 0000000..4394817 --- /dev/null +++ b/worker/src/archive/ranged/rar-ranged.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect } from "vitest"; +import { readVint, detectRarSignature, parseRar5BlockExtent, parseRar4BlockExtent, walkRarVolume, readRarListingRanged } from "./rar-ranged.js"; +import type { RangeReader } from "./range-reader.js"; + +describe("readVint", () => { + it("reads single-byte and multi-byte values (base-128 LE)", () => { + expect(readVint(Buffer.from([0x08]), 0)).toEqual({ value: 8, bytes: 1 }); + // 0x80,0x01 => 0 | (1<<7) = 128 + expect(readVint(Buffer.from([0x80, 0x01]), 0)).toEqual({ value: 128, bytes: 2 }); + }); +}); + +describe("detectRarSignature", () => { + it("detects RAR5 and RAR4", () => { + expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]))).toEqual({ version: 5, sigLen: 8 }); + expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x00]))).toEqual({ version: 4, sigLen: 7 }); + expect(detectRarSignature(Buffer.alloc(8))).toBeNull(); + }); +}); + +describe("parseRar5BlockExtent", () => { + it("computes header+data extent and flags end-of-archive", () => { + // CRC32(4) | HeaderSize vint=5 | Type vint=2 (file) | Flags vint=2 (data present) | DataSize vint=100 | (pad to headerSize) + const b = Buffer.concat([ + Buffer.from([0,0,0,0]), // CRC + Buffer.from([0x05]), // HeaderSize = 5 (bytes after this vint) + Buffer.from([0x02]), // Type = 2 (file) + Buffer.from([0x02]), // Flags = 0x02 -> data present + Buffer.from([0x64]), // DataSize = 100 + Buffer.from([0x00, 0x00]), // padding to fill HeaderSize(5): Type+Flags+DataSize=3, +2 pad =5 + ]); + const ext = parseRar5BlockExtent(b, 0); + // headerBytes = 4 (CRC) + 1 (HeaderSize vint) + 5 (HeaderSize) = 10 + expect(ext.headerBytes).toBe(10); + expect(ext.dataSize).toBe(100); + expect(ext.isEnd).toBe(false); + + const endBlk = Buffer.from([0,0,0,0, 0x02, 0x05, 0x00]); // HeaderSize=2, Type=5(end), Flags=0 + const e2 = parseRar5BlockExtent(endBlk, 0); + expect(e2.isEnd).toBe(true); + }); +}); + +describe("parseRar4BlockExtent", () => { + it("computes extent with ADD_SIZE when flag 0x8000 is set", () => { + // CRC(2) TYPE(1)=0x74 FLAGS(2)=0x8000 HEAD_SIZE(2)=11 ADD_SIZE(4)=200 + const b = Buffer.alloc(11); + b.writeUInt8(0x74, 2); + b.writeUInt16LE(0x8000, 3); + b.writeUInt16LE(11, 5); + b.writeUInt32LE(200, 7); + const ext = parseRar4BlockExtent(b, 0); + expect(ext.headerBytes).toBe(11); + expect(ext.dataSize).toBe(200); + expect(ext.isEnd).toBe(false); + }); +}); + +// Build a synthetic RAR5 volume: signature + main header + 2 file blocks (each +// with data) + end block. We only need extents to be walkable. +function buildRar5Volume(): Buffer { + const sig = Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]); + const block = (type: number, flags: number, dataSize: number, pad = 0) => { + const body = [Buffer.from([type]), Buffer.from([flags])]; + if (flags & 0x0002) body.push(Buffer.from([dataSize])); // DataSize (<=127 for test) + if (pad) body.push(Buffer.alloc(pad)); + const bodyBuf = Buffer.concat(body); + const hs = Buffer.from([bodyBuf.length]); // HeaderSize vint (<=127) + const header = Buffer.concat([Buffer.alloc(4), hs, bodyBuf]); // CRC(4)+HeaderSize+body + const data = Buffer.alloc(flags & 0x0002 ? dataSize : 0, 0xEE); + return Buffer.concat([header, data]); + }; + const main = block(1, 0, 0); // main archive header, no data + const f1 = block(2, 0x02, 20); // file header + 20 bytes data + const f2 = block(2, 0x02, 30); // file header + 30 bytes data + const end = block(5, 0, 0); // end of archive + return Buffer.concat([sig, main, f1, f2, end]); +} + +describe("walkRarVolume", () => { + it("harvests every block header and stops at end-of-archive", async () => { + const vol = buildRar5Volume(); + const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length); + const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }, 5, 8); + expect(regions).not.toBeNull(); + // main + 2 files + end = 4 header regions + expect(regions!).toHaveLength(4); + // First region starts right after the 8-byte signature + expect(regions![0].offset).toBe(8); + }); + + it("returns null when a block claims an absurd header size (corrupt/desynced)", async () => { + // RAR5 block with HeaderSize vint encoding a value > 8MB. + // Encode 9_000_000 as RAR vint: bytes little-endian 7-bit groups with continuation bit. + function encodeVint(n: number): number[] { + const out: number[] = []; + while (n >= 0x80) { + out.push((n & 0x7f) | 0x80); + n = Math.floor(n / 128); + } + out.push(n); + return out; + } + const sig = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); // RAR5 signature + const hsVint = encodeVint(9_000_000); + // Block = CRC(4) + HeaderSize vint(9MB) + Type(1 byte) + Flags(1 byte) + const block = Buffer.concat([Buffer.alloc(4), Buffer.from(hsVint), Buffer.from([0x02, 0x00])]); + const vol = Buffer.concat([sig, block]); + const size = 20 * 1024 * 1024; + const read = async (_id: string, offset: number, length: number) => { + if (offset >= vol.length) return Buffer.alloc(0); + return vol.subarray(offset, Math.min(offset + length, vol.length)); + }; + const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(size), fileName: "c.rar" }, 5, 8); + expect(regions).toBeNull(); + }); +}); + +describe("readRarListingRanged (single part)", () => { + it("returns null cleanly when the reconstructed file isn't a real RAR", async () => { + const vol = buildRar5Volume(); + const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length); + const res = await readRarListingRanged([{ fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }], read); + expect(res === null || Array.isArray(res)).toBe(true); // real unrar parse covered live + }); +}); + +describe("readRarListingRanged (multipart)", () => { + it("walks each volume from its own signature and reconstructs all parts", async () => { + const vol = buildRar5Volume(); // reuse from Task 6 test + // Two volumes with identical structure; each RangeReader read is scoped by fileId. + const byId: Record = { p1: vol, p2: vol }; + const reads: Record = { p1: 0, p2: 0 }; + const read: RangeReader = async (fileId, offset, length) => { + reads[fileId]++; + return byId[fileId].subarray(offset, offset + length); + }; + const res = await readRarListingRanged( + [ + { fileId: "p1", fileSize: BigInt(vol.length), fileName: "x.part1.rar" }, + { fileId: "p2", fileSize: BigInt(vol.length), fileName: "x.part2.rar" }, + ], + read, + ); + // Both volumes were walked (each read at least its signature + blocks). + expect(reads.p1).toBeGreaterThan(0); + expect(reads.p2).toBeGreaterThan(0); + expect(res === null || Array.isArray(res)).toBe(true); + }); +}); diff --git a/worker/src/archive/ranged/rar-ranged.ts b/worker/src/archive/ranged/rar-ranged.ts new file mode 100644 index 0000000..1e87194 --- /dev/null +++ b/worker/src/archive/ranged/rar-ranged.ts @@ -0,0 +1,110 @@ +const RAR4_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]); +const RAR5_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); + +export function readVint(buf: Buffer, pos: number): { value: number; bytes: number } { + let value = 0, shift = 0, bytes = 0; + while (pos + bytes < buf.length) { + const b = buf[pos + bytes]; + value += (b & 0x7f) * Math.pow(2, shift); // Math.pow keeps >32-bit sizes exact up to 2^53 + bytes++; + if ((b & 0x80) === 0) return { value, bytes }; + shift += 7; + if (shift > 63) break; + } + throw new RangeError("incomplete RAR vint"); +} + +export function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null { + if (buf.length >= 8 && buf.subarray(0, 8).equals(RAR5_SIG)) return { version: 5, sigLen: 8 }; + if (buf.length >= 7 && buf.subarray(0, 7).equals(RAR4_SIG)) return { version: 4, sigLen: 7 }; + return null; +} + +export interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean } + +// RAR5: CRC32(4) | HeaderSize(vint) | HeaderType(vint) | HeaderFlags(vint) +// [ExtraAreaSize(vint) if flags&0x0001] [DataSize(vint) if flags&0x0002] ... +export function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent { + let p = pos + 4; // skip CRC32 + const hs = readVint(buf, p); p += hs.bytes; + const headerBytes = 4 + hs.bytes + hs.value; // CRC + HeaderSize-vint + HeaderSize + const type = readVint(buf, p); p += type.bytes; + const flags = readVint(buf, p); p += flags.bytes; + if (flags.value & 0x0001) { const ea = readVint(buf, p); p += ea.bytes; } // extra area size (skip) + let dataSize = 0; + if (flags.value & 0x0002) { const ds = readVint(buf, p); p += ds.bytes; dataSize = ds.value; } + return { headerBytes, dataSize, isEnd: type.value === 5 }; +} + +// RAR4: HEAD_CRC(2) | HEAD_TYPE(1) | HEAD_FLAGS(2) | HEAD_SIZE(2) [ADD_SIZE(4) if flags&0x8000] +export function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent { + const type = buf.readUInt8(pos + 2); + const flags = buf.readUInt16LE(pos + 3); + const headSize = buf.readUInt16LE(pos + 5); + const dataSize = (flags & 0x8000) ? buf.readUInt32LE(pos + 7) : 0; + return { headerBytes: headSize, dataSize, isEnd: type === 0x7b }; +} + +import type { FileEntry } from "../zip-reader.js"; +import type { RangeReader } from "./range-reader.js"; +import type { RangedPart } from "./sevenz-ranged.js"; +import { listFromSparse, type SparsePart } from "./sparse-list.js"; +import { readRarContents } from "../rar-reader.js"; +import { childLogger } from "../../util/logger.js"; + +const rlog = childLogger("rar-ranged"); +const MAX_RAR_BLOCKS = 50000; +const MAX_RAR_HEADER_BYTES = 8 * 1024 * 1024; // 8 MB — real RAR block headers are far smaller; guards against a corrupt/desynced HeaderSize +const HEADER_CHUNK = 8192; + +export async function walkRarVolume( + read: RangeReader, + part: RangedPart, + version: 4 | 5, + sigLen: number, +): Promise<{ offset: number; bytes: Buffer }[] | null> { + const size = Number(part.fileSize); + const regions: { offset: number; bytes: Buffer }[] = []; + let pos = sigLen; + let blocks = 0; + try { + while (pos < size) { + if (++blocks > MAX_RAR_BLOCKS) return null; + const chunkLen = Math.min(HEADER_CHUNK, size - pos); + let chunk = await read(part.fileId, pos, chunkLen, part.fileSize); + const ext = version === 5 ? parseRar5BlockExtent(chunk, 0) : parseRar4BlockExtent(chunk, 0); + if (ext.headerBytes > MAX_RAR_HEADER_BYTES) return null; + // Ensure we have the full header bytes to harvest (long filenames). + let headerBuf = chunk; + if (ext.headerBytes > chunk.length) { + headerBuf = await read(part.fileId, pos, Math.min(ext.headerBytes, size - pos), part.fileSize); + } + regions.push({ offset: pos, bytes: headerBuf.subarray(0, Math.min(ext.headerBytes, size - pos)) }); + if (ext.isEnd) break; + const advance = ext.headerBytes + ext.dataSize; + if (advance <= 0) return null; + if (pos + advance > size) break; // data clamped at the volume boundary (multipart continuation) + pos += advance; + } + return regions; + } catch (err) { + rlog.warn({ err, fileId: part.fileId }, "RAR volume walk failed"); + return null; + } +} + +export async function readRarListingRanged( + parts: RangedPart[], + read: RangeReader, +): Promise { + const sparseParts: SparsePart[] = []; + for (const part of parts) { + const head = await read(part.fileId, 0, 16, part.fileSize); + const sig = detectRarSignature(head); + if (!sig) return null; + const regions = await walkRarVolume(read, part, sig.version, sig.sigLen); + if (!regions) return null; + sparseParts.push({ fileName: part.fileName, size: Number(part.fileSize), regions }); + } + return listFromSparse(sparseParts, readRarContents); +} diff --git a/worker/src/archive/ranged/sevenz-ranged.test.ts b/worker/src/archive/ranged/sevenz-ranged.test.ts new file mode 100644 index 0000000..b489f32 --- /dev/null +++ b/worker/src/archive/ranged/sevenz-ranged.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect } from "vitest"; +import { parseSevenZSignatureHeader } from "./sevenz-ranged.js"; + +const MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + +function buildSignatureHeader(nextOffset: bigint, nextSize: bigint): Buffer { + const buf = Buffer.alloc(32); + MAGIC.copy(buf, 0); + buf.writeUInt8(0, 6); buf.writeUInt8(4, 7); // version 0.4 + buf.writeUInt32LE(0, 8); // StartHeaderCRC (unused here) + buf.writeBigUInt64LE(nextOffset, 12); + buf.writeBigUInt64LE(nextSize, 20); + buf.writeUInt32LE(0, 28); // NextHeaderCRC (unused here) + return buf; +} + +describe("parseSevenZSignatureHeader", () => { + it("reads NextHeaderOffset and NextHeaderSize", () => { + const buf = buildSignatureHeader(1_000_000n, 4096n); + expect(parseSevenZSignatureHeader(buf)).toEqual({ nextHeaderOffset: 1_000_000, nextHeaderSize: 4096 }); + }); + + it("returns null on bad magic", () => { + expect(parseSevenZSignatureHeader(Buffer.alloc(32))).toBeNull(); + }); + + it("returns null when shorter than 32 bytes", () => { + expect(parseSevenZSignatureHeader(MAGIC)).toBeNull(); + }); +}); + +import { readSevenZListingRanged } from "./sevenz-ranged.js"; +import type { RangeReader } from "./range-reader.js"; + +describe("readSevenZListingRanged", () => { + it("reads the signature + end-header regions and reconstructs for 7z l", async () => { + const size = 5_000_000; + const endHeaderOffset = 4_900_000; // absolute + const nextHeaderOffset = endHeaderOffset - 32; + const sig = Buffer.alloc(32); + Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0); + sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12); + sig.writeBigUInt64LE(100n, 20); + + const reads: { offset: number; length: number }[] = []; + const read: RangeReader = async (_id, offset, length) => { + reads.push({ offset, length }); + if (offset === 0) return sig.subarray(0, length); + return Buffer.alloc(length, 0xAB); // stand-in end-header bytes + }; + + // Inject a fake lister via the module boundary: readSevenZListingRanged + // calls listFromSparse(parts, read7zContents). We assert the ranged reads + // it issued; the sparse file + real 7z is covered by live verification. + const entries = await readSevenZListingRanged( + [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }], + read, + ); + // entries may be null here because the stand-in bytes aren't a real 7z; + // the contract under test is the ranged-read offsets: + expect(reads[0]).toEqual({ offset: 0, length: 32 }); + expect(reads[1]).toEqual({ offset: endHeaderOffset, length: 100 }); + expect(entries === null || Array.isArray(entries)).toBe(true); + }); +}); + +import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js"; + +describe("read7zNumber", () => { + it("reads a single-byte number", () => { + expect(read7zNumber(Buffer.from([0x2a]), 0)).toEqual({ value: 42, next: 1 }); + }); + it("reads a two-byte number (first-byte length mask + LE trailing byte)", () => { + // 1000 = 0x03E8: low byte 0xE8, high nibble 0x03 -> first = 0x80|0x03 = 0x83, trailing 0xE8 + expect(read7zNumber(Buffer.from([0x83, 0xe8]), 0)).toEqual({ value: 1000, next: 2 }); + // 500 = 0x01F4 -> first 0x81, trailing 0xF4 + expect(read7zNumber(Buffer.from([0x81, 0xf4]), 0)).toEqual({ value: 500, next: 2 }); + }); + it("throws when pos starts past the buffer end (short read)", () => { + expect(() => read7zNumber(Buffer.from([0x2a]), 5)).toThrow(RangeError); + expect(() => read7zNumber(Buffer.alloc(0), 0)).toThrow(RangeError); + }); +}); + +describe("locate7zEncodedHeaderPack", () => { + it("parses PackPos and summed PackSize from an encoded header", () => { + // kEncodedHeader, kPackInfo, PackPos=1000([0x83,0xe8]), NumStreams=1([0x01]), + // kSize, PackSize=500([0x81,0xf4]) + const enc = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]); + expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 1000, packSize: 500 }); + }); + it("returns null for a plain (kHeader 0x01) header", () => { + expect(locate7zEncodedHeaderPack(Buffer.from([0x01, 0x04]))).toBeNull(); + }); + it("sums multiple pack streams", () => { + // PackPos=0([0x00]), NumStreams=2([0x02]), kSize, sizes 10([0x0a]) + 20([0x14]) + const enc = Buffer.from([0x17, 0x06, 0x00, 0x02, 0x09, 0x0a, 0x14]); + expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 0, packSize: 30 }); + }); +}); + +describe("readSevenZListingRanged (encoded header)", () => { + it("fetches the mid-file packed-header region as a third read", async () => { + const size = 5_000_000; + const nextHeaderOffset = 4_000_000; // relative to end of 32-byte sig header + const endStart = 32 + nextHeaderOffset; // absolute + const packPos = 1000; // relative to end of sig header + const packStart = 32 + packPos; // absolute + const packSize = 500; + const sig = Buffer.alloc(32); + Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0); + sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12); + sig.writeBigUInt64LE(8n, 20); // NextHeaderSize = 8 (the encoded-header descriptor below) + const encHeader = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]); + + const reads: { offset: number; length: number }[] = []; + const read = async (_id: string, offset: number, length: number) => { + reads.push({ offset, length }); + if (offset === 0) return sig.subarray(0, length); + if (offset === endStart) return encHeader.subarray(0, length); + return Buffer.alloc(length, 0xcd); // stand-in packed-header bytes + }; + + const entries = await readSevenZListingRanged( + [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }], + read, + ); + expect(reads[0]).toEqual({ offset: 0, length: 32 }); + expect(reads[1]).toEqual({ offset: endStart, length: 8 }); + expect(reads[2]).toEqual({ offset: packStart, length: packSize }); + expect(entries === null || Array.isArray(entries)).toBe(true); + }); +}); diff --git a/worker/src/archive/ranged/sevenz-ranged.ts b/worker/src/archive/ranged/sevenz-ranged.ts new file mode 100644 index 0000000..0ca9d7a --- /dev/null +++ b/worker/src/archive/ranged/sevenz-ranged.ts @@ -0,0 +1,112 @@ +import type { FileEntry } from "../zip-reader.js"; +import { read7zContents } from "../sevenz-reader.js"; +import { listFromSparse } from "./sparse-list.js"; +import type { RangeReader } from "./range-reader.js"; +import { childLogger } from "../../util/logger.js"; + +const log = childLogger("sevenz-ranged"); + +const SEVENZ_MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + +const K_HEADER = 0x01; +const K_ENCODED_HEADER = 0x17; +const K_PACK_INFO = 0x06; +const K_SIZE = 0x09; + +/** Read a 7z variable-length number: first byte is a length mask, followed by + * little-endian bytes. Math.pow keeps values exact above 2^31. */ +export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } { + if (pos >= buf.length) throw new RangeError("7z number reads past buffer end"); + const first = buf[pos]; + let mask = 0x80; + let value = 0; + let p = pos + 1; + for (let i = 0; i < 8; i++) { + if ((first & mask) === 0) { + value += (first & (mask - 1)) * Math.pow(2, 8 * i); + return { value, next: p }; + } + if (p >= buf.length) throw new RangeError("7z number overruns buffer"); + value += buf[p] * Math.pow(2, 8 * i); + p++; + mask >>= 1; + } + return { value, next: p }; +} + +/** For an encoded (kEncodedHeader) 7z next-header, return the absolute-ish + * location of the packed header stream(s): PackPos (relative to end of the + * 32-byte signature header) and the summed PackSize. Null if not encoded or + * the StreamsInfo doesn't start with PackInfo as expected. */ +export function locate7zEncodedHeaderPack( + nextHeader: Buffer, +): { packPos: number; packSize: number } | null { + let p = 0; + if (nextHeader[p] !== K_ENCODED_HEADER) return null; + p++; + if (nextHeader[p] !== K_PACK_INFO) return null; + p++; + const packPos = read7zNumber(nextHeader, p); p = packPos.next; + const numStreams = read7zNumber(nextHeader, p); p = numStreams.next; + if (nextHeader[p] !== K_SIZE) return null; + p++; + let total = 0; + for (let i = 0; i < numStreams.value; i++) { + const s = read7zNumber(nextHeader, p); p = s.next; + total += s.value; + } + return { packPos: packPos.value, packSize: total }; +} + +export function parseSevenZSignatureHeader( + buf: Buffer, +): { nextHeaderOffset: number; nextHeaderSize: number } | null { + if (buf.length < 32) return null; + if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null; + return { + nextHeaderOffset: Number(buf.readBigUInt64LE(12)), + nextHeaderSize: Number(buf.readBigUInt64LE(20)), + }; +} + +export interface RangedPart { fileId: string; fileSize: bigint; fileName: string } + +export async function readSevenZListingRanged( + parts: RangedPart[], + read: RangeReader, +): Promise { + const part = parts[0]; + if (!part) return null; + const size = Number(part.fileSize); + try { + const sig = await read(part.fileId, 0, 32, part.fileSize); + const parsed = parseSevenZSignatureHeader(sig); + if (!parsed) return null; + const endStart = 32 + parsed.nextHeaderOffset; + if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null; + const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize); + + const regions = [ + { offset: 0, bytes: sig }, + { offset: endStart, bytes: endHeader }, + ]; + + const headerType = endHeader[0]; + if (headerType === K_ENCODED_HEADER) { + // Compressed header: its packed bytes live mid-file, not at EOF. Fetch them. + const pack = locate7zEncodedHeaderPack(endHeader); + if (!pack) return null; + const packStart = 32 + pack.packPos; + if (packStart < 0 || packStart + pack.packSize > size) return null; + const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize); + regions.push({ offset: packStart, bytes: packBytes }); + } else if (headerType !== K_HEADER) { + return null; // unknown next-header type + } + + return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents); + } catch (err) { + log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed"); + return null; + } +} diff --git a/worker/src/archive/ranged/sparse-list.test.ts b/worker/src/archive/ranged/sparse-list.test.ts new file mode 100644 index 0000000..08545c0 --- /dev/null +++ b/worker/src/archive/ranged/sparse-list.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from "vitest"; +import { open } from "fs/promises"; +import { listFromSparse } from "./sparse-list.js"; + +describe("listFromSparse", () => { + it("writes each region at its offset into a sparse file and passes the path to the lister", async () => { + const size = 1_000_000; + const regions = [ + { offset: 0, bytes: Buffer.from("HEAD") }, + { offset: size - 4, bytes: Buffer.from("TAIL") }, + ]; + let seenPath = ""; + const entries = await listFromSparse( + [{ fileName: "sample.7z", size, regions }], + async (firstPartPath) => { + seenPath = firstPartPath; + const fh = await open(firstPartPath, "r"); + try { + const head = Buffer.alloc(4); await fh.read(head, 0, 4, 0); + const tail = Buffer.alloc(4); await fh.read(tail, 0, 4, size - 4); + const hole = Buffer.alloc(4); await fh.read(hole, 0, 4, 500_000); + expect(head.toString()).toBe("HEAD"); + expect(tail.toString()).toBe("TAIL"); + expect(hole.equals(Buffer.alloc(4))).toBe(true); // gap is zero + } finally { await fh.close(); } + return [{ path: "a/b.stl", fileName: "b.stl", extension: "stl", compressedSize: 1n, uncompressedSize: 1n, crc32: null }]; + }, + ); + expect(seenPath.endsWith("sample.7z")).toBe(true); + expect(entries).not.toBeNull(); + expect(entries!).toHaveLength(1); + }); + + it("returns null when the lister yields no entries", async () => { + const res = await listFromSparse( + [{ fileName: "x.7z", size: 100, regions: [{ offset: 0, bytes: Buffer.from("A") }] }], + async () => [], + ); + expect(res).toBeNull(); + }); +}); diff --git a/worker/src/archive/ranged/sparse-list.ts b/worker/src/archive/ranged/sparse-list.ts new file mode 100644 index 0000000..f96f0e4 --- /dev/null +++ b/worker/src/archive/ranged/sparse-list.ts @@ -0,0 +1,52 @@ +import { mkdtemp, open, rm } from "fs/promises"; +import path from "path"; +import { config } from "../../util/config.js"; +import { childLogger } from "../../util/logger.js"; +import type { FileEntry } from "../zip-reader.js"; + +const log = childLogger("sparse-list"); + +export interface SparsePart { + fileName: string; + size: number; + regions: { offset: number; bytes: Buffer }[]; +} + +export type SparseLister = (firstPartPath: string) => Promise; + +/** + * Reconstruct archive header bytes into sparse temp files (data areas left as + * zero holes), run `lister` on the first part, return its entries. + * Returns null on any error or when the lister finds nothing. + */ +export async function listFromSparse( + parts: SparsePart[], + lister: SparseLister, +): Promise { + if (parts.length === 0) return null; + const dir = await mkdtemp(path.join(config.tempDir, "ranged-")); + try { + let firstPath = ""; + for (let i = 0; i < parts.length; i++) { + const p = parts[i]; + const filePath = path.join(dir, p.fileName); + if (i === 0) firstPath = filePath; + const fh = await open(filePath, "w"); + try { + await fh.truncate(p.size); // create the sparse hole + for (const r of p.regions) { + await fh.write(r.bytes, 0, r.bytes.length, r.offset); + } + } finally { + await fh.close(); + } + } + const entries = await lister(firstPath); + return entries.length > 0 ? entries : null; + } catch (err) { + log.warn({ err }, "sparse listing failed"); + return null; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/worker/src/db/queries.ts b/worker/src/db/queries.ts index 09590ad..3f7f63f 100644 --- a/worker/src/db/queries.ts +++ b/worker/src/db/queries.ts @@ -1,5 +1,6 @@ import { db } from "./client.js"; import type { ArchiveType, FetchStatus } from "@prisma/client"; +import type { FileEntry } from "../archive/zip-reader.js"; export async function getActiveAccounts() { return db.telegramAccount.findMany({ @@ -375,6 +376,7 @@ export interface ActivityUpdate { zipsFound?: number; zipsDuplicate?: number; zipsIngested?: number; + zipsBackfilled?: number; } export async function updateRunActivity( @@ -398,6 +400,7 @@ export async function updateRunActivity( ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }), ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }), ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }), + ...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }), ...(activity.currentTopicId !== undefined && { currentTopicId: activity.currentTopicId }), ...(activity.currentAccountChannelMapId !== undefined && { currentAccountChannelMapId: activity.currentAccountChannelMapId, @@ -428,6 +431,7 @@ export async function completeIngestionRun( zipsFound: number; zipsDuplicate: number; zipsIngested: number; + zipsBackfilled: number; } ) { return db.ingestionRun.update({ @@ -1005,3 +1009,142 @@ export async function createAutoGroup(input: { return group.id; } + +// ── Provenance backfill ── + +export interface PlaceholderCandidate { + id: string; + archiveType: string; + fileName: string; + fileCount: number; + fileSize: bigint; + destMessageId: bigint | null; + destMessageIds: bigint[]; + destChannel: { telegramId: bigint } | null; +} + +/** + * Find every placeholder Package matching name+size (oldest first). Package + * has no direct `destChannel` relation (only the scalar `destChannelId`), so + * each row's destination TelegramChannel telegramId is resolved with a + * follow-up lookup rather than a Prisma include. + */ +export async function findPlaceholderCandidates( + destChannelId: string, + fileName: string, + fileSize: bigint, +): Promise { + const rows = await db.package.findMany({ + where: { + fileName, + fileSize, + destMessageId: { not: null }, + // Placeholder provenance (spec §1): manual-upload (source == destination) + // OR rebuild record (sourceMessageId == 0 "unknown" sentinel). + OR: [ + { sourceChannelId: destChannelId }, + { sourceMessageId: 0n }, + ], + }, + select: { + id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, destChannelId: true, + }, + orderBy: { indexedAt: "asc" }, + }); + if (rows.length === 0) return []; + + const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))]; + const channels = destChannelIds.length + ? await db.telegramChannel.findMany({ + where: { id: { in: destChannelIds } }, + select: { id: true, telegramId: true }, + }) + : []; + const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId])); + + return rows.map((row) => ({ + id: row.id, + archiveType: row.archiveType, + fileName: row.fileName, + fileCount: row.fileCount, + fileSize: row.fileSize, + destMessageId: row.destMessageId, + destMessageIds: row.destMessageIds, + destChannel: row.destChannelId && telegramIdById.has(row.destChannelId) + ? { telegramId: telegramIdById.get(row.destChannelId)! } + : null, + })); +} + +export async function findPlaceholderCandidate( + destChannelId: string, + fileName: string, + fileSize: bigint, +): Promise { + return (await findPlaceholderCandidates(destChannelId, fileName, fileSize))[0] ?? null; +} + +export async function getPackageFileCrcs(packageId: string): Promise<(string | null)[]> { + const rows = await db.packageFile.findMany({ + where: { packageId }, + select: { crc32: true }, + }); + return rows.map((r) => r.crc32); +} + +export interface BackfillProvenanceInput { + packageId: string; + destChannelId: string; // to re-check placeholder status in-txn + sourceChannelId: string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + sourceCaption: string | null; + remoteUniqueId: string | null; + creator: string | null; // always set (re-derived by caller) + entries?: FileEntry[]; // set only if candidate had fileCount === 0 + previewData?: Buffer | null; // set only if provided and candidate lacks one + previewMsgId?: bigint | null; +} + +export async function backfillProvenance(input: BackfillProvenanceInput): Promise { + return db.$transaction(async (tx) => { + const current = await tx.package.findUnique({ + where: { id: input.packageId }, + select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true }, + }); + // Re-check placeholder status inside the txn (another worker may have won). + // Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0). + const stillPlaceholder = + !!current && + (current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n); + if (!stillPlaceholder) return false; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data: any = { + sourceChannelId: input.sourceChannelId, + sourceMessageId: input.sourceMessageId, + sourceTopicId: input.sourceTopicId, + sourceCaption: input.sourceCaption, + remoteUniqueId: input.remoteUniqueId, + creator: input.creator, + }; + if (input.entries && current.fileCount === 0) { + await tx.packageFile.deleteMany({ where: { packageId: input.packageId } }); + await tx.packageFile.createMany({ + data: input.entries.map((e) => ({ + packageId: input.packageId, + path: e.path, fileName: e.fileName, extension: e.extension, + compressedSize: e.compressedSize, uncompressedSize: e.uncompressedSize, crc32: e.crc32, + })), + }); + data.fileCount = input.entries.length; + } + if (input.previewData && !current.previewData) { + data.previewData = input.previewData; + data.previewMsgId = input.previewMsgId ?? null; + } + await tx.package.update({ where: { id: input.packageId }, data }); + return true; + }); +} diff --git a/worker/src/provenance-backfill.ts b/worker/src/provenance-backfill.ts new file mode 100644 index 0000000..313282a --- /dev/null +++ b/worker/src/provenance-backfill.ts @@ -0,0 +1,306 @@ +import { db } from "./db/client.js"; +import { childLogger } from "./util/logger.js"; +import { downloadFileRange } from "./tdlib/range-download.js"; +import { invokeWithTimeout } from "./tdlib/download.js"; +import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js"; +import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js"; +import { + findPlaceholderCandidates, + getPackageFileCrcs, + backfillProvenance, + type PlaceholderCandidate, +} from "./db/queries.js"; +import type { FileEntry } from "./archive/zip-reader.js"; +import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js"; +import { readRarListingRanged } from "./archive/ranged/rar-ranged.js"; +import { tdlibRangeReader } from "./archive/ranged/range-reader.js"; +import { fullDownloadListing } from "./archive/ranged/fallback.js"; +import type { Client } from "tdl"; + +const log = childLogger("provenance-backfill"); + +export interface BackfillArgs { + client: Client; + destChannelId: string; + scannedSourceChannelId: string; + fileName: string; + fileSize: bigint; + archiveType: string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + sourceCaption: string | null; + remoteUniqueId: string | null; + creator: string | null; + scannedParts: RangedPart[]; + previewData?: Buffer | null; + previewMsgId?: bigint | null; +} + +/** + * Read a ZIP central directory from the tail of a (possibly multipart) + * archive. `parts` is ordered; only the LAST part carries the EOCD record. + * `fileSize` on each part is that part's own size (NOT the whole-archive + * total) so the download offset stays within that part's bounds, while + * `tailStart` passed to the parser is the logical whole-archive offset + * (preceding parts' sizes + the offset within the last part). + */ +async function readScannedZipListing( + client: Client, + parts: { fileId: string; fileSize: bigint }[], +): Promise { + if (parts.length === 0) return null; + const lastPart = parts[parts.length - 1]; + const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0); + const lastSize = Number(lastPart.fileSize); + for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) { + const partOffset = Math.max(0, lastSize - tailBytes); + const downloadLen = Math.min(tailBytes, lastSize); + try { + const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize); + const tailStart = precedingSize + partOffset; + return parseZipCentralDirectoryFromTail(buf, tailStart); + } catch (err) { + if (err instanceof RangeError) continue; // try a larger tail + log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed"); + return null; + } + } + return null; +} + +/** + * Resolve the destination copy's message(s) into ranged parts (file id + + * size + name), in order, so a multipart destination copy is reconstructed + * with correct per-part sizes and names (the last message carries the + * EOCD-bearing tail part for ZIP; multipart RAR needs correctly-suffixed + * `.partN.rar` names for `unrar` sibling discovery). Cheap-only: any TDLib + * failure here degrades the caller to name-size confidence rather than + * falling back to a full download. + */ +async function resolveDestParts( + client: Client, + destChatTelegramId: bigint, + destMessageIds: bigint[], + destMessageId: bigint | null, + fallbackFileName: string, +): Promise { + const messageIds = destMessageIds.length > 0 ? destMessageIds : destMessageId ? [destMessageId] : []; + if (messageIds.length === 0) return null; + try { + const parts: RangedPart[] = []; + for (const msgId of messageIds) { + const msg = (await invokeWithTimeout(client, { + _: "getMessage", + chat_id: Number(destChatTelegramId), + message_id: Number(msgId), + })) as { content?: { document?: { document?: { id: number; size?: number }; file_name?: string } } }; + const doc = msg?.content?.document?.document; + if (!doc?.id) return null; + const fileName = msg?.content?.document?.file_name || fallbackFileName; + parts.push({ fileId: String(doc.id), fileSize: BigInt(doc.size ?? 0), fileName }); + } + return parts; + } catch (err) { + log.warn({ err, destMessageIds: messageIds.map(Number) }, "destination archive part resolution failed"); + return null; + } +} + +async function readScannedListingRanged( + archiveType: string, + client: Client, + parts: RangedPart[], +): Promise { + const read = tdlibRangeReader(client); + if (archiveType === "ZIP") return readScannedZipListing(client, parts); + if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read); + if (archiveType === "RAR") return readRarListingRanged(parts, read); + return null; +} + +/** + * Build the CRC fingerprint entries for a placeholder candidate: start from + * its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild + * candidate with fileCount === 0), fall back to a fresh ranged read of the + * candidate's own copy in the destination channel (Task 9). + */ +async function resolveCandidateFingerprintEntries( + client: Client, + candidate: PlaceholderCandidate, +): Promise { + const candidateCrcs = await getPackageFileCrcs(candidate.id); + let candidateEntries: FileEntry[] = candidateCrcs.map((crc) => ({ + path: "", fileName: "", extension: null, compressedSize: 0n, uncompressedSize: 0n, crc32: crc, + })); + const hasDestMessage = candidate.destMessageIds.length > 0 || candidate.destMessageId != null; + if (!crcFingerprint(candidateEntries).complete && hasDestMessage && candidate.destChannel) { + const destParts = await resolveDestParts( + client, + candidate.destChannel.telegramId, + candidate.destMessageIds, + candidate.destMessageId, + candidate.fileName, + ); + let destEntries: FileEntry[] | null = null; + if (destParts) { + const read = tdlibRangeReader(client); + destEntries = + candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts) + : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read) + : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read) + : null; + } + if (destEntries) { + candidateEntries = destEntries; + } + } + return candidateEntries; +} + +/** + * Classify a fingerprint comparison between two entry sets. "incomplete" + * means at least one side is missing CRCs (e.g. an empty file → CRC32 of + * zero-length data → null) and the comparison CANNOT be used to confirm or + * refute a match — callers must fall back to name+size confidence rather + * than treating this as a mismatch. + */ +function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" { + const fa = crcFingerprint(a); + const fb = crcFingerprint(b); + if (!fa.complete || !fb.complete) return "incomplete"; + return fingerprintsMatch(a, b) ? "match" : "mismatch"; +} + +export async function tryProvenanceBackfill( + args: BackfillArgs, +): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }> { + const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize); + if (candidates.length === 0) return { backfilled: false }; + + let scannedEntries: FileEntry[] | null = await readScannedListingRanged( + args.archiveType, args.client, args.scannedParts, + ); + // Cheap ranged read failed — fall back to a size-capped full download so the + // listing still gets indexed. Only worth it when the candidate lacks a listing. + if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) { + const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n); + scannedEntries = await fullDownloadListing({ + client: args.client, parts: args.scannedParts, archiveType: args.archiveType, + totalSize, fileName: args.fileName, + }); + } + + let chosen = candidates[0]; + let confidence: "fingerprint" | "name-size" = "name-size"; + + if (candidates.length > 1) { + // Multiple placeholder packages share this name+size. Try to + // disambiguate by fingerprint (ZIP only); if we can't uniquely resolve + // it, notify instead of guessing which one is the real match. + if (args.archiveType === "ZIP" && scannedEntries) { + const matches: PlaceholderCandidate[] = []; + // Candidates NOT ruled out as a definite (both-complete) mismatch — + // used as the name+size fallback pool when the fingerprint can't + // confirm a match (e.g. incomplete CRCs on either side). + const nonMismatches: PlaceholderCandidate[] = []; + for (const c of candidates) { + const candidateEntries = await resolveCandidateFingerprintEntries(args.client, c); + const comparison = compareFingerprints(scannedEntries, candidateEntries); + if (comparison === "match") { + matches.push(c); + nonMismatches.push(c); + } else if (comparison === "incomplete") { + nonMismatches.push(c); + } + // comparison === "mismatch": both sides complete and differ — excluded. + } + if (matches.length === 1) { + chosen = matches[0]; + confidence = "fingerprint"; + } else if (matches.length === 0 && nonMismatches.length === 1) { + // Fingerprint couldn't confirm (incomplete CRCs), but exactly one + // candidate wasn't ruled out as a definite mismatch — fall back to + // name+size confidence rather than treating this as unresolved. + chosen = nonMismatches[0]; + confidence = "name-size"; + } else { + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "WARNING", + title: `Ambiguous provenance match: ${args.fileName}`, + message: `${candidates.length} placeholder packages share this name+size and the fingerprint did not uniquely disambiguate. No provenance was backfilled.`, + context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) }, + }, + }); + return { backfilled: false }; + } + } else { + // Can't disambiguate without a fingerprint — notify, don't guess. + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "WARNING", + title: `Ambiguous provenance match: ${args.fileName}`, + message: `${candidates.length} placeholder packages share this name+size (archive type ${args.archiveType} — no cheap fingerprint). No provenance was backfilled.`, + context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) }, + }, + }); + return { backfilled: false }; + } + } else if (scannedEntries) { + const candidateEntries = await resolveCandidateFingerprintEntries(args.client, chosen); + const comparison = compareFingerprints(scannedEntries, candidateEntries); + if (comparison === "match") { + confidence = "fingerprint"; + } else if (comparison === "mismatch") { + // Both sides' CRCs are complete and differ: NOT the same content + // despite name+size. Do not backfill. + log.info({ candidateId: chosen.id, fileName: args.fileName }, "fingerprint mismatch — not backfilling"); + return { backfilled: false }; + } + // comparison === "incomplete": can't confirm or refute by fingerprint — + // fall through and backfill on name+size confidence instead. + } + + const ok = await backfillProvenance({ + packageId: chosen.id, + destChannelId: args.destChannelId, + sourceChannelId: args.scannedSourceChannelId, + sourceMessageId: args.sourceMessageId, + sourceTopicId: args.sourceTopicId, + sourceCaption: args.sourceCaption, + remoteUniqueId: args.remoteUniqueId, + creator: args.creator, + entries: chosen.fileCount === 0 && scannedEntries ? scannedEntries : undefined, + previewData: args.previewData ?? undefined, + previewMsgId: args.previewMsgId ?? undefined, + }); + + if (!ok) return { backfilled: false }; + + if (confidence === "name-size") { + // Lower-confidence backfill: no CRC fingerprint guard confirmed this + // match. Record it as an auditable event so name+size-only backfills + // can be reviewed after the fact. + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "INFO", + title: `Provenance backfilled by name+size: ${args.fileName}`, + message: `Package ${chosen.id} was matched to a scanned source message by file name and size only (no CRC fingerprint confirmation).`, + context: { + packageId: chosen.id, + fileName: args.fileName, + sourceChannelId: args.scannedSourceChannelId, + }, + }, + }); + } + + log.info( + { candidateId: chosen.id, fileName: args.fileName, confidence, source: args.scannedSourceChannelId }, + "provenance backfilled", + ); + return { backfilled: true, confidence }; +} diff --git a/worker/src/tdlib/range-download.ts b/worker/src/tdlib/range-download.ts new file mode 100644 index 0000000..b1434c6 --- /dev/null +++ b/worker/src/tdlib/range-download.ts @@ -0,0 +1,57 @@ +import { open } from "fs/promises"; +import type { Client } from "tdl"; +import { childLogger } from "../util/logger.js"; +import { withFloodWait } from "../util/retry.js"; + +const log = childLogger("range-download"); +const RANGE_TIMEOUT_MS = 120_000; + +// NOTE (from Task 4 spike): TDLib writes the requested region into +// file.local.path at its ABSOLUTE file offset; file.local.downloaded_prefix_size +// counts contiguous bytes from download_offset. We request a 1KB-aligned offset +// so downloaded_prefix_size covers our whole [offset, offset+limit) window. +// This absolute-offset assumption is PENDING LIVE VERIFICATION ON DEPLOY — +// the authenticated TDLib session could not be spiked in this environment. +export async function downloadFileRange( + client: Client, + fileId: string, + offset: number, + limit: number, + expectedSize: bigint, +): Promise { + const numericId = parseInt(fileId, 10); + const alignedOffset = Math.max(0, offset - (offset % 1024)); + const alignedLimit = limit + (offset - alignedOffset); + + const file = await withFloodWait( + () => + new Promise<{ local: { path: string; download_offset: number; downloaded_prefix_size: number } }>( + (resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Range download timed out for ${fileId}`)), RANGE_TIMEOUT_MS); + client + .invoke({ + _: "downloadFile", + file_id: numericId, + priority: 1, + offset: alignedOffset, + limit: alignedLimit, + synchronous: true, + } as never) + .then((f) => { clearTimeout(timer); resolve(f as never); }) + .catch((e) => { clearTimeout(timer); reject(e); }); + }, + ), + `downloadFileRange:${fileId}`, + ); + + const start = offset; + const fh = await open(file.local.path, "r"); + try { + const buf = Buffer.alloc(limit); + const { bytesRead } = await fh.read(buf, 0, limit, start); + log.debug({ fileId, offset, limit, bytesRead }, "range read"); + return bytesRead < limit ? buf.subarray(0, bytesRead) : buf; + } finally { + await fh.close(); + } +} diff --git a/worker/src/worker.ts b/worker/src/worker.ts index 793ba0b..0017de0 100644 --- a/worker/src/worker.ts +++ b/worker/src/worker.ts @@ -63,9 +63,10 @@ import { hashParts } from "./archive/hash.js"; import { readZipCentralDirectory } from "./archive/zip-reader.js"; import { readRarContents } from "./archive/rar-reader.js"; import { read7zContents } from "./archive/sevenz-reader.js"; +import { tryProvenanceBackfill } from "./provenance-backfill.js"; import { byteLevelSplit, concatenateFiles } from "./archive/split.js"; import { uploadToChannel, UploadStallError } from "./upload/channel.js"; -import { processAlbumGroups, processRuleBasedGroups, processTimeWindowGroups, processPatternGroups, processCreatorGroups, processZipPathGroups, processReplyChainGroups, processCaptionGroups, detectGroupingConflicts, type IndexedPackageRef } from "./grouping.js"; +import { processAlbumGroups, detectGroupingConflicts, type IndexedPackageRef } from "./grouping.js"; import { db } from "./db/client.js"; import type { TelegramAccount, TelegramChannel } from "@prisma/client"; import type { Client } from "tdl"; @@ -314,6 +315,7 @@ interface PipelineContext { zipsFound: number; zipsDuplicate: number; zipsIngested: number; + zipsBackfilled: number; }; /** Creator from forum topic name (null for non-forum). */ topicCreator: string | null; @@ -422,6 +424,7 @@ export async function runWorkerForAccount( zipsFound: 0, zipsDuplicate: 0, zipsIngested: 0, + zipsBackfilled: 0, }; try { @@ -1479,34 +1482,13 @@ async function processArchiveSets( scanResult.photos ); - // Auto-grouping passes (gated by per-channel flag) - const channelRecord = await db.telegramChannel.findUnique({ - where: { id: channel.id }, - select: { autoGroupEnabled: true }, - }); - - if (channelRecord?.autoGroupEnabled !== false) { - // Learned rule-based grouping (from manual overrides) - await processRuleBasedGroups(channel.id, indexedPackageRefs); - - // Time-window grouping for remaining ungrouped packages - await processTimeWindowGroups(channel.id, indexedPackageRefs); - - // Pattern-based grouping (date patterns, project slugs) - await processPatternGroups(channel.id, indexedPackageRefs); - - // Creator-based grouping (3+ files from same creator) - await processCreatorGroups(channel.id, indexedPackageRefs); - - // ZIP path prefix grouping (shared root folder inside archives) - await processZipPathGroups(channel.id, indexedPackageRefs); - - // Reply chain grouping (messages replying to same root) - await processReplyChainGroups(channel.id, indexedPackageRefs); - - // Caption fuzzy match grouping - await processCaptionGroups(channel.id, indexedPackageRefs); - } + // Heuristic auto-grouping passes (rule/time/pattern/creator/zip-path/ + // reply-chain/caption) were removed: the STL view is now a flat list + // organized by the creator filter, so automatically inventing groups at + // ingestion is no longer wanted. Album grouping above is kept because it + // reflects real upload structure (files posted together as one Telegram + // album), not a heuristic guess. Existing groups and the manual grouping + // actions in the UI are unaffected. // Check for potential grouping conflicts await detectGroupingConflicts(channel.id, indexedPackageRefs); @@ -1666,6 +1648,56 @@ async function processOneArchiveSet( return null; } + // ── Cross-channel provenance backfill ── + // The same-channel checks above missed. Before downloading, see if this + // archive is the true origin of a placeholder-source package (manual upload + // / rebuild record whose sourceChannelId == destChannelId). If so, backfill + // its real provenance and skip the download entirely. + const archType = archiveSet.type === "7Z" ? "SEVEN_Z" : archiveSet.type; + if (destChannelId && (archType === "ZIP" || archType === "RAR" || archType === "SEVEN_Z")) { + try { + const derivedCreator = + topicCreator && topicCreator !== "General" + ? topicCreator + : (extractCreatorFromFileName(archiveName) ?? topicCreator ?? null); + const preview = previewMatches.get(archiveSet.baseName); + const result = await tryProvenanceBackfill({ + client, + destChannelId, + scannedSourceChannelId: channel.id, + fileName: archiveName, + fileSize: totalArchiveSize, + archiveType: archType, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId, + sourceCaption: archiveSet.parts[0].caption ?? null, + remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null, + creator: derivedCreator, + scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize, fileName: p.fileName })), + previewData: null, + previewMsgId: preview?.id ?? null, + }); + if (result.backfilled) { + counters.zipsBackfilled++; + accountLog.info( + { fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id), confidence: result.confidence }, + "Backfilled provenance for placeholder package — skipping download", + ); + await updateRunActivity(runId, { + currentActivity: `Backfilled provenance for ${archiveName}`, + currentStep: "backfilling", + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + zipsBackfilled: counters.zipsBackfilled, + }); + return null; + } + } catch (err) { + accountLog.warn({ err, fileName: archiveName }, "Provenance backfill attempt failed (non-fatal), continuing to normal ingestion"); + } + } + // ── Size guard: skip archives that exceed WORKER_MAX_ZIP_SIZE_MB ── const maxSizeBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n; if (totalArchiveSize > maxSizeBytes) { diff --git a/worker/vitest.config.ts b/worker/vitest.config.ts new file mode 100644 index 0000000..c1433e6 --- /dev/null +++ b/worker/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + }, +});