mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-09-21 05:21:43 +00:00
Merge branch 'feat/ranged-archive-listing' into main
# Conflicts: # .env.example # backup/Dockerfile # docker-compose.yml
This commit is contained in:
@@ -89,7 +89,14 @@
|
||||
"Bash(wait:*)",
|
||||
"WebSearch",
|
||||
"Bash(SKILL_CREATOR_PATH=\"C:\\\\Users\\\\A00963355\\\\.claude\\\\plugins\\\\cache\\\\claude-plugins-official\\\\skill-creator\\\\d5c15b861cd2\\\\skills\\\\skill-creator\" && WORKSPACE=\"C:\\\\Users\\\\A00963355\\\\OneDrive - Amaris Zorggroep\\\\Documents\\\\VScodeProjects\\\\DragonsStash\\\\.claude\\\\skills\\\\tdlib-telegram-workspace\\\\iteration-1\" && python \"$SKILL_CREATOR_PATH/eval-viewer/generate_review.py\" \"$WORKSPACE\" --skill-name \"tdlib-telegram\" --benchmark \"$WORKSPACE/benchmark.json\" --static \"$WORKSPACE/review.html\" 2>&1)",
|
||||
"Bash(start:*)"
|
||||
"Bash(start:*)",
|
||||
"Bash(npm run:*)",
|
||||
"Bash(DATABASE_URL=\"postgresql://dragons:stash@localhost:5432/dragonsstash\" npx prisma migrate dev --name add-skipped-packages)",
|
||||
"Bash(git checkout:*)",
|
||||
"Bash(DATABASE_URL=\"postgresql://dragons:stash@localhost:5432/dragonsstash?schema=public\" npx prisma migrate dev --name add_package_groups 2>&1)",
|
||||
"Bash(psql:*)",
|
||||
"Bash(git log:*)",
|
||||
"Bash(git merge:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+16
-1
@@ -54,9 +54,24 @@ steps:
|
||||
password:
|
||||
from_secret: gitea_password
|
||||
|
||||
- name: build-backup
|
||||
image: plugins/docker
|
||||
depends_on: [clone]
|
||||
settings:
|
||||
repo: git.samagsteribbe.nl/admin/dragonsstash-backup
|
||||
registry: git.samagsteribbe.nl
|
||||
dockerfile: backup/Dockerfile
|
||||
tags:
|
||||
- latest
|
||||
- "${DRONE_COMMIT_SHA:0:8}"
|
||||
username:
|
||||
from_secret: gitea_username
|
||||
password:
|
||||
from_secret: gitea_password
|
||||
|
||||
- name: deploy
|
||||
image: alpine
|
||||
depends_on: [build-app, build-worker, build-bot]
|
||||
depends_on: [build-app, build-worker, build-bot, build-backup]
|
||||
environment:
|
||||
SSH_KEY:
|
||||
from_secret: ssh_key
|
||||
|
||||
+8
-7
@@ -37,10 +37,11 @@ WORKER_MAX_ZIP_SIZE_MB=4096
|
||||
MULTIPART_TIMEOUT_HOURS=0
|
||||
LOG_LEVEL="info"
|
||||
|
||||
# Disaster recovery backups
|
||||
BACKUP_MOUNT_PATH="/mnt/dragonsstash-backups"
|
||||
BACKUP_STAGING_PATH="/var/lib/dragons-stash/backup-staging"
|
||||
BACKUP_REPOSITORY="/backup/restic"
|
||||
BACKUP_RESTIC_PASSWORD_FILE="/etc/dragons-stash/restic-password"
|
||||
BACKUP_RETENTION_DAYS=30
|
||||
BACKUP_APP_VERSION="unknown"
|
||||
# Backup (NAS via SMB/CIFS + restic)
|
||||
NAS_HOST="" # Synology NAS IP or hostname reachable from this host
|
||||
NAS_SHARE="" # SMB share name, e.g. dragonsstash_backups
|
||||
NAS_USERNAME="" # SMB user with read/write on the share
|
||||
NAS_PASSWORD="" # SMB user password (avoid commas — they delimit cifs mount opts)
|
||||
RESTIC_PASSWORD="" # generate with: openssl rand -base64 32
|
||||
KUMA_PUSH_URL="" # optional: Uptime Kuma Push monitor URL; leave empty to disable alerting
|
||||
TZ="Etc/UTC"
|
||||
|
||||
+10
-5
@@ -1,8 +1,13 @@
|
||||
FROM postgres:16-alpine
|
||||
FROM alpine:3.20
|
||||
|
||||
RUN apk add --no-cache bash restic coreutils
|
||||
# Note: use busybox's built-in crond (Alpine base), NOT the dcron package —
|
||||
# dcron's crond fails with "setpgid: Operation not permitted" in this runtime.
|
||||
RUN apk add --no-cache restic postgresql16-client curl tzdata tar bash
|
||||
|
||||
COPY scripts/backup/container-entrypoint.sh /usr/local/bin/dragons-stash-backup
|
||||
RUN chmod 0755 /usr/local/bin/dragons-stash-backup
|
||||
COPY backup/backup.sh /backup.sh
|
||||
COPY backup/entrypoint.sh /entrypoint.sh
|
||||
COPY backup/crontab /etc/crontabs/root
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/dragons-stash-backup"]
|
||||
RUN chmod +x /backup.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
report_failure() {
|
||||
[ -n "${KUMA_PUSH_URL:-}" ] || return 0
|
||||
curl -fsS "$KUMA_PUSH_URL" --get \
|
||||
--data-urlencode "status=down" \
|
||||
--data-urlencode "msg=$BASH_COMMAND failed" || true
|
||||
}
|
||||
trap report_failure ERR
|
||||
|
||||
DUMP_FILE=/tmp/dragonsstash.dump
|
||||
TAR_FILE=/tmp/tdlib.tar.gz
|
||||
|
||||
trap 'rm -f "$DUMP_FILE" "$TAR_FILE"' EXIT
|
||||
|
||||
pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
|
||||
|
||||
# TDLib volumes are tarred live (best-effort, per design). A file changing
|
||||
# mid-read makes GNU tar exit 1 (warning) — that is expected here and must not
|
||||
# abort the backup. Only a genuine error (exit >= 2) is fatal.
|
||||
tar --warning=no-file-changed -czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot \
|
||||
|| { rc=$?; [ "$rc" -le 1 ] || exit "$rc"; }
|
||||
|
||||
restic backup "$DUMP_FILE" "$TAR_FILE"
|
||||
restic forget --keep-daily 14 --prune
|
||||
|
||||
if [ -n "${KUMA_PUSH_URL:-}" ]; then
|
||||
curl -fsS "$KUMA_PUSH_URL" --get \
|
||||
--data-urlencode "status=up" \
|
||||
--data-urlencode "msg=OK"
|
||||
fi
|
||||
@@ -0,0 +1,2 @@
|
||||
0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
|
||||
0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
set -uo pipefail
|
||||
|
||||
# Ensure the repo exists, but never crash-loop on it: a transient error reading
|
||||
# the repo (CIFS hiccup, stale lock) must not kill PID 1. `restic init` failing
|
||||
# because the repo already exists is expected and harmless here.
|
||||
if ! restic cat config >/dev/null 2>&1; then
|
||||
restic init || echo "restic init skipped (repo already exists or temporarily unreachable)"
|
||||
fi
|
||||
|
||||
exec crond -f -l 2
|
||||
+24
-12
@@ -98,26 +98,31 @@ services:
|
||||
- backend
|
||||
|
||||
backup:
|
||||
profiles: ["backup"]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: backup/Dockerfile
|
||||
pull_policy: never
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-dragons}:${POSTGRES_PASSWORD:-stash}@db:5432/${POSTGRES_DB:-dragonsstash}
|
||||
RESTIC_REPOSITORY: ${BACKUP_REPOSITORY:-/backup/restic}
|
||||
RESTIC_PASSWORD_FILE: /run/secrets/restic-password
|
||||
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30}
|
||||
BACKUP_APP_VERSION: ${BACKUP_APP_VERSION:-unknown}
|
||||
user: "0:0"
|
||||
- POSTGRES_USER=${POSTGRES_USER:-dragons}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- PGPASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
|
||||
- RESTIC_REPOSITORY=/backups/restic-repo
|
||||
- RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
|
||||
- KUMA_PUSH_URL=${KUMA_PUSH_URL:-}
|
||||
- TZ=${TZ:-Etc/UTC}
|
||||
volumes:
|
||||
- tdlib_state:/data/tdlib-worker:ro
|
||||
- tdlib_bot_state:/data/tdlib-bot:ro
|
||||
- ${BACKUP_MOUNT_PATH:?Set BACKUP_MOUNT_PATH to the mounted Synology share}:/backup:rw
|
||||
- ${BACKUP_STAGING_PATH:?Set BACKUP_STAGING_PATH to a local staging directory}:/staging:rw
|
||||
- ${BACKUP_RESTIC_PASSWORD_FILE:?Set BACKUP_RESTIC_PASSWORD_FILE to a root-readable secret file}:/run/secrets/restic-password:ro
|
||||
- nas_backups:/backups
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
networks:
|
||||
- backend
|
||||
|
||||
@@ -140,8 +145,10 @@ services:
|
||||
limits:
|
||||
memory: 1G
|
||||
networks:
|
||||
- frontend
|
||||
- backend
|
||||
frontend: {}
|
||||
backend:
|
||||
aliases:
|
||||
- dragonsstash-db
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -149,6 +156,11 @@ volumes:
|
||||
tdlib_bot_state:
|
||||
tmp_zips:
|
||||
manual_uploads:
|
||||
nas_backups:
|
||||
driver_opts:
|
||||
type: cifs
|
||||
o: "username=${NAS_USERNAME},password=${NAS_PASSWORD},vers=3.0,uid=0,gid=0,file_mode=0660,dir_mode=0770"
|
||||
device: "//${NAS_HOST}/${NAS_SHARE}"
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
|
||||
@@ -0,0 +1,555 @@
|
||||
# NAS Backup for Postgres + TDLib State Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a `backup` container to the DragonsStash stack that takes daily, encrypted, deduplicated backups of the Postgres database and both TDLib state volumes, and ships them to a Synology NAS over NFS.
|
||||
|
||||
**Architecture:** A small Alpine-based image (restic + postgresql16-client + curl + dcron) runs as its own compose service. A crontab fires `backup.sh` daily at 03:00, which dumps Postgres, tars the TDLib volumes, hands both to `restic backup` against an NFS-backed Docker volume, prunes with `restic forget --keep-daily 14`, and reports success/failure to an Uptime Kuma push monitor. Matches the existing `worker`/`bot` pattern: build context in the repo's `docker-compose.yml`, prebuilt image in `/opt/stacks/DragonsStash/docker-compose.yml`, built and pushed by `.drone.yml`.
|
||||
|
||||
**Tech Stack:** Alpine 3.20, restic 0.16, postgresql16-client, dcron, bash, Docker Compose NFS volume driver.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Retention: `restic forget --keep-daily 14 --prune` (14-day window, per approved spec).
|
||||
- Schedule: daily backup at 03:00, weekly `restic check` at 04:00 Sunday.
|
||||
- No Docker socket mount, no `privileged: true` — the backup container must not be able to control sibling containers.
|
||||
- No host-level mount — NFS access only via Docker's native `driver_opts: type: nfs` volume, never `/etc/fstab`.
|
||||
- Encryption and retention are restic's job — no hand-rolled `age`/`gpg`/`find -mtime` logic.
|
||||
- TDLib volumes are tarred live (best-effort) — never pause `worker`/`bot` for the backup.
|
||||
- Restore is a manual, documented procedure only — never scripted/automated.
|
||||
|
||||
**Required user input before Task 5 can run:** `NAS_HOST` and `NAS_EXPORT_PATH` (the Synology NFS share details) and a Kuma Push-monitor URL (`KUMA_PUSH_URL`, created manually in the existing Uptime Kuma instance, ~26h expected heartbeat interval). Tasks 1–4 need none of these and can proceed immediately; do not substitute placeholder values for them in Task 5 — stop and ask the user instead.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Backup image (Dockerfile + entrypoint)
|
||||
|
||||
**Files:**
|
||||
- Create: `backup/Dockerfile`
|
||||
- Create: `backup/entrypoint.sh`
|
||||
- Create: `backup/crontab`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: a buildable image tagged `dragonsstash-backup:test` locally, with `/entrypoint.sh` as `ENTRYPOINT`, `/backup.sh` present at the image root (written in Task 2 — this task only needs the `COPY` line and a placeholder-free stub isn't acceptable, so create an empty‑body-but-real `backup/backup.sh` here containing just `#!/bin/bash` + `exit 0`, and Task 2 replaces its contents), `restic`, `pg_dump`/`pg_restore`, `curl`, `tar`, `bash`, `dcron` all on `PATH`.
|
||||
- Consumes: nothing from earlier tasks.
|
||||
|
||||
- [ ] **Step 1: Write `backup/backup.sh` stub**
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
exit 0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write `backup/entrypoint.sh`**
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
if ! restic snapshots >/dev/null 2>&1; then
|
||||
restic init
|
||||
fi
|
||||
|
||||
exec crond -f -l 2
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Write `backup/crontab`**
|
||||
|
||||
```
|
||||
0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
|
||||
0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write `backup/Dockerfile`**
|
||||
|
||||
```dockerfile
|
||||
FROM alpine:3.20
|
||||
|
||||
RUN apk add --no-cache restic postgresql16-client curl tzdata dcron tar bash
|
||||
|
||||
COPY backup/backup.sh /backup.sh
|
||||
COPY backup/entrypoint.sh /entrypoint.sh
|
||||
COPY backup/crontab /etc/crontabs/root
|
||||
|
||||
RUN chmod +x /backup.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Build the image**
|
||||
|
||||
Run: `cd /home/sam/Documents/DragonsStash && docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
|
||||
Expected: build completes with `Successfully tagged dragonsstash-backup:test` (or Buildkit's equivalent final `naming to docker.io/library/dragonsstash-backup:test done`), no errors.
|
||||
|
||||
- [ ] **Step 6: Verify the tools are present**
|
||||
|
||||
Run: `docker run --rm dragonsstash-backup:test restic version && docker run --rm dragonsstash-backup:test pg_dump --version`
|
||||
Expected: `restic 0.16.x ...` and `pg_dump (PostgreSQL) 16.x` printed, both commands exit 0.
|
||||
|
||||
- [ ] **Step 7: Verify the entrypoint initializes an empty repo and starts cron**
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/backup-repo-smoke
|
||||
docker run -d --name backup-smoke \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=smoketest \
|
||||
-v /tmp/backup-repo-smoke:/backups \
|
||||
dragonsstash-backup:test
|
||||
sleep 2
|
||||
docker logs backup-smoke
|
||||
docker exec backup-smoke restic snapshots
|
||||
docker rm -f backup-smoke
|
||||
rm -rf /tmp/backup-repo-smoke
|
||||
```
|
||||
|
||||
Expected: `docker logs` shows no errors (restic init ran silently); `restic snapshots` prints an empty snapshot list (repo exists, header row only, no error).
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add backup/Dockerfile backup/entrypoint.sh backup/backup.sh backup/crontab
|
||||
git commit -m "Add backup service image (Dockerfile, entrypoint, crontab)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `backup.sh` script
|
||||
|
||||
**Files:**
|
||||
- Modify: `backup/backup.sh` (replace Task 1's stub with the real script)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the image built in Task 1 (`dragonsstash-backup:test`), rebuilt after this change.
|
||||
- Produces: `/backup.sh`, invoked by cron in Task 1's `crontab` and manually in Task 6's verification. Reads env vars `POSTGRES_USER`, `PGPASSWORD`, `POSTGRES_DB`, `RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, `KUMA_PUSH_URL`. Assumes network hostname `dragonsstash-db:5432` for Postgres and mounts `/data/tdlib-worker`, `/data/tdlib-bot` (read-only) for TDLib state.
|
||||
|
||||
- [ ] **Step 1: Replace `backup/backup.sh` with the real script**
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
report_failure() {
|
||||
curl -fsS "$KUMA_PUSH_URL" --get \
|
||||
--data-urlencode "status=down" \
|
||||
--data-urlencode "msg=$BASH_COMMAND failed" || true
|
||||
}
|
||||
trap report_failure ERR
|
||||
|
||||
DUMP_FILE=/tmp/dragonsstash.dump
|
||||
TAR_FILE=/tmp/tdlib.tar.gz
|
||||
|
||||
pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
|
||||
|
||||
tar czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot
|
||||
|
||||
restic backup "$DUMP_FILE" "$TAR_FILE"
|
||||
restic forget --keep-daily 14 --prune
|
||||
|
||||
rm -f "$DUMP_FILE" "$TAR_FILE"
|
||||
|
||||
curl -fsS "$KUMA_PUSH_URL" --get \
|
||||
--data-urlencode "status=up" \
|
||||
--data-urlencode "msg=OK"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Rebuild the image**
|
||||
|
||||
Run: `docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
|
||||
Expected: build succeeds.
|
||||
|
||||
- [ ] **Step 3: Stand up a scratch Postgres aliased as `dragonsstash-db`**
|
||||
|
||||
```bash
|
||||
docker network create backup-test-net 2>/dev/null || true
|
||||
docker run -d --name test-pg --network backup-test-net --network-alias dragonsstash-db \
|
||||
-e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e POSTGRES_DB=dragonsstash \
|
||||
postgres:16-alpine
|
||||
sleep 5
|
||||
docker exec test-pg pg_isready -U dragons -d dragonsstash
|
||||
```
|
||||
|
||||
Expected: `accepting connections`.
|
||||
|
||||
- [ ] **Step 4: Stand up a mock Kuma push endpoint**
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/mock-kuma-root && touch /tmp/mock-kuma-root/push
|
||||
docker run -d --name mock-kuma --network backup-test-net \
|
||||
-v /tmp/mock-kuma-root:/srv -w /srv python:3-alpine \
|
||||
python3 -m http.server 8000
|
||||
sleep 1
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Prepare fake TDLib state and a local restic repo dir**
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/backup-test/tdlib-worker /tmp/backup-test/tdlib-bot /tmp/backup-test/repo
|
||||
echo "fake-session" > /tmp/backup-test/tdlib-worker/state.bin
|
||||
echo "fake-session" > /tmp/backup-test/tdlib-bot/state.bin
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Initialize the test restic repo and run `backup.sh` (success path)**
|
||||
|
||||
```bash
|
||||
docker run --rm --network backup-test-net \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
|
||||
-v /tmp/backup-test/repo:/backups \
|
||||
--entrypoint restic dragonsstash-backup:test init
|
||||
|
||||
docker run --rm --network backup-test-net \
|
||||
-e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e PGPASSWORD=stash -e POSTGRES_DB=dragonsstash \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
|
||||
-e KUMA_PUSH_URL=http://mock-kuma:8000/push \
|
||||
-v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
|
||||
-v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
|
||||
-v /tmp/backup-test/repo:/backups \
|
||||
--entrypoint /backup.sh dragonsstash-backup:test
|
||||
echo "exit code: $?"
|
||||
```
|
||||
|
||||
Expected: exit code `0`, restic prints a line like `snapshot xxxxxxxx saved`, no error output.
|
||||
|
||||
- [ ] **Step 7: Verify the snapshot landed and contains both files**
|
||||
|
||||
```bash
|
||||
docker run --rm -v /tmp/backup-test/repo:/backups \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
|
||||
--entrypoint restic dragonsstash-backup:test snapshots
|
||||
|
||||
docker run --rm -v /tmp/backup-test/repo:/backups \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
|
||||
--entrypoint restic dragonsstash-backup:test ls latest
|
||||
```
|
||||
|
||||
Expected: `snapshots` shows exactly one entry; `ls latest` lists `/tmp/dragonsstash.dump` and `/tmp/tdlib.tar.gz`.
|
||||
|
||||
- [ ] **Step 8: Verify the failure path reports to Kuma**
|
||||
|
||||
```bash
|
||||
docker run --rm --network backup-test-net \
|
||||
-e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=wrongpass -e PGPASSWORD=wrongpass -e POSTGRES_DB=dragonsstash \
|
||||
-e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
|
||||
-e KUMA_PUSH_URL=http://mock-kuma:8000/push \
|
||||
-v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
|
||||
-v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
|
||||
-v /tmp/backup-test/repo:/backups \
|
||||
--entrypoint /backup.sh dragonsstash-backup:test
|
||||
echo "exit code: $?"
|
||||
docker logs mock-kuma | tail -5
|
||||
```
|
||||
|
||||
Expected: exit code nonzero (pg_dump auth failure trips `set -e`); `docker logs mock-kuma` shows a GET request line containing `status=down`.
|
||||
|
||||
- [ ] **Step 9: Clean up test resources**
|
||||
|
||||
```bash
|
||||
docker rm -f test-pg mock-kuma
|
||||
docker network rm backup-test-net
|
||||
rm -rf /tmp/backup-test /tmp/mock-kuma-root
|
||||
```
|
||||
|
||||
- [ ] **Step 10: Commit**
|
||||
|
||||
```bash
|
||||
git add backup/backup.sh
|
||||
git commit -m "Implement backup.sh: pg_dump + tdlib tar + restic backup/forget + Kuma reporting"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Wire the `backup` service into the repo's `docker-compose.yml`
|
||||
|
||||
**Files:**
|
||||
- Modify: `docker-compose.yml`
|
||||
- Modify: `.env.example`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backup/Dockerfile` (Task 1), `backup/backup.sh` (Task 2).
|
||||
- Produces: a `backup` compose service buildable via `docker compose build backup`, and a `nas_backups` named volume other tasks (5) will mirror into the production compose file.
|
||||
|
||||
- [ ] **Step 1: Add the `backup` service to `docker-compose.yml`**
|
||||
|
||||
Insert after the existing `bot` service (before `db`):
|
||||
|
||||
```yaml
|
||||
backup:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: backup/Dockerfile
|
||||
pull_policy: never
|
||||
environment:
|
||||
- POSTGRES_USER=${POSTGRES_USER:-dragons}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- PGPASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
|
||||
- RESTIC_REPOSITORY=/backups/restic-repo
|
||||
- RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
|
||||
- KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
|
||||
- TZ=${TZ:-Etc/UTC}
|
||||
volumes:
|
||||
- tdlib_state:/data/tdlib-worker:ro
|
||||
- tdlib_bot_state:/data/tdlib-bot:ro
|
||||
- nas_backups:/backups
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
networks:
|
||||
- backend
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add the `nas_backups` volume to the `volumes:` block**
|
||||
|
||||
```yaml
|
||||
nas_backups:
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
|
||||
device: ":${NAS_EXPORT_PATH}"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Document the new env vars in `.env.example`**
|
||||
|
||||
Append:
|
||||
|
||||
```
|
||||
# Backup (NAS via NFS + restic)
|
||||
NAS_HOST="" # Synology NAS IP or hostname reachable from this host
|
||||
NAS_EXPORT_PATH="" # NFS export path, e.g. /volume1/dragonsstash-backups
|
||||
RESTIC_PASSWORD="" # generate with: openssl rand -base64 32
|
||||
KUMA_PUSH_URL="" # Uptime Kuma Push monitor URL (create the monitor first)
|
||||
TZ="Etc/UTC"
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Validate the compose file parses**
|
||||
|
||||
Run (with dummy values so the `:?` guards don't fail parsing — `AUTH_SECRET` is required by the existing `app` service, not by this change, but `config` validates the whole file):
|
||||
```bash
|
||||
RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
|
||||
docker compose -f docker-compose.yml config --quiet
|
||||
```
|
||||
Expected: no output, exit code 0 (a syntax/interpolation error would print to stderr and exit nonzero).
|
||||
|
||||
- [ ] **Step 5: Validate the service actually builds through Compose**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
|
||||
docker compose -f docker-compose.yml build backup
|
||||
```
|
||||
Expected: build succeeds (reuses Task 1's image layers).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add docker-compose.yml .env.example
|
||||
git commit -m "Add backup service and nas_backups volume to docker-compose.yml"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: CI — build and push the backup image
|
||||
|
||||
**Files:**
|
||||
- Modify: `.drone.yml`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backup/Dockerfile` (Task 1).
|
||||
- Produces: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (and `:<short-sha>`), pushed on every push to `main`. Task 5's production compose file references this image tag.
|
||||
|
||||
- [ ] **Step 1: Add a `build-backup` step, mirroring `build-worker`/`build-bot`**
|
||||
|
||||
Insert after the existing `build-bot` step in `.drone.yml`:
|
||||
|
||||
```yaml
|
||||
- name: build-backup
|
||||
image: plugins/docker
|
||||
depends_on: [clone]
|
||||
settings:
|
||||
repo: git.samagsteribbe.nl/admin/dragonsstash-backup
|
||||
registry: git.samagsteribbe.nl
|
||||
dockerfile: backup/Dockerfile
|
||||
tags:
|
||||
- latest
|
||||
- "${DRONE_COMMIT_SHA:0:8}"
|
||||
username:
|
||||
from_secret: gitea_username
|
||||
password:
|
||||
from_secret: gitea_password
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add `build-backup` to the `deploy` step's `depends_on`**
|
||||
|
||||
Change:
|
||||
```yaml
|
||||
- name: deploy
|
||||
image: alpine
|
||||
depends_on: [build-app, build-worker, build-bot]
|
||||
```
|
||||
to:
|
||||
```yaml
|
||||
- name: deploy
|
||||
image: alpine
|
||||
depends_on: [build-app, build-worker, build-bot, build-backup]
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Validate YAML syntax**
|
||||
|
||||
Run: `python3 -c "import yaml; yaml.safe_load(open('.drone.yml')); print('OK')"`
|
||||
Expected: `OK` printed, no exception.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add .drone.yml
|
||||
git commit -m "Add build-backup CI step, include it in deploy dependencies"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Wire production (`/opt/stacks/DragonsStash`) — requires NAS + Kuma details from the user
|
||||
|
||||
**Do not start this task until the user has supplied `NAS_HOST` and `NAS_EXPORT_PATH` for the Synology NFS share, and has created an Uptime Kuma Push monitor (name it `dragonsstash-backup`, ~26h expected heartbeat interval) and shared its push URL. If any of these are missing, stop and ask — do not substitute placeholder values here, since this file drives the real deployment.**
|
||||
|
||||
**Files:**
|
||||
- Modify: `/opt/stacks/DragonsStash/docker-compose.yml`
|
||||
- Modify: `/opt/stacks/DragonsStash/.env`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (published by Task 4's CI step once merged/pushed), the real `NAS_HOST`/`NAS_EXPORT_PATH`/`KUMA_PUSH_URL` values gathered above.
|
||||
- Produces: a running `dragonsstash-backup` container on the production host, verified in Task 6.
|
||||
|
||||
- [ ] **Step 1: Generate `RESTIC_PASSWORD` and add all four new vars to `/opt/stacks/DragonsStash/.env`**
|
||||
|
||||
```bash
|
||||
cd /opt/stacks/DragonsStash
|
||||
printf '\n# Backup (NAS via NFS + restic)\nNAS_HOST="<value from user>"\nNAS_EXPORT_PATH="<value from user>"\nRESTIC_PASSWORD="%s"\nKUMA_PUSH_URL="<value from user>"\nTZ="Etc/UTC"\n' "$(openssl rand -base64 32)" >> .env
|
||||
```
|
||||
|
||||
Replace the two `<value from user>` placeholders with the real NAS details and Kuma push URL before saving — this step cannot be completed with the literal placeholder text left in place.
|
||||
|
||||
- [ ] **Step 2: Add the `backup` service to `/opt/stacks/DragonsStash/docker-compose.yml`**
|
||||
|
||||
Insert after the existing `bot` service (before `db`):
|
||||
|
||||
```yaml
|
||||
backup:
|
||||
image: git.samagsteribbe.nl/admin/dragonsstash-backup:latest
|
||||
container_name: dragonsstash-backup
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- POSTGRES_USER=${POSTGRES_USER:-dragons}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- PGPASSWORD=${POSTGRES_PASSWORD:-stash}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
|
||||
- RESTIC_REPOSITORY=/backups/restic-repo
|
||||
- RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
|
||||
- KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
|
||||
- TZ=${TZ:-Etc/UTC}
|
||||
volumes:
|
||||
- tdlib_state:/data/tdlib-worker:ro
|
||||
- tdlib_bot_state:/data/tdlib-bot:ro
|
||||
- nas_backups:/backups
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 256M
|
||||
networks:
|
||||
- internal
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add the `nas_backups` volume**
|
||||
|
||||
```yaml
|
||||
nas_backups:
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
|
||||
device: ":${NAS_EXPORT_PATH}"
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Validate the production compose file parses with the real `.env`**
|
||||
|
||||
Run: `cd /opt/stacks/DragonsStash && docker compose config --quiet`
|
||||
Expected: no output, exit code 0.
|
||||
|
||||
- [ ] **Step 5: Commit is not applicable here** — `/opt/stacks/DragonsStash` is a deployed copy, not the git repo (confirm with `git -C /opt/stacks/DragonsStash status` — expect "not a git repository"). Skip committing; Task 6 deploys these file changes directly.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Deploy and verify
|
||||
|
||||
**Files:** none (operational task)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: everything from Tasks 1–5.
|
||||
- Produces: a running, verified backup on the real NAS, and one completed restore drill.
|
||||
|
||||
- [ ] **Step 1: Confirm with the user before pushing/deploying**
|
||||
|
||||
Pushing to `main` triggers Drone CI to build all four images and deploy to the production host via SSH (`docker compose pull && docker compose up -d`). Confirm the user wants this to happen now before proceeding — this affects the live stack.
|
||||
|
||||
- [ ] **Step 2: Push to `main`**
|
||||
|
||||
```bash
|
||||
cd /home/sam/Documents/DragonsStash
|
||||
git push origin main
|
||||
```
|
||||
|
||||
Expected: Drone pipeline runs `build-app`, `build-worker`, `build-bot`, `build-backup`, then `deploy`, all green. Check via the Drone UI or `drone build info admin/DragonsStash <build-number>` if the `drone` CLI is configured.
|
||||
|
||||
- [ ] **Step 3: Confirm the container is up on the production host**
|
||||
|
||||
```bash
|
||||
ssh sam@192.168.68.68 "docker ps --filter name=dragonsstash-backup --format '{{.Names}}\t{{.Status}}'"
|
||||
```
|
||||
|
||||
Expected: `dragonsstash-backup Up ...`.
|
||||
|
||||
- [ ] **Step 4: Trigger one manual backup run and confirm a snapshot lands on the NAS**
|
||||
|
||||
```bash
|
||||
ssh sam@192.168.68.68 "docker exec dragonsstash-backup /backup.sh"
|
||||
ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic snapshots"
|
||||
```
|
||||
|
||||
Expected: `backup.sh` exits 0; `restic snapshots` lists exactly one entry.
|
||||
|
||||
- [ ] **Step 5: Confirm the Uptime Kuma push monitor shows green**
|
||||
|
||||
Open the Uptime Kuma dashboard and check the `dragonsstash-backup` monitor's status is up with a recent heartbeat.
|
||||
|
||||
- [ ] **Step 6: Restore drill — prove the backup is actually restorable**
|
||||
|
||||
```bash
|
||||
ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic restore latest --target /tmp/restore-drill"
|
||||
ssh sam@192.168.68.68 "docker exec dragonsstash-backup ls -la /tmp/restore-drill/tmp"
|
||||
```
|
||||
|
||||
Expected: `/tmp/restore-drill/tmp/dragonsstash.dump` and `/tmp/restore-drill/tmp/tdlib.tar.gz` both present with nonzero size. Then, on a scratch Postgres (not the live `dragonsstash-db`), confirm the dump restores cleanly:
|
||||
|
||||
```bash
|
||||
ssh sam@192.168.68.68 "docker run -d --rm --name restore-drill-pg --network dragonsstash_internal \
|
||||
-e POSTGRES_USER=drill -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill postgres:16-alpine"
|
||||
ssh sam@192.168.68.68 "docker cp dragonsstash-backup:/tmp/restore-drill/tmp/dragonsstash.dump /tmp/dragonsstash.dump"
|
||||
ssh sam@192.168.68.68 "docker cp /tmp/dragonsstash.dump restore-drill-pg:/tmp/dragonsstash.dump"
|
||||
ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg pg_restore -U drill -d drill --clean --if-exists /tmp/dragonsstash.dump"
|
||||
ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg psql -U drill -d drill -c '\\dt' | head -20"
|
||||
ssh sam@192.168.68.68 "docker rm -f restore-drill-pg"
|
||||
```
|
||||
|
||||
Expected: `pg_restore` completes without fatal errors; `\dt` lists the app's tables (e.g. `Package`, `User`, `TelegramLink`).
|
||||
|
||||
- [ ] **Step 7: Clean up drill artifacts**
|
||||
|
||||
```bash
|
||||
ssh sam@192.168.68.68 "docker exec dragonsstash-backup rm -rf /tmp/restore-drill"
|
||||
ssh sam@192.168.68.68 "rm -f /tmp/dragonsstash.dump"
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,205 @@
|
||||
# NAS Backup for Postgres + TDLib State — Design
|
||||
|
||||
**Date:** 2026-07-23
|
||||
**Status:** Approved for planning
|
||||
|
||||
## Summary
|
||||
|
||||
Add a dedicated `backup` container to the DragonsStash stack that takes daily,
|
||||
encrypted, deduplicated backups of the two things that can't be regenerated —
|
||||
the Postgres database (inventory/STL metadata, users, everything the app
|
||||
manages) and the two TDLib state volumes (Telegram session/auth state for the
|
||||
worker and bot) — and ships them to a Synology NAS over NFS. STL archive
|
||||
contents themselves are explicitly out of scope: they only live on this host
|
||||
temporarily and are not backed up.
|
||||
|
||||
Backups are stored via [restic](https://restic.net/), which provides
|
||||
encryption-at-rest, block-level dedup, and retention pruning natively, so no
|
||||
custom encryption or pruning scripts need to be written or maintained.
|
||||
|
||||
## Context
|
||||
|
||||
Current state (as of this design):
|
||||
|
||||
- Production stack runs from `/opt/stacks/DragonsStash/docker-compose.yml` on
|
||||
this Dockge-managed host, pulling prebuilt images from
|
||||
`git.samagsteribbe.nl`. The `docker-compose.yml` in this repo is the
|
||||
build/dev reference and should be kept in sync.
|
||||
- Named volumes in use: `postgres_data` (Postgres 16 data directory),
|
||||
`tdlib_state` (worker's TDLib session), `tdlib_bot_state` (bot's TDLib
|
||||
session), `tmp_zips` and `manual_uploads` (both transient, explicitly out of
|
||||
scope here).
|
||||
- No backup mechanism, NFS mount, or host cron currently exists anywhere in
|
||||
this deployment.
|
||||
- The host already runs Uptime Kuma (used here for backup alerting) and Loki
|
||||
(container logs are presumably already collected there).
|
||||
|
||||
## Requirements
|
||||
|
||||
1. Daily backup of the Postgres database and both TDLib state volumes.
|
||||
2. Backups stored on a Synology NAS via NFS, not on local disk.
|
||||
3. 14-day retention, oldest snapshots pruned automatically.
|
||||
4. Backups encrypted at rest (Postgres dumps and TDLib session files both
|
||||
contain sensitive material — password hashes, Telegram API secrets, live
|
||||
session state).
|
||||
5. Postgres backups must be transactionally consistent regardless of live app
|
||||
traffic. TDLib state backups are best-effort (see Decisions below) — this
|
||||
is an accepted trade-off, not a defect.
|
||||
6. Alert (via existing Uptime Kuma) if a backup run fails or doesn't happen.
|
||||
7. No new host-level state (no `/etc/fstab` entries, no host crontab) — the
|
||||
backup mechanism should be a container, consistent with how everything
|
||||
else on this host is deployed and versioned.
|
||||
8. No new privileged access — specifically, the backup container must not
|
||||
have Docker socket access or any ability to control sibling containers.
|
||||
|
||||
## Decisions
|
||||
|
||||
- **NFS mounted via Docker's native NFS volume driver** (`driver_opts: type:
|
||||
nfs`), not a host-level mount. Keeps all backup-related state inside the
|
||||
compose file instead of split across host config.
|
||||
- **Restic, not hand-rolled tar+age+find.** Restic already solves encryption,
|
||||
dedup, and retention correctly; hand-rolled scripts would be reinventing
|
||||
that logic with more room for bugs.
|
||||
- **TDLib state is tarred live (best-effort), not paused.** Pausing the
|
||||
worker/bot for a clean snapshot would require mounting the Docker socket
|
||||
into the backup container so it could stop/start sibling containers — a
|
||||
real privilege escalation (a compromised backup container could then
|
||||
control any container on the host). The downside of a best-effort tar is
|
||||
bounded: worst case, a bad TDLib restore means redoing the Telegram SMS
|
||||
auth flow, which is the same outcome as having no backup at all. That
|
||||
bounded, low-severity downside doesn't justify the privilege escalation.
|
||||
- **Fixed-time cron (`crond`), not a sleep-loop.** A `sleep 86400` loop drifts
|
||||
on every container restart; a real crontab entry fires at a fixed time of
|
||||
day regardless of restarts, for negligible extra complexity.
|
||||
- **Restore is manual, not automated.** A script capable of restoring can
|
||||
overwrite live state; that should always require a human deliberately
|
||||
running it, not run unattended.
|
||||
|
||||
## Design
|
||||
|
||||
### New service: `backup`
|
||||
|
||||
Added to both `/opt/stacks/DragonsStash/docker-compose.yml` (production) and
|
||||
this repo's `docker-compose.yml` (dev/build reference).
|
||||
|
||||
- **Image**: custom, `FROM alpine:3.20`, `apk add --no-cache restic
|
||||
postgresql16-client curl tzdata dcron tar bash`. No dependency on app
|
||||
source — independent Dockerfile, e.g. `backup/Dockerfile`.
|
||||
- **Scheduling**: `crond -f` in the foreground as the container's entrypoint,
|
||||
with a crontab installed at build time:
|
||||
```
|
||||
0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
|
||||
0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
|
||||
```
|
||||
(daily dump/backup at 03:00, weekly repo integrity check at 04:00 Sunday).
|
||||
`restic init` runs once at container startup (entrypoint, before `crond`
|
||||
starts), swallowing the "already initialized" error on subsequent
|
||||
container (re)starts.
|
||||
- **Network**: `internal` only — reaches `dragonsstash-db:5432` for
|
||||
`pg_dump`. No ports exposed.
|
||||
- **Volumes**:
|
||||
- `tdlib_state:/data/tdlib-worker:ro`
|
||||
- `tdlib_bot_state:/data/tdlib-bot:ro`
|
||||
- `nas_backups:/backups`, a named volume defined with:
|
||||
```yaml
|
||||
nas_backups:
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
|
||||
device: ":${NAS_EXPORT_PATH}"
|
||||
```
|
||||
- **New `.env` entries**: `NAS_HOST`, `NAS_EXPORT_PATH` (NFS share details),
|
||||
`RESTIC_PASSWORD` (repo encryption key), `KUMA_PUSH_URL` (Uptime Kuma push
|
||||
monitor URL). All four are inputs to gather during implementation, not
|
||||
hardcoded.
|
||||
- `restart: unless-stopped`, no `privileged`, no Docker socket mount.
|
||||
|
||||
### `backup.sh`
|
||||
|
||||
```
|
||||
set -euo pipefail
|
||||
trap 'curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=down" \
|
||||
--data-urlencode "msg=$BASH_COMMAND failed"' ERR
|
||||
|
||||
pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
-Fc -f /tmp/dragonsstash.dump
|
||||
|
||||
tar czf /tmp/tdlib.tar.gz -C /data tdlib-worker tdlib-bot
|
||||
|
||||
restic backup /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
|
||||
restic forget --keep-daily 14 --prune
|
||||
|
||||
rm -f /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
|
||||
|
||||
curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=up" \
|
||||
--data-urlencode "msg=OK"
|
||||
```
|
||||
|
||||
`PGPASSWORD` and `RESTIC_REPOSITORY=/backups/restic-repo` are set as
|
||||
container environment variables (from `.env`), not inline in the script.
|
||||
|
||||
### Data flow
|
||||
|
||||
```
|
||||
crond (daily 03:00)
|
||||
→ pg_dump (consistent snapshot via Postgres MVCC) → /tmp/dragonsstash.dump
|
||||
→ tar tdlib_state + tdlib_bot_state (best-effort, live) → /tmp/tdlib.tar.gz
|
||||
→ restic backup (encrypt + dedup) → NFS-mounted repo on Synology NAS
|
||||
→ restic forget --keep-daily 14 --prune
|
||||
→ curl Uptime Kuma push monitor (up on success, down + reason on any failure)
|
||||
```
|
||||
|
||||
### Restore (manual, documented procedure — not scripted/automated)
|
||||
|
||||
```
|
||||
restic -r /backups/restic-repo restore latest --target /tmp/restore
|
||||
pg_restore -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
--clean --if-exists /tmp/restore/tmp/dragonsstash.dump
|
||||
# untar /tmp/restore/tmp/tdlib.tar.gz back into the tdlib_state /
|
||||
# tdlib_bot_state volumes (via a throwaway container mounting both)
|
||||
```
|
||||
|
||||
## Alerting
|
||||
|
||||
- One Uptime Kuma **Push** monitor, created manually in the existing Kuma
|
||||
instance, with an expected heartbeat interval of ~26 hours (slack past the
|
||||
24h schedule so one slow run doesn't false-positive). Whatever notification
|
||||
channels are already configured on that monitor fire automatically — no new
|
||||
alerting integration.
|
||||
- `backup.sh` pushes `status=up` on success and `status=down` (with the
|
||||
failing command in `msg`) on any failure, via the `ERR` trap.
|
||||
- Container logs go to stdout, collected the same way every other container's
|
||||
logs already are on this host.
|
||||
|
||||
## Testing
|
||||
|
||||
This repo has no automated test framework (documented convention: manual
|
||||
testing). For this infra change:
|
||||
|
||||
- After deploy: manually run `docker exec dragonsstash-backup /backup.sh`
|
||||
once, confirm a snapshot appears (`restic snapshots`), confirm the Kuma
|
||||
monitor goes green.
|
||||
- **Restore drill** (once, during setup): actually restore the dump into a
|
||||
scratch Postgres and untar the TDLib archive into scratch volumes, to prove
|
||||
the backup is really restorable. Not automated or recurring for now.
|
||||
|
||||
## Out of scope / non-goals
|
||||
|
||||
- Backing up `tmp_zips` or `manual_uploads` — both transient by design.
|
||||
- Automated/scheduled restore testing.
|
||||
- Backing up any other stack on this host (this design is DragonsStash-only,
|
||||
though the pattern — Docker-native NFS volume + restic — could be reused
|
||||
for other stacks later).
|
||||
- Pausing worker/bot for a guaranteed-consistent TDLib snapshot (see
|
||||
Decisions).
|
||||
|
||||
## Files touched
|
||||
|
||||
- `docker-compose.yml` (this repo) and
|
||||
`/opt/stacks/DragonsStash/docker-compose.yml` (production) — add `backup`
|
||||
service, `nas_backups` volume.
|
||||
- `backup/Dockerfile` — new.
|
||||
- `backup/backup.sh` — new.
|
||||
- `backup/crontab` — new.
|
||||
- `.env.example` / `.env` — add `NAS_HOST`, `NAS_EXPORT_PATH`,
|
||||
`RESTIC_PASSWORD`, `KUMA_PUSH_URL`.
|
||||
@@ -0,0 +1,271 @@
|
||||
# Provenance Backfill on Re-Index — Design
|
||||
|
||||
**Date:** 2026-07-23
|
||||
**Status:** Approved for planning
|
||||
|
||||
## Summary
|
||||
|
||||
Recover the true origin of packages whose recorded "source" is a placeholder —
|
||||
specifically the manual-upload and `rebuild.ts`-created records whose
|
||||
`sourceChannelId` points at the destination (archive) channel itself. When a
|
||||
real source channel is re-indexed and the worker encounters an archive that
|
||||
matches such a placeholder package, it backfills the real
|
||||
`sourceChannelId` / `sourceMessageId` / `sourceTopicId` / `sourceCaption` /
|
||||
`creator` (and, for records that never had one, the file listing and preview)
|
||||
onto the existing package — **without downloading the full archive**.
|
||||
|
||||
Matching is a two-stage process: a zero-download candidate lookup by
|
||||
`fileName` + total `fileSize`, confirmed by a CRC32 fingerprint read from just
|
||||
the archive's central directory via a **ranged (tail) download** of a few
|
||||
KB–MB, instead of the multi-GB whole file.
|
||||
|
||||
This is the first of four linked sub-projects. The others (creator
|
||||
normalization, provenance display, missing-files) are out of scope here and get
|
||||
their own spec → plan cycles. "Missing files" is explicitly deferred until this
|
||||
lands.
|
||||
|
||||
## Context
|
||||
|
||||
Current state (as of this design):
|
||||
|
||||
- `Package` records their origin via `sourceChannelId`, `sourceMessageId`,
|
||||
`sourceTopicId`, `sourceCaption`, and (for content dedup) `contentHash` and
|
||||
`remoteUniqueId`. `creator` is a free-text string extracted at ingestion.
|
||||
- Two ingestion paths create records with **placeholder provenance**, where the
|
||||
"source" is the destination channel, not a real origin:
|
||||
- **Manual uploads** (`worker/src/manual-upload.ts`) set
|
||||
`sourceChannelId = destChannel.id` and `sourceMessageId = destResult.messageId`.
|
||||
They *do* populate `PackageFile` (with `crc32`) from a local central-directory
|
||||
read at upload time.
|
||||
- **`rebuild.ts`** scans the destination channel and creates minimal records
|
||||
with `fileCount == 0` and **no** `PackageFile` rows.
|
||||
- The main worker upload path (`worker/src/worker.ts` → `uploadToChannel`) sends
|
||||
archives to the destination channel with **no caption**, so provenance cannot
|
||||
be recovered by re-reading destination messages — it has to come from matching
|
||||
against real source channels.
|
||||
- The scan/dedup ladder in `processOneArchiveSet` (`worker/src/worker.ts:1521`)
|
||||
is entirely **source-channel-scoped**:
|
||||
1. `findPackageByRemoteUniqueId(channel.id, …)` — same channel only
|
||||
2. `packageExistsBySourceMessage(channel.id, …)` — same channel only
|
||||
3. `findRepostedPackage(channel.id, fileName, size)` — same channel only
|
||||
4. …then full download → `packageExistsByHash(contentHash)` — global, but only
|
||||
*after* downloading the whole archive.
|
||||
- Because a placeholder package's `sourceChannelId` is the destination, checks
|
||||
1–3 never match it when a real source channel is scanned. Today the worker
|
||||
therefore downloads the entire archive, hits check 4, finds it is a duplicate,
|
||||
and skips — **wasting the download and leaving the wrong provenance in place.**
|
||||
- There is already an in-production precedent for "match an already-known
|
||||
archive during scan and enrich its metadata": when `findRepostedPackage`
|
||||
matches, the worker backfills richer *topic* context onto the existing package
|
||||
via `updatePackageTopicContext` (`worker/src/worker.ts:1608`+). Provenance
|
||||
backfill is the same move, widened from same-channel to cross-channel.
|
||||
- `remote.unique_id` is **not** a reliable cross-channel key: it identifies a
|
||||
stored file object on Telegram's servers, not the content. The same archive
|
||||
independently uploaded to two channels gets different `unique_id`s; it only
|
||||
matches for forwarded messages (same underlying file object). This is why the
|
||||
existing dedup scopes it to a single channel.
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals**
|
||||
|
||||
- Attribute true origin to placeholder-provenance packages during normal
|
||||
re-index scans, opportunistically (no separate pass).
|
||||
- Do it without downloading whole archives (ranged central-directory read only).
|
||||
- Be non-destructive: only ever touch packages that currently have placeholder
|
||||
provenance; never overwrite a real, non-placeholder source.
|
||||
- Be idempotent: re-running a re-index does not re-mutate or duplicate.
|
||||
|
||||
**Non-Goals (separate sub-projects / deferred)**
|
||||
|
||||
- Creator name normalization / canonical creator entity.
|
||||
- UI changes to display provenance.
|
||||
- Recovering "missing files."
|
||||
- Choosing a *preferred* origin when an archive genuinely exists in several
|
||||
source channels (first confirmed source wins).
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Candidate definition
|
||||
|
||||
> A package is a backfill candidate iff **`sourceChannelId == destChannelId`
|
||||
> OR `sourceMessageId == 0`**.
|
||||
|
||||
Two placeholder shapes exist (verified against live data 2026-07-23):
|
||||
- **Manual uploads** (`manual-upload.ts`): `sourceChannelId == destChannelId`,
|
||||
real `contentHash`, real `sourceMessageId`, has a `PackageFile` listing.
|
||||
- **Rebuild records** (`rebuild.ts`): `sourceMessageId == 0n` (deliberate
|
||||
"unknown" sentinel), synthetic `contentHash = "rebuild:<destChannelId>:<destMessageId>"`,
|
||||
`fileCount == 0`, and `sourceChannelId` set to an **arbitrary fallback source
|
||||
channel** (`sourceChannels[0]`) — NOT the destination. (This is the common
|
||||
case: e.g. 59,893 records after a destination rebuild.)
|
||||
|
||||
Normal ingestion always sets a real `sourceMessageId` (> 0) and a real source
|
||||
channel, so neither marker matches a genuinely-sourced package. Both markers are
|
||||
overwritten on backfill (source channel + message become real), so a record
|
||||
stops being a candidate once fixed — this is what makes re-scans idempotent.
|
||||
|
||||
**Known limitation:** backfill does NOT rewrite a rebuild record's synthetic
|
||||
`"rebuild:"` `contentHash` (the true content hash would require a full download,
|
||||
which this feature avoids). That is acceptable — dedup after backfill relies on
|
||||
`remoteUniqueId` + name/size within the source channel, not on `contentHash`.
|
||||
|
||||
### 2. Where it hooks
|
||||
|
||||
A new step is inserted into `processOneArchiveSet` **between check #3
|
||||
(`findRepostedPackage`) and the full download**. It runs only after the
|
||||
same-channel checks have missed (so genuine same-channel reposts keep their
|
||||
existing fast paths).
|
||||
|
||||
Flow for the scanned archive set:
|
||||
|
||||
1. Stage A — **candidate lookup (zero download).** Query for a package where
|
||||
`(sourceChannelId == destChannelId OR sourceMessageId == 0)` AND
|
||||
`fileName == archiveName` AND `fileSize == totalArchiveSize`.
|
||||
(`Package` has `@@index([fileName])`.)
|
||||
- No candidate → fall through to normal ingestion unchanged.
|
||||
2. Stage B — **fingerprint confirmation (tiny download).** See §3.
|
||||
3. On confirmation → **backfill** (see §4) and return `null` (treated as a
|
||||
duplicate; no full download, no new package). Increment a `zipsBackfilled`
|
||||
counter.
|
||||
4. On mismatch / failure / ambiguity → fall through to normal ingestion (see §6).
|
||||
|
||||
### 3. Fingerprint confirmation
|
||||
|
||||
The confirmation signal is the **multiset of internal CRC32s** of the archive's
|
||||
entries (CRC32 of each entry's *uncompressed* data). This value is a property of
|
||||
the archive contents and is identical regardless of which channel hosts the
|
||||
file.
|
||||
|
||||
- **Candidate side:** for manual uploads, `PackageFile.crc32` is already
|
||||
populated from the local central-directory read — zero cost. For **rebuild
|
||||
records** (`fileCount == 0`, no `crc32`), there is nothing stored to compare
|
||||
against; obtain the candidate's fingerprint with a **second ranged tail read
|
||||
of its destination copy** (ZIP/7z — still no full download). RAR candidates
|
||||
cannot be tail-read on either side, so they fall back to name+size (see §5).
|
||||
- **Scanned-source side:** read via a **ranged (tail) download** of the central
|
||||
directory:
|
||||
- **ZIP** (incl. multipart): the End-of-Central-Directory record + central
|
||||
directory live at the tail of the last part. Download only that tail and
|
||||
parse entries. (Reuses the lightweight-listing mechanism that is
|
||||
sub-project 4's core.)
|
||||
- **7z**: a start header at byte 0 points to an end header at the tail; fetch
|
||||
both small pieces.
|
||||
- **RAR**: headers are scattered through the file — no cheap tail read. See §5.
|
||||
- **Match rule:** confirmed iff the sorted CRC32 multisets are equal **and** file
|
||||
counts are equal.
|
||||
|
||||
The comparison logic (CRC32 multiset equality) and the candidate-match predicate
|
||||
are implemented as **pure functions** with no TDLib dependency, so they are unit
|
||||
testable (see §7).
|
||||
|
||||
### 4. What gets written
|
||||
|
||||
On a confirmed match, update the candidate package in a single transaction,
|
||||
overwriting only placeholder/empty fields:
|
||||
|
||||
| Field | New value | Condition |
|
||||
|---|---|---|
|
||||
| `sourceChannelId` | scanned `channel.id` | always |
|
||||
| `sourceMessageId` | scanned `parts[0].id` | always |
|
||||
| `sourceTopicId` | `ctx.sourceTopicId` | always |
|
||||
| `sourceCaption` | scanned message caption | always |
|
||||
| `remoteUniqueId` | scanned `firstRemoteUniqueId` | always (enables future same-channel dedup via check #1) |
|
||||
| `creator` | re-derived from source (topic > filename > channel) | **always** — un-normalized; the later creator-normalization sub-project cleans it up |
|
||||
| `fileCount` + `PackageFile[]` | from the central-directory read | only if candidate had none (`fileCount == 0`) — folds in the sub-project 4 outcome for rebuild records |
|
||||
| `previewData` / `previewMsgId` | matched preview from scan | only if candidate has none |
|
||||
|
||||
**Left untouched:** `contentHash`, `destChannelId`, `destMessageId`,
|
||||
`destMessageIds` — the bytes physically live in the destination channel; that is
|
||||
correct and must not change.
|
||||
|
||||
**Transaction safety:** re-check `sourceChannelId == destChannelId` *inside* the
|
||||
transaction before writing, so a concurrent worker that already backfilled the
|
||||
record causes this one to no-op (mirrors the existing `backfill.ts` guard).
|
||||
|
||||
### 5. RAR handling
|
||||
|
||||
RAR sources cannot be tail-read, so no cheap CRC fingerprint is available.
|
||||
Decision: **backfill RAR matches on `fileName` + `fileSize` alone**, flagged as
|
||||
lower-confidence in logs and via a `SystemNotification`, so they can be audited.
|
||||
No full download.
|
||||
|
||||
This name+size-only fallback applies to any candidate that cannot produce a
|
||||
CRC32 fingerprint cheaply: a RAR **source**, a RAR **candidate**, or a rebuild
|
||||
candidate whose destination copy is RAR. ZIP/7z rebuild candidates are still
|
||||
confirmed by fingerprint via the second tail read described in §3.
|
||||
|
||||
### 6. Conflict, mismatch, ambiguity
|
||||
|
||||
- **Fingerprint mismatch** → the scanned archive is genuinely different content
|
||||
that merely shares name+size. Fall through to **normal ingestion** (download +
|
||||
index) — it is a new package for this source.
|
||||
- **Ambiguous candidates** (2+ match name+size and the fingerprint cannot
|
||||
disambiguate) → log a `SystemNotification`, backfill nothing, and fall through
|
||||
to normal ingestion; the post-download `packageExistsByHash` check still
|
||||
dedups it safely.
|
||||
- **Same archive in multiple source channels** → the **first re-indexed source
|
||||
that confirms wins.** After backfill the package has a real source, so it is no
|
||||
longer a candidate; later scans treat it as an ordinary duplicate via checks #1
|
||||
(the `remoteUniqueId` we set) or #3.
|
||||
|
||||
### 7. Idempotency
|
||||
|
||||
Falls out of the candidate definition. Once backfilled:
|
||||
- `sourceChannelId` is the real channel → no longer a candidate.
|
||||
- A re-scan of that source hits check #1 (`remoteUniqueId`, now set) or check #3
|
||||
(`findRepostedPackage`) → normal dedup skip. No re-mutation, no duplicate.
|
||||
|
||||
### 8. Error handling
|
||||
|
||||
- **Tail-download failure** (network / `FLOOD_WAIT`) → cannot confirm this round.
|
||||
Do **not** fall back to a full download and do **not** guess: leave the
|
||||
candidate untouched and let the next re-index retry. Wrap the ranged read in
|
||||
`withFloodWait` (per the TDLib skill).
|
||||
- All new TDLib calls follow the skill's patterns: `withFloodWait`, listener
|
||||
attached before the async op, client closed in `finally`.
|
||||
|
||||
### 9. Visibility
|
||||
|
||||
- Add a `zipsBackfilled` counter to `IngestionRun` activity so a re-index run
|
||||
reports "N provenance backfills" instead of silently mutating records.
|
||||
- Info log per backfill (candidate id, old vs new source, confidence:
|
||||
fingerprint | name+size-RAR).
|
||||
- `SystemNotification` for ambiguous-candidate cases.
|
||||
|
||||
## Testing
|
||||
|
||||
The repo currently has no test framework (`CLAUDE.md`: "testing is manual"). This
|
||||
work introduces a **lightweight test harness** for the worker (e.g. `vitest` or
|
||||
node's built-in `node:test`) and unit-tests the correctness-critical pure logic:
|
||||
|
||||
- **Unit (automated):**
|
||||
- CRC32 multiset fingerprint equality (equal sets, different order, differing
|
||||
counts, disjoint sets).
|
||||
- Candidate-match predicate (name+size+placeholder true/false cases).
|
||||
- Field-merge rules (which fields overwrite, which only fill-if-empty).
|
||||
- **Manual integration checklist:**
|
||||
1. Re-index a real source channel containing a known manually-uploaded pack →
|
||||
verify `sourceChannelId`/`sourceMessageId`/`sourceCaption`/`creator` are
|
||||
backfilled and no full download occurs.
|
||||
2. A rebuild-created record (`fileCount == 0`) → verify listing + provenance
|
||||
are both populated from the single tail read.
|
||||
3. A RAR pack → verify name+size backfill with the lower-confidence log/notice.
|
||||
4. Re-run the same re-index → verify it is a no-op (idempotent).
|
||||
5. A genuine name+size collision (different content) → verify it ingests as a
|
||||
new package rather than being mis-attributed.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None blocking. Preferred-origin selection among multiple real sources is
|
||||
deliberately out of scope (first confirmed wins).
|
||||
|
||||
## Affected Code (indicative, for planning)
|
||||
|
||||
- `worker/src/worker.ts` — `processOneArchiveSet`: new Stage A/B step; new counter.
|
||||
- `worker/src/archive/` — ranged central-directory reader (ZIP/7z tail); shared
|
||||
with sub-project 4. Pure CRC32-fingerprint compare helper.
|
||||
- `worker/src/tdlib/download.ts` — ranged/partial download support (`offset`/`limit`).
|
||||
- `worker/src/db/queries.ts` — candidate lookup + transactional backfill update.
|
||||
- `prisma/schema.prisma` — `IngestionRun.zipsBackfilled` (and run-counter plumbing).
|
||||
- Worker test harness + first unit tests.
|
||||
@@ -0,0 +1,180 @@
|
||||
# Ranged inner-file listing for RAR & 7z — design
|
||||
|
||||
**Date:** 2026-07-27
|
||||
**Status:** Approved (design), pending spec review → implementation plan
|
||||
|
||||
## Problem
|
||||
|
||||
The reindex/provenance-backfill path (`worker/src/provenance-backfill.ts`) can index an
|
||||
archive's inner files *without* re-downloading it, by reading the file listing from a small
|
||||
ranged read of the copy already in the source/destination channel. This works **only for
|
||||
ZIP** today (ZIP keeps its central directory in a tail that `parseZipCentralDirectoryFromTail`
|
||||
reads). For **RAR and 7z**, `tryProvenanceBackfill` backfills provenance (creator, source
|
||||
channel, `remoteUniqueId`) so the file is skipped on re-scan and never re-downloaded — but it
|
||||
leaves the inner listing empty (`fileCount = 0`), because `scannedEntries` is only computed for
|
||||
ZIP.
|
||||
|
||||
Scope of the gap (rebuild placeholders with `fileCount = 0`, as of 2026-07-27):
|
||||
|
||||
| Type | Count | Total | Avg | Max | Multipart |
|
||||
|---|---|---|---|---|---|
|
||||
| 7z | 19,646 | 14 TB | 0.73 GB | 3.9 GB | 0 |
|
||||
| RAR | 12,780 | 13 TB | 1.04 GB | 116 GB | 1,016 |
|
||||
|
||||
A "just download the whole archive" fallback for all of these means ~27 TB of re-downloads —
|
||||
the exact cost this path exists to avoid.
|
||||
|
||||
## Goals
|
||||
|
||||
- Index the inner files (names + sizes; CRCs where cheaply available) of RAR and 7z
|
||||
placeholders **without** downloading the whole archive in the common case.
|
||||
- Reuse the existing, battle-tested CLI listing parsers (`parse7zOutput`,
|
||||
`parseUnrarTechnical`) rather than reimplementing filename/size/CRC extraction.
|
||||
- Keep cost proportional to **file count**, not archive size (so even the 116 GB RAR is cheap).
|
||||
- Guarantee a listing for the rare archives the cheap path can't handle, via a full-download
|
||||
fallback that respects the existing max-size guard.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No change to ingestion of genuinely-new files (those are downloaded in full to be re-uploaded
|
||||
regardless, so a cheap listing does not help there). This feature only affects the
|
||||
provenance-backfill / skip path.
|
||||
- No new ZIP behaviour — the existing ZIP tail reader stays as-is.
|
||||
- Not attempting to list password-encrypted-header archives from ranged reads (no password);
|
||||
those take the fallback and, if oversized, are flagged.
|
||||
|
||||
## Approach (chosen: "harvest header regions → sparse file → native CLI")
|
||||
|
||||
Do the *minimum* binary parsing needed to locate an archive's header bytes, fetch only those
|
||||
via ranged reads, write them into a sparse temp file at their true offsets (data regions left
|
||||
as unwritten zero holes → ~no disk use), then run the real `7z l` / `unrar lt` and reuse the
|
||||
existing parsers. The native tools do the hard parsing (7z's LZMA-encoded headers, RAR's two
|
||||
format versions, Unicode names) — we only compute where the headers are.
|
||||
|
||||
Rejected alternatives: full native TS parsers (most custom binary code, highest risk);
|
||||
RAR5-quick-open-only (most community RARs lack it → collapses to ~13 TB of RAR downloads).
|
||||
|
||||
## Components
|
||||
|
||||
New directory `worker/src/archive/ranged/`, one focused module per concern, all returning the
|
||||
existing `FileEntry[]` type from `zip-reader.ts`:
|
||||
|
||||
- `sparse-list.ts` — `listFromSparse(parts, runner, parse) → FileEntry[] | null`, where each
|
||||
`part` is `{ fileName, size, regions: {offset, bytes}[] }`. For each part it writes a sparse
|
||||
temp file (`truncate` to `size`, then write only the header `regions` at their offsets),
|
||||
co-locates all parts in one temp dir under their real names, invokes the supplied CLI runner
|
||||
(`7z l` / `unrar lt`) on the first part, feeds stdout to the supplied `parse` fn
|
||||
(`parse7zOutput` / `parseUnrarTechnical`), and cleans up. Single-part archives are just the
|
||||
one-element case. Returns `null` on CLI error / empty parse.
|
||||
- `sevenz-ranged.ts` — `readSevenZListingRanged(client, parts) → FileEntry[] | null`.
|
||||
- `rar-ranged.ts` — `readRarListingRanged(client, parts) → FileEntry[] | null`.
|
||||
- Dispatcher in `provenance-backfill.ts`: `readScannedListingRanged(archiveType, client, parts)`
|
||||
replacing the current `if (archiveType === "ZIP")` branch; the destination-copy read in
|
||||
`resolveCandidateFingerprintEntries` gets the same dispatch.
|
||||
|
||||
`FileEntry` shape (unchanged): `{ path, fileName, extension, compressedSize, uncompressedSize, crc32 }`.
|
||||
|
||||
### 7z ranged listing
|
||||
|
||||
7z layout: 32-byte signature header at offset 0 → packed streams → end header (lists files) at
|
||||
the end; the signature header stores the end header's location.
|
||||
|
||||
1. Ranged-read `[0, 32)`; validate magic `37 7A BC AF 27 1C`. Read LE `uint64`
|
||||
`NextHeaderOffset` (byte 12) and `NextHeaderSize` (byte 20). End header is at absolute offset
|
||||
`32 + NextHeaderOffset`, length `NextHeaderSize`.
|
||||
2. Ranged-read `[32 + NextHeaderOffset, NextHeaderSize)` — the "next header".
|
||||
3. Branch on the next header's first byte (a 7z property id):
|
||||
- **`0x01` (kHeader, plain/uncompressed header):** two regions suffice —
|
||||
`{0: sigHeader}` and `{32+NextHeaderOffset: endHeader}`.
|
||||
- **`0x17` (kEncodedHeader, LZMA-compressed header):** the next header is only a *descriptor*
|
||||
whose `PackInfo` points at a packed header stream stored **in the middle** of the file (not
|
||||
at EOF). Parse the descriptor's `StreamsInfo → kPackInfo (0x06)` to read `PackPos` and the
|
||||
`PackSize`s (7z variable-length "numbers"; sum them). Ranged-read the contiguous packed
|
||||
region `[32 + PackPos, Σ PackSize)` and add it as a **third** sparse region. `7z l` then
|
||||
decodes the header from that region.
|
||||
- **anything else:** return `null` (→ fallback).
|
||||
4. `listFromSparse` with the 2 or 3 regions, runner = `7z l`. `parse7zOutput` yields
|
||||
names+sizes (`crc32: null`, as today). Return `null` on bad magic / read failure / CLI error.
|
||||
|
||||
**Why the third region is required (spike finding, 2026-07-27):** the original two-region
|
||||
(start+end) reconstruction was proven insufficient in a live test — `7z l` rejected it with
|
||||
"Cannot open the file as [7z] archive" because these archives use an *encoded* header whose
|
||||
compressed bytes live in a packed stream in the file body (a sparse hole), not at EOF. The
|
||||
`0x17` branch fetches exactly that packed region. `7z l` still never touches the file-data
|
||||
packed streams (it only lists), so those gaps stay sparse. The `read7zNumber` reader (7z's
|
||||
base-128-ish variable-length integer with a first-byte length mask) and the minimal
|
||||
`kPackInfo` walk are the only new 7z binary parsing; `7z l` still does the actual file listing.
|
||||
All 7z placeholders are single-part.
|
||||
|
||||
### RAR ranged listing
|
||||
|
||||
RAR has no index; walk the block chain, parsing only each block's **size fields** to step
|
||||
forward and harvest header bytes.
|
||||
|
||||
1. Read first ~16 bytes; detect **RAR4** (`52 61 72 21 1A 07 00`) vs **RAR5** (`…07 01 00`) and
|
||||
the signature length.
|
||||
2. From just after the signature, loop:
|
||||
- Ranged-read a header chunk (start 8 KB; if parsed `HeaderSize` exceeds it — long filenames
|
||||
— re-read exactly).
|
||||
- Minimal block-extent parse:
|
||||
- RAR5: `CRC32(4)` + vint `HeaderSize` + vint `HeaderType` + vint `HeaderFlags`; if the
|
||||
"extra area" flag (`0x0001`) → vint `ExtraAreaSize`; if the "data present" flag
|
||||
(`0x0002`) → vint `DataSize`. Next block = `pos + 4 + len(HeaderSize vint) + HeaderSize
|
||||
+ DataSize`.
|
||||
- RAR4: `HEAD_CRC(2)` + `HEAD_TYPE(1)` + `HEAD_FLAGS(2)` + `HEAD_SIZE(2)`; if flag `0x8000`
|
||||
→ `ADD_SIZE(4)`. Next block = `pos + HEAD_SIZE + ADD_SIZE`.
|
||||
- Harvest `[blockOffset, blockOffset + HeaderSize)` into the regions list.
|
||||
- Stop at the end-of-archive block or EOF.
|
||||
3. `listFromSparse` (headers present, data sparse) → `unrar lt` → `parseUnrarTechnical`. RAR
|
||||
headers carry CRC32, so RAR contributes CRCs (fingerprint disambiguation keeps working).
|
||||
|
||||
**Multipart RAR** (1,016): each volume starts with its own signature + headers. Walk **each
|
||||
part from its own signature**, reconstruct one sparse temp file per part with correct names
|
||||
(`name.part1.rar`, `.part2.rar`, …) co-located in a temp dir, and run `unrar lt` on part 1 —
|
||||
`unrar` auto-discovers co-located siblings (per the existing reader's note). The global-offset →
|
||||
`(part, offsetInPart)` mapping reuses the multipart size math the ZIP path already uses.
|
||||
|
||||
## Fallback & integration
|
||||
|
||||
- A ranged reader returning `null` = cheap read failed (bad magic, read error, walk gave up, or
|
||||
CLI error on the sparse file) → **full-download fallback**: download the whole archive, run the
|
||||
existing `readRarContents` / `read7zContents`, backfill.
|
||||
- The fallback is gated by `config.maxZipSizeMB` (the same guard used at ingest). Over the cap →
|
||||
no download; write a `SystemNotification` (`INTEGRITY_AUDIT`, WARNING) and leave the listing
|
||||
empty for manual review. This ensures nothing pathological (e.g. the 116 GB RAR) is pulled.
|
||||
- Downstream is unchanged: `compareFingerprints` already treats null/incomplete CRCs as
|
||||
"incomplete" (name-size path), and `backfillProvenance` writes entries when the candidate's
|
||||
`fileCount === 0`.
|
||||
- Observability: reuse the `zipsBackfilled` counter; add structured logs with
|
||||
`confidence: "ranged" | "full-download-fallback"` and a WARN on fallback so miss-rate is
|
||||
visible.
|
||||
|
||||
## Risks & de-risking spike (do before the full build)
|
||||
|
||||
On 3–4 real placeholder archives per format:
|
||||
|
||||
1. Confirm `downloadFileRange` returns correct bytes at **arbitrary (non-tail) offsets** —
|
||||
currently only tail-verified in production. Underpins everything; if it fails, stop and
|
||||
rethink. (Note: `range-download.ts` flags absolute-offset behaviour as pending live
|
||||
verification; tail reads are proven by the 43 ZIP backfills done 2026-07-26.)
|
||||
2. Confirm `7z l` and `unrar lt` list correctly from a **sparse reconstructed file** — single
|
||||
part first, then multipart RAR (the highest-risk case).
|
||||
|
||||
If multipart-RAR sparse reconstruction proves unreliable in the spike, multipart RAR uses the
|
||||
full-download fallback (respecting the size cap → oversized ones flagged, not downloaded).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit (vitest, alongside `central-directory.test.ts`):** 7z signature-header parse; RAR4 &
|
||||
RAR5 block-extent walk against committed small fixtures; `sparse-list` writes the correct
|
||||
regions. Pure logic, no TDLib.
|
||||
- **Live post-deploy:** watch `zipsBackfilled` climb for RAR/7z via the ranged path; spot-check
|
||||
a handful of backfilled packages' `package_files` against a real `unrar lt` / `7z l` on a full
|
||||
download of the same file; confirm the fallback/flag path fires on a deliberately-broken case.
|
||||
|
||||
## Rollout
|
||||
|
||||
Local build + deploy (no GitHub push required), per the established recipe: build
|
||||
`worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image
|
||||
(no `pull`). No new DB migration. The scheduler re-runs hourly and will backfill RAR/7z
|
||||
placeholders on subsequent cycles.
|
||||
@@ -0,0 +1,211 @@
|
||||
# Forward-priority ingestion — design
|
||||
|
||||
**Date:** 2026-07-30
|
||||
**Status:** Approved (design), pending spec review → implementation plan
|
||||
|
||||
## Problem
|
||||
|
||||
The worker ingests every archive the same way regardless of whether it needs to: download the
|
||||
full file from the source channel, then re-upload the full file to the destination (archive)
|
||||
channel. That download+reupload round-trip was originally necessary because some source channels
|
||||
have "restrict saving content" (protected content) enabled, which blocks Telegram-native
|
||||
forwarding — for those channels there is no alternative to moving the bytes through the worker.
|
||||
|
||||
But most source channels do NOT restrict forwarding. For those, the round-trip is pure waste:
|
||||
Telegram can copy the message from source chat to destination chat server-side, with no bytes
|
||||
ever passing through the worker. The worker still needs to end up with the same outcome it has
|
||||
today — a destination-channel copy, a dedup-safe identity, and a full inner-file listing — just
|
||||
without paying for a download and re-upload to get there.
|
||||
|
||||
Separately, `feat/ranged-archive-listing` (merged to master ahead of this feature) already built
|
||||
exactly the missing piece: reading a ZIP/RAR/7z archive's inner-file listing via small ranged
|
||||
reads against the file wherever it currently lives (source channel, destination channel — doesn't
|
||||
matter), with no full download. It was built for backfilling listings onto already-deduped
|
||||
placeholder packages. This feature generalizes that same capability to fresh ingestion, and pairs
|
||||
it with a new native-forward upload path.
|
||||
|
||||
## Goals
|
||||
|
||||
- For channels that allow forwarding: skip download and re-upload entirely for new archives. Use
|
||||
Telegram-native forwarding from source chat to destination chat, and the existing ranged-listing
|
||||
readers to index inner files, with no full download in the common case.
|
||||
- For channels that block forwarding (or when forwarding isn't yet known): keep today's
|
||||
download+reupload pipeline exactly as-is.
|
||||
- Every ingested package — regardless of path — ends up with the same outcome as today: a
|
||||
`Package` row with a valid dedup identity, `destMessageId`/`destMessageIds`, creator, tags, and a
|
||||
full inner-file listing (`PackageFile` rows). Indexing completeness must not regress.
|
||||
- If the cheap ranged listing fails for a specific archive (bad/unsupported header, CLI error,
|
||||
etc.) in an otherwise-forwarding-eligible channel, fall back to today's full download+reupload
|
||||
pipeline for that one archive — never forward with an empty or partial listing.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No ranged single-entry preview extraction. Forward-path packages still get a preview when a
|
||||
channel photo message matches (cheap, unrelated to archive bytes); when there's no matching
|
||||
photo, forward-path packages simply have no preview, same as any package where preview
|
||||
extraction fails today. In-archive preview extraction (unzip/unrar/7z against a local file) stays
|
||||
as a download-path-only feature. May be revisited as a follow-up if it turns out to matter.
|
||||
- No reprocessing of already-ingested packages. This only changes behavior for newly-scanned
|
||||
archives going forward.
|
||||
- No change to the bot's user-delivery leg (`bot/src/tdlib/client.ts` `copyMessageToUser`) — it
|
||||
already sends via `inputFileRemote` with no download, and is unaffected by this feature.
|
||||
- No change to `config.maxZipSizeMB` or the multipart byte-level split/repack logic. The existing
|
||||
size guard runs before either path is chosen, so nothing above the cap reaches the forward path's
|
||||
fallback-to-download step either. Splitting simply never engages on the forward path — a
|
||||
forwarded message is already within whatever size Telegram accepted when it was first uploaded.
|
||||
|
||||
## Approaches considered
|
||||
|
||||
**A — Branch inside the existing pipeline (chosen).** Add one fork point in
|
||||
`processOneArchiveSet`, immediately after the existing pre-download dedup checks: if the channel
|
||||
allows forwarding, attempt the ranged-listing + forward path; on any failure, fall through into
|
||||
today's download-based code for that one archive, unchanged. Smallest diff; reuses the existing
|
||||
dedup/retry/watermark machinery as-is; matches the file's existing forum-vs-non-forum branching
|
||||
style.
|
||||
|
||||
**B — Separate pipeline per channel.** Decide once per channel and route the whole channel through
|
||||
either a "forward module" or the existing "download module." Cleaner separation on paper, but
|
||||
duplicates the SkippedPackage/stall/watermark bookkeeping that currently lives once in
|
||||
`processArchiveSets`/`processOneArchiveSet` — higher regression risk in a large orchestration file
|
||||
with no tests at that level. Rejected.
|
||||
|
||||
**C — Strategy-object refactor.** Extract an `IngestStrategy` interface (`download` / `forward`)
|
||||
and slim `processOneArchiveSet` to delegate to it. The more "proper" abstraction, but it's a
|
||||
structural refactor of already-battle-tested code that doesn't need it for this feature to work.
|
||||
Rejected — can revisit later if a third strategy ever appears.
|
||||
|
||||
## Sequencing
|
||||
|
||||
`feat/ranged-archive-listing` merges to master first, as-is (it's complete and serves a different
|
||||
purpose already). This feature is built on a fresh branch off master afterward.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. `TelegramChannel.allowsForwarding` (new column, new migration)
|
||||
|
||||
`Boolean?` — nullable, `null` means "not yet checked". Refreshed from TDLib's chat
|
||||
protected-content flag (exact field name to be confirmed against the pinned `tdl`/TDLib version
|
||||
via docs lookup during implementation — expected to be `chat.has_protected_content`) at the same
|
||||
point the worker already calls `getChat` per channel per cycle, mirroring the existing
|
||||
`isForum`/`setChannelForum` read-and-persist pattern precisely. `null` or `false` both route to the
|
||||
download path — a channel never uses the forward path on unverified permission.
|
||||
|
||||
### 2. Shared ranged-listing dispatcher
|
||||
|
||||
`readScannedListingRanged` (plus `RangedPart`, `tdlibRangeReader`, and the format-specific
|
||||
ZIP/RAR/7z readers) currently live inside `provenance-backfill.ts`. Promote the dispatcher (and
|
||||
whatever it depends on) into a shared module (e.g. `worker/src/archive/ranged/dispatch.ts`) so
|
||||
`worker.ts` can call the same no-download listing logic for fresh ingestion without a circular
|
||||
import. `provenance-backfill.ts` switches to importing from the new shared location; behavior
|
||||
unchanged for the existing backfill path.
|
||||
|
||||
### 3. `forwardArchiveToChannel` (new, `worker/src/upload/forward.ts`)
|
||||
|
||||
Mirrors `uploadToChannel`'s shape and return type (`{ messageId, messageIds }`). Uses TDLib
|
||||
`forwardMessages` to copy all parts of an archive set from the source chat to the destination chat
|
||||
in one batch call (message IDs in original order), wrapped in the same flood-wait/retry handling
|
||||
style as `uploadToChannel`. Followed by the same destination read-back verification style as
|
||||
today's post-upload check (`getMessage` on each new destination message ID, confirm a document is
|
||||
present).
|
||||
|
||||
### 4. Dedup identity for forward-path packages
|
||||
|
||||
`Package.contentHash` stays a required unique string, but forward-path packages can't hash real
|
||||
bytes. Derivation order:
|
||||
1. If the ranged listing's CRC32s are complete (ZIP/RAR today) — hash the sorted CRC32 list into a
|
||||
synthetic `fingerprint:<hash>` value, reusing `archive/fingerprint.ts`'s existing
|
||||
`crcFingerprint`.
|
||||
2. Otherwise (7z, or any incomplete-CRC case) — synthesize `forward:<remoteUniqueId>`, following
|
||||
the existing `rebuild:`-prefixed placeholder-hash precedent in `rebuild.ts`.
|
||||
|
||||
Additionally, extend repost detection: before committing to the forward path, compare the new
|
||||
listing's CRC fingerprint (via the existing `compareFingerprints`/`fingerprintsMatch` logic already
|
||||
used in `provenance-backfill.ts`'s ambiguous-candidate disambiguation) against recent Packages
|
||||
sharing the same file name + size. A fingerprint match is treated as a duplicate and skipped, same
|
||||
as today's `findRepostedPackage` handling — this is what lets a forwarded copy and a previously
|
||||
fully-downloaded copy of the same archive still dedupe against each other, despite never sharing a
|
||||
byte-hash-derived `contentHash`.
|
||||
|
||||
### 5. Fork point in `processOneArchiveSet`
|
||||
|
||||
All existing pre-download checks run first, completely unchanged, in the same order:
|
||||
`remote.unique_id` match → `packageExistsBySourceMessage` → `findRepostedPackage` (name+size) →
|
||||
cross-channel provenance backfill → size guard (`maxZipSizeMB`).
|
||||
|
||||
Then:
|
||||
|
||||
```
|
||||
if channel.allowsForwarding === true:
|
||||
entries = readScannedListingRanged(archiveType, client, scannedParts)
|
||||
if entries is not null:
|
||||
contentHash = deriveForwardContentHash(entries, remoteUniqueId)
|
||||
if fingerprintRepostCheck(entries, fileName, fileSize) finds a match:
|
||||
→ treat as duplicate, skip (same bookkeeping as today's dup path)
|
||||
destResult = forwardArchiveToChannel(client, sourceChatId, partMessageIds, destChatId)
|
||||
creator, tags ← derived from entries/filename/channel/topic, same as today
|
||||
preview ← channel-photo match only (no in-archive extraction)
|
||||
createPackageStub(...) + updatePackageWithMetadata(...), same as today
|
||||
counters.zipsForwarded++
|
||||
→ done
|
||||
else:
|
||||
→ fall through into the existing download/hash/split/upload flow below, unchanged
|
||||
(log the fallback for observability)
|
||||
else:
|
||||
→ existing download/hash/split/upload flow, completely unchanged
|
||||
```
|
||||
|
||||
### 6. Observability
|
||||
|
||||
New `zipsForwarded` counter alongside the existing `zipsFound`/`zipsDuplicate`/`zipsIngested`/
|
||||
`zipsBackfilled` counters, surfaced the same way (run activity, ingestion run summary). A WARN-level
|
||||
log line when a forwarding-eligible archive falls back to download (mirrors the existing
|
||||
`confidence: "ranged" | "full-download-fallback"` logging convention from the ranged-listing
|
||||
backfill work), so the fallback rate is visible without digging through debug logs.
|
||||
|
||||
## Data flow
|
||||
|
||||
```
|
||||
scan → pre-download dedup + size guard (unchanged)
|
||||
→ channel.allowsForwarding?
|
||||
true → ranged listing
|
||||
ok → fingerprint dedup check → forward → stub + entries + tags (no in-archive preview) → done
|
||||
null → [fall through] existing download pipeline
|
||||
false/unknown → existing download pipeline (unchanged)
|
||||
```
|
||||
|
||||
## Error handling
|
||||
|
||||
- `forwardMessages` failure (permission revoked mid-run, rate limit, transient Telegram error) —
|
||||
same `SkippedPackage`/`SystemNotification` bookkeeping as today's upload failures. Extend
|
||||
`inferSkipReason` to recognize forward-specific error text the same way it already recognizes
|
||||
upload errors.
|
||||
- Fingerprint-repost check finds multiple ambiguous same-name/size candidates that can't be
|
||||
uniquely disambiguated — same `INTEGRITY_AUDIT` notification pattern already used in
|
||||
`provenance-backfill.ts`: don't guess, surface for manual triage.
|
||||
- `allowsForwarding` unknown (channel just linked, not yet scanned by the refresh point) — treated
|
||||
as `false`; the download path runs. No channel uses an unverified forwarding permission.
|
||||
- Ranged listing throwing instead of returning `null` — treated identically to returning `null`
|
||||
(fall through to download), consistent with how the existing ranged readers already treat
|
||||
internal errors (they catch and return `null` themselves).
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit tests (vitest, alongside the existing `archive/*.test.ts` and `archive/ranged/*.test.ts`
|
||||
files): the dedup-identity derivation function (fingerprint-hash vs remoteUniqueId-fallback
|
||||
branches), the extended fingerprint-based repost check, and `forwardArchiveToChannel`'s
|
||||
request-building logic against a mocked TDLib client — same style as the existing ranged-reader
|
||||
tests (pure logic, no live TDLib).
|
||||
- Live verification (manual — matches this repo's existing convention that the large
|
||||
`worker.ts`/`worker.py`-equivalent orchestration function has no automated test coverage and is
|
||||
verified live post-deploy): one forwarding-enabled test channel and one protected-content test
|
||||
channel. Confirm forward-path packages land with correct entries/tags/dedup identity and
|
||||
`destMessageIds`; confirm the protected channel still goes through the unchanged full pipeline;
|
||||
confirm a deliberately-unparseable archive in a forwarding-enabled channel correctly falls back
|
||||
to download+reupload and still ends up fully indexed.
|
||||
|
||||
## Rollout
|
||||
|
||||
Local build + deploy, following the same recipe as the ranged-archive-listing work: build
|
||||
`worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image (no
|
||||
`pull`, no GitHub push required). New DB migration for `TelegramChannel.allowsForwarding`. No
|
||||
changes required to the bot or app services.
|
||||
@@ -0,0 +1,5 @@
|
||||
-- AlterTable: count of packages whose provenance was backfilled during a run
|
||||
-- (opportunistic cross-channel provenance backfill). Additive, non-null with a
|
||||
-- default of 0 — no data change for existing rows.
|
||||
ALTER TABLE "ingestion_runs"
|
||||
ADD COLUMN "zipsBackfilled" INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -569,6 +569,7 @@ model IngestionRun {
|
||||
zipsFound Int @default(0)
|
||||
zipsDuplicate Int @default(0)
|
||||
zipsIngested Int @default(0)
|
||||
zipsBackfilled Int @default(0)
|
||||
errorMessage String?
|
||||
|
||||
// Live activity tracking — written by worker in real-time
|
||||
|
||||
@@ -1,415 +0,0 @@
|
||||
# Production backup and recovery
|
||||
|
||||
Dragon's Stash backs up its PostgreSQL database and the worker/bot Telegram
|
||||
(TDLib) session volumes to an encrypted Restic repository on a Synology NFS
|
||||
share. Docker volumes are local runtime storage; they are not, by themselves,
|
||||
backups.
|
||||
|
||||
The backup job runs as `root` because it reads the Restic password file and
|
||||
mounts host paths into the backup container. Keep the Synology share on the
|
||||
private network. Do **not** expose NFS to the Internet.
|
||||
|
||||
## Protected data and limits
|
||||
|
||||
Each snapshot contains:
|
||||
|
||||
- a custom-format PostgreSQL dump;
|
||||
- a backup manifest with dump checksum, migration metadata, version metadata,
|
||||
and captured source paths;
|
||||
- the worker TDLib session volume; and
|
||||
- the bot TDLib session volume.
|
||||
|
||||
The `manual_uploads` and `tmp_zips` volumes are excluded. Completed STL
|
||||
binaries are not retained as a local recovery set by this backup feature; they
|
||||
remain in Telegram. Recovery preserves the PostgreSQL archive, message,
|
||||
package, file, Telegram channel, and Telegram message ID mappings needed for
|
||||
normal lookup and delivery after restore.
|
||||
|
||||
Future Telegram channel-forwarding behavior and archive/STL-content integrity
|
||||
auditing are intentionally outside this backup and restore procedure.
|
||||
|
||||
## 1. Configure the Synology NFS share
|
||||
|
||||
On the Synology DSM host:
|
||||
|
||||
1. In **Control Panel > Shared Folder**, create a shared folder named
|
||||
`dragonsstash-backups`.
|
||||
2. In **Control Panel > File Services > NFS**, enable NFS.
|
||||
3. Edit the new shared folder and add an **NFS Permissions** entry for the
|
||||
Docker host's fixed private-network IP address. Grant read/write access.
|
||||
Use the least permissive squash and authentication settings that work for
|
||||
the root-run backup job, and do not use a broad subnet or public address.
|
||||
4. Record the NFS export path shown by DSM, for example
|
||||
`/volume1/dragonsstash-backups`.
|
||||
|
||||
On the Docker host, install the NFS client package for its distribution, create
|
||||
the mountpoint, and mount the export. Replace `NAS_IP` and the export path with
|
||||
the values for the Synology:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /mnt/dragonsstash-backups
|
||||
sudo mount -t nfs -o nfsvers=4 NAS_IP:/volume1/dragonsstash-backups /mnt/dragonsstash-backups
|
||||
```
|
||||
|
||||
For a persistent mount, add an `/etc/fstab` entry using the DSM export path:
|
||||
|
||||
```fstab
|
||||
NAS_IP:/volume1/dragonsstash-backups /mnt/dragonsstash-backups nfs nfsvers=4,_netdev,nofail,x-systemd.automount 0 0
|
||||
```
|
||||
|
||||
Then verify that the host sees an actual writable mount rather than an empty
|
||||
local directory:
|
||||
|
||||
```bash
|
||||
mountpoint /mnt/dragonsstash-backups
|
||||
touch /mnt/dragonsstash-backups/.write-test
|
||||
rm /mnt/dragonsstash-backups/.write-test
|
||||
```
|
||||
|
||||
The scheduled backup refuses to run if `BACKUP_MOUNT_PATH` is not a mountpoint
|
||||
or is not writable.
|
||||
|
||||
## 2. Create secrets and host configuration
|
||||
|
||||
Create a root-readable Restic password file. Generate and store a strong,
|
||||
unique repository password by your approved secret-management process; never
|
||||
commit it, place it in the Compose file, or paste it into shell history.
|
||||
|
||||
```bash
|
||||
sudo install -d -m 0700 -o root -g root /etc/dragons-stash
|
||||
sudo install -m 0600 -o root -g root /dev/null /etc/dragons-stash/restic-password
|
||||
sudoedit /etc/dragons-stash/restic-password
|
||||
```
|
||||
|
||||
Create a local staging directory. It must be on host-local storage, not the NFS
|
||||
share, and must have enough room for a PostgreSQL dump and restore staging
|
||||
data.
|
||||
|
||||
```bash
|
||||
sudo install -d -m 0700 -o root -g root /var/lib/dragons-stash/backup-staging
|
||||
```
|
||||
|
||||
Create the systemd environment file at `/etc/dragons-stash/backup.env` and keep
|
||||
it root-only. It supplies both the host wrapper and the profile-gated backup
|
||||
container:
|
||||
|
||||
```bash
|
||||
sudo install -m 0600 -o root -g root /dev/null /etc/dragons-stash/backup.env
|
||||
sudoedit /etc/dragons-stash/backup.env
|
||||
```
|
||||
|
||||
Set these production values, replacing paths and retention as needed:
|
||||
|
||||
```dotenv
|
||||
BACKUP_MOUNT_PATH=/mnt/dragonsstash-backups
|
||||
BACKUP_STAGING_PATH=/var/lib/dragons-stash/backup-staging
|
||||
BACKUP_RESTIC_PASSWORD_FILE=/etc/dragons-stash/restic-password
|
||||
BACKUP_REPOSITORY=/backup/restic
|
||||
BACKUP_RETENTION_DAYS=30
|
||||
BACKUP_APP_VERSION=unknown
|
||||
```
|
||||
|
||||
`BACKUP_REPOSITORY` is evaluated inside the backup container, where the NFS
|
||||
share is mounted at `/backup`; `/backup/restic` keeps the Restic repository in
|
||||
the Synology share. `BACKUP_APP_VERSION` is optional metadata. The production
|
||||
Compose environment must also retain its existing database and application
|
||||
secrets; do not add secrets to Git.
|
||||
|
||||
## 3. Initialize the Restic repository once
|
||||
|
||||
Before enabling the scheduled job, initialize a new repository explicitly:
|
||||
|
||||
```bash
|
||||
docker compose --profile backup run --rm backup init
|
||||
```
|
||||
|
||||
The backup container preflight refuses to create a repository implicitly. If a
|
||||
scheduled backup reports that the repository is unavailable or uninitialized,
|
||||
verify the NFS mount, `BACKUP_REPOSITORY`, and password file before running the
|
||||
explicit initialization command for an intended new repository.
|
||||
|
||||
## 4. Install the nightly systemd job
|
||||
|
||||
The supplied unit assumes the production Compose checkout is
|
||||
`/opt/stacks/DragonsStash`. If your deployment lives elsewhere, update the
|
||||
installed service's `WorkingDirectory` and `ExecStart` to the corresponding
|
||||
absolute paths before enabling it.
|
||||
|
||||
Install and start the units:
|
||||
|
||||
```bash
|
||||
sudo install -m 0644 deploy/systemd/dragons-stash-backup.service /etc/systemd/system/
|
||||
sudo install -m 0644 deploy/systemd/dragons-stash-backup.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now dragons-stash-backup.timer
|
||||
sudo systemctl start dragons-stash-backup.service
|
||||
sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager
|
||||
```
|
||||
|
||||
The timer runs nightly at 03:00 with up to 15 minutes of randomized delay and
|
||||
catches up after downtime. The first run captures PostgreSQL and both TDLib
|
||||
session volumes. Later Restic snapshots deduplicate unchanged session state and
|
||||
database dump content.
|
||||
|
||||
## 5. Monitor and maintain backups
|
||||
|
||||
Inspect the next scheduled run and the last service result:
|
||||
|
||||
```bash
|
||||
systemctl list-timers dragons-stash-backup.timer
|
||||
systemctl status dragons-stash-backup.timer
|
||||
systemctl status dragons-stash-backup.service
|
||||
sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager
|
||||
sudo systemctl --failed
|
||||
```
|
||||
|
||||
List snapshots through the configured backup container:
|
||||
|
||||
```bash
|
||||
./scripts/backup/restore.sh list
|
||||
```
|
||||
|
||||
Check a selected repository snapshot before restoring it:
|
||||
|
||||
```bash
|
||||
./scripts/backup/restore.sh verify SNAPSHOT_ID
|
||||
```
|
||||
|
||||
`verify` runs `restic snapshots SNAPSHOT_ID` and `restic check`. Routine backup
|
||||
runs apply retention with `restic forget --keep-daily "$BACKUP_RETENTION_DAYS"
|
||||
--prune`; with the example configuration, this retains 30 daily snapshots.
|
||||
Choose `BACKUP_RETENTION_DAYS` based on storage capacity and the recovery
|
||||
window you need. Watch Synology capacity and investigate any failed timer or
|
||||
service promptly.
|
||||
|
||||
At least monthly, perform a full repository read check:
|
||||
|
||||
```bash
|
||||
docker compose --profile backup run --rm backup check --read-data
|
||||
```
|
||||
|
||||
## 6. Monthly disposable recovery rehearsal
|
||||
|
||||
Perform the following procedure at least monthly. It restores one snapshot into
|
||||
a unique, disposable Compose project, so the database and Compose volumes are
|
||||
isolated from production. **Never use the production Compose project name,
|
||||
production volume names, or `restore-live` for this rehearsal.** Do not run
|
||||
`docker compose down -v` without the explicit rehearsal `--project-name` shown
|
||||
below.
|
||||
|
||||
Run these commands from the production Compose checkout as an operator allowed
|
||||
to read `/etc/dragons-stash/backup.env`. They use a separate application stack;
|
||||
do not expose its published port beyond the host. The worker and bot are
|
||||
deliberately replaced with inert processes, so this validates restored images,
|
||||
database state, and TDLib volume layout without executing Telegram clients or
|
||||
using production Telegram credentials.
|
||||
|
||||
```bash
|
||||
set -Eeuo pipefail
|
||||
set -a
|
||||
. /etc/dragons-stash/backup.env
|
||||
set +a
|
||||
|
||||
./scripts/backup/restore.sh list
|
||||
SNAPSHOT_ID=SNAPSHOT_ID_FROM_LIST
|
||||
REHEARSAL_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
REHEARSAL_PROJECT="dragonsstash-rehearsal-$REHEARSAL_ID"
|
||||
REHEARSAL_DIR="$BACKUP_STAGING_PATH/monthly-rehearsal-$REHEARSAL_ID"
|
||||
REHEARSAL_ENV="$REHEARSAL_DIR/compose.env"
|
||||
REHEARSAL_OVERRIDE="$REHEARSAL_DIR/compose.rehearsal.override.yml"
|
||||
REHEARSAL_APP_PORT=13000 # Choose an unused host-local port.
|
||||
|
||||
./scripts/backup/restore.sh verify "$SNAPSHOT_ID"
|
||||
./scripts/backup/restore.sh restore-to-staging "$SNAPSHOT_ID" "$REHEARSAL_DIR"
|
||||
RESTORE_ROOT="$(printf '%s\n' "$REHEARSAL_DIR"/staging/backup-*)"
|
||||
sha256sum --check "$RESTORE_ROOT/manifest/database.dump.sha256"
|
||||
|
||||
test -s "$RESTORE_ROOT/database.dump"
|
||||
test -s "$RESTORE_ROOT/manifest/backup-manifest.json"
|
||||
test -d "$REHEARSAL_DIR/data/tdlib-worker"
|
||||
test -d "$REHEARSAL_DIR/data/tdlib-bot"
|
||||
|
||||
umask 077
|
||||
cp .env "$REHEARSAL_ENV"
|
||||
printf '\nAPP_PORT=%s\nNEXT_PUBLIC_APP_URL=http://localhost:%s\n' \
|
||||
"$REHEARSAL_APP_PORT" "$REHEARSAL_APP_PORT" >> "$REHEARSAL_ENV"
|
||||
|
||||
cat > "$REHEARSAL_OVERRIDE" <<'EOF'
|
||||
services:
|
||||
worker:
|
||||
environment:
|
||||
TELEGRAM_API_ID: ""
|
||||
TELEGRAM_API_HASH: ""
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: ["exec sleep infinity"]
|
||||
bot:
|
||||
environment:
|
||||
BOT_TOKEN: ""
|
||||
TELEGRAM_API_ID: ""
|
||||
TELEGRAM_API_HASH: ""
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: ["exec sleep infinity"]
|
||||
EOF
|
||||
```
|
||||
|
||||
Confirm that `RESTORE_ROOT` names exactly one `backup-*` directory before
|
||||
continuing. The staging restore has already verified the custom PostgreSQL dump
|
||||
and restored `data/tdlib-worker` and `data/tdlib-bot`.
|
||||
|
||||
Create the isolated project and volumes, start only its database, then import
|
||||
the dump. `compose_rehearsal()` always applies the disposable override created
|
||||
above: worker and bot retain their restored images and volumes but have their
|
||||
Telegram credential variables blanked and run only `sleep infinity`, never
|
||||
Telegram clients. The `create` command makes project-scoped application
|
||||
volumes without starting app, worker, or bot.
|
||||
|
||||
```bash
|
||||
compose_rehearsal() {
|
||||
docker compose --project-name "$REHEARSAL_PROJECT" --env-file "$REHEARSAL_ENV" \
|
||||
-f docker-compose.yml -f "$REHEARSAL_OVERRIDE" "$@"
|
||||
}
|
||||
rehearsal_volume() {
|
||||
docker volume ls \
|
||||
--filter "label=com.docker.compose.project=$REHEARSAL_PROJECT" \
|
||||
--filter "label=com.docker.compose.volume=$1" \
|
||||
--format '{{.Name}}'
|
||||
}
|
||||
|
||||
compose_rehearsal --profile full create app worker bot
|
||||
compose_rehearsal --profile full up -d db
|
||||
compose_rehearsal exec -T db dropdb --if-exists --force \
|
||||
--username "${POSTGRES_USER:-dragons}" "${POSTGRES_DB:-dragonsstash}"
|
||||
compose_rehearsal exec -T db createdb --username "${POSTGRES_USER:-dragons}" \
|
||||
"${POSTGRES_DB:-dragonsstash}"
|
||||
compose_rehearsal exec -T db pg_restore --no-owner --exit-on-error \
|
||||
--username "${POSTGRES_USER:-dragons}" --dbname "${POSTGRES_DB:-dragonsstash}" \
|
||||
< "$RESTORE_ROOT/database.dump"
|
||||
```
|
||||
|
||||
Copy each restored TDLib tree into its matching **rehearsal** volume. Each
|
||||
target is new and empty; the function rejects an ambiguous volume lookup.
|
||||
|
||||
```bash
|
||||
restore_rehearsal_volume() {
|
||||
local logical_name="$1"
|
||||
local source_dir="$2"
|
||||
local volume_name
|
||||
volume_name="$(rehearsal_volume "$logical_name")"
|
||||
test -n "$volume_name"
|
||||
test "$(printf '%s\n' "$volume_name" | wc -l)" -eq 1
|
||||
compose_rehearsal --profile backup run --rm --no-deps \
|
||||
--entrypoint bash \
|
||||
-v "$source_dir:/restore-source:ro" \
|
||||
-v "$volume_name:/restore-target" \
|
||||
backup -ceu 'cp -a /restore-source/. /restore-target/'
|
||||
}
|
||||
|
||||
restore_rehearsal_volume tdlib_state "$REHEARSAL_DIR/data/tdlib-worker"
|
||||
restore_rehearsal_volume tdlib_bot_state "$REHEARSAL_DIR/data/tdlib-bot"
|
||||
```
|
||||
|
||||
Start the disposable app, worker, and bot containers. The app and database
|
||||
perform their normal health checks; the worker and bot remain inert `sleep
|
||||
infinity` processes, so this does not execute Telegram clients or send
|
||||
messages. Check the health endpoint, then retain the `ps` and log output as
|
||||
rehearsal evidence.
|
||||
|
||||
```bash
|
||||
compose_rehearsal --profile full up -d app worker bot
|
||||
curl --fail --silent --show-error \
|
||||
"http://localhost:$REHEARSAL_APP_PORT/api/health"
|
||||
compose_rehearsal --profile full ps
|
||||
compose_rehearsal --profile full logs --tail=100 app worker bot
|
||||
```
|
||||
|
||||
Optionally spot-check the restored database metadata that lets Dragon's Stash
|
||||
locate Telegram-hosted STL binaries after restore. This checks database
|
||||
metadata and Telegram IDs only; it does not assert local STL file presence,
|
||||
STL checksums, Telegram forwarding behavior, or archive/STL binary integrity.
|
||||
|
||||
```bash
|
||||
compose_rehearsal exec -T db psql --no-psqlrc --tuples-only --no-align --quiet \
|
||||
--username "${POSTGRES_USER:-dragons}" \
|
||||
--dbname "${POSTGRES_DB:-dragonsstash}" \
|
||||
--command 'SELECT count(*) FROM "packages" WHERE "destChannelId" IS NOT NULL AND "destMessageId" IS NOT NULL;'
|
||||
```
|
||||
|
||||
After recording the evidence, destroy only the explicitly named disposable
|
||||
project and its project-labeled volumes. Do not remove `REHEARSAL_DIR` until
|
||||
the evidence is recorded; then remove only that generated child directory by
|
||||
your approved host cleanup procedure. Production services, volumes, and backup
|
||||
snapshots must remain untouched.
|
||||
|
||||
```bash
|
||||
compose_rehearsal --profile full down --volumes --remove-orphans
|
||||
docker volume ls --filter "label=com.docker.compose.project=$REHEARSAL_PROJECT"
|
||||
test -z "$(docker volume ls --quiet --filter "label=com.docker.compose.project=$REHEARSAL_PROJECT")"
|
||||
```
|
||||
|
||||
Record this evidence for every rehearsal; do not include secrets, database
|
||||
dumps, Telegram session contents, or private NAS details.
|
||||
|
||||
```text
|
||||
Disposable restore rehearsal
|
||||
Date/time (UTC):
|
||||
Operator:
|
||||
Snapshot ID:
|
||||
Snapshot backup date:
|
||||
Disposable Compose project:
|
||||
Full restic check --read-data result:
|
||||
Database dump manifest checksum result:
|
||||
TDLib worker tree restored:
|
||||
TDLib bot tree restored:
|
||||
Health endpoint result (HTTP/body):
|
||||
docker compose ps result:
|
||||
app/worker/bot log review result:
|
||||
Database Telegram metadata/mapping spot-check result:
|
||||
Cleanup result (project and rehearsal volumes absent):
|
||||
Notes/caveats:
|
||||
```
|
||||
|
||||
This document describes the procedure only; it has not been run by this
|
||||
documentation update.
|
||||
|
||||
## 7. Restore modes
|
||||
|
||||
Run restore commands from the production Compose checkout after loading the
|
||||
same backup environment used by systemd, for example as root with
|
||||
`/etc/dragons-stash/backup.env` exported. All restore staging directories must
|
||||
be children of `BACKUP_STAGING_PATH`.
|
||||
|
||||
| Mode | Command | Effect |
|
||||
| --- | --- | --- |
|
||||
| List snapshots | `./scripts/backup/restore.sh list` | Lists available Restic snapshots. Does not stop services or change volumes. |
|
||||
| Verify a snapshot | `./scripts/backup/restore.sh verify SNAPSHOT_ID` | Confirms the snapshot exists and runs a Restic repository check. Does not change live data. |
|
||||
| Restore to staging | `./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR` | Restores and validates a snapshot in a new or empty child directory of `BACKUP_STAGING_PATH`. Does not stop services or change volumes. |
|
||||
| Replace live data | `./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data` | Stops application services and replaces the PostgreSQL database plus both TDLib session volumes after validation. |
|
||||
|
||||
`restore-live` is destructive. It requires the exact
|
||||
`--confirm-replace-live-data` flag and should be used only after a successful
|
||||
staging restore has been inspected. It creates a safety database dump and
|
||||
archives of the current TDLib volumes in local staging before replacement. If a
|
||||
live restore fails, it leaves the application services stopped, retains the
|
||||
safety artifacts and staging directory, and attempts rollback after replacement
|
||||
has begun. Review the reported paths and service health before manually
|
||||
starting services.
|
||||
|
||||
After a successful live restore, confirm the services are running and inspect
|
||||
their recent logs before declaring recovery complete:
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
docker compose logs --tail=100 app worker bot
|
||||
```
|
||||
|
||||
For a non-destructive recovery rehearsal, choose a snapshot from `list` and
|
||||
restore it to a fresh staging directory, for example:
|
||||
|
||||
```bash
|
||||
./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID /var/lib/dragons-stash/backup-staging/rehearsal-SNAPSHOT_ID
|
||||
```
|
||||
|
||||
The restored tree must contain a non-empty PostgreSQL dump, its manifest,
|
||||
worker TDLib state, and bot TDLib state. STL binaries remain in Telegram, and
|
||||
the restored database mappings and Telegram IDs are what recovery preserves for
|
||||
normal lookup and delivery.
|
||||
@@ -1,202 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly BACKUP_ROOT="/backup"
|
||||
readonly STAGING_ROOT="/staging"
|
||||
readonly TDLIB_WORKER_PATH="/data/tdlib-worker"
|
||||
readonly TDLIB_BOT_PATH="/data/tdlib-bot"
|
||||
|
||||
RUN_DIR=""
|
||||
|
||||
require_value() {
|
||||
local name="$1"
|
||||
|
||||
if [[ -z "${!name:-}" ]]; then
|
||||
printf 'Required environment variable %s is not set.\n' "$name" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_directory() {
|
||||
local path="$1"
|
||||
|
||||
if [[ ! -d "$path" ]]; then
|
||||
printf 'Required mounted directory %s is unavailable.\n' "$path" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_restic_repository() {
|
||||
local canonical_repository
|
||||
|
||||
canonical_repository="$(realpath -m -- "$RESTIC_REPOSITORY")"
|
||||
if [[ "$canonical_repository" == "$BACKUP_ROOT" || "$canonical_repository" != "$BACKUP_ROOT"/* ]]; then
|
||||
printf 'RESTIC_REPOSITORY must be strictly below /backup; got %s.\n' "$RESTIC_REPOSITORY" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
RESTIC_REPOSITORY="$canonical_repository"
|
||||
export RESTIC_REPOSITORY
|
||||
}
|
||||
|
||||
validate_restic_configuration() {
|
||||
require_value RESTIC_REPOSITORY
|
||||
require_value RESTIC_PASSWORD_FILE
|
||||
|
||||
if [[ ! -r "$RESTIC_PASSWORD_FILE" ]]; then
|
||||
printf 'Restic password file %s is not readable.\n' "$RESTIC_PASSWORD_FILE" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
require_directory "$BACKUP_ROOT"
|
||||
validate_restic_repository
|
||||
}
|
||||
|
||||
ensure_repository_initialized() {
|
||||
if ! restic cat config >/dev/null; then
|
||||
printf 'Restic repository %s is not initialized or could not be opened. Verify the backup mount, repository path, and password.\n' "$RESTIC_REPOSITORY" >&2
|
||||
printf 'For an intended first-time repository, initialize it explicitly with: docker compose --profile backup run --rm backup init\n' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
json_escape() {
|
||||
local input="$1"
|
||||
local output=""
|
||||
local character
|
||||
local code
|
||||
|
||||
while [[ -n "$input" ]]; do
|
||||
character="${input:0:1}"
|
||||
input="${input:1}"
|
||||
|
||||
case "$character" in
|
||||
'"') output+='\"' ;;
|
||||
'\\') output+='\\' ;;
|
||||
$'\b') output+='\b' ;;
|
||||
$'\f') output+='\f' ;;
|
||||
$'\n') output+='\n' ;;
|
||||
$'\r') output+='\r' ;;
|
||||
$'\t') output+='\t' ;;
|
||||
*)
|
||||
printf -v code '%d' "'$character"
|
||||
if (( code < 32 )); then
|
||||
printf -v character '\u%04x' "$code"
|
||||
fi
|
||||
output+="$character"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
printf '%s' "$output"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
|
||||
if [[ -n "$RUN_DIR" && "$RUN_DIR" == "$STAGING_ROOT"/* ]]; then
|
||||
rm -rf -- "$RUN_DIR"
|
||||
fi
|
||||
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
local timestamp
|
||||
local checksum
|
||||
local app_version
|
||||
local applied_migrations
|
||||
|
||||
require_value DATABASE_URL
|
||||
require_value BACKUP_RETENTION_DAYS
|
||||
validate_restic_configuration
|
||||
require_directory "$STAGING_ROOT"
|
||||
require_directory "$TDLIB_WORKER_PATH"
|
||||
require_directory "$TDLIB_BOT_PATH"
|
||||
ensure_repository_initialized
|
||||
|
||||
timestamp="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
RUN_DIR="$STAGING_ROOT/backup-${timestamp//[:]/}-$$"
|
||||
mkdir -- "$RUN_DIR"
|
||||
mkdir -- "$RUN_DIR/manifest"
|
||||
trap cleanup EXIT
|
||||
|
||||
pg_dump --format=custom --file="$RUN_DIR/database.dump" "$DATABASE_URL"
|
||||
|
||||
if [[ ! -f "$RUN_DIR/database.dump" || ! -s "$RUN_DIR/database.dump" ]]; then
|
||||
printf 'PostgreSQL dump is missing or empty.\n' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
sha256sum "$RUN_DIR/database.dump" > "$RUN_DIR/manifest/database.dump.sha256"
|
||||
checksum="$(awk '{print $1}' "$RUN_DIR/manifest/database.dump.sha256")"
|
||||
app_version="${BACKUP_APP_VERSION:-unknown}"
|
||||
applied_migrations="$(
|
||||
psql --dbname "$DATABASE_URL" --no-psqlrc --tuples-only --no-align --quiet \
|
||||
--set ON_ERROR_STOP=on <<'SQL'
|
||||
SELECT COALESCE(
|
||||
json_agg(
|
||||
json_build_object(
|
||||
'name', "migration_name",
|
||||
'finishedAtUtc', to_char(
|
||||
"finished_at" AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'
|
||||
)
|
||||
)
|
||||
ORDER BY "finished_at", "migration_name"
|
||||
),
|
||||
'[]'::json
|
||||
)::text
|
||||
FROM "_prisma_migrations"
|
||||
WHERE "finished_at" IS NOT NULL AND "rolled_back_at" IS NULL;
|
||||
SQL
|
||||
)"
|
||||
|
||||
cat > "$RUN_DIR/manifest/backup-manifest.json" <<EOF
|
||||
{
|
||||
"backupTimestampUtc": "$(json_escape "$timestamp")",
|
||||
"repository": "$(json_escape "$RESTIC_REPOSITORY")",
|
||||
"retentionDays": "$(json_escape "$BACKUP_RETENTION_DAYS")",
|
||||
"applicationVersion": "$(json_escape "$app_version")",
|
||||
"appliedPrismaMigrations": $applied_migrations,
|
||||
"databaseDump": {
|
||||
"filename": "database.dump",
|
||||
"sha256": "$(json_escape "$checksum")"
|
||||
},
|
||||
"volumePaths": [
|
||||
"$(json_escape "$TDLIB_WORKER_PATH")",
|
||||
"$(json_escape "$TDLIB_BOT_PATH")"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
restic backup \
|
||||
--tag "application:dragons-stash" \
|
||||
--tag "source:database" \
|
||||
--tag "source:tdlib-worker" \
|
||||
--tag "source:tdlib-bot" \
|
||||
"$RUN_DIR/database.dump" \
|
||||
"$RUN_DIR/manifest" \
|
||||
"$TDLIB_WORKER_PATH" \
|
||||
"$TDLIB_BOT_PATH"
|
||||
restic snapshots --latest 1
|
||||
restic check
|
||||
restic forget --keep-daily "$BACKUP_RETENTION_DAYS" --prune
|
||||
}
|
||||
|
||||
run_restore() {
|
||||
shift
|
||||
validate_restic_configuration
|
||||
exec restic restore "$@"
|
||||
}
|
||||
|
||||
run_restic() {
|
||||
validate_restic_configuration
|
||||
exec restic "$@"
|
||||
}
|
||||
|
||||
case "${1:-backup}" in
|
||||
backup) run_backup ;;
|
||||
restore) run_restore "$@" ;;
|
||||
*) run_restic "$@" ;;
|
||||
esac
|
||||
@@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
PROJECT_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)"
|
||||
RESTORE_SCRIPT="$PROJECT_ROOT/scripts/backup/restore.sh"
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
|
||||
cleanup() {
|
||||
rm -rf -- "$TMP_ROOT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
printf 'ASSERTION FAILED: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
setup_fake_environment() {
|
||||
local fake_bin="$TMP_ROOT/bin"
|
||||
|
||||
mkdir -p -- "$fake_bin" "$TMP_ROOT/mount" "$TMP_ROOT/staging"
|
||||
printf 'not-secret\n' > "$TMP_ROOT/restic-password"
|
||||
|
||||
cat > "$fake_bin/docker" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
printf '%s\t' "$@" >> "$DRAGONS_STASH_DOCKER_CALLS"
|
||||
printf '\n' >> "$DRAGONS_STASH_DOCKER_CALLS"
|
||||
|
||||
target=""
|
||||
previous=""
|
||||
safety_output_host=""
|
||||
safety_archive=""
|
||||
for argument in "$@"; do
|
||||
if [[ "$previous" == "--target" ]]; then
|
||||
target="$argument"
|
||||
fi
|
||||
if [[ "$previous" == "-v" && "$argument" == *":/safety-output" ]]; then
|
||||
safety_output_host="${argument%:/safety-output}"
|
||||
fi
|
||||
if [[ "$argument" == /safety-output/pre-restore-*.tar ]]; then
|
||||
safety_archive="${argument#/safety-output/}"
|
||||
fi
|
||||
previous="$argument"
|
||||
done
|
||||
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "config" && "${3:-}" == "--format" && "${4:-}" == "json" ]]; then
|
||||
printf '{\n "name": "dragonsstash"\n}\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "--profile" && "${3:-}" == "full" && "${4:-}" == "ps" && "${5:-}" == "--status" && "${6:-}" == "running" && "${7:-}" == "-q" ]]; then
|
||||
case " ${RESTORE_ASSERT_RUNNING_SERVICES:-app worker bot} " in
|
||||
*" ${8:-} "*) printf '%s-container\n' "${8:-}" ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "volume" && "${2:-}" == "ls" ]]; then
|
||||
if [[ " $* " == *"com.docker.compose.volume=tdlib_state"* ]]; then
|
||||
printf 'dragonsstash_tdlib_state\n'
|
||||
elif [[ " $* " == *"com.docker.compose.volume=tdlib_bot_state"* ]]; then
|
||||
printf 'dragonsstash_tdlib_bot_state\n'
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ " $* " == *" exec -T db pg_dump "* ]]; then
|
||||
printf 'custom dump\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ " $* " == *" backup restore "* ]]; then
|
||||
if [[ "$target" != /staging/* ]]; then
|
||||
printf 'Unexpected restore target: %s\n' "$target" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
host_target="$BACKUP_STAGING_PATH/${target#/staging/}"
|
||||
mkdir -p \
|
||||
"$host_target/staging/backup-legacy/manifest" \
|
||||
"$host_target/data/tdlib-worker" \
|
||||
"$host_target/data/tdlib-bot"
|
||||
printf 'custom dump\n' > "$host_target/staging/backup-legacy/database.dump"
|
||||
printf '{}\n' > "$host_target/staging/backup-legacy/manifest/backup-manifest.json"
|
||||
|
||||
if [[ "${RESTORE_ASSERT_CREATE_UNEXPECTED:-0}" == "1" ]]; then
|
||||
mkdir -p "$host_target/data/uploads" "$host_target/data/tmp-zips" "$host_target/data/postgres"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "$safety_output_host" && -n "$safety_archive" ]]; then
|
||||
printf 'archive\n' > "$safety_output_host/$safety_archive"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/docker"
|
||||
|
||||
cat > "$fake_bin/mountpoint" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/mountpoint"
|
||||
|
||||
cat > "$fake_bin/curl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
printf '%s\t' "$@" >> "$DRAGONS_STASH_CURL_CALLS"
|
||||
printf '\n' >> "$DRAGONS_STASH_CURL_CALLS"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/curl"
|
||||
|
||||
export PATH="$fake_bin:$PATH"
|
||||
export BACKUP_MOUNT_PATH="$TMP_ROOT/mount"
|
||||
export BACKUP_STAGING_PATH="$TMP_ROOT/staging"
|
||||
export BACKUP_RESTIC_PASSWORD_FILE="$TMP_ROOT/restic-password"
|
||||
export BACKUP_REPOSITORY="/backup/restic"
|
||||
}
|
||||
|
||||
run_restore_to_staging() {
|
||||
local name="$1"
|
||||
shift
|
||||
|
||||
DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \
|
||||
"$@" "$RESTORE_SCRIPT" restore-to-staging snapshot-scope "$TMP_ROOT/staging/$name"
|
||||
}
|
||||
|
||||
run_restore_live() {
|
||||
local name="$1"
|
||||
shift
|
||||
|
||||
DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \
|
||||
DRAGONS_STASH_CURL_CALLS="$TMP_ROOT/curl-calls-$name.log" \
|
||||
"$@" "$RESTORE_SCRIPT" restore-live snapshot-scope --confirm-replace-live-data
|
||||
}
|
||||
|
||||
assert_restore_call_is_filtered() {
|
||||
local calls="$1"
|
||||
local restore_call
|
||||
|
||||
restore_call="$(grep $'backup\trestore\tsnapshot-scope' "$calls" || true)"
|
||||
[[ -n "$restore_call" ]] || fail "restore command was not invoked"
|
||||
[[ "$restore_call" == *$'--include\t/staging/backup-*/database.dump'* ]] || fail "database dump include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/staging/backup-*/manifest\t'* ]] || fail "manifest include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/staging/backup-*/manifest/**'* ]] || fail "manifest subtree include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/data/tdlib-worker\t'* ]] || fail "worker TDLib root include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/data/tdlib-worker/**'* ]] || fail "worker TDLib subtree include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/data/tdlib-bot\t'* ]] || fail "bot TDLib root include filter missing"
|
||||
[[ "$restore_call" == *$'--include\t/data/tdlib-bot/**'* ]] || fail "bot TDLib subtree include filter missing"
|
||||
[[ "$restore_call" != *"/data/uploads"* ]] || fail "restore includes uploads path"
|
||||
}
|
||||
|
||||
assert_restore_uses_include_filters() {
|
||||
local calls="$TMP_ROOT/docker-calls-safe.log"
|
||||
|
||||
setup_fake_environment
|
||||
run_restore_to_staging safe bash >/dev/null
|
||||
|
||||
assert_restore_call_is_filtered "$calls"
|
||||
}
|
||||
|
||||
assert_live_restore_uses_include_filters() {
|
||||
local calls="$TMP_ROOT/docker-calls-live.log"
|
||||
|
||||
setup_fake_environment
|
||||
run_restore_live live bash >/dev/null
|
||||
|
||||
assert_restore_call_is_filtered "$calls"
|
||||
}
|
||||
|
||||
assert_live_restore_restarts_only_previously_running_services() {
|
||||
local calls="$TMP_ROOT/docker-calls-live-running-subset.log"
|
||||
local stop_call
|
||||
local start_call
|
||||
|
||||
setup_fake_environment
|
||||
RESTORE_ASSERT_RUNNING_SERVICES="app worker" run_restore_live live-running-subset bash >/dev/null
|
||||
|
||||
stop_call="$(grep $'compose\t--profile\tfull\tstop' "$calls" || true)"
|
||||
start_call="$(grep $'compose\t--profile\tfull\tup\t-d' "$calls" || true)"
|
||||
|
||||
[[ "$stop_call" == $'compose\t--profile\tfull\tstop\tapp\tworker\t' ]] \
|
||||
|| fail "restore-live stopped services other than the running app/worker subset: $stop_call"
|
||||
[[ "$start_call" == $'compose\t--profile\tfull\tup\t-d\tapp\tworker\t' ]] \
|
||||
|| fail "restore-live started services other than the running app/worker subset: $start_call"
|
||||
}
|
||||
|
||||
assert_live_restore_skips_health_when_app_was_not_running() {
|
||||
local calls="$TMP_ROOT/curl-calls-live-without-app.log"
|
||||
|
||||
setup_fake_environment
|
||||
RESTORE_ASSERT_RUNNING_SERVICES="worker" run_restore_live live-without-app bash >/dev/null
|
||||
|
||||
[[ ! -s "$calls" ]] || fail "restore-live waited for app health even though app was not previously running"
|
||||
}
|
||||
|
||||
assert_unexpected_volume_content_is_rejected() {
|
||||
local output="$TMP_ROOT/unexpected-output.log"
|
||||
|
||||
setup_fake_environment
|
||||
if RESTORE_ASSERT_CREATE_UNEXPECTED=1 run_restore_to_staging unexpected bash >"$output" 2>&1; then
|
||||
fail "restore accepted unexpected restored data volume content"
|
||||
fi
|
||||
grep -Eq 'Unexpected restored data volume content|Refusing restore' "$output" \
|
||||
|| fail "restore rejected unexpected content without an explicit guard message"
|
||||
}
|
||||
|
||||
assert_restore_uses_include_filters
|
||||
assert_live_restore_uses_include_filters
|
||||
assert_live_restore_restarts_only_previously_running_services
|
||||
assert_live_restore_skips_health_when_app_was_not_running
|
||||
assert_unexpected_volume_content_is_rejected
|
||||
|
||||
printf 'restore-path assertions passed\n'
|
||||
@@ -1,477 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly CONFIRM_REPLACE_LIVE_DATA="--confirm-replace-live-data"
|
||||
readonly STAGING_CONTAINER_ROOT="/staging"
|
||||
readonly BACKUP_CONTAINER_ROOT="/backup"
|
||||
readonly -a LIVE_SERVICES=(app worker bot)
|
||||
readonly -a RESTORE_INCLUDE_FILTERS=(
|
||||
--include "/staging/backup-*/database.dump"
|
||||
--include "/staging/backup-*/manifest"
|
||||
--include "/staging/backup-*/manifest/**"
|
||||
--include "/data/tdlib-worker"
|
||||
--include "/data/tdlib-worker/**"
|
||||
--include "/data/tdlib-bot"
|
||||
--include "/data/tdlib-bot/**"
|
||||
)
|
||||
|
||||
RESTORED_DUMP=""
|
||||
RESTORED_TDLIB_WORKER=""
|
||||
RESTORED_TDLIB_BOT=""
|
||||
LIVE_STAGING_DIR=""
|
||||
SAFETY_DUMP=""
|
||||
LIVE_RESTORE_ACTIVE=0
|
||||
LIVE_REPLACEMENT_STARTED=0
|
||||
LIVE_WORKER_VOLUME=""
|
||||
LIVE_BOT_VOLUME=""
|
||||
declare -a RUNNING_LIVE_SERVICES=()
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/backup/restore.sh list
|
||||
./scripts/backup/restore.sh verify SNAPSHOT_ID
|
||||
./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR
|
||||
./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data
|
||||
EOF
|
||||
}
|
||||
|
||||
require_value() {
|
||||
local name="$1"
|
||||
if [[ -z "${!name:-}" ]]; then
|
||||
printf 'Required environment variable %s is not set.\n' "$name" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_environment() {
|
||||
require_value BACKUP_MOUNT_PATH
|
||||
require_value BACKUP_STAGING_PATH
|
||||
require_value BACKUP_RESTIC_PASSWORD_FILE
|
||||
require_value BACKUP_REPOSITORY
|
||||
if [[ ! -r "$BACKUP_RESTIC_PASSWORD_FILE" ]]; then
|
||||
printf 'Restic password file %s is not readable.\n' "$BACKUP_RESTIC_PASSWORD_FILE" >&2
|
||||
return 1
|
||||
fi
|
||||
validate_backup_repository
|
||||
}
|
||||
|
||||
validate_backup_repository() {
|
||||
local repository="$BACKUP_REPOSITORY"
|
||||
local canonical_repository
|
||||
|
||||
canonical_repository="$(realpath -ms -- "$repository")"
|
||||
if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then
|
||||
printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_backup_mount() {
|
||||
local probe_file
|
||||
|
||||
if ! mountpoint --q "$BACKUP_MOUNT_PATH"; then
|
||||
printf 'Backup mount %s is not an active mountpoint.\n' "$BACKUP_MOUNT_PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! probe_file="$(mktemp "$BACKUP_MOUNT_PATH/.dragons-stash-backup-write-probe.XXXXXX")"; then
|
||||
printf 'Backup mount %s is not writable.\n' "$BACKUP_MOUNT_PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! rm -f -- "$probe_file"; then
|
||||
printf 'Unable to remove writable probe %s.\n' "$probe_file" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
backup_restic() {
|
||||
docker compose --profile backup run --rm --no-deps backup "$@"
|
||||
}
|
||||
|
||||
restore_snapshot_subset() {
|
||||
local snapshot_id="$1"
|
||||
local container_staging_dir="$2"
|
||||
|
||||
backup_restic restore "$snapshot_id" --target "$container_staging_dir" "${RESTORE_INCLUDE_FILTERS[@]}"
|
||||
}
|
||||
|
||||
canonical_staging_root() {
|
||||
realpath -m -- "$BACKUP_STAGING_PATH"
|
||||
}
|
||||
|
||||
validate_staging_directory() {
|
||||
local staging_dir="$1"
|
||||
local staging_root
|
||||
local canonical_staging_dir
|
||||
|
||||
staging_root="$(canonical_staging_root)"
|
||||
canonical_staging_dir="$(realpath -m -- "$staging_dir")"
|
||||
if [[ "$canonical_staging_dir" == "$staging_root" || "$canonical_staging_dir" != "$staging_root"/* ]]; then
|
||||
printf 'Staging directory %s must be inside BACKUP_STAGING_PATH (%s).\n' "$staging_dir" "$staging_root" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$canonical_staging_dir"
|
||||
}
|
||||
|
||||
prepare_fresh_staging_directory() {
|
||||
local staging_dir="$1"
|
||||
|
||||
if [[ -e "$staging_dir" ]]; then
|
||||
if [[ ! -d "$staging_dir" ]]; then
|
||||
printf 'Staging target %s exists but is not a directory.\n' "$staging_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "$(find "$staging_dir" -mindepth 1 -print -quit)" ]]; then
|
||||
printf 'Staging target %s must be fresh and empty; remove stale artifacts first.\n' "$staging_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
mkdir -p -- "$staging_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
container_staging_directory() {
|
||||
local host_staging_dir="$1"
|
||||
local staging_root
|
||||
|
||||
staging_root="$(canonical_staging_root)"
|
||||
printf '%s/%s\n' "$STAGING_CONTAINER_ROOT" "${host_staging_dir#"$staging_root"/}"
|
||||
}
|
||||
|
||||
verify_custom_dump() {
|
||||
local dump_path="$1"
|
||||
docker compose --profile backup run --rm --no-deps \
|
||||
--entrypoint pg_restore \
|
||||
-v "$dump_path:/restore/database.dump:ro" \
|
||||
backup --list /restore/database.dump >/dev/null
|
||||
}
|
||||
|
||||
reject_unexpected_restored_content() {
|
||||
local staging_dir="$1"
|
||||
local backup_directory="$2"
|
||||
local entry
|
||||
local name
|
||||
local -a unexpected_entries=()
|
||||
|
||||
while IFS= read -r -d '' entry; do
|
||||
name="$(basename -- "$entry")"
|
||||
case "$name" in
|
||||
data|staging) ;;
|
||||
*) unexpected_entries+=("$entry") ;;
|
||||
esac
|
||||
done < <(find "$staging_dir" -mindepth 1 -maxdepth 1 -type d -print0)
|
||||
|
||||
if [[ -d "$staging_dir/data" ]]; then
|
||||
while IFS= read -r -d '' entry; do
|
||||
name="$(basename -- "$entry")"
|
||||
case "$name" in
|
||||
tdlib-worker|tdlib-bot) ;;
|
||||
*) unexpected_entries+=("$entry") ;;
|
||||
esac
|
||||
done < <(find "$staging_dir/data" -mindepth 1 -maxdepth 1 -print0)
|
||||
fi
|
||||
|
||||
while IFS= read -r -d '' entry; do
|
||||
name="$(basename -- "$entry")"
|
||||
case "$name" in
|
||||
database.dump|manifest) ;;
|
||||
*) unexpected_entries+=("$entry") ;;
|
||||
esac
|
||||
done < <(find "$backup_directory" -mindepth 1 -maxdepth 1 -print0)
|
||||
|
||||
if ((${#unexpected_entries[@]})); then
|
||||
printf 'Refusing restore: unexpected restored data volume content found under %s:\n' "$staging_dir" >&2
|
||||
for entry in "${unexpected_entries[@]}"; do
|
||||
printf ' - %s\n' "${entry#"$staging_dir"/}" >&2
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_restored_tree() {
|
||||
local staging_dir="$1"
|
||||
local -a backup_directories=("$staging_dir"/staging/backup-*)
|
||||
local backup_directory
|
||||
local manifest
|
||||
|
||||
if ((${#backup_directories[@]} != 1)) || [[ ! -d "${backup_directories[0]:-}" ]]; then
|
||||
printf 'Restored tree %s must contain exactly one staging/backup-* directory.\n' "$staging_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
backup_directory="${backup_directories[0]}"
|
||||
reject_unexpected_restored_content "$staging_dir" "$backup_directory"
|
||||
RESTORED_DUMP="$backup_directory/database.dump"
|
||||
manifest="$backup_directory/manifest/backup-manifest.json"
|
||||
RESTORED_TDLIB_WORKER="$staging_dir/data/tdlib-worker"
|
||||
RESTORED_TDLIB_BOT="$staging_dir/data/tdlib-bot"
|
||||
if [[ ! -s "$RESTORED_DUMP" || ! -s "$manifest" || ! -d "$RESTORED_TDLIB_WORKER" || ! -d "$RESTORED_TDLIB_BOT" ]]; then
|
||||
printf 'Restored tree %s is incomplete; require staging/backup-*/database.dump, its manifest, data/tdlib-worker, and data/tdlib-bot.\n' "$staging_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
verify_custom_dump "$RESTORED_DUMP"
|
||||
}
|
||||
|
||||
verify_snapshot() {
|
||||
local snapshot_id="$1"
|
||||
backup_restic snapshots "$snapshot_id"
|
||||
backup_restic check
|
||||
}
|
||||
|
||||
restore_to_staging() {
|
||||
local snapshot_id="$1"
|
||||
local requested_staging_dir="$2"
|
||||
local staging_dir
|
||||
local container_staging_dir
|
||||
validate_environment
|
||||
staging_dir="$(validate_staging_directory "$requested_staging_dir")"
|
||||
container_staging_dir="$(container_staging_directory "$staging_dir")"
|
||||
prepare_fresh_staging_directory "$staging_dir"
|
||||
verify_snapshot "$snapshot_id"
|
||||
restore_snapshot_subset "$snapshot_id" "$container_staging_dir"
|
||||
validate_restored_tree "$staging_dir"
|
||||
printf 'Staging restore verified: %s\n' "$staging_dir"
|
||||
}
|
||||
|
||||
compose_project_name() {
|
||||
local project_name
|
||||
project_name="$(docker compose config --format json | awk -F'"' '/^[[:space:]]*"name"[[:space:]]*:/ { print $4; exit }')"
|
||||
if [[ -z "$project_name" ]]; then
|
||||
printf '%s\n' 'Unable to determine the Docker Compose project name.' >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$project_name"
|
||||
}
|
||||
|
||||
compose_volume_name() {
|
||||
local project_name="$1"
|
||||
local logical_name="$2"
|
||||
local -a matches=()
|
||||
mapfile -t matches < <(docker volume ls \
|
||||
--filter "label=com.docker.compose.project=$project_name" \
|
||||
--filter "label=com.docker.compose.volume=$logical_name" \
|
||||
--format '{{.Name}}')
|
||||
if ((${#matches[@]} != 1)); then
|
||||
printf 'Expected exactly one %s volume for Compose project %s; found %s.\n' "$logical_name" "$project_name" "${#matches[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "${matches[0]}"
|
||||
}
|
||||
|
||||
capture_running_live_services() {
|
||||
local service
|
||||
local container_ids
|
||||
|
||||
RUNNING_LIVE_SERVICES=()
|
||||
for service in "${LIVE_SERVICES[@]}"; do
|
||||
if ! container_ids="$(docker compose --profile full ps --status running -q "$service")"; then
|
||||
printf 'Unable to determine whether Compose service %s is running.\n' "$service" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "$container_ids" ]]; then
|
||||
RUNNING_LIVE_SERVICES+=("$service")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
live_service_was_running() {
|
||||
local requested_service="$1"
|
||||
local service
|
||||
|
||||
for service in "${RUNNING_LIVE_SERVICES[@]}"; do
|
||||
[[ "$service" == "$requested_service" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
replace_volume() {
|
||||
local source_dir="$1"
|
||||
local volume_name="$2"
|
||||
docker compose --profile backup run --rm --no-deps \
|
||||
--entrypoint bash \
|
||||
-v "$source_dir:/restore-source:ro" \
|
||||
-v "$volume_name:/restore-target" \
|
||||
backup -ceu '
|
||||
find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
cp -a /restore-source/. /restore-target/
|
||||
'
|
||||
}
|
||||
|
||||
create_safety_dump() {
|
||||
local database_user="${POSTGRES_USER:-dragons}"
|
||||
local database_name="${POSTGRES_DB:-dragonsstash}"
|
||||
SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump"
|
||||
docker compose exec -T db pg_dump --format=custom --no-owner \
|
||||
--username "$database_user" --dbname "$database_name" > "$SAFETY_DUMP"
|
||||
if [[ ! -s "$SAFETY_DUMP" ]]; then
|
||||
printf 'Safety database dump %s is missing or empty.\n' "$SAFETY_DUMP" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restore_database_dump() {
|
||||
local dump_path="$1"
|
||||
local database_user="${POSTGRES_USER:-dragons}"
|
||||
local database_name="${POSTGRES_DB:-dragonsstash}"
|
||||
docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$database_name"
|
||||
docker compose exec -T db createdb --username "$database_user" "$database_name"
|
||||
docker compose exec -T db pg_restore --no-owner --exit-on-error \
|
||||
--username "$database_user" --dbname "$database_name" < "$dump_path"
|
||||
}
|
||||
|
||||
restore_database() {
|
||||
restore_database_dump "$RESTORED_DUMP"
|
||||
}
|
||||
|
||||
archive_live_volume() {
|
||||
local volume_name="$1"
|
||||
local logical_name="$2"
|
||||
local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar"
|
||||
|
||||
docker compose --profile backup run --rm --no-deps \
|
||||
--entrypoint bash \
|
||||
-v "$volume_name:/safety-source:ro" \
|
||||
-v "$LIVE_STAGING_DIR:/safety-output" \
|
||||
backup -ceu 'tar -C /safety-source -cf "$1" .' bash \
|
||||
"/safety-output/pre-restore-$logical_name.tar"
|
||||
if [[ ! -s "$archive_path" ]]; then
|
||||
printf 'Safety archive %s is missing or empty.\n' "$archive_path" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restore_live_volume_archive() {
|
||||
local volume_name="$1"
|
||||
local logical_name="$2"
|
||||
local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar"
|
||||
|
||||
docker compose --profile backup run --rm --no-deps \
|
||||
--entrypoint bash \
|
||||
-v "$archive_path:/safety-archive:ro" \
|
||||
-v "$volume_name:/restore-target" \
|
||||
backup -ceu '
|
||||
find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
tar -C /restore-target -xf /safety-archive
|
||||
'
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local health_url="${RESTORE_HEALTH_URL:-http://localhost:${APP_PORT:-3000}/api/health}"
|
||||
local attempt
|
||||
for attempt in {1..30}; do
|
||||
if curl --fail --silent --show-error "$health_url" >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
printf 'Application health endpoint did not become ready: %s\n' "$health_url" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
live_restore_failure() {
|
||||
local exit_code=$?
|
||||
local rollback_ok=1
|
||||
trap - EXIT
|
||||
if ((LIVE_RESTORE_ACTIVE)); then
|
||||
if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then
|
||||
docker compose --profile full stop "${RUNNING_LIVE_SERVICES[@]}" || true
|
||||
fi
|
||||
if ((LIVE_REPLACEMENT_STARTED)); then
|
||||
restore_live_volume_archive "$LIVE_WORKER_VOLUME" tdlib_state || rollback_ok=0
|
||||
restore_live_volume_archive "$LIVE_BOT_VOLUME" tdlib_bot_state || rollback_ok=0
|
||||
restore_database_dump "$SAFETY_DUMP" || rollback_ok=0
|
||||
if ((rollback_ok)); then
|
||||
printf 'Rollback restored the pre-restore database and both TDLib Docker volumes. Services remain stopped.\n' >&2
|
||||
else
|
||||
printf 'Rollback failed; services remain stopped. Restore the safety archives and database dump manually.\n' >&2
|
||||
fi
|
||||
fi
|
||||
printf 'Live restore failed (exit %s). Services remain stopped. Staging directory: %s\n' "$exit_code" "$LIVE_STAGING_DIR" >&2
|
||||
printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP" >&2
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
restore_live() {
|
||||
local snapshot_id="$1"
|
||||
local project_name
|
||||
local worker_volume
|
||||
local bot_volume
|
||||
local timestamp
|
||||
local container_staging_dir
|
||||
validate_environment
|
||||
validate_backup_mount
|
||||
docker compose --profile backup config --quiet
|
||||
project_name="$(compose_project_name)"
|
||||
verify_snapshot "$snapshot_id"
|
||||
printf 'Live restore target confirmed: Compose project %s, configured Restic repository, snapshot %s.\n' "$project_name" "$snapshot_id"
|
||||
timestamp="$(date -u +'%Y-%m-%dT%H-%M-%SZ')"
|
||||
LIVE_STAGING_DIR="$(validate_staging_directory "$BACKUP_STAGING_PATH/live-restore-$timestamp-$$")"
|
||||
container_staging_dir="$(container_staging_directory "$LIVE_STAGING_DIR")"
|
||||
prepare_fresh_staging_directory "$LIVE_STAGING_DIR"
|
||||
SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump"
|
||||
trap live_restore_failure EXIT
|
||||
LIVE_RESTORE_ACTIVE=1
|
||||
capture_running_live_services
|
||||
if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then
|
||||
docker compose --profile full stop "${RUNNING_LIVE_SERVICES[@]}"
|
||||
fi
|
||||
create_safety_dump
|
||||
restore_snapshot_subset "$snapshot_id" "$container_staging_dir"
|
||||
validate_restored_tree "$LIVE_STAGING_DIR"
|
||||
worker_volume="$(compose_volume_name "$project_name" tdlib_state)"
|
||||
bot_volume="$(compose_volume_name "$project_name" tdlib_bot_state)"
|
||||
LIVE_WORKER_VOLUME="$worker_volume"
|
||||
LIVE_BOT_VOLUME="$bot_volume"
|
||||
archive_live_volume "$worker_volume" tdlib_state
|
||||
archive_live_volume "$bot_volume" tdlib_bot_state
|
||||
LIVE_REPLACEMENT_STARTED=1
|
||||
replace_volume "$RESTORED_TDLIB_WORKER" "$worker_volume"
|
||||
replace_volume "$RESTORED_TDLIB_BOT" "$bot_volume"
|
||||
restore_database
|
||||
if ((${#RUNNING_LIVE_SERVICES[@]} > 0)); then
|
||||
docker compose --profile full up -d "${RUNNING_LIVE_SERVICES[@]}"
|
||||
fi
|
||||
if live_service_was_running app; then
|
||||
wait_for_health
|
||||
fi
|
||||
LIVE_RESTORE_ACTIVE=0
|
||||
trap - EXIT
|
||||
printf 'Live restore completed. Staging directory retained at: %s\n' "$LIVE_STAGING_DIR"
|
||||
printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP"
|
||||
}
|
||||
|
||||
main() {
|
||||
local command="${1:-}"
|
||||
case "$command" in
|
||||
list)
|
||||
[[ $# -eq 1 ]] || { usage >&2; return 2; }
|
||||
validate_environment
|
||||
backup_restic snapshots
|
||||
;;
|
||||
verify)
|
||||
[[ $# -eq 2 ]] || { usage >&2; return 2; }
|
||||
validate_environment
|
||||
verify_snapshot "$2"
|
||||
;;
|
||||
restore-to-staging)
|
||||
[[ $# -eq 3 ]] || { usage >&2; return 2; }
|
||||
restore_to_staging "$2" "$3"
|
||||
;;
|
||||
restore-live)
|
||||
if [[ $# -ne 3 || "$3" != "$CONFIRM_REPLACE_LIVE_DATA" ]]; then
|
||||
printf 'restore-live requires the exact confirmation flag: %s\n' "$CONFIRM_REPLACE_LIVE_DATA" >&2
|
||||
return 2
|
||||
fi
|
||||
restore_live "$2"
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
PROJECT_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)"
|
||||
BACKUP_SCRIPT="$PROJECT_ROOT/scripts/backup/run-backup.sh"
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
|
||||
cleanup() {
|
||||
rm -rf -- "$TMP_ROOT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
printf 'ASSERTION FAILED: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
setup_fake_environment() {
|
||||
local fake_bin="$TMP_ROOT/bin"
|
||||
|
||||
mkdir -p -- "$fake_bin" "$TMP_ROOT/mount" "$TMP_ROOT/staging"
|
||||
printf 'not-secret\n' > "$TMP_ROOT/restic-password"
|
||||
|
||||
cat > "$fake_bin/docker" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
printf '%s\t' "$@" >> "$DRAGONS_STASH_DOCKER_CALLS"
|
||||
printf '\n' >> "$DRAGONS_STASH_DOCKER_CALLS"
|
||||
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "ps" && "${3:-}" == "--status" && "${4:-}" == "running" && "${5:-}" == "-q" ]]; then
|
||||
case "${6:-}" in
|
||||
app) printf 'app-container\n' ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "stop" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "start" ]]; then
|
||||
exit "${BACKUP_ASSERT_START_EXIT:-0}"
|
||||
fi
|
||||
|
||||
if [[ " $* " == *" backup backup "* ]]; then
|
||||
exit "${BACKUP_ASSERT_BACKUP_EXIT:-0}"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/docker"
|
||||
|
||||
cat > "$fake_bin/mountpoint" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/mountpoint"
|
||||
|
||||
cat > "$fake_bin/flock" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$fake_bin/flock"
|
||||
|
||||
export PATH="$fake_bin:$PATH"
|
||||
export BACKUP_MOUNT_PATH="$TMP_ROOT/mount"
|
||||
export BACKUP_STAGING_PATH="$TMP_ROOT/staging"
|
||||
export BACKUP_RESTIC_PASSWORD_FILE="$TMP_ROOT/restic-password"
|
||||
export BACKUP_RETENTION_DAYS="30"
|
||||
export BACKUP_REPOSITORY="/backup/restic"
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
local name="$1"
|
||||
shift
|
||||
|
||||
DRAGONS_STASH_DOCKER_CALLS="$TMP_ROOT/docker-calls-$name.log" \
|
||||
"$@" "$BACKUP_SCRIPT"
|
||||
}
|
||||
|
||||
assert_restart_failure_makes_successful_backup_fail() {
|
||||
local output="$TMP_ROOT/restart-fails-output.log"
|
||||
|
||||
setup_fake_environment
|
||||
if BACKUP_ASSERT_START_EXIT=9 run_backup restart-fails bash >"$output" 2>&1; then
|
||||
fail "backup reported success when restart failed"
|
||||
fi
|
||||
grep -q 'Failed to restart one or more previously running services' "$output" \
|
||||
|| fail "restart failure did not produce an explicit error"
|
||||
}
|
||||
|
||||
assert_backup_failure_exit_is_preserved_when_restart_also_fails() {
|
||||
local output="$TMP_ROOT/backup-and-restart-fail-output.log"
|
||||
local exit_code
|
||||
|
||||
setup_fake_environment
|
||||
set +e
|
||||
BACKUP_ASSERT_BACKUP_EXIT=42 BACKUP_ASSERT_START_EXIT=9 run_backup backup-and-restart-fail bash >"$output" 2>&1
|
||||
exit_code=$?
|
||||
set -e
|
||||
|
||||
[[ "$exit_code" -eq 42 ]] || fail "expected backup failure exit 42 to be preserved; got $exit_code"
|
||||
grep -q 'Failed to restart one or more previously running services' "$output" \
|
||||
|| fail "restart failure did not produce an explicit error"
|
||||
}
|
||||
|
||||
assert_restart_failure_makes_successful_backup_fail
|
||||
assert_backup_failure_exit_is_preserved_when_restart_also_fails
|
||||
|
||||
printf 'run-backup assertions passed\n'
|
||||
@@ -1,112 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly LOCK_FILE="/run/lock/dragons-stash-backup.lock"
|
||||
readonly BACKUP_CONTAINER_ROOT="/backup"
|
||||
readonly -a MANAGED_SERVICES=(app worker bot)
|
||||
|
||||
declare -a RUNNING_SERVICES=()
|
||||
|
||||
require_value() {
|
||||
local name="$1"
|
||||
|
||||
if [[ -z "${!name:-}" ]]; then
|
||||
printf 'Required environment variable %s is not set.\n' "$name" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_environment() {
|
||||
require_value BACKUP_MOUNT_PATH
|
||||
require_value BACKUP_STAGING_PATH
|
||||
require_value BACKUP_RESTIC_PASSWORD_FILE
|
||||
require_value BACKUP_RETENTION_DAYS
|
||||
}
|
||||
|
||||
validate_backup_repository() {
|
||||
local repository="${BACKUP_REPOSITORY:-/backup/restic}"
|
||||
local canonical_repository
|
||||
|
||||
canonical_repository="$(realpath -ms -- "$repository")"
|
||||
if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then
|
||||
printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_backup_mount() {
|
||||
local probe_file
|
||||
|
||||
if ! mountpoint --q "$BACKUP_MOUNT_PATH"; then
|
||||
printf 'Backup mount %s is not an active mountpoint.\n' "$BACKUP_MOUNT_PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! probe_file="$(mktemp "$BACKUP_MOUNT_PATH/.dragons-stash-backup-write-probe.XXXXXX")"; then
|
||||
printf 'Backup mount %s is not writable.\n' "$BACKUP_MOUNT_PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! rm -f -- "$probe_file"; then
|
||||
printf 'Unable to remove writable probe %s.\n' "$probe_file" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
capture_running_services() {
|
||||
local service
|
||||
local container_ids
|
||||
|
||||
for service in "${MANAGED_SERVICES[@]}"; do
|
||||
if ! container_ids="$(docker compose ps --status running -q "$service")"; then
|
||||
printf 'Unable to determine whether Compose service %s is running.\n' "$service" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "$container_ids" ]]; then
|
||||
RUNNING_SERVICES+=("$service")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
restart_running_services() {
|
||||
local exit_code=$?
|
||||
local restart_exit=0
|
||||
|
||||
trap - EXIT
|
||||
|
||||
if ((${#RUNNING_SERVICES[@]} > 0)); then
|
||||
docker compose start "${RUNNING_SERVICES[@]}" || restart_exit=$?
|
||||
if ((restart_exit != 0)); then
|
||||
printf 'Failed to restart one or more previously running services: %s\n' "${RUNNING_SERVICES[*]}" >&2
|
||||
if ((exit_code == 0)); then
|
||||
exit_code="$restart_exit"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
main() {
|
||||
validate_environment
|
||||
validate_backup_repository
|
||||
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
printf "%s\n" "Another Dragon's Stash backup is already running." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
validate_backup_mount
|
||||
capture_running_services
|
||||
trap restart_running_services EXIT
|
||||
|
||||
if ((${#RUNNING_SERVICES[@]} > 0)); then
|
||||
docker compose stop "${RUNNING_SERVICES[@]}"
|
||||
fi
|
||||
|
||||
docker compose --profile backup run --rm backup backup
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,7 +1,23 @@
|
||||
import { redirect } from "next/navigation";
|
||||
import { auth } from "@/lib/auth";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { Sidebar } from "@/components/layout/sidebar";
|
||||
import { Header } from "@/components/layout/header";
|
||||
|
||||
export default function AppLayout({ children }: { children: React.ReactNode }) {
|
||||
export default async function AppLayout({ children }: { children: React.ReactNode }) {
|
||||
// Guard against a stale JWT session whose user no longer exists in the
|
||||
// database (e.g. after a DB reset). The signed cookie still passes edge
|
||||
// middleware, but every downstream query keyed on session.user.id would fail.
|
||||
// Send such sessions to /logout, which clears the cookie and returns to login.
|
||||
const session = await auth();
|
||||
if (session?.user?.id) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: session.user.id },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!user) redirect("/logout");
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-screen overflow-hidden">
|
||||
<div className="hidden lg:block">
|
||||
|
||||
@@ -500,14 +500,9 @@ export function StlTable({
|
||||
</Badge>
|
||||
)}
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="ungrouped" className="gap-1.5">
|
||||
Ungrouped
|
||||
{ungroupedTotalCount > 0 && (
|
||||
<Badge variant="secondary" className="h-5 px-1.5 text-[10px]">
|
||||
{ungroupedTotalCount}
|
||||
</Badge>
|
||||
)}
|
||||
</TabsTrigger>
|
||||
{/* "Ungrouped" tab hidden: the STL list is now flat and grouping is
|
||||
no longer surfaced here. The tab content below is kept (unreachable)
|
||||
to avoid churn; remove it and its data fetch if grouping is dropped. */}
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="packages" className="space-y-4">
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { signOut } from "@/lib/auth";
|
||||
|
||||
// Server-side sign-out that clears the JWT session cookie and redirects to the
|
||||
// login page. Used to recover from a stale session whose user no longer exists
|
||||
// in the database (e.g. after a DB reset), which a client-only signOut can't
|
||||
// reach because the app crashes before rendering the user menu.
|
||||
export async function GET() {
|
||||
await signOut({ redirectTo: "/login" });
|
||||
}
|
||||
@@ -1,20 +1,37 @@
|
||||
import { Prisma } from "@prisma/client";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
const DEFAULT_SETTINGS = {
|
||||
lowStockThreshold: 20,
|
||||
currency: "EUR",
|
||||
theme: "dark",
|
||||
units: "metric",
|
||||
} as const;
|
||||
|
||||
export async function getUserSettings(userId: string) {
|
||||
let settings = await prisma.userSettings.findUnique({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!settings) {
|
||||
settings = await prisma.userSettings.create({
|
||||
data: {
|
||||
userId,
|
||||
lowStockThreshold: 20,
|
||||
currency: "EUR",
|
||||
theme: "dark",
|
||||
units: "metric",
|
||||
},
|
||||
});
|
||||
try {
|
||||
settings = await prisma.userSettings.create({
|
||||
data: { userId, ...DEFAULT_SETTINGS },
|
||||
});
|
||||
} catch (err) {
|
||||
// The session's user may no longer exist (e.g. a stale JWT cookie after a
|
||||
// database reset). Creating settings then hits a foreign-key violation
|
||||
// (P2003). Don't crash the Server Component render — return unsaved
|
||||
// defaults. The (app) layout guard redirects such stale sessions to
|
||||
// sign-out, so this fallback is only ever momentarily visible.
|
||||
if (
|
||||
err instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
err.code === "P2003"
|
||||
) {
|
||||
return { id: "", userId, ...DEFAULT_SETTINGS };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
return settings;
|
||||
|
||||
+12
-12
@@ -135,31 +135,31 @@ export async function listDisplayItems(options: {
|
||||
const sortCol = sortBy === "fileName" ? `"fileName"` : sortBy === "fileSize" ? `"fileSize"` : `"indexedAt"`;
|
||||
const sortDir = order === "asc" ? "ASC" : "DESC";
|
||||
|
||||
// Step 1: Count display items
|
||||
// NOTE: The STL list is intentionally FLAT — every package is its own display
|
||||
// row regardless of packageGroupId. Grouping is no longer surfaced in this
|
||||
// view (the creator column/filter organizes the list instead). PackageGroup
|
||||
// rows and the manual grouping actions still exist in the DB/UI; they just
|
||||
// don't drive this list's layout anymore.
|
||||
|
||||
// Step 1: Count display items (one per package)
|
||||
const countResult = await prisma.$queryRawUnsafe<[{ count: bigint }]>(
|
||||
`SELECT COUNT(*) AS count FROM (
|
||||
SELECT DISTINCT COALESCE(p."packageGroupId", p."id") AS display_id
|
||||
FROM packages p
|
||||
${whereClause}
|
||||
) AS display_items`,
|
||||
`SELECT COUNT(*) AS count FROM packages p ${whereClause}`,
|
||||
...params
|
||||
);
|
||||
const total = Number(countResult[0].count);
|
||||
|
||||
// Step 2: Get display item IDs for this page
|
||||
// Step 2: Get package IDs for this page
|
||||
const limitParam = paramIdx++;
|
||||
const offsetParam = paramIdx++;
|
||||
const displayRows = await prisma.$queryRawUnsafe<
|
||||
{ display_id: string; display_type: string }[]
|
||||
>(
|
||||
`SELECT
|
||||
COALESCE(p."packageGroupId", p."id") AS display_id,
|
||||
CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END AS display_type,
|
||||
MAX(p.${sortCol}) AS sort_value
|
||||
p."id" AS display_id,
|
||||
'package' AS display_type,
|
||||
p.${sortCol} AS sort_value
|
||||
FROM packages p
|
||||
${whereClause}
|
||||
GROUP BY COALESCE(p."packageGroupId", p."id"),
|
||||
CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END
|
||||
ORDER BY sort_value ${sortDir}
|
||||
LIMIT $${limitParam} OFFSET $${offsetParam}`,
|
||||
...params, limit, (page - 1) * limit
|
||||
|
||||
Generated
+1034
-1
File diff suppressed because it is too large
Load Diff
+5
-2
@@ -6,7 +6,9 @@
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "tsx watch src/index.ts"
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@prisma/adapter-pg": "^7.4.0",
|
||||
@@ -23,6 +25,7 @@
|
||||
"@types/yauzl": "^2.10.3",
|
||||
"prisma": "^7.4.0",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5"
|
||||
"typescript": "^5",
|
||||
"vitest": "^3.2.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { parseZipCentralDirectoryFromTail } from "./central-directory.js";
|
||||
import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
|
||||
|
||||
// Build a minimal STORE (no compression) ZIP in-memory with the given files.
|
||||
function buildStoreZip(files: { name: string; data: Buffer }[]): Buffer {
|
||||
const chunks: Buffer[] = [];
|
||||
const central: Buffer[] = [];
|
||||
let offset = 0;
|
||||
for (const f of files) {
|
||||
const crc = crc32(f.data) >>> 0;
|
||||
const nameBuf = Buffer.from(f.name, "utf8");
|
||||
const local = Buffer.alloc(30);
|
||||
local.writeUInt32LE(0x04034b50, 0);
|
||||
local.writeUInt16LE(20, 4); // version needed
|
||||
local.writeUInt16LE(0, 6); // flags
|
||||
local.writeUInt16LE(0, 8); // method = store
|
||||
local.writeUInt32LE(crc, 14);
|
||||
local.writeUInt32LE(f.data.length, 18); // compressed
|
||||
local.writeUInt32LE(f.data.length, 22); // uncompressed
|
||||
local.writeUInt16LE(nameBuf.length, 26);
|
||||
local.writeUInt16LE(0, 28); // extra len
|
||||
const localHeader = Buffer.concat([local, nameBuf, f.data]);
|
||||
chunks.push(localHeader);
|
||||
|
||||
const cd = Buffer.alloc(46);
|
||||
cd.writeUInt32LE(0x02014b50, 0);
|
||||
cd.writeUInt16LE(20, 4); cd.writeUInt16LE(20, 6);
|
||||
cd.writeUInt16LE(0, 8); cd.writeUInt16LE(0, 10);
|
||||
cd.writeUInt32LE(crc, 16);
|
||||
cd.writeUInt32LE(f.data.length, 20);
|
||||
cd.writeUInt32LE(f.data.length, 24);
|
||||
cd.writeUInt16LE(nameBuf.length, 28);
|
||||
cd.writeUInt32LE(offset, 42); // local header offset
|
||||
central.push(Buffer.concat([cd, nameBuf]));
|
||||
offset += localHeader.length;
|
||||
}
|
||||
const cdBuf = Buffer.concat(central);
|
||||
const cdOffset = offset;
|
||||
const eocd = Buffer.alloc(22);
|
||||
eocd.writeUInt32LE(0x06054b50, 0);
|
||||
eocd.writeUInt16LE(files.length, 8);
|
||||
eocd.writeUInt16LE(files.length, 10);
|
||||
eocd.writeUInt32LE(cdBuf.length, 12);
|
||||
eocd.writeUInt32LE(cdOffset, 16);
|
||||
return Buffer.concat([...chunks, cdBuf, eocd]);
|
||||
}
|
||||
|
||||
describe("parseZipCentralDirectoryFromTail", () => {
|
||||
it("lists entries with correct names, sizes, and crc32", () => {
|
||||
const zip = buildStoreZip([
|
||||
{ name: "models/dragon.stl", data: Buffer.from("DRAGON") },
|
||||
{ name: "readme.txt", data: Buffer.from("hello world") },
|
||||
]);
|
||||
const entries = parseZipCentralDirectoryFromTail(zip, 0);
|
||||
expect(entries.map((e) => e.fileName).sort()).toEqual(["dragon.stl", "readme.txt"]);
|
||||
const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
|
||||
expect(dragon.path).toBe("models/dragon.stl");
|
||||
expect(dragon.uncompressedSize).toBe(6n);
|
||||
expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
|
||||
});
|
||||
|
||||
it("throws when the central directory begins before the tail window", () => {
|
||||
const zip = buildStoreZip([{ name: "a.txt", data: Buffer.alloc(100) }]);
|
||||
// Provide only the last 30 bytes but claim they start at offset (len-30):
|
||||
const tail = zip.subarray(zip.length - 30);
|
||||
expect(() => parseZipCentralDirectoryFromTail(tail, zip.length - 30)).toThrow(RangeError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import path from "path";
|
||||
import type { FileEntry } from "./zip-reader.js";
|
||||
|
||||
export const MIN_ZIP_TAIL_BYTES = 65_557;
|
||||
|
||||
const EOCD_SIG = 0x06054b50;
|
||||
const CD_SIG = 0x02014b50;
|
||||
|
||||
function extOf(name: string): string | null {
|
||||
const e = path.extname(name).replace(/^\./, "").toLowerCase();
|
||||
return e === "" ? null : e;
|
||||
}
|
||||
|
||||
/** Parse a ZIP central directory from the tail of an archive. */
|
||||
export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
|
||||
// 1. Find EOCD by scanning backward for its signature.
|
||||
let eocd = -1;
|
||||
for (let i = tail.length - 22; i >= 0; i--) {
|
||||
if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
|
||||
}
|
||||
if (eocd < 0) throw new RangeError("EOCD not found in tail");
|
||||
|
||||
let cdSize = tail.readUInt32LE(eocd + 12);
|
||||
let cdOffset = tail.readUInt32LE(eocd + 16);
|
||||
|
||||
// ZIP64: sizes/offsets of 0xFFFFFFFF mean "see ZIP64 EOCD".
|
||||
if (cdOffset === 0xffffffff || cdSize === 0xffffffff) {
|
||||
const locSig = 0x07064b50;
|
||||
let loc = -1;
|
||||
for (let i = eocd - 20; i >= 0; i--) {
|
||||
if (tail.readUInt32LE(i) === locSig) { loc = i; break; }
|
||||
}
|
||||
if (loc < 0) throw new RangeError("ZIP64 EOCD locator not in tail");
|
||||
const z64Abs = Number(tail.readBigUInt64LE(loc + 8)); // absolute offset of ZIP64 EOCD
|
||||
const z64 = z64Abs - tailStart;
|
||||
if (z64 < 0) throw new RangeError("ZIP64 EOCD before tail window");
|
||||
cdSize = Number(tail.readBigUInt64LE(z64 + 40));
|
||||
cdOffset = Number(tail.readBigUInt64LE(z64 + 48));
|
||||
}
|
||||
|
||||
// 2. Map the absolute central-directory offset into the tail buffer.
|
||||
const cdLocal = cdOffset - tailStart;
|
||||
if (cdLocal < 0 || cdLocal + cdSize > tail.length) {
|
||||
throw new RangeError("Central directory begins before tail window");
|
||||
}
|
||||
|
||||
// 3. Walk central-directory headers.
|
||||
const entries: FileEntry[] = [];
|
||||
let p = cdLocal;
|
||||
const end = cdLocal + cdSize;
|
||||
while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
|
||||
let crc = tail.readUInt32LE(p + 16) >>> 0;
|
||||
let comp = BigInt(tail.readUInt32LE(p + 20));
|
||||
let uncomp = BigInt(tail.readUInt32LE(p + 24));
|
||||
const nameLen = tail.readUInt16LE(p + 28);
|
||||
const extraLen = tail.readUInt16LE(p + 30);
|
||||
const commentLen = tail.readUInt16LE(p + 32);
|
||||
const name = tail.toString("utf8", p + 46, p + 46 + nameLen);
|
||||
|
||||
// ZIP64 extra field overrides 0xFFFFFFFF sizes.
|
||||
if (comp === 0xffffffffn || uncomp === 0xffffffffn) {
|
||||
let ep = p + 46 + nameLen;
|
||||
const extraEnd = ep + extraLen;
|
||||
while (ep + 4 <= extraEnd) {
|
||||
const id = tail.readUInt16LE(ep);
|
||||
const sz = tail.readUInt16LE(ep + 2);
|
||||
if (id === 0x0001) {
|
||||
let fp = ep + 4;
|
||||
if (uncomp === 0xffffffffn) { uncomp = tail.readBigUInt64LE(fp); fp += 8; }
|
||||
if (comp === 0xffffffffn) { comp = tail.readBigUInt64LE(fp); fp += 8; }
|
||||
}
|
||||
ep += 4 + sz;
|
||||
}
|
||||
}
|
||||
|
||||
const isDir = name.endsWith("/");
|
||||
if (!isDir) {
|
||||
entries.push({
|
||||
path: name,
|
||||
fileName: path.basename(name),
|
||||
extension: extOf(name),
|
||||
compressedSize: comp,
|
||||
uncompressedSize: uncomp,
|
||||
crc32: crc !== 0 ? crc.toString(16).padStart(8, "0") : null,
|
||||
});
|
||||
}
|
||||
p += 46 + nameLen + extraLen + commentLen;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { crcFingerprint, fingerprintsMatch } from "./fingerprint.js";
|
||||
import type { FileEntry } from "./zip-reader.js";
|
||||
|
||||
const fe = (crc: string | null): FileEntry => ({
|
||||
path: "a", fileName: "a", extension: null,
|
||||
compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
|
||||
});
|
||||
|
||||
describe("crcFingerprint", () => {
|
||||
it("sorts crcs and marks complete", () => {
|
||||
expect(crcFingerprint([fe("00ff"), fe("00aa")])).toEqual({ crcs: ["00aa", "00ff"], complete: true });
|
||||
});
|
||||
it("is incomplete when any crc is null", () => {
|
||||
expect(crcFingerprint([fe("00aa"), fe(null)]).complete).toBe(false);
|
||||
});
|
||||
it("is incomplete when empty", () => {
|
||||
expect(crcFingerprint([]).complete).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fingerprintsMatch", () => {
|
||||
it("matches identical crc multisets regardless of order", () => {
|
||||
expect(fingerprintsMatch([fe("01"), fe("02")], [fe("02"), fe("01")])).toBe(true);
|
||||
});
|
||||
it("rejects different counts", () => {
|
||||
expect(fingerprintsMatch([fe("01")], [fe("01"), fe("02")])).toBe(false);
|
||||
});
|
||||
it("rejects disjoint sets", () => {
|
||||
expect(fingerprintsMatch([fe("01")], [fe("09")])).toBe(false);
|
||||
});
|
||||
it("rejects when either side is incomplete", () => {
|
||||
expect(fingerprintsMatch([fe("01"), fe(null)], [fe("01"), fe("02")])).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { FileEntry } from "./zip-reader.js";
|
||||
|
||||
export function crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean } {
|
||||
if (entries.length === 0) return { crcs: [], complete: false };
|
||||
const crcs: string[] = [];
|
||||
let complete = true;
|
||||
for (const e of entries) {
|
||||
if (e.crc32 == null) { complete = false; continue; }
|
||||
crcs.push(e.crc32.toLowerCase());
|
||||
}
|
||||
crcs.sort();
|
||||
return { crcs, complete };
|
||||
}
|
||||
|
||||
export function fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean {
|
||||
const fa = crcFingerprint(a);
|
||||
const fb = crcFingerprint(b);
|
||||
if (!fa.complete || !fb.complete) return false;
|
||||
if (fa.crcs.length !== fb.crcs.length) return false;
|
||||
return fa.crcs.every((c, i) => c === fb.crcs[i]);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
|
||||
// logLevel is required here too (not just maxZipSizeMB/tempDir) because this
|
||||
// mock replaces the config module for the whole test-file graph, including
|
||||
// util/logger.ts's module-level `pino({ level: config.logLevel })` call —
|
||||
// pino throws at import time if level is undefined.
|
||||
vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } }));
|
||||
const created: unknown[] = [];
|
||||
vi.mock("../../db/client.js", () => ({
|
||||
db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } },
|
||||
}));
|
||||
|
||||
import { fullDownloadListing } from "./fallback.js";
|
||||
|
||||
describe("fullDownloadListing", () => {
|
||||
it("refuses to download over the size cap and records a notification", async () => {
|
||||
const res = await fullDownloadListing({
|
||||
client: {} as never,
|
||||
parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }],
|
||||
archiveType: "RAR",
|
||||
totalSize: 2n * 1024n * 1024n * 1024n,
|
||||
fileName: "big.rar",
|
||||
});
|
||||
expect(res).toBeNull();
|
||||
expect(created).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { mkdtemp, rm } from "fs/promises";
|
||||
import path from "path";
|
||||
import type { Client } from "tdl";
|
||||
import { config } from "../../util/config.js";
|
||||
import { db } from "../../db/client.js";
|
||||
import { childLogger } from "../../util/logger.js";
|
||||
import { downloadFile } from "../../tdlib/download.js";
|
||||
import { read7zContents } from "../sevenz-reader.js";
|
||||
import { readRarContents } from "../rar-reader.js";
|
||||
import type { FileEntry } from "../zip-reader.js";
|
||||
import type { RangedPart } from "./sevenz-ranged.js";
|
||||
|
||||
const log = childLogger("ranged-fallback");
|
||||
|
||||
export async function fullDownloadListing(args: {
|
||||
client: Client;
|
||||
parts: RangedPart[];
|
||||
archiveType: string;
|
||||
totalSize: bigint;
|
||||
fileName: string;
|
||||
}): Promise<FileEntry[] | null> {
|
||||
const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n;
|
||||
if (args.totalSize > capBytes) {
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "INTEGRITY_AUDIT",
|
||||
severity: "WARNING",
|
||||
title: `Listing skipped (over size cap): ${args.fileName}`,
|
||||
message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`,
|
||||
context: { fileName: args.fileName, archiveType: args.archiveType },
|
||||
},
|
||||
});
|
||||
log.warn({ fileName: args.fileName }, "fallback skipped — over size cap");
|
||||
return null;
|
||||
}
|
||||
const dir = await mkdtemp(path.join(config.tempDir, "fallback-"));
|
||||
const paths: string[] = [];
|
||||
try {
|
||||
for (const p of args.parts) {
|
||||
const dest = path.join(dir, p.fileName);
|
||||
await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {});
|
||||
paths.push(dest);
|
||||
}
|
||||
const entries =
|
||||
args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0])
|
||||
: args.archiveType === "RAR" ? await readRarContents(paths[0])
|
||||
: [];
|
||||
return entries.length > 0 ? entries : null;
|
||||
} catch (err) {
|
||||
log.warn({ err, fileName: args.fileName }, "full-download fallback failed");
|
||||
return null;
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { Client } from "tdl";
|
||||
import { downloadFileRange } from "../../tdlib/range-download.js";
|
||||
|
||||
export type RangeReader = (
|
||||
fileId: string,
|
||||
offset: number,
|
||||
length: number,
|
||||
partSize: bigint,
|
||||
) => Promise<Buffer>;
|
||||
|
||||
export function tdlibRangeReader(client: Client): RangeReader {
|
||||
return (fileId, offset, length, partSize) =>
|
||||
downloadFileRange(client, fileId, offset, length, partSize);
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readVint, detectRarSignature, parseRar5BlockExtent, parseRar4BlockExtent, walkRarVolume, readRarListingRanged } from "./rar-ranged.js";
|
||||
import type { RangeReader } from "./range-reader.js";
|
||||
|
||||
describe("readVint", () => {
|
||||
it("reads single-byte and multi-byte values (base-128 LE)", () => {
|
||||
expect(readVint(Buffer.from([0x08]), 0)).toEqual({ value: 8, bytes: 1 });
|
||||
// 0x80,0x01 => 0 | (1<<7) = 128
|
||||
expect(readVint(Buffer.from([0x80, 0x01]), 0)).toEqual({ value: 128, bytes: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectRarSignature", () => {
|
||||
it("detects RAR5 and RAR4", () => {
|
||||
expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]))).toEqual({ version: 5, sigLen: 8 });
|
||||
expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x00]))).toEqual({ version: 4, sigLen: 7 });
|
||||
expect(detectRarSignature(Buffer.alloc(8))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseRar5BlockExtent", () => {
|
||||
it("computes header+data extent and flags end-of-archive", () => {
|
||||
// CRC32(4) | HeaderSize vint=5 | Type vint=2 (file) | Flags vint=2 (data present) | DataSize vint=100 | (pad to headerSize)
|
||||
const b = Buffer.concat([
|
||||
Buffer.from([0,0,0,0]), // CRC
|
||||
Buffer.from([0x05]), // HeaderSize = 5 (bytes after this vint)
|
||||
Buffer.from([0x02]), // Type = 2 (file)
|
||||
Buffer.from([0x02]), // Flags = 0x02 -> data present
|
||||
Buffer.from([0x64]), // DataSize = 100
|
||||
Buffer.from([0x00, 0x00]), // padding to fill HeaderSize(5): Type+Flags+DataSize=3, +2 pad =5
|
||||
]);
|
||||
const ext = parseRar5BlockExtent(b, 0);
|
||||
// headerBytes = 4 (CRC) + 1 (HeaderSize vint) + 5 (HeaderSize) = 10
|
||||
expect(ext.headerBytes).toBe(10);
|
||||
expect(ext.dataSize).toBe(100);
|
||||
expect(ext.isEnd).toBe(false);
|
||||
|
||||
const endBlk = Buffer.from([0,0,0,0, 0x02, 0x05, 0x00]); // HeaderSize=2, Type=5(end), Flags=0
|
||||
const e2 = parseRar5BlockExtent(endBlk, 0);
|
||||
expect(e2.isEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseRar4BlockExtent", () => {
|
||||
it("computes extent with ADD_SIZE when flag 0x8000 is set", () => {
|
||||
// CRC(2) TYPE(1)=0x74 FLAGS(2)=0x8000 HEAD_SIZE(2)=11 ADD_SIZE(4)=200
|
||||
const b = Buffer.alloc(11);
|
||||
b.writeUInt8(0x74, 2);
|
||||
b.writeUInt16LE(0x8000, 3);
|
||||
b.writeUInt16LE(11, 5);
|
||||
b.writeUInt32LE(200, 7);
|
||||
const ext = parseRar4BlockExtent(b, 0);
|
||||
expect(ext.headerBytes).toBe(11);
|
||||
expect(ext.dataSize).toBe(200);
|
||||
expect(ext.isEnd).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// Build a synthetic RAR5 volume: signature + main header + 2 file blocks (each
|
||||
// with data) + end block. We only need extents to be walkable.
|
||||
function buildRar5Volume(): Buffer {
|
||||
const sig = Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]);
|
||||
const block = (type: number, flags: number, dataSize: number, pad = 0) => {
|
||||
const body = [Buffer.from([type]), Buffer.from([flags])];
|
||||
if (flags & 0x0002) body.push(Buffer.from([dataSize])); // DataSize (<=127 for test)
|
||||
if (pad) body.push(Buffer.alloc(pad));
|
||||
const bodyBuf = Buffer.concat(body);
|
||||
const hs = Buffer.from([bodyBuf.length]); // HeaderSize vint (<=127)
|
||||
const header = Buffer.concat([Buffer.alloc(4), hs, bodyBuf]); // CRC(4)+HeaderSize+body
|
||||
const data = Buffer.alloc(flags & 0x0002 ? dataSize : 0, 0xEE);
|
||||
return Buffer.concat([header, data]);
|
||||
};
|
||||
const main = block(1, 0, 0); // main archive header, no data
|
||||
const f1 = block(2, 0x02, 20); // file header + 20 bytes data
|
||||
const f2 = block(2, 0x02, 30); // file header + 30 bytes data
|
||||
const end = block(5, 0, 0); // end of archive
|
||||
return Buffer.concat([sig, main, f1, f2, end]);
|
||||
}
|
||||
|
||||
describe("walkRarVolume", () => {
|
||||
it("harvests every block header and stops at end-of-archive", async () => {
|
||||
const vol = buildRar5Volume();
|
||||
const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
|
||||
const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }, 5, 8);
|
||||
expect(regions).not.toBeNull();
|
||||
// main + 2 files + end = 4 header regions
|
||||
expect(regions!).toHaveLength(4);
|
||||
// First region starts right after the 8-byte signature
|
||||
expect(regions![0].offset).toBe(8);
|
||||
});
|
||||
|
||||
it("returns null when a block claims an absurd header size (corrupt/desynced)", async () => {
|
||||
// RAR5 block with HeaderSize vint encoding a value > 8MB.
|
||||
// Encode 9_000_000 as RAR vint: bytes little-endian 7-bit groups with continuation bit.
|
||||
function encodeVint(n: number): number[] {
|
||||
const out: number[] = [];
|
||||
while (n >= 0x80) {
|
||||
out.push((n & 0x7f) | 0x80);
|
||||
n = Math.floor(n / 128);
|
||||
}
|
||||
out.push(n);
|
||||
return out;
|
||||
}
|
||||
const sig = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); // RAR5 signature
|
||||
const hsVint = encodeVint(9_000_000);
|
||||
// Block = CRC(4) + HeaderSize vint(9MB) + Type(1 byte) + Flags(1 byte)
|
||||
const block = Buffer.concat([Buffer.alloc(4), Buffer.from(hsVint), Buffer.from([0x02, 0x00])]);
|
||||
const vol = Buffer.concat([sig, block]);
|
||||
const size = 20 * 1024 * 1024;
|
||||
const read = async (_id: string, offset: number, length: number) => {
|
||||
if (offset >= vol.length) return Buffer.alloc(0);
|
||||
return vol.subarray(offset, Math.min(offset + length, vol.length));
|
||||
};
|
||||
const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(size), fileName: "c.rar" }, 5, 8);
|
||||
expect(regions).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("readRarListingRanged (single part)", () => {
|
||||
it("returns null cleanly when the reconstructed file isn't a real RAR", async () => {
|
||||
const vol = buildRar5Volume();
|
||||
const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
|
||||
const res = await readRarListingRanged([{ fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }], read);
|
||||
expect(res === null || Array.isArray(res)).toBe(true); // real unrar parse covered live
|
||||
});
|
||||
});
|
||||
|
||||
describe("readRarListingRanged (multipart)", () => {
|
||||
it("walks each volume from its own signature and reconstructs all parts", async () => {
|
||||
const vol = buildRar5Volume(); // reuse from Task 6 test
|
||||
// Two volumes with identical structure; each RangeReader read is scoped by fileId.
|
||||
const byId: Record<string, Buffer> = { p1: vol, p2: vol };
|
||||
const reads: Record<string, number> = { p1: 0, p2: 0 };
|
||||
const read: RangeReader = async (fileId, offset, length) => {
|
||||
reads[fileId]++;
|
||||
return byId[fileId].subarray(offset, offset + length);
|
||||
};
|
||||
const res = await readRarListingRanged(
|
||||
[
|
||||
{ fileId: "p1", fileSize: BigInt(vol.length), fileName: "x.part1.rar" },
|
||||
{ fileId: "p2", fileSize: BigInt(vol.length), fileName: "x.part2.rar" },
|
||||
],
|
||||
read,
|
||||
);
|
||||
// Both volumes were walked (each read at least its signature + blocks).
|
||||
expect(reads.p1).toBeGreaterThan(0);
|
||||
expect(reads.p2).toBeGreaterThan(0);
|
||||
expect(res === null || Array.isArray(res)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
const RAR4_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]);
|
||||
const RAR5_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]);
|
||||
|
||||
export function readVint(buf: Buffer, pos: number): { value: number; bytes: number } {
|
||||
let value = 0, shift = 0, bytes = 0;
|
||||
while (pos + bytes < buf.length) {
|
||||
const b = buf[pos + bytes];
|
||||
value += (b & 0x7f) * Math.pow(2, shift); // Math.pow keeps >32-bit sizes exact up to 2^53
|
||||
bytes++;
|
||||
if ((b & 0x80) === 0) return { value, bytes };
|
||||
shift += 7;
|
||||
if (shift > 63) break;
|
||||
}
|
||||
throw new RangeError("incomplete RAR vint");
|
||||
}
|
||||
|
||||
export function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null {
|
||||
if (buf.length >= 8 && buf.subarray(0, 8).equals(RAR5_SIG)) return { version: 5, sigLen: 8 };
|
||||
if (buf.length >= 7 && buf.subarray(0, 7).equals(RAR4_SIG)) return { version: 4, sigLen: 7 };
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean }
|
||||
|
||||
// RAR5: CRC32(4) | HeaderSize(vint) | HeaderType(vint) | HeaderFlags(vint)
|
||||
// [ExtraAreaSize(vint) if flags&0x0001] [DataSize(vint) if flags&0x0002] ...
|
||||
export function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent {
|
||||
let p = pos + 4; // skip CRC32
|
||||
const hs = readVint(buf, p); p += hs.bytes;
|
||||
const headerBytes = 4 + hs.bytes + hs.value; // CRC + HeaderSize-vint + HeaderSize
|
||||
const type = readVint(buf, p); p += type.bytes;
|
||||
const flags = readVint(buf, p); p += flags.bytes;
|
||||
if (flags.value & 0x0001) { const ea = readVint(buf, p); p += ea.bytes; } // extra area size (skip)
|
||||
let dataSize = 0;
|
||||
if (flags.value & 0x0002) { const ds = readVint(buf, p); p += ds.bytes; dataSize = ds.value; }
|
||||
return { headerBytes, dataSize, isEnd: type.value === 5 };
|
||||
}
|
||||
|
||||
// RAR4: HEAD_CRC(2) | HEAD_TYPE(1) | HEAD_FLAGS(2) | HEAD_SIZE(2) [ADD_SIZE(4) if flags&0x8000]
|
||||
export function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent {
|
||||
const type = buf.readUInt8(pos + 2);
|
||||
const flags = buf.readUInt16LE(pos + 3);
|
||||
const headSize = buf.readUInt16LE(pos + 5);
|
||||
const dataSize = (flags & 0x8000) ? buf.readUInt32LE(pos + 7) : 0;
|
||||
return { headerBytes: headSize, dataSize, isEnd: type === 0x7b };
|
||||
}
|
||||
|
||||
import type { FileEntry } from "../zip-reader.js";
|
||||
import type { RangeReader } from "./range-reader.js";
|
||||
import type { RangedPart } from "./sevenz-ranged.js";
|
||||
import { listFromSparse, type SparsePart } from "./sparse-list.js";
|
||||
import { readRarContents } from "../rar-reader.js";
|
||||
import { childLogger } from "../../util/logger.js";
|
||||
|
||||
const rlog = childLogger("rar-ranged");
|
||||
const MAX_RAR_BLOCKS = 50000;
|
||||
const MAX_RAR_HEADER_BYTES = 8 * 1024 * 1024; // 8 MB — real RAR block headers are far smaller; guards against a corrupt/desynced HeaderSize
|
||||
const HEADER_CHUNK = 8192;
|
||||
|
||||
export async function walkRarVolume(
|
||||
read: RangeReader,
|
||||
part: RangedPart,
|
||||
version: 4 | 5,
|
||||
sigLen: number,
|
||||
): Promise<{ offset: number; bytes: Buffer }[] | null> {
|
||||
const size = Number(part.fileSize);
|
||||
const regions: { offset: number; bytes: Buffer }[] = [];
|
||||
let pos = sigLen;
|
||||
let blocks = 0;
|
||||
try {
|
||||
while (pos < size) {
|
||||
if (++blocks > MAX_RAR_BLOCKS) return null;
|
||||
const chunkLen = Math.min(HEADER_CHUNK, size - pos);
|
||||
let chunk = await read(part.fileId, pos, chunkLen, part.fileSize);
|
||||
const ext = version === 5 ? parseRar5BlockExtent(chunk, 0) : parseRar4BlockExtent(chunk, 0);
|
||||
if (ext.headerBytes > MAX_RAR_HEADER_BYTES) return null;
|
||||
// Ensure we have the full header bytes to harvest (long filenames).
|
||||
let headerBuf = chunk;
|
||||
if (ext.headerBytes > chunk.length) {
|
||||
headerBuf = await read(part.fileId, pos, Math.min(ext.headerBytes, size - pos), part.fileSize);
|
||||
}
|
||||
regions.push({ offset: pos, bytes: headerBuf.subarray(0, Math.min(ext.headerBytes, size - pos)) });
|
||||
if (ext.isEnd) break;
|
||||
const advance = ext.headerBytes + ext.dataSize;
|
||||
if (advance <= 0) return null;
|
||||
if (pos + advance > size) break; // data clamped at the volume boundary (multipart continuation)
|
||||
pos += advance;
|
||||
}
|
||||
return regions;
|
||||
} catch (err) {
|
||||
rlog.warn({ err, fileId: part.fileId }, "RAR volume walk failed");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readRarListingRanged(
|
||||
parts: RangedPart[],
|
||||
read: RangeReader,
|
||||
): Promise<FileEntry[] | null> {
|
||||
const sparseParts: SparsePart[] = [];
|
||||
for (const part of parts) {
|
||||
const head = await read(part.fileId, 0, 16, part.fileSize);
|
||||
const sig = detectRarSignature(head);
|
||||
if (!sig) return null;
|
||||
const regions = await walkRarVolume(read, part, sig.version, sig.sigLen);
|
||||
if (!regions) return null;
|
||||
sparseParts.push({ fileName: part.fileName, size: Number(part.fileSize), regions });
|
||||
}
|
||||
return listFromSparse(sparseParts, readRarContents);
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { parseSevenZSignatureHeader } from "./sevenz-ranged.js";
|
||||
|
||||
const MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
|
||||
|
||||
function buildSignatureHeader(nextOffset: bigint, nextSize: bigint): Buffer {
|
||||
const buf = Buffer.alloc(32);
|
||||
MAGIC.copy(buf, 0);
|
||||
buf.writeUInt8(0, 6); buf.writeUInt8(4, 7); // version 0.4
|
||||
buf.writeUInt32LE(0, 8); // StartHeaderCRC (unused here)
|
||||
buf.writeBigUInt64LE(nextOffset, 12);
|
||||
buf.writeBigUInt64LE(nextSize, 20);
|
||||
buf.writeUInt32LE(0, 28); // NextHeaderCRC (unused here)
|
||||
return buf;
|
||||
}
|
||||
|
||||
describe("parseSevenZSignatureHeader", () => {
|
||||
it("reads NextHeaderOffset and NextHeaderSize", () => {
|
||||
const buf = buildSignatureHeader(1_000_000n, 4096n);
|
||||
expect(parseSevenZSignatureHeader(buf)).toEqual({ nextHeaderOffset: 1_000_000, nextHeaderSize: 4096 });
|
||||
});
|
||||
|
||||
it("returns null on bad magic", () => {
|
||||
expect(parseSevenZSignatureHeader(Buffer.alloc(32))).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when shorter than 32 bytes", () => {
|
||||
expect(parseSevenZSignatureHeader(MAGIC)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
import { readSevenZListingRanged } from "./sevenz-ranged.js";
|
||||
import type { RangeReader } from "./range-reader.js";
|
||||
|
||||
describe("readSevenZListingRanged", () => {
|
||||
it("reads the signature + end-header regions and reconstructs for 7z l", async () => {
|
||||
const size = 5_000_000;
|
||||
const endHeaderOffset = 4_900_000; // absolute
|
||||
const nextHeaderOffset = endHeaderOffset - 32;
|
||||
const sig = Buffer.alloc(32);
|
||||
Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
|
||||
sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
|
||||
sig.writeBigUInt64LE(100n, 20);
|
||||
|
||||
const reads: { offset: number; length: number }[] = [];
|
||||
const read: RangeReader = async (_id, offset, length) => {
|
||||
reads.push({ offset, length });
|
||||
if (offset === 0) return sig.subarray(0, length);
|
||||
return Buffer.alloc(length, 0xAB); // stand-in end-header bytes
|
||||
};
|
||||
|
||||
// Inject a fake lister via the module boundary: readSevenZListingRanged
|
||||
// calls listFromSparse(parts, read7zContents). We assert the ranged reads
|
||||
// it issued; the sparse file + real 7z is covered by live verification.
|
||||
const entries = await readSevenZListingRanged(
|
||||
[{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
|
||||
read,
|
||||
);
|
||||
// entries may be null here because the stand-in bytes aren't a real 7z;
|
||||
// the contract under test is the ranged-read offsets:
|
||||
expect(reads[0]).toEqual({ offset: 0, length: 32 });
|
||||
expect(reads[1]).toEqual({ offset: endHeaderOffset, length: 100 });
|
||||
expect(entries === null || Array.isArray(entries)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js";
|
||||
|
||||
describe("read7zNumber", () => {
|
||||
it("reads a single-byte number", () => {
|
||||
expect(read7zNumber(Buffer.from([0x2a]), 0)).toEqual({ value: 42, next: 1 });
|
||||
});
|
||||
it("reads a two-byte number (first-byte length mask + LE trailing byte)", () => {
|
||||
// 1000 = 0x03E8: low byte 0xE8, high nibble 0x03 -> first = 0x80|0x03 = 0x83, trailing 0xE8
|
||||
expect(read7zNumber(Buffer.from([0x83, 0xe8]), 0)).toEqual({ value: 1000, next: 2 });
|
||||
// 500 = 0x01F4 -> first 0x81, trailing 0xF4
|
||||
expect(read7zNumber(Buffer.from([0x81, 0xf4]), 0)).toEqual({ value: 500, next: 2 });
|
||||
});
|
||||
it("throws when pos starts past the buffer end (short read)", () => {
|
||||
expect(() => read7zNumber(Buffer.from([0x2a]), 5)).toThrow(RangeError);
|
||||
expect(() => read7zNumber(Buffer.alloc(0), 0)).toThrow(RangeError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("locate7zEncodedHeaderPack", () => {
|
||||
it("parses PackPos and summed PackSize from an encoded header", () => {
|
||||
// kEncodedHeader, kPackInfo, PackPos=1000([0x83,0xe8]), NumStreams=1([0x01]),
|
||||
// kSize, PackSize=500([0x81,0xf4])
|
||||
const enc = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
|
||||
expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 1000, packSize: 500 });
|
||||
});
|
||||
it("returns null for a plain (kHeader 0x01) header", () => {
|
||||
expect(locate7zEncodedHeaderPack(Buffer.from([0x01, 0x04]))).toBeNull();
|
||||
});
|
||||
it("sums multiple pack streams", () => {
|
||||
// PackPos=0([0x00]), NumStreams=2([0x02]), kSize, sizes 10([0x0a]) + 20([0x14])
|
||||
const enc = Buffer.from([0x17, 0x06, 0x00, 0x02, 0x09, 0x0a, 0x14]);
|
||||
expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 0, packSize: 30 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("readSevenZListingRanged (encoded header)", () => {
|
||||
it("fetches the mid-file packed-header region as a third read", async () => {
|
||||
const size = 5_000_000;
|
||||
const nextHeaderOffset = 4_000_000; // relative to end of 32-byte sig header
|
||||
const endStart = 32 + nextHeaderOffset; // absolute
|
||||
const packPos = 1000; // relative to end of sig header
|
||||
const packStart = 32 + packPos; // absolute
|
||||
const packSize = 500;
|
||||
const sig = Buffer.alloc(32);
|
||||
Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
|
||||
sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
|
||||
sig.writeBigUInt64LE(8n, 20); // NextHeaderSize = 8 (the encoded-header descriptor below)
|
||||
const encHeader = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
|
||||
|
||||
const reads: { offset: number; length: number }[] = [];
|
||||
const read = async (_id: string, offset: number, length: number) => {
|
||||
reads.push({ offset, length });
|
||||
if (offset === 0) return sig.subarray(0, length);
|
||||
if (offset === endStart) return encHeader.subarray(0, length);
|
||||
return Buffer.alloc(length, 0xcd); // stand-in packed-header bytes
|
||||
};
|
||||
|
||||
const entries = await readSevenZListingRanged(
|
||||
[{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
|
||||
read,
|
||||
);
|
||||
expect(reads[0]).toEqual({ offset: 0, length: 32 });
|
||||
expect(reads[1]).toEqual({ offset: endStart, length: 8 });
|
||||
expect(reads[2]).toEqual({ offset: packStart, length: packSize });
|
||||
expect(entries === null || Array.isArray(entries)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { FileEntry } from "../zip-reader.js";
|
||||
import { read7zContents } from "../sevenz-reader.js";
|
||||
import { listFromSparse } from "./sparse-list.js";
|
||||
import type { RangeReader } from "./range-reader.js";
|
||||
import { childLogger } from "../../util/logger.js";
|
||||
|
||||
const log = childLogger("sevenz-ranged");
|
||||
|
||||
const SEVENZ_MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
|
||||
|
||||
const K_HEADER = 0x01;
|
||||
const K_ENCODED_HEADER = 0x17;
|
||||
const K_PACK_INFO = 0x06;
|
||||
const K_SIZE = 0x09;
|
||||
|
||||
/** Read a 7z variable-length number: first byte is a length mask, followed by
|
||||
* little-endian bytes. Math.pow keeps values exact above 2^31. */
|
||||
export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } {
|
||||
if (pos >= buf.length) throw new RangeError("7z number reads past buffer end");
|
||||
const first = buf[pos];
|
||||
let mask = 0x80;
|
||||
let value = 0;
|
||||
let p = pos + 1;
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * Math.pow(2, 8 * i);
|
||||
return { value, next: p };
|
||||
}
|
||||
if (p >= buf.length) throw new RangeError("7z number overruns buffer");
|
||||
value += buf[p] * Math.pow(2, 8 * i);
|
||||
p++;
|
||||
mask >>= 1;
|
||||
}
|
||||
return { value, next: p };
|
||||
}
|
||||
|
||||
/** For an encoded (kEncodedHeader) 7z next-header, return the absolute-ish
|
||||
* location of the packed header stream(s): PackPos (relative to end of the
|
||||
* 32-byte signature header) and the summed PackSize. Null if not encoded or
|
||||
* the StreamsInfo doesn't start with PackInfo as expected. */
|
||||
export function locate7zEncodedHeaderPack(
|
||||
nextHeader: Buffer,
|
||||
): { packPos: number; packSize: number } | null {
|
||||
let p = 0;
|
||||
if (nextHeader[p] !== K_ENCODED_HEADER) return null;
|
||||
p++;
|
||||
if (nextHeader[p] !== K_PACK_INFO) return null;
|
||||
p++;
|
||||
const packPos = read7zNumber(nextHeader, p); p = packPos.next;
|
||||
const numStreams = read7zNumber(nextHeader, p); p = numStreams.next;
|
||||
if (nextHeader[p] !== K_SIZE) return null;
|
||||
p++;
|
||||
let total = 0;
|
||||
for (let i = 0; i < numStreams.value; i++) {
|
||||
const s = read7zNumber(nextHeader, p); p = s.next;
|
||||
total += s.value;
|
||||
}
|
||||
return { packPos: packPos.value, packSize: total };
|
||||
}
|
||||
|
||||
export function parseSevenZSignatureHeader(
|
||||
buf: Buffer,
|
||||
): { nextHeaderOffset: number; nextHeaderSize: number } | null {
|
||||
if (buf.length < 32) return null;
|
||||
if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null;
|
||||
return {
|
||||
nextHeaderOffset: Number(buf.readBigUInt64LE(12)),
|
||||
nextHeaderSize: Number(buf.readBigUInt64LE(20)),
|
||||
};
|
||||
}
|
||||
|
||||
export interface RangedPart { fileId: string; fileSize: bigint; fileName: string }
|
||||
|
||||
export async function readSevenZListingRanged(
|
||||
parts: RangedPart[],
|
||||
read: RangeReader,
|
||||
): Promise<FileEntry[] | null> {
|
||||
const part = parts[0];
|
||||
if (!part) return null;
|
||||
const size = Number(part.fileSize);
|
||||
try {
|
||||
const sig = await read(part.fileId, 0, 32, part.fileSize);
|
||||
const parsed = parseSevenZSignatureHeader(sig);
|
||||
if (!parsed) return null;
|
||||
const endStart = 32 + parsed.nextHeaderOffset;
|
||||
if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null;
|
||||
const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize);
|
||||
|
||||
const regions = [
|
||||
{ offset: 0, bytes: sig },
|
||||
{ offset: endStart, bytes: endHeader },
|
||||
];
|
||||
|
||||
const headerType = endHeader[0];
|
||||
if (headerType === K_ENCODED_HEADER) {
|
||||
// Compressed header: its packed bytes live mid-file, not at EOF. Fetch them.
|
||||
const pack = locate7zEncodedHeaderPack(endHeader);
|
||||
if (!pack) return null;
|
||||
const packStart = 32 + pack.packPos;
|
||||
if (packStart < 0 || packStart + pack.packSize > size) return null;
|
||||
const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize);
|
||||
regions.push({ offset: packStart, bytes: packBytes });
|
||||
} else if (headerType !== K_HEADER) {
|
||||
return null; // unknown next-header type
|
||||
}
|
||||
|
||||
return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents);
|
||||
} catch (err) {
|
||||
log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { open } from "fs/promises";
|
||||
import { listFromSparse } from "./sparse-list.js";
|
||||
|
||||
describe("listFromSparse", () => {
|
||||
it("writes each region at its offset into a sparse file and passes the path to the lister", async () => {
|
||||
const size = 1_000_000;
|
||||
const regions = [
|
||||
{ offset: 0, bytes: Buffer.from("HEAD") },
|
||||
{ offset: size - 4, bytes: Buffer.from("TAIL") },
|
||||
];
|
||||
let seenPath = "";
|
||||
const entries = await listFromSparse(
|
||||
[{ fileName: "sample.7z", size, regions }],
|
||||
async (firstPartPath) => {
|
||||
seenPath = firstPartPath;
|
||||
const fh = await open(firstPartPath, "r");
|
||||
try {
|
||||
const head = Buffer.alloc(4); await fh.read(head, 0, 4, 0);
|
||||
const tail = Buffer.alloc(4); await fh.read(tail, 0, 4, size - 4);
|
||||
const hole = Buffer.alloc(4); await fh.read(hole, 0, 4, 500_000);
|
||||
expect(head.toString()).toBe("HEAD");
|
||||
expect(tail.toString()).toBe("TAIL");
|
||||
expect(hole.equals(Buffer.alloc(4))).toBe(true); // gap is zero
|
||||
} finally { await fh.close(); }
|
||||
return [{ path: "a/b.stl", fileName: "b.stl", extension: "stl", compressedSize: 1n, uncompressedSize: 1n, crc32: null }];
|
||||
},
|
||||
);
|
||||
expect(seenPath.endsWith("sample.7z")).toBe(true);
|
||||
expect(entries).not.toBeNull();
|
||||
expect(entries!).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("returns null when the lister yields no entries", async () => {
|
||||
const res = await listFromSparse(
|
||||
[{ fileName: "x.7z", size: 100, regions: [{ offset: 0, bytes: Buffer.from("A") }] }],
|
||||
async () => [],
|
||||
);
|
||||
expect(res).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { mkdtemp, open, rm } from "fs/promises";
|
||||
import path from "path";
|
||||
import { config } from "../../util/config.js";
|
||||
import { childLogger } from "../../util/logger.js";
|
||||
import type { FileEntry } from "../zip-reader.js";
|
||||
|
||||
const log = childLogger("sparse-list");
|
||||
|
||||
export interface SparsePart {
|
||||
fileName: string;
|
||||
size: number;
|
||||
regions: { offset: number; bytes: Buffer }[];
|
||||
}
|
||||
|
||||
export type SparseLister = (firstPartPath: string) => Promise<FileEntry[]>;
|
||||
|
||||
/**
|
||||
* Reconstruct archive header bytes into sparse temp files (data areas left as
|
||||
* zero holes), run `lister` on the first part, return its entries.
|
||||
* Returns null on any error or when the lister finds nothing.
|
||||
*/
|
||||
export async function listFromSparse(
|
||||
parts: SparsePart[],
|
||||
lister: SparseLister,
|
||||
): Promise<FileEntry[] | null> {
|
||||
if (parts.length === 0) return null;
|
||||
const dir = await mkdtemp(path.join(config.tempDir, "ranged-"));
|
||||
try {
|
||||
let firstPath = "";
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const p = parts[i];
|
||||
const filePath = path.join(dir, p.fileName);
|
||||
if (i === 0) firstPath = filePath;
|
||||
const fh = await open(filePath, "w");
|
||||
try {
|
||||
await fh.truncate(p.size); // create the sparse hole
|
||||
for (const r of p.regions) {
|
||||
await fh.write(r.bytes, 0, r.bytes.length, r.offset);
|
||||
}
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
const entries = await lister(firstPath);
|
||||
return entries.length > 0 ? entries : null;
|
||||
} catch (err) {
|
||||
log.warn({ err }, "sparse listing failed");
|
||||
return null;
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { db } from "./client.js";
|
||||
import type { ArchiveType, FetchStatus } from "@prisma/client";
|
||||
import type { FileEntry } from "../archive/zip-reader.js";
|
||||
|
||||
export async function getActiveAccounts() {
|
||||
return db.telegramAccount.findMany({
|
||||
@@ -375,6 +376,7 @@ export interface ActivityUpdate {
|
||||
zipsFound?: number;
|
||||
zipsDuplicate?: number;
|
||||
zipsIngested?: number;
|
||||
zipsBackfilled?: number;
|
||||
}
|
||||
|
||||
export async function updateRunActivity(
|
||||
@@ -398,6 +400,7 @@ export async function updateRunActivity(
|
||||
...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
|
||||
...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
|
||||
...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
|
||||
...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }),
|
||||
...(activity.currentTopicId !== undefined && { currentTopicId: activity.currentTopicId }),
|
||||
...(activity.currentAccountChannelMapId !== undefined && {
|
||||
currentAccountChannelMapId: activity.currentAccountChannelMapId,
|
||||
@@ -428,6 +431,7 @@ export async function completeIngestionRun(
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
zipsBackfilled: number;
|
||||
}
|
||||
) {
|
||||
return db.ingestionRun.update({
|
||||
@@ -1005,3 +1009,142 @@ export async function createAutoGroup(input: {
|
||||
|
||||
return group.id;
|
||||
}
|
||||
|
||||
// ── Provenance backfill ──
|
||||
|
||||
export interface PlaceholderCandidate {
|
||||
id: string;
|
||||
archiveType: string;
|
||||
fileName: string;
|
||||
fileCount: number;
|
||||
fileSize: bigint;
|
||||
destMessageId: bigint | null;
|
||||
destMessageIds: bigint[];
|
||||
destChannel: { telegramId: bigint } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find every placeholder Package matching name+size (oldest first). Package
|
||||
* has no direct `destChannel` relation (only the scalar `destChannelId`), so
|
||||
* each row's destination TelegramChannel telegramId is resolved with a
|
||||
* follow-up lookup rather than a Prisma include.
|
||||
*/
|
||||
export async function findPlaceholderCandidates(
|
||||
destChannelId: string,
|
||||
fileName: string,
|
||||
fileSize: bigint,
|
||||
): Promise<PlaceholderCandidate[]> {
|
||||
const rows = await db.package.findMany({
|
||||
where: {
|
||||
fileName,
|
||||
fileSize,
|
||||
destMessageId: { not: null },
|
||||
// Placeholder provenance (spec §1): manual-upload (source == destination)
|
||||
// OR rebuild record (sourceMessageId == 0 "unknown" sentinel).
|
||||
OR: [
|
||||
{ sourceChannelId: destChannelId },
|
||||
{ sourceMessageId: 0n },
|
||||
],
|
||||
},
|
||||
select: {
|
||||
id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true,
|
||||
destMessageId: true, destMessageIds: true, destChannelId: true,
|
||||
},
|
||||
orderBy: { indexedAt: "asc" },
|
||||
});
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))];
|
||||
const channels = destChannelIds.length
|
||||
? await db.telegramChannel.findMany({
|
||||
where: { id: { in: destChannelIds } },
|
||||
select: { id: true, telegramId: true },
|
||||
})
|
||||
: [];
|
||||
const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId]));
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
archiveType: row.archiveType,
|
||||
fileName: row.fileName,
|
||||
fileCount: row.fileCount,
|
||||
fileSize: row.fileSize,
|
||||
destMessageId: row.destMessageId,
|
||||
destMessageIds: row.destMessageIds,
|
||||
destChannel: row.destChannelId && telegramIdById.has(row.destChannelId)
|
||||
? { telegramId: telegramIdById.get(row.destChannelId)! }
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function findPlaceholderCandidate(
|
||||
destChannelId: string,
|
||||
fileName: string,
|
||||
fileSize: bigint,
|
||||
): Promise<PlaceholderCandidate | null> {
|
||||
return (await findPlaceholderCandidates(destChannelId, fileName, fileSize))[0] ?? null;
|
||||
}
|
||||
|
||||
export async function getPackageFileCrcs(packageId: string): Promise<(string | null)[]> {
|
||||
const rows = await db.packageFile.findMany({
|
||||
where: { packageId },
|
||||
select: { crc32: true },
|
||||
});
|
||||
return rows.map((r) => r.crc32);
|
||||
}
|
||||
|
||||
export interface BackfillProvenanceInput {
|
||||
packageId: string;
|
||||
destChannelId: string; // to re-check placeholder status in-txn
|
||||
sourceChannelId: string;
|
||||
sourceMessageId: bigint;
|
||||
sourceTopicId: bigint | null;
|
||||
sourceCaption: string | null;
|
||||
remoteUniqueId: string | null;
|
||||
creator: string | null; // always set (re-derived by caller)
|
||||
entries?: FileEntry[]; // set only if candidate had fileCount === 0
|
||||
previewData?: Buffer | null; // set only if provided and candidate lacks one
|
||||
previewMsgId?: bigint | null;
|
||||
}
|
||||
|
||||
export async function backfillProvenance(input: BackfillProvenanceInput): Promise<boolean> {
|
||||
return db.$transaction(async (tx) => {
|
||||
const current = await tx.package.findUnique({
|
||||
where: { id: input.packageId },
|
||||
select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true },
|
||||
});
|
||||
// Re-check placeholder status inside the txn (another worker may have won).
|
||||
// Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0).
|
||||
const stillPlaceholder =
|
||||
!!current &&
|
||||
(current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n);
|
||||
if (!stillPlaceholder) return false;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const data: any = {
|
||||
sourceChannelId: input.sourceChannelId,
|
||||
sourceMessageId: input.sourceMessageId,
|
||||
sourceTopicId: input.sourceTopicId,
|
||||
sourceCaption: input.sourceCaption,
|
||||
remoteUniqueId: input.remoteUniqueId,
|
||||
creator: input.creator,
|
||||
};
|
||||
if (input.entries && current.fileCount === 0) {
|
||||
await tx.packageFile.deleteMany({ where: { packageId: input.packageId } });
|
||||
await tx.packageFile.createMany({
|
||||
data: input.entries.map((e) => ({
|
||||
packageId: input.packageId,
|
||||
path: e.path, fileName: e.fileName, extension: e.extension,
|
||||
compressedSize: e.compressedSize, uncompressedSize: e.uncompressedSize, crc32: e.crc32,
|
||||
})),
|
||||
});
|
||||
data.fileCount = input.entries.length;
|
||||
}
|
||||
if (input.previewData && !current.previewData) {
|
||||
data.previewData = input.previewData;
|
||||
data.previewMsgId = input.previewMsgId ?? null;
|
||||
}
|
||||
await tx.package.update({ where: { id: input.packageId }, data });
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
import { db } from "./db/client.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { downloadFileRange } from "./tdlib/range-download.js";
|
||||
import { invokeWithTimeout } from "./tdlib/download.js";
|
||||
import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js";
|
||||
import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js";
|
||||
import {
|
||||
findPlaceholderCandidates,
|
||||
getPackageFileCrcs,
|
||||
backfillProvenance,
|
||||
type PlaceholderCandidate,
|
||||
} from "./db/queries.js";
|
||||
import type { FileEntry } from "./archive/zip-reader.js";
|
||||
import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js";
|
||||
import { readRarListingRanged } from "./archive/ranged/rar-ranged.js";
|
||||
import { tdlibRangeReader } from "./archive/ranged/range-reader.js";
|
||||
import { fullDownloadListing } from "./archive/ranged/fallback.js";
|
||||
import type { Client } from "tdl";
|
||||
|
||||
const log = childLogger("provenance-backfill");
|
||||
|
||||
export interface BackfillArgs {
|
||||
client: Client;
|
||||
destChannelId: string;
|
||||
scannedSourceChannelId: string;
|
||||
fileName: string;
|
||||
fileSize: bigint;
|
||||
archiveType: string;
|
||||
sourceMessageId: bigint;
|
||||
sourceTopicId: bigint | null;
|
||||
sourceCaption: string | null;
|
||||
remoteUniqueId: string | null;
|
||||
creator: string | null;
|
||||
scannedParts: RangedPart[];
|
||||
previewData?: Buffer | null;
|
||||
previewMsgId?: bigint | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a ZIP central directory from the tail of a (possibly multipart)
|
||||
* archive. `parts` is ordered; only the LAST part carries the EOCD record.
|
||||
* `fileSize` on each part is that part's own size (NOT the whole-archive
|
||||
* total) so the download offset stays within that part's bounds, while
|
||||
* `tailStart` passed to the parser is the logical whole-archive offset
|
||||
* (preceding parts' sizes + the offset within the last part).
|
||||
*/
|
||||
async function readScannedZipListing(
|
||||
client: Client,
|
||||
parts: { fileId: string; fileSize: bigint }[],
|
||||
): Promise<FileEntry[] | null> {
|
||||
if (parts.length === 0) return null;
|
||||
const lastPart = parts[parts.length - 1];
|
||||
const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
|
||||
const lastSize = Number(lastPart.fileSize);
|
||||
for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
|
||||
const partOffset = Math.max(0, lastSize - tailBytes);
|
||||
const downloadLen = Math.min(tailBytes, lastSize);
|
||||
try {
|
||||
const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
|
||||
const tailStart = precedingSize + partOffset;
|
||||
return parseZipCentralDirectoryFromTail(buf, tailStart);
|
||||
} catch (err) {
|
||||
if (err instanceof RangeError) continue; // try a larger tail
|
||||
log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the destination copy's message(s) into ranged parts (file id +
|
||||
* size + name), in order, so a multipart destination copy is reconstructed
|
||||
* with correct per-part sizes and names (the last message carries the
|
||||
* EOCD-bearing tail part for ZIP; multipart RAR needs correctly-suffixed
|
||||
* `.partN.rar` names for `unrar` sibling discovery). Cheap-only: any TDLib
|
||||
* failure here degrades the caller to name-size confidence rather than
|
||||
* falling back to a full download.
|
||||
*/
|
||||
async function resolveDestParts(
|
||||
client: Client,
|
||||
destChatTelegramId: bigint,
|
||||
destMessageIds: bigint[],
|
||||
destMessageId: bigint | null,
|
||||
fallbackFileName: string,
|
||||
): Promise<RangedPart[] | null> {
|
||||
const messageIds = destMessageIds.length > 0 ? destMessageIds : destMessageId ? [destMessageId] : [];
|
||||
if (messageIds.length === 0) return null;
|
||||
try {
|
||||
const parts: RangedPart[] = [];
|
||||
for (const msgId of messageIds) {
|
||||
const msg = (await invokeWithTimeout(client, {
|
||||
_: "getMessage",
|
||||
chat_id: Number(destChatTelegramId),
|
||||
message_id: Number(msgId),
|
||||
})) as { content?: { document?: { document?: { id: number; size?: number }; file_name?: string } } };
|
||||
const doc = msg?.content?.document?.document;
|
||||
if (!doc?.id) return null;
|
||||
const fileName = msg?.content?.document?.file_name || fallbackFileName;
|
||||
parts.push({ fileId: String(doc.id), fileSize: BigInt(doc.size ?? 0), fileName });
|
||||
}
|
||||
return parts;
|
||||
} catch (err) {
|
||||
log.warn({ err, destMessageIds: messageIds.map(Number) }, "destination archive part resolution failed");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readScannedListingRanged(
|
||||
archiveType: string,
|
||||
client: Client,
|
||||
parts: RangedPart[],
|
||||
): Promise<FileEntry[] | null> {
|
||||
const read = tdlibRangeReader(client);
|
||||
if (archiveType === "ZIP") return readScannedZipListing(client, parts);
|
||||
if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
|
||||
if (archiveType === "RAR") return readRarListingRanged(parts, read);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the CRC fingerprint entries for a placeholder candidate: start from
|
||||
* its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild
|
||||
* candidate with fileCount === 0), fall back to a fresh ranged read of the
|
||||
* candidate's own copy in the destination channel (Task 9).
|
||||
*/
|
||||
async function resolveCandidateFingerprintEntries(
|
||||
client: Client,
|
||||
candidate: PlaceholderCandidate,
|
||||
): Promise<FileEntry[]> {
|
||||
const candidateCrcs = await getPackageFileCrcs(candidate.id);
|
||||
let candidateEntries: FileEntry[] = candidateCrcs.map((crc) => ({
|
||||
path: "", fileName: "", extension: null, compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
|
||||
}));
|
||||
const hasDestMessage = candidate.destMessageIds.length > 0 || candidate.destMessageId != null;
|
||||
if (!crcFingerprint(candidateEntries).complete && hasDestMessage && candidate.destChannel) {
|
||||
const destParts = await resolveDestParts(
|
||||
client,
|
||||
candidate.destChannel.telegramId,
|
||||
candidate.destMessageIds,
|
||||
candidate.destMessageId,
|
||||
candidate.fileName,
|
||||
);
|
||||
let destEntries: FileEntry[] | null = null;
|
||||
if (destParts) {
|
||||
const read = tdlibRangeReader(client);
|
||||
destEntries =
|
||||
candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
|
||||
: candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
|
||||
: candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
|
||||
: null;
|
||||
}
|
||||
if (destEntries) {
|
||||
candidateEntries = destEntries;
|
||||
}
|
||||
}
|
||||
return candidateEntries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a fingerprint comparison between two entry sets. "incomplete"
|
||||
* means at least one side is missing CRCs (e.g. an empty file → CRC32 of
|
||||
* zero-length data → null) and the comparison CANNOT be used to confirm or
|
||||
* refute a match — callers must fall back to name+size confidence rather
|
||||
* than treating this as a mismatch.
|
||||
*/
|
||||
function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" {
|
||||
const fa = crcFingerprint(a);
|
||||
const fb = crcFingerprint(b);
|
||||
if (!fa.complete || !fb.complete) return "incomplete";
|
||||
return fingerprintsMatch(a, b) ? "match" : "mismatch";
|
||||
}
|
||||
|
||||
export async function tryProvenanceBackfill(
|
||||
args: BackfillArgs,
|
||||
): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }> {
|
||||
const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize);
|
||||
if (candidates.length === 0) return { backfilled: false };
|
||||
|
||||
let scannedEntries: FileEntry[] | null = await readScannedListingRanged(
|
||||
args.archiveType, args.client, args.scannedParts,
|
||||
);
|
||||
// Cheap ranged read failed — fall back to a size-capped full download so the
|
||||
// listing still gets indexed. Only worth it when the candidate lacks a listing.
|
||||
if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) {
|
||||
const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n);
|
||||
scannedEntries = await fullDownloadListing({
|
||||
client: args.client, parts: args.scannedParts, archiveType: args.archiveType,
|
||||
totalSize, fileName: args.fileName,
|
||||
});
|
||||
}
|
||||
|
||||
let chosen = candidates[0];
|
||||
let confidence: "fingerprint" | "name-size" = "name-size";
|
||||
|
||||
if (candidates.length > 1) {
|
||||
// Multiple placeholder packages share this name+size. Try to
|
||||
// disambiguate by fingerprint (ZIP only); if we can't uniquely resolve
|
||||
// it, notify instead of guessing which one is the real match.
|
||||
if (args.archiveType === "ZIP" && scannedEntries) {
|
||||
const matches: PlaceholderCandidate[] = [];
|
||||
// Candidates NOT ruled out as a definite (both-complete) mismatch —
|
||||
// used as the name+size fallback pool when the fingerprint can't
|
||||
// confirm a match (e.g. incomplete CRCs on either side).
|
||||
const nonMismatches: PlaceholderCandidate[] = [];
|
||||
for (const c of candidates) {
|
||||
const candidateEntries = await resolveCandidateFingerprintEntries(args.client, c);
|
||||
const comparison = compareFingerprints(scannedEntries, candidateEntries);
|
||||
if (comparison === "match") {
|
||||
matches.push(c);
|
||||
nonMismatches.push(c);
|
||||
} else if (comparison === "incomplete") {
|
||||
nonMismatches.push(c);
|
||||
}
|
||||
// comparison === "mismatch": both sides complete and differ — excluded.
|
||||
}
|
||||
if (matches.length === 1) {
|
||||
chosen = matches[0];
|
||||
confidence = "fingerprint";
|
||||
} else if (matches.length === 0 && nonMismatches.length === 1) {
|
||||
// Fingerprint couldn't confirm (incomplete CRCs), but exactly one
|
||||
// candidate wasn't ruled out as a definite mismatch — fall back to
|
||||
// name+size confidence rather than treating this as unresolved.
|
||||
chosen = nonMismatches[0];
|
||||
confidence = "name-size";
|
||||
} else {
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "INTEGRITY_AUDIT",
|
||||
severity: "WARNING",
|
||||
title: `Ambiguous provenance match: ${args.fileName}`,
|
||||
message: `${candidates.length} placeholder packages share this name+size and the fingerprint did not uniquely disambiguate. No provenance was backfilled.`,
|
||||
context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
|
||||
},
|
||||
});
|
||||
return { backfilled: false };
|
||||
}
|
||||
} else {
|
||||
// Can't disambiguate without a fingerprint — notify, don't guess.
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "INTEGRITY_AUDIT",
|
||||
severity: "WARNING",
|
||||
title: `Ambiguous provenance match: ${args.fileName}`,
|
||||
message: `${candidates.length} placeholder packages share this name+size (archive type ${args.archiveType} — no cheap fingerprint). No provenance was backfilled.`,
|
||||
context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
|
||||
},
|
||||
});
|
||||
return { backfilled: false };
|
||||
}
|
||||
} else if (scannedEntries) {
|
||||
const candidateEntries = await resolveCandidateFingerprintEntries(args.client, chosen);
|
||||
const comparison = compareFingerprints(scannedEntries, candidateEntries);
|
||||
if (comparison === "match") {
|
||||
confidence = "fingerprint";
|
||||
} else if (comparison === "mismatch") {
|
||||
// Both sides' CRCs are complete and differ: NOT the same content
|
||||
// despite name+size. Do not backfill.
|
||||
log.info({ candidateId: chosen.id, fileName: args.fileName }, "fingerprint mismatch — not backfilling");
|
||||
return { backfilled: false };
|
||||
}
|
||||
// comparison === "incomplete": can't confirm or refute by fingerprint —
|
||||
// fall through and backfill on name+size confidence instead.
|
||||
}
|
||||
|
||||
const ok = await backfillProvenance({
|
||||
packageId: chosen.id,
|
||||
destChannelId: args.destChannelId,
|
||||
sourceChannelId: args.scannedSourceChannelId,
|
||||
sourceMessageId: args.sourceMessageId,
|
||||
sourceTopicId: args.sourceTopicId,
|
||||
sourceCaption: args.sourceCaption,
|
||||
remoteUniqueId: args.remoteUniqueId,
|
||||
creator: args.creator,
|
||||
entries: chosen.fileCount === 0 && scannedEntries ? scannedEntries : undefined,
|
||||
previewData: args.previewData ?? undefined,
|
||||
previewMsgId: args.previewMsgId ?? undefined,
|
||||
});
|
||||
|
||||
if (!ok) return { backfilled: false };
|
||||
|
||||
if (confidence === "name-size") {
|
||||
// Lower-confidence backfill: no CRC fingerprint guard confirmed this
|
||||
// match. Record it as an auditable event so name+size-only backfills
|
||||
// can be reviewed after the fact.
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "INTEGRITY_AUDIT",
|
||||
severity: "INFO",
|
||||
title: `Provenance backfilled by name+size: ${args.fileName}`,
|
||||
message: `Package ${chosen.id} was matched to a scanned source message by file name and size only (no CRC fingerprint confirmation).`,
|
||||
context: {
|
||||
packageId: chosen.id,
|
||||
fileName: args.fileName,
|
||||
sourceChannelId: args.scannedSourceChannelId,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ candidateId: chosen.id, fileName: args.fileName, confidence, source: args.scannedSourceChannelId },
|
||||
"provenance backfilled",
|
||||
);
|
||||
return { backfilled: true, confidence };
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { open } from "fs/promises";
|
||||
import type { Client } from "tdl";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { withFloodWait } from "../util/retry.js";
|
||||
|
||||
const log = childLogger("range-download");
|
||||
const RANGE_TIMEOUT_MS = 120_000;
|
||||
|
||||
// NOTE (from Task 4 spike): TDLib writes the requested region into
|
||||
// file.local.path at its ABSOLUTE file offset; file.local.downloaded_prefix_size
|
||||
// counts contiguous bytes from download_offset. We request a 1KB-aligned offset
|
||||
// so downloaded_prefix_size covers our whole [offset, offset+limit) window.
|
||||
// This absolute-offset assumption is PENDING LIVE VERIFICATION ON DEPLOY —
|
||||
// the authenticated TDLib session could not be spiked in this environment.
|
||||
export async function downloadFileRange(
|
||||
client: Client,
|
||||
fileId: string,
|
||||
offset: number,
|
||||
limit: number,
|
||||
expectedSize: bigint,
|
||||
): Promise<Buffer> {
|
||||
const numericId = parseInt(fileId, 10);
|
||||
const alignedOffset = Math.max(0, offset - (offset % 1024));
|
||||
const alignedLimit = limit + (offset - alignedOffset);
|
||||
|
||||
const file = await withFloodWait(
|
||||
() =>
|
||||
new Promise<{ local: { path: string; download_offset: number; downloaded_prefix_size: number } }>(
|
||||
(resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`Range download timed out for ${fileId}`)), RANGE_TIMEOUT_MS);
|
||||
client
|
||||
.invoke({
|
||||
_: "downloadFile",
|
||||
file_id: numericId,
|
||||
priority: 1,
|
||||
offset: alignedOffset,
|
||||
limit: alignedLimit,
|
||||
synchronous: true,
|
||||
} as never)
|
||||
.then((f) => { clearTimeout(timer); resolve(f as never); })
|
||||
.catch((e) => { clearTimeout(timer); reject(e); });
|
||||
},
|
||||
),
|
||||
`downloadFileRange:${fileId}`,
|
||||
);
|
||||
|
||||
const start = offset;
|
||||
const fh = await open(file.local.path, "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(limit);
|
||||
const { bytesRead } = await fh.read(buf, 0, limit, start);
|
||||
log.debug({ fileId, offset, limit, bytesRead }, "range read");
|
||||
return bytesRead < limit ? buf.subarray(0, bytesRead) : buf;
|
||||
} finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
+61
-29
@@ -63,9 +63,10 @@ import { hashParts } from "./archive/hash.js";
|
||||
import { readZipCentralDirectory } from "./archive/zip-reader.js";
|
||||
import { readRarContents } from "./archive/rar-reader.js";
|
||||
import { read7zContents } from "./archive/sevenz-reader.js";
|
||||
import { tryProvenanceBackfill } from "./provenance-backfill.js";
|
||||
import { byteLevelSplit, concatenateFiles } from "./archive/split.js";
|
||||
import { uploadToChannel, UploadStallError } from "./upload/channel.js";
|
||||
import { processAlbumGroups, processRuleBasedGroups, processTimeWindowGroups, processPatternGroups, processCreatorGroups, processZipPathGroups, processReplyChainGroups, processCaptionGroups, detectGroupingConflicts, type IndexedPackageRef } from "./grouping.js";
|
||||
import { processAlbumGroups, detectGroupingConflicts, type IndexedPackageRef } from "./grouping.js";
|
||||
import { db } from "./db/client.js";
|
||||
import type { TelegramAccount, TelegramChannel } from "@prisma/client";
|
||||
import type { Client } from "tdl";
|
||||
@@ -314,6 +315,7 @@ interface PipelineContext {
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
zipsBackfilled: number;
|
||||
};
|
||||
/** Creator from forum topic name (null for non-forum). */
|
||||
topicCreator: string | null;
|
||||
@@ -422,6 +424,7 @@ export async function runWorkerForAccount(
|
||||
zipsFound: 0,
|
||||
zipsDuplicate: 0,
|
||||
zipsIngested: 0,
|
||||
zipsBackfilled: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -1479,34 +1482,13 @@ async function processArchiveSets(
|
||||
scanResult.photos
|
||||
);
|
||||
|
||||
// Auto-grouping passes (gated by per-channel flag)
|
||||
const channelRecord = await db.telegramChannel.findUnique({
|
||||
where: { id: channel.id },
|
||||
select: { autoGroupEnabled: true },
|
||||
});
|
||||
|
||||
if (channelRecord?.autoGroupEnabled !== false) {
|
||||
// Learned rule-based grouping (from manual overrides)
|
||||
await processRuleBasedGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// Time-window grouping for remaining ungrouped packages
|
||||
await processTimeWindowGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// Pattern-based grouping (date patterns, project slugs)
|
||||
await processPatternGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// Creator-based grouping (3+ files from same creator)
|
||||
await processCreatorGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// ZIP path prefix grouping (shared root folder inside archives)
|
||||
await processZipPathGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// Reply chain grouping (messages replying to same root)
|
||||
await processReplyChainGroups(channel.id, indexedPackageRefs);
|
||||
|
||||
// Caption fuzzy match grouping
|
||||
await processCaptionGroups(channel.id, indexedPackageRefs);
|
||||
}
|
||||
// Heuristic auto-grouping passes (rule/time/pattern/creator/zip-path/
|
||||
// reply-chain/caption) were removed: the STL view is now a flat list
|
||||
// organized by the creator filter, so automatically inventing groups at
|
||||
// ingestion is no longer wanted. Album grouping above is kept because it
|
||||
// reflects real upload structure (files posted together as one Telegram
|
||||
// album), not a heuristic guess. Existing groups and the manual grouping
|
||||
// actions in the UI are unaffected.
|
||||
|
||||
// Check for potential grouping conflicts
|
||||
await detectGroupingConflicts(channel.id, indexedPackageRefs);
|
||||
@@ -1666,6 +1648,56 @@ async function processOneArchiveSet(
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Cross-channel provenance backfill ──
|
||||
// The same-channel checks above missed. Before downloading, see if this
|
||||
// archive is the true origin of a placeholder-source package (manual upload
|
||||
// / rebuild record whose sourceChannelId == destChannelId). If so, backfill
|
||||
// its real provenance and skip the download entirely.
|
||||
const archType = archiveSet.type === "7Z" ? "SEVEN_Z" : archiveSet.type;
|
||||
if (destChannelId && (archType === "ZIP" || archType === "RAR" || archType === "SEVEN_Z")) {
|
||||
try {
|
||||
const derivedCreator =
|
||||
topicCreator && topicCreator !== "General"
|
||||
? topicCreator
|
||||
: (extractCreatorFromFileName(archiveName) ?? topicCreator ?? null);
|
||||
const preview = previewMatches.get(archiveSet.baseName);
|
||||
const result = await tryProvenanceBackfill({
|
||||
client,
|
||||
destChannelId,
|
||||
scannedSourceChannelId: channel.id,
|
||||
fileName: archiveName,
|
||||
fileSize: totalArchiveSize,
|
||||
archiveType: archType,
|
||||
sourceMessageId: archiveSet.parts[0].id,
|
||||
sourceTopicId,
|
||||
sourceCaption: archiveSet.parts[0].caption ?? null,
|
||||
remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null,
|
||||
creator: derivedCreator,
|
||||
scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize, fileName: p.fileName })),
|
||||
previewData: null,
|
||||
previewMsgId: preview?.id ?? null,
|
||||
});
|
||||
if (result.backfilled) {
|
||||
counters.zipsBackfilled++;
|
||||
accountLog.info(
|
||||
{ fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id), confidence: result.confidence },
|
||||
"Backfilled provenance for placeholder package — skipping download",
|
||||
);
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Backfilled provenance for ${archiveName}`,
|
||||
currentStep: "backfilling",
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsBackfilled: counters.zipsBackfilled,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
} catch (err) {
|
||||
accountLog.warn({ err, fileName: archiveName }, "Provenance backfill attempt failed (non-fatal), continuing to normal ingestion");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Size guard: skip archives that exceed WORKER_MAX_ZIP_SIZE_MB ──
|
||||
const maxSizeBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n;
|
||||
if (totalArchiveSize > maxSizeBytes) {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user