diff --git a/.superpowers/sdd/scope-correction-implementation-report.md b/.superpowers/sdd/scope-correction-implementation-report.md new file mode 100644 index 0000000..adcbccf --- /dev/null +++ b/.superpowers/sdd/scope-correction-implementation-report.md @@ -0,0 +1,88 @@ +# Scope correction implementation report + +## Summary + +Implemented the approved backup scope correction for Dragon's Stash. Backups and restores now cover only: + +- PostgreSQL logical custom-format dump plus manifest/migration metadata. +- `tdlib_state` worker Telegram session volume. +- `tdlib_bot_state` bot Telegram session volume. + +The implementation no longer treats `manual_uploads`, completed local STL binaries, or `tmp_zips` as protected backup data. Future channel forwarding and archive/STL-content integrity auditing remain out of scope. + +## Changed files + +- `docker-compose.yml` + - Removed the backup service's read-only `manual_uploads:/data/uploads` mount. + - Kept the normal operational app/worker `manual_uploads` mounts. + - Kept both TDLib backup mounts. + +- `scripts/backup/container-entrypoint.sh` + - Removed `/data/uploads` as a required mounted directory. + - Removed uploads from the manifest `volumePaths`. + - Removed the `source:uploads` Restic tag. + - Removed uploads from the Restic source list. + - Kept database dump, manifest, worker TDLib, and bot TDLib sources/tags. + +- `scripts/backup/restore.sh` + - Removed restored uploads variables. + - Removed uploads staging validation. + - Removed manual uploads volume discovery, safety archive, replacement, and rollback. + - Removed retained/manual upload file-path verification. + - Removed temporary database verification that existed only for local upload file references. + - Preserved guarded `restore-live` confirmation, backup mount/repository checks, service stop/start handling, safety PostgreSQL dump, TDLib volume safety archives, TDLib volume replacement/rollback, and `pg_restore --list`/`pg_restore --exit-on-error` validation. + +- `prisma/schema.prisma` + - Removed `ManualUploadFile.retainedAt`. + +- `prisma/migrations/20260722100000_remove_retained_manual_files/migration.sql` + - Added forward migration: `ALTER TABLE "manual_upload_files" DROP COLUMN IF EXISTS "retainedAt";` + - Preserved the existing committed migration that added `retainedAt`. + +- `src/app/api/uploads/route.ts` + - Removed `retainedAt: new Date()` from manual upload file creation. + +- `worker/src/manual-upload.ts` + - Restored final best-effort cleanup of `/data/uploads/` using the older `path.join("/data/uploads", uploadId)` behavior. + +- `scripts/backup/README.md` + - Rewrote backup set and restore rehearsal docs around PostgreSQL plus both TDLib volumes only. + - Removed local STL file, retainedAt, upload path, retained file reference, and restored checksum checks. + - Clarified that STL binaries remain in Telegram and recovery preserves database mappings/Telegram IDs. + - Kept monthly `restic check --read-data` and disposable restore rehearsal runbook. + - Explicitly left future channel forwarding and archive/STL-content integrity auditing out of scope. + +- `README.md` + - Updated the production backup summary to name PostgreSQL logical dump plus Telegram session volumes as the protected set. + - Clarified that `manual_uploads` and temporary ZIPs are excluded and STL binaries remain in Telegram. + +## Verification + +- `git diff --check` + - Passed. + +- `bash -n scripts/backup/container-entrypoint.sh scripts/backup/run-backup.sh scripts/backup/restore.sh` + - Local `bash` failed because Windows only had the WSL shim and no installed WSL distribution. + - Passed via Docker fallback: + `docker run --rm --entrypoint bash -v E:\Projects\DragonsStash:/work:ro -w /work postgres:16-alpine -n scripts/backup/container-entrypoint.sh scripts/backup/run-backup.sh scripts/backup/restore.sh` + +- `npx prisma validate` + - Passed. + +- `npm run build` + - Passed. + +- `cd worker && npm run build` + - Passed. + +- Focused backup/restore scope assertions + - Backup shell paths assertion passed: no `manual_uploads`, `/data/uploads`, `retainedAt`, upload source tag, or upload-restore helper references in `scripts/backup/*.sh`. + - Compose backup service assertion passed: no `manual_uploads`, `/data/uploads`, `retainedAt`, or `source:uploads` in the `backup` service block. + - Active retainedAt assertion passed: no `retainedAt` in active Prisma schema, upload API, worker source, or backup shell scripts. + - Active backup/restore upload-source assertion passed: no `manual_uploads`, `/data/uploads`, or `data/uploads` in backup/restore shell scripts. + - Remaining expected matches are limited to normal operational app/worker upload mounts and paths, docs stating exclusions, and the historical add/drop migrations. + +## Concerns + +- None for implementation scope. +- Environment note: local Bash is unavailable because WSL has no installed distribution; Bash syntax was verified inside Docker instead. diff --git a/README.md b/README.md index 26400ba..8efb7a2 100644 --- a/README.md +++ b/README.md @@ -142,11 +142,14 @@ docker compose --profile bot up -d ### Production Backups -Docker volumes are not backups. Production backups protect PostgreSQL, durable -STL uploads, and the worker and bot Telegram session volumes in an encrypted -Restic repository on a Synology NFS share. See the [backup and recovery -guide](scripts/backup/README.md) for Synology setup, secrets, systemd -installation, monitoring, retention, and guarded restore procedures. +Docker volumes are not backups. Production backups protect a PostgreSQL +logical dump plus the worker and bot Telegram session volumes in an encrypted +Restic repository on a Synology NFS share. `manual_uploads` and temporary ZIP +processing data are excluded; STL binaries remain in Telegram, while the +database mappings and Telegram IDs are what recovery preserves for lookup and +delivery. See the [backup and recovery guide](scripts/backup/README.md) for +Synology setup, secrets, systemd installation, monitoring, retention, and +guarded restore procedures. ### Seeding the Database diff --git a/docker-compose.yml b/docker-compose.yml index 9364fe5..2e7363a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,7 +110,6 @@ services: BACKUP_APP_VERSION: ${BACKUP_APP_VERSION:-unknown} user: "0:0" volumes: - - manual_uploads:/data/uploads:ro - tdlib_state:/data/tdlib-worker:ro - tdlib_bot_state:/data/tdlib-bot:ro - ${BACKUP_MOUNT_PATH:?Set BACKUP_MOUNT_PATH to the mounted Synology share}:/backup:rw diff --git a/prisma/migrations/20260722100000_remove_retained_manual_files/migration.sql b/prisma/migrations/20260722100000_remove_retained_manual_files/migration.sql new file mode 100644 index 0000000..7d5363d --- /dev/null +++ b/prisma/migrations/20260722100000_remove_retained_manual_files/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "manual_upload_files" DROP COLUMN IF EXISTS "retainedAt"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index ce14213..679a74e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -937,7 +937,6 @@ model ManualUploadFile { filePath String // Path on shared volume fileSize BigInt packageId String? // Set after processing - retainedAt DateTime? upload ManualUpload @relation(fields: [uploadId], references: [id], onDelete: Cascade) diff --git a/scripts/backup/README.md b/scripts/backup/README.md index 1dd5154..00f2d95 100644 --- a/scripts/backup/README.md +++ b/scripts/backup/README.md @@ -1,9 +1,9 @@ # Production backup and recovery -Dragon's Stash backs up its PostgreSQL database, durable STL uploads, and the -worker and bot Telegram (TDLib) session volumes to an encrypted Restic -repository on a Synology NFS share. Docker volumes are local runtime storage; -they are not, by themselves, backups. +Dragon's Stash backs up its PostgreSQL database and the worker/bot Telegram +(TDLib) session volumes to an encrypted Restic repository on a Synology NFS +share. Docker volumes are local runtime storage; they are not, by themselves, +backups. The backup job runs as `root` because it reads the Restic password file and mounts host paths into the backup container. Keep the Synology share on the @@ -14,16 +14,19 @@ private network. Do **not** expose NFS to the Internet. Each snapshot contains: - a custom-format PostgreSQL dump; -- the `manual_uploads` volume at `/data/uploads` (including newly completed - manual STL uploads); +- a backup manifest with dump checksum, migration metadata, version metadata, + and captured source paths; - the worker TDLib session volume; and - the bot TDLib session volume. -New manual-upload STL files are retained in `manual_uploads` and can therefore -be included in future backups. STL files that an older worker run already -deleted cannot be recovered by this backup feature. Their legacy database rows -are reported as warnings during restore validation; only files marked as -retained must be present in a restored snapshot. +The `manual_uploads` and `tmp_zips` volumes are excluded. Completed STL +binaries are not retained as a local recovery set by this backup feature; they +remain in Telegram. Recovery preserves the PostgreSQL archive, message, +package, file, Telegram channel, and Telegram message ID mappings needed for +normal lookup and delivery after restore. + +Future Telegram channel-forwarding behavior and archive/STL-content integrity +auditing are intentionally outside this backup and restore procedure. ## 1. Configure the Synology NFS share @@ -36,8 +39,8 @@ On the Synology DSM host: Docker host's fixed private-network IP address. Grant read/write access. Use the least permissive squash and authentication settings that work for the root-run backup job, and do not use a broad subnet or public address. -4. Record the NFS export path shown by DSM (for example, - `/volume1/dragonsstash-backups`). +4. Record the NFS export path shown by DSM, for example + `/volume1/dragonsstash-backups`. On the Docker host, install the NFS client package for its distribution, create the mountpoint, and mount the export. Replace `NAS_IP` and the export path with @@ -95,8 +98,7 @@ sudo install -m 0600 -o root -g root /dev/null /etc/dragons-stash/backup.env sudoedit /etc/dragons-stash/backup.env ``` -Set these production values (replace the example paths and retention period as -needed): +Set these production values, replacing paths and retention as needed: ```dotenv BACKUP_MOUNT_PATH=/mnt/dragonsstash-backups @@ -111,7 +113,7 @@ BACKUP_APP_VERSION=unknown share is mounted at `/backup`; `/backup/restic` keeps the Restic repository in the Synology share. `BACKUP_APP_VERSION` is optional metadata. The production Compose environment must also retain its existing database and application -secrets; do not add any secrets to Git. +secrets; do not add secrets to Git. ## 3. Initialize the Restic repository once @@ -145,9 +147,9 @@ sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager ``` The timer runs nightly at 03:00 with up to 15 minutes of randomized delay and -catches up after downtime. The first run can take a long time because it uploads -all existing STL and session data. Later Restic snapshots deduplicate unchanged -data. +catches up after downtime. The first run captures PostgreSQL and both TDLib +session volumes. Later Restic snapshots deduplicate unchanged session state and +database dump content. ## 5. Monitor and maintain backups @@ -161,7 +163,7 @@ sudo journalctl -u dragons-stash-backup.service -n 100 --no-pager sudo systemctl --failed ``` -List the snapshots through the configured backup container: +List snapshots through the configured backup container: ```bash ./scripts/backup/restore.sh list @@ -186,24 +188,21 @@ At least monthly, perform a full repository read check: docker compose --profile backup run --rm backup check --read-data ``` -### Monthly disposable recovery rehearsal +## 6. Monthly disposable recovery rehearsal Perform the following procedure at least monthly. It restores one snapshot into -a unique, disposable Compose project, so the database and all Compose volumes -are isolated from production. **Never use the production Compose project name, -production volume names, or `restore-live` for this rehearsal.** In particular, -do not run `docker compose down -v` without the explicit rehearsal -`--project-name` shown below. +a unique, disposable Compose project, so the database and Compose volumes are +isolated from production. **Never use the production Compose project name, +production volume names, or `restore-live` for this rehearsal.** Do not run +`docker compose down -v` without the explicit rehearsal `--project-name` shown +below. Run these commands from the production Compose checkout as an operator allowed -to read `/etc/dragons-stash/backup.env`. They use the configuration required -for a separate application stack; do not expose its published port beyond the -host. The worker and bot are deliberately replaced with inert processes, so -this validates their restored images and volumes without executing Telegram -clients or using production Telegram credentials. First select a snapshot and -set the expected values for a known retained STL that was recorded when the -backup was made. `EXPECTED_FILE_PATH` must be the database value under -`/data/uploads`, and `EXPECTED_FILE_SIZE` is bytes. +to read `/etc/dragons-stash/backup.env`. They use a separate application stack; +do not expose its published port beyond the host. The worker and bot are +deliberately replaced with inert processes, so this validates restored images, +database state, and TDLib volume layout without executing Telegram clients or +using production Telegram credentials. ```bash set -Eeuo pipefail @@ -220,18 +219,16 @@ REHEARSAL_ENV="$REHEARSAL_DIR/compose.env" REHEARSAL_OVERRIDE="$REHEARSAL_DIR/compose.rehearsal.override.yml" REHEARSAL_APP_PORT=13000 # Choose an unused host-local port. -EXPECTED_UPLOAD_ID=RECORDED_UPLOAD_ID -EXPECTED_UPLOAD_STATUS=COMPLETED -EXPECTED_FILE_NAME=RECORDED_FILENAME.stl -EXPECTED_FILE_PATH=/data/uploads/RECORDED_RELATIVE_PATH.stl -EXPECTED_FILE_SIZE=RECORDED_SIZE_IN_BYTES -EXPECTED_SHA256=RECORDED_SHA256 - ./scripts/backup/restore.sh verify "$SNAPSHOT_ID" ./scripts/backup/restore.sh restore-to-staging "$SNAPSHOT_ID" "$REHEARSAL_DIR" RESTORE_ROOT="$(printf '%s\n' "$REHEARSAL_DIR"/staging/backup-*)" sha256sum --check "$RESTORE_ROOT/manifest/database.dump.sha256" +test -s "$RESTORE_ROOT/database.dump" +test -s "$RESTORE_ROOT/manifest/backup-manifest.json" +test -d "$REHEARSAL_DIR/data/tdlib-worker" +test -d "$REHEARSAL_DIR/data/tdlib-bot" + umask 077 cp .env "$REHEARSAL_ENV" printf '\nAPP_PORT=%s\nNEXT_PUBLIC_APP_URL=http://localhost:%s\n' \ @@ -257,13 +254,13 @@ EOF Confirm that `RESTORE_ROOT` names exactly one `backup-*` directory before continuing. The staging restore has already verified the custom PostgreSQL dump -and restored `data/uploads`, `data/tdlib-worker`, and `data/tdlib-bot`. +and restored `data/tdlib-worker` and `data/tdlib-bot`. Create the isolated project and volumes, start only its database, then import the dump. `compose_rehearsal()` always applies the disposable override created above: worker and bot retain their restored images and volumes but have their Telegram credential variables blanked and run only `sleep infinity`, never -Telegram clients. The `create` command makes the project-scoped application +Telegram clients. The `create` command makes project-scoped application volumes without starting app, worker, or bot. ```bash @@ -289,7 +286,7 @@ compose_rehearsal exec -T db pg_restore --no-owner --exit-on-error \ < "$RESTORE_ROOT/database.dump" ``` -Copy each restored file tree into its matching **rehearsal** volume. Each +Copy each restored TDLib tree into its matching **rehearsal** volume. Each target is new and empty; the function rejects an ambiguous volume lookup. ```bash @@ -307,16 +304,15 @@ restore_rehearsal_volume() { backup -ceu 'cp -a /restore-source/. /restore-target/' } -restore_rehearsal_volume manual_uploads "$REHEARSAL_DIR/data/uploads" restore_rehearsal_volume tdlib_state "$REHEARSAL_DIR/data/tdlib-worker" restore_rehearsal_volume tdlib_bot_state "$REHEARSAL_DIR/data/tdlib-bot" ``` Start the disposable app, worker, and bot containers. The app and database -perform their normal health and restored-data checks; the worker and bot remain -inert `sleep infinity` processes, so this does not execute Telegram clients or -send messages. Check the health endpoint, then retain the `ps` and log output -as rehearsal evidence. +perform their normal health checks; the worker and bot remain inert `sleep +infinity` processes, so this does not execute Telegram clients or send +messages. Check the health endpoint, then retain the `ps` and log output as +rehearsal evidence. ```bash compose_rehearsal --profile full up -d app worker bot @@ -326,40 +322,16 @@ compose_rehearsal --profile full ps compose_rehearsal --profile full logs --tail=100 app worker bot ``` -Validate that every retained database file reference has a restored file. Rows -whose `retainedAt` is `NULL` are legacy references and are warnings, not -failures. Then compare the recorded STL checksum and metadata with the -disposable database and volume; both commands must succeed. +Optionally spot-check the restored database metadata that lets Dragon's Stash +locate Telegram-hosted STL binaries after restore. This checks database +metadata and Telegram IDs only; it does not assert local STL file presence, +STL checksums, Telegram forwarding behavior, or archive/STL binary integrity. ```bash compose_rehearsal exec -T db psql --no-psqlrc --tuples-only --no-align --quiet \ - --field-separator=$'\t' --username "${POSTGRES_USER:-dragons}" \ + --username "${POSTGRES_USER:-dragons}" \ --dbname "${POSTGRES_DB:-dragonsstash}" \ - --command "SELECT CASE WHEN \"retainedAt\" IS NULL THEN 'legacy' ELSE 'retained' END, \"filePath\" FROM \"manual_upload_files\" ORDER BY 2" | - compose_rehearsal --profile backup run --rm --no-deps -T --entrypoint bash backup -ceu ' - missing=0 - while IFS="$(printf "\\t")" read -r retention file_path; do - if [[ "$retention" == legacy ]]; then - printf "Warning: legacy reference is not required: %s\\n" "$file_path" >&2 - elif [[ "$retention" != retained || "$file_path" != /data/uploads/* || ! -f "/data/uploads/${file_path#/data/uploads/}" ]]; then - printf "Missing or invalid retained upload: %s\\n" "$file_path" >&2 - missing=1 - fi - done - exit "$missing" - ' - -actual_sha256="$(compose_rehearsal --profile backup run --rm --no-deps \ - --entrypoint sha256sum backup "$EXPECTED_FILE_PATH" | awk '{print $1}')" -test "$actual_sha256" = "$EXPECTED_SHA256" - -metadata_rows="$(compose_rehearsal exec -T db psql --no-psqlrc --tuples-only \ - --no-align --quiet --username "${POSTGRES_USER:-dragons}" \ - --dbname "${POSTGRES_DB:-dragonsstash}" \ - --set="upload_id=$EXPECTED_UPLOAD_ID" --set="upload_status=$EXPECTED_UPLOAD_STATUS" \ - --set="file_name=$EXPECTED_FILE_NAME" --set="file_path=$EXPECTED_FILE_PATH" \ - --set="file_size=$EXPECTED_FILE_SIZE" --command "SELECT count(*) FROM \"manual_uploads\" u JOIN \"manual_upload_files\" f ON f.\"uploadId\" = u.id WHERE u.id = :'upload_id' AND u.status::text = :'upload_status' AND f.\"fileName\" = :'file_name' AND f.\"filePath\" = :'file_path' AND f.\"fileSize\" = :'file_size'::bigint AND f.\"retainedAt\" IS NOT NULL;")" -test "$metadata_rows" = 1 + --command 'SELECT count(*) FROM "packages" WHERE "destChannelId" IS NOT NULL AND "destMessageId" IS NOT NULL;' ``` After recording the evidence, destroy only the explicitly named disposable @@ -384,14 +356,14 @@ Operator: Snapshot ID: Snapshot backup date: Disposable Compose project: +Full restic check --read-data result: +Database dump manifest checksum result: +TDLib worker tree restored: +TDLib bot tree restored: Health endpoint result (HTTP/body): docker compose ps result: app/worker/bot log review result: -Retained-file reference validation result: -Known retained STL upload ID/path: -Expected SHA-256 / restored SHA-256: -Expected metadata (status, filename, size, retainedAt) / restored result: -Database dump manifest checksum result: +Database Telegram metadata/mapping spot-check result: Cleanup result (project and rehearsal volumes absent): Notes/caveats: ``` @@ -399,11 +371,11 @@ Notes/caveats: This document describes the procedure only; it has not been run by this documentation update. -## 6. Restore modes +## 7. Restore modes Run restore commands from the production Compose checkout after loading the -same backup environment used by systemd (for example, as root with -`/etc/dragons-stash/backup.env` exported). All restore staging directories must +same backup environment used by systemd, for example as root with +`/etc/dragons-stash/backup.env` exported. All restore staging directories must be children of `BACKUP_STAGING_PATH`. | Mode | Command | Effect | @@ -411,13 +383,13 @@ be children of `BACKUP_STAGING_PATH`. | List snapshots | `./scripts/backup/restore.sh list` | Lists available Restic snapshots. Does not stop services or change volumes. | | Verify a snapshot | `./scripts/backup/restore.sh verify SNAPSHOT_ID` | Confirms the snapshot exists and runs a Restic repository check. Does not change live data. | | Restore to staging | `./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR` | Restores and validates a snapshot in a new or empty child directory of `BACKUP_STAGING_PATH`. Does not stop services or change volumes. | -| Replace live data | `./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data` | Stops application services and replaces the PostgreSQL database plus all protected volumes after validation. | +| Replace live data | `./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data` | Stops application services and replaces the PostgreSQL database plus both TDLib session volumes after validation. | `restore-live` is destructive. It requires the exact `--confirm-replace-live-data` flag and should be used only after a successful staging restore has been inspected. It creates a safety database dump and -archives of the current protected volumes in local staging before replacement. -If a live restore fails, it leaves the application services stopped, retains the +archives of the current TDLib volumes in local staging before replacement. If a +live restore fails, it leaves the application services stopped, retains the safety artifacts and staging directory, and attempts rollback after replacement has begun. Review the reported paths and service health before manually starting services. @@ -438,6 +410,6 @@ restore it to a fresh staging directory, for example: ``` The restored tree must contain a non-empty PostgreSQL dump, its manifest, -uploads, worker TDLib state, and bot TDLib state. Live restore additionally -checks every retained manual-upload file reference against the staged uploads -before replacing any live volume. +worker TDLib state, and bot TDLib state. STL binaries remain in Telegram, and +the restored database mappings and Telegram IDs are what recovery preserves for +normal lookup and delivery. diff --git a/scripts/backup/container-entrypoint.sh b/scripts/backup/container-entrypoint.sh index 9a8bbd2..f27ccb9 100644 --- a/scripts/backup/container-entrypoint.sh +++ b/scripts/backup/container-entrypoint.sh @@ -3,7 +3,6 @@ set -Eeuo pipefail readonly BACKUP_ROOT="/backup" readonly STAGING_ROOT="/staging" -readonly UPLOADS_PATH="/data/uploads" readonly TDLIB_WORKER_PATH="/data/tdlib-worker" readonly TDLIB_BOT_PATH="/data/tdlib-bot" @@ -112,7 +111,6 @@ run_backup() { require_value BACKUP_RETENTION_DAYS validate_restic_configuration require_directory "$STAGING_ROOT" - require_directory "$UPLOADS_PATH" require_directory "$TDLIB_WORKER_PATH" require_directory "$TDLIB_BOT_PATH" ensure_repository_initialized @@ -166,7 +164,6 @@ SQL "sha256": "$(json_escape "$checksum")" }, "volumePaths": [ - "$(json_escape "$UPLOADS_PATH")", "$(json_escape "$TDLIB_WORKER_PATH")", "$(json_escape "$TDLIB_BOT_PATH")" ] @@ -176,12 +173,10 @@ EOF restic backup \ --tag "application:dragons-stash" \ --tag "source:database" \ - --tag "source:uploads" \ --tag "source:tdlib-worker" \ --tag "source:tdlib-bot" \ "$RUN_DIR/database.dump" \ "$RUN_DIR/manifest" \ - "$UPLOADS_PATH" \ "$TDLIB_WORKER_PATH" \ "$TDLIB_BOT_PATH" restic snapshots --latest 1 diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index b8e2164..8963aad 100755 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -7,15 +7,12 @@ readonly BACKUP_CONTAINER_ROOT="/backup" readonly -a LIVE_SERVICES=(app worker bot) RESTORED_DUMP="" -RESTORED_UPLOADS="" RESTORED_TDLIB_WORKER="" RESTORED_TDLIB_BOT="" LIVE_STAGING_DIR="" SAFETY_DUMP="" LIVE_RESTORE_ACTIVE=0 LIVE_REPLACEMENT_STARTED=0 -TEMP_VERIFY_DATABASE="" -LIVE_UPLOADS_VOLUME="" LIVE_WORKER_VOLUME="" LIVE_BOT_VOLUME="" @@ -147,11 +144,10 @@ validate_restored_tree() { backup_directory="${backup_directories[0]}" RESTORED_DUMP="$backup_directory/database.dump" manifest="$backup_directory/manifest/backup-manifest.json" - RESTORED_UPLOADS="$staging_dir/data/uploads" RESTORED_TDLIB_WORKER="$staging_dir/data/tdlib-worker" RESTORED_TDLIB_BOT="$staging_dir/data/tdlib-bot" - if [[ ! -s "$RESTORED_DUMP" || ! -s "$manifest" || ! -d "$RESTORED_UPLOADS" || ! -d "$RESTORED_TDLIB_WORKER" || ! -d "$RESTORED_TDLIB_BOT" ]]; then - printf 'Restored tree %s is incomplete; require staging/backup-*/database.dump, its manifest, data/uploads, data/tdlib-worker, and data/tdlib-bot.\n' "$staging_dir" >&2 + if [[ ! -s "$RESTORED_DUMP" || ! -s "$manifest" || ! -d "$RESTORED_TDLIB_WORKER" || ! -d "$RESTORED_TDLIB_BOT" ]]; then + printf 'Restored tree %s is incomplete; require staging/backup-*/database.dump, its manifest, data/tdlib-worker, and data/tdlib-bot.\n' "$staging_dir" >&2 return 1 fi verify_custom_dump "$RESTORED_DUMP" @@ -242,71 +238,6 @@ restore_database() { restore_database_dump "$RESTORED_DUMP" } -verify_file_references() { - local database_name="$1" - local uploads_source="$2" - local database_user="${POSTGRES_USER:-dragons}" - docker compose exec -T db psql --no-psqlrc --tuples-only --no-align --quiet \ - --field-separator=$'\t' \ - --username "$database_user" --dbname "$database_name" \ - --command "SELECT 'legacy', \"filePath\" FROM \"manual_upload_files\" WHERE \"retainedAt\" IS NULL - UNION ALL - SELECT 'retained', \"filePath\" FROM \"manual_upload_files\" WHERE \"retainedAt\" IS NOT NULL - ORDER BY 2" | - docker compose --profile backup run --rm --no-deps -T --entrypoint bash \ - -v "$uploads_source:/data/uploads:ro" backup -ceu ' - missing=0 - while IFS="$(printf "\t")" read -r retention file_path; do - if [[ "$retention" == "legacy" ]]; then - printf "Warning: legacy manual-upload file reference is not required because retainedAt is NULL: %s\\n" "$file_path" >&2 - continue - fi - if [[ "$retention" != "retained" ]]; then - printf "Unexpected retention status for database reference: %s\\n" "$file_path" >&2 - missing=1 - continue - fi - case "$file_path" in - /data/uploads/*) relative_path="${file_path#/data/uploads/}" ;; - *) - printf "Database reference is outside /data/uploads: %s\\n" "$file_path" >&2 - missing=1 - continue - ;; - esac - if [[ -z "$relative_path" || "$relative_path" == .. || "$relative_path" == ../* || "$relative_path" == */../* ]]; then - printf "Database reference has an invalid uploads path: %s\\n" "$file_path" >&2 - missing=1 - elif [[ ! -f "/data/uploads/$relative_path" ]]; then - printf "Missing restored upload for database reference: %s\\n" "$file_path" >&2 - missing=1 - fi - done - exit "$missing" - ' -} - -drop_temporary_verification_database() { - local database_user="${POSTGRES_USER:-dragons}" - - if [[ -n "$TEMP_VERIFY_DATABASE" ]]; then - docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$TEMP_VERIFY_DATABASE" - TEMP_VERIFY_DATABASE="" - fi -} - -verify_staged_snapshot_file_references() { - local database_user="${POSTGRES_USER:-dragons}" - - TEMP_VERIFY_DATABASE="dragons_restore_verify_$$_$(date +%s)" - docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$TEMP_VERIFY_DATABASE" - docker compose exec -T db createdb --username "$database_user" "$TEMP_VERIFY_DATABASE" - docker compose exec -T db pg_restore --no-owner --exit-on-error \ - --username "$database_user" --dbname "$TEMP_VERIFY_DATABASE" < "$RESTORED_DUMP" - verify_file_references "$TEMP_VERIFY_DATABASE" "$RESTORED_UPLOADS" - drop_temporary_verification_database -} - archive_live_volume() { local volume_name="$1" local logical_name="$2" @@ -358,14 +289,12 @@ live_restore_failure() { trap - EXIT if ((LIVE_RESTORE_ACTIVE)); then docker compose --profile full stop "${LIVE_SERVICES[@]}" || true - drop_temporary_verification_database || true if ((LIVE_REPLACEMENT_STARTED)); then - restore_live_volume_archive "$LIVE_UPLOADS_VOLUME" manual_uploads || rollback_ok=0 restore_live_volume_archive "$LIVE_WORKER_VOLUME" tdlib_state || rollback_ok=0 restore_live_volume_archive "$LIVE_BOT_VOLUME" tdlib_bot_state || rollback_ok=0 restore_database_dump "$SAFETY_DUMP" || rollback_ok=0 if ((rollback_ok)); then - printf 'Rollback restored the pre-restore database and all three Docker volumes. Services remain stopped.\n' >&2 + printf 'Rollback restored the pre-restore database and both TDLib Docker volumes. Services remain stopped.\n' >&2 else printf 'Rollback failed; services remain stopped. Restore the safety archives and database dump manually.\n' >&2 fi @@ -379,7 +308,6 @@ live_restore_failure() { restore_live() { local snapshot_id="$1" local project_name - local uploads_volume local worker_volume local bot_volume local timestamp @@ -401,18 +329,13 @@ restore_live() { create_safety_dump backup_restic restore "$snapshot_id" --target "$container_staging_dir" validate_restored_tree "$LIVE_STAGING_DIR" - uploads_volume="$(compose_volume_name "$project_name" manual_uploads)" worker_volume="$(compose_volume_name "$project_name" tdlib_state)" bot_volume="$(compose_volume_name "$project_name" tdlib_bot_state)" - LIVE_UPLOADS_VOLUME="$uploads_volume" LIVE_WORKER_VOLUME="$worker_volume" LIVE_BOT_VOLUME="$bot_volume" - archive_live_volume "$uploads_volume" manual_uploads archive_live_volume "$worker_volume" tdlib_state archive_live_volume "$bot_volume" tdlib_bot_state - verify_staged_snapshot_file_references LIVE_REPLACEMENT_STARTED=1 - replace_volume "$RESTORED_UPLOADS" "$uploads_volume" replace_volume "$RESTORED_TDLIB_WORKER" "$worker_volume" replace_volume "$RESTORED_TDLIB_BOT" "$bot_volume" restore_database diff --git a/src/app/api/uploads/route.ts b/src/app/api/uploads/route.ts index 4f9cc30..225abee 100644 --- a/src/app/api/uploads/route.ts +++ b/src/app/api/uploads/route.ts @@ -55,7 +55,6 @@ export async function POST(request: Request) { fileName: file.name, filePath, fileSize: BigInt(file.size), - retainedAt: new Date(), }, }); } diff --git a/worker/src/manual-upload.ts b/worker/src/manual-upload.ts index 1757261..3cef614 100644 --- a/worker/src/manual-upload.ts +++ b/worker/src/manual-upload.ts @@ -1,3 +1,4 @@ +import path from "path"; import { rm } from "fs/promises"; import { db } from "./db/client.js"; import { childLogger } from "./util/logger.js"; @@ -199,4 +200,12 @@ export async function processManualUpload(uploadId: string): Promise { data: { status: "FAILED", errorMessage: message }, }); } + + // Clean up uploaded files + try { + const uploadDir = path.join("/data/uploads", uploadId); + await rm(uploadDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup + } }