diff --git a/.superpowers/sdd/final-review-fix-report.md b/.superpowers/sdd/final-review-fix-report.md new file mode 100644 index 0000000..489b639 --- /dev/null +++ b/.superpowers/sdd/final-review-fix-report.md @@ -0,0 +1,37 @@ +# Backup Final Review Fix Report + +## Implemented findings + +- The backup container now validates that `RESTIC_REPOSITORY` resolves strictly + below `/backup`, verifies that the Restic repository already has a readable + configuration before a scheduled backup, and gives the explicit first-run + initialization command when that preflight fails. +- The backup manifest now records every successfully applied Prisma migration + as a JSON object with its name and UTC completion timestamp. The `psql` + query uses the existing `DATABASE_URL` connection configuration and stops on + query errors. +- Restore now requires `BACKUP_REPOSITORY`, validates that it resolves strictly + below `/backup`, and validates the configured backup mount (including the + existing writable probe) before `restore-live` can stop services or replace + live data. +- The backup runbook now documents explicit repository initialization, monthly + full-read Restic checks, a disposable restore/checksum rehearsal, and + post-restore Compose status/log checks. + +## Verification + +- `bash -n scripts/backup/container-entrypoint.sh` +- `bash -n scripts/backup/restore.sh` +- Focused `rg` assertions for repository validation/preflight, migration + timestamp metadata, restore mount validation, initialization, maintenance, + and post-restore runbook commands. +- `npx prisma validate` +- `git diff --check` + +## Scope and concerns + +- No Docker, NAS, systemd, Restic repository, or live restore was run, per the + bounded review scope. The command-level behavior is therefore statically + validated only. +- Existing durable STL handling, the exact live-restore confirmation flag, and + rollback/safety-artifact behavior were retained. diff --git a/scripts/backup/README.md b/scripts/backup/README.md index 00f51b1..de82a69 100644 --- a/scripts/backup/README.md +++ b/scripts/backup/README.md @@ -113,7 +113,20 @@ the Synology share. `BACKUP_APP_VERSION` is optional metadata. The production Compose environment must also retain its existing database and application secrets; do not add any secrets to Git. -## 3. Install the nightly systemd job +## 3. Initialize the Restic repository once + +Before enabling the scheduled job, initialize a new repository explicitly: + +```bash +docker compose --profile backup run --rm backup init +``` + +The backup container preflight refuses to create a repository implicitly. If a +scheduled backup reports that the repository is unavailable or uninitialized, +verify the NFS mount, `BACKUP_REPOSITORY`, and password file before running the +explicit initialization command for an intended new repository. + +## 4. Install the nightly systemd job The supplied unit assumes the production Compose checkout is `/opt/stacks/DragonsStash`. If your deployment lives elsewhere, update the @@ -136,7 +149,7 @@ catches up after downtime. The first run can take a long time because it uploads all existing STL and session data. Later Restic snapshots deduplicate unchanged data. -## 4. Monitor and maintain backups +## 5. Monitor and maintain backups Inspect the next scheduled run and the last service result: @@ -167,7 +180,29 @@ Choose `BACKUP_RETENTION_DAYS` based on storage capacity and the recovery window you need. Watch Synology capacity and investigate any failed timer or service promptly. -## 5. Restore modes +At least monthly, perform a full repository read check: + +```bash +docker compose --profile backup run --rm backup check --read-data +``` + +Also rehearse recovery using a disposable staging directory and verify the +database dump checksum before deleting the rehearsal directory: + +```bash +REHEARSAL_DIR=/var/lib/dragons-stash/backup-staging/monthly-rehearsal-SNAPSHOT_ID +./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID "$REHEARSAL_DIR" +( + cd "$REHEARSAL_DIR"/staging/backup-* + sha256sum --check manifest/database.dump.sha256 +) +``` + +The staging restore also verifies the custom PostgreSQL dump and required data +directories. Inspect the restored data as appropriate, then remove the +disposable directory using your approved host cleanup procedure. + +## 6. Restore modes Run restore commands from the production Compose checkout after loading the same backup environment used by systemd (for example, as root with @@ -190,6 +225,14 @@ safety artifacts and staging directory, and attempts rollback after replacement has begun. Review the reported paths and service health before manually starting services. +After a successful live restore, confirm the services are running and inspect +their recent logs before declaring recovery complete: + +```bash +docker compose ps +docker compose logs --tail=100 app worker bot +``` + For a non-destructive recovery rehearsal, choose a snapshot from `list` and restore it to a fresh staging directory, for example: diff --git a/scripts/backup/container-entrypoint.sh b/scripts/backup/container-entrypoint.sh index b6571a1..9a8bbd2 100644 --- a/scripts/backup/container-entrypoint.sh +++ b/scripts/backup/container-entrypoint.sh @@ -27,6 +27,40 @@ require_directory() { fi } +validate_restic_repository() { + local canonical_repository + + canonical_repository="$(realpath -m -- "$RESTIC_REPOSITORY")" + if [[ "$canonical_repository" == "$BACKUP_ROOT" || "$canonical_repository" != "$BACKUP_ROOT"/* ]]; then + printf 'RESTIC_REPOSITORY must be strictly below /backup; got %s.\n' "$RESTIC_REPOSITORY" >&2 + return 1 + fi + + RESTIC_REPOSITORY="$canonical_repository" + export RESTIC_REPOSITORY +} + +validate_restic_configuration() { + require_value RESTIC_REPOSITORY + require_value RESTIC_PASSWORD_FILE + + if [[ ! -r "$RESTIC_PASSWORD_FILE" ]]; then + printf 'Restic password file %s is not readable.\n' "$RESTIC_PASSWORD_FILE" >&2 + return 1 + fi + + require_directory "$BACKUP_ROOT" + validate_restic_repository +} + +ensure_repository_initialized() { + if ! restic cat config >/dev/null; then + printf 'Restic repository %s is not initialized or could not be opened. Verify the backup mount, repository path, and password.\n' "$RESTIC_REPOSITORY" >&2 + printf 'For an intended first-time repository, initialize it explicitly with: docker compose --profile backup run --rm backup init\n' >&2 + return 1 + fi +} + json_escape() { local input="$1" local output="" @@ -72,22 +106,16 @@ run_backup() { local timestamp local checksum local app_version + local applied_migrations require_value DATABASE_URL - require_value RESTIC_REPOSITORY - require_value RESTIC_PASSWORD_FILE require_value BACKUP_RETENTION_DAYS - - if [[ ! -r "$RESTIC_PASSWORD_FILE" ]]; then - printf 'Restic password file %s is not readable.\n' "$RESTIC_PASSWORD_FILE" >&2 - return 1 - fi - - require_directory "$BACKUP_ROOT" + validate_restic_configuration require_directory "$STAGING_ROOT" require_directory "$UPLOADS_PATH" require_directory "$TDLIB_WORKER_PATH" require_directory "$TDLIB_BOT_PATH" + ensure_repository_initialized timestamp="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" RUN_DIR="$STAGING_ROOT/backup-${timestamp//[:]/}-$$" @@ -105,6 +133,26 @@ run_backup() { sha256sum "$RUN_DIR/database.dump" > "$RUN_DIR/manifest/database.dump.sha256" checksum="$(awk '{print $1}' "$RUN_DIR/manifest/database.dump.sha256")" app_version="${BACKUP_APP_VERSION:-unknown}" + applied_migrations="$( + psql --dbname "$DATABASE_URL" --no-psqlrc --tuples-only --no-align --quiet \ + --set ON_ERROR_STOP=on <<'SQL' + SELECT COALESCE( + json_agg( + json_build_object( + 'name', "migration_name", + 'finishedAtUtc', to_char( + "finished_at" AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) + ) + ORDER BY "finished_at", "migration_name" + ), + '[]'::json + )::text + FROM "_prisma_migrations" + WHERE "finished_at" IS NOT NULL AND "rolled_back_at" IS NULL; +SQL + )" cat > "$RUN_DIR/manifest/backup-manifest.json" <&2 return 1 fi + validate_backup_repository +} + +validate_backup_repository() { + local repository="$BACKUP_REPOSITORY" + local canonical_repository + + canonical_repository="$(realpath -ms -- "$repository")" + if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then + printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2 + return 1 + fi +} + +validate_backup_mount() { + local probe_file + + if ! mountpoint --q "$BACKUP_MOUNT_PATH"; then + printf 'Backup mount %s is not an active mountpoint.\n' "$BACKUP_MOUNT_PATH" >&2 + return 1 + fi + + if ! probe_file="$(mktemp "$BACKUP_MOUNT_PATH/.dragons-stash-backup-write-probe.XXXXXX")"; then + printf 'Backup mount %s is not writable.\n' "$BACKUP_MOUNT_PATH" >&2 + return 1 + fi + + if ! rm -f -- "$probe_file"; then + printf 'Unable to remove writable probe %s.\n' "$probe_file" >&2 + return 1 + fi } backup_restic() { @@ -352,6 +385,7 @@ restore_live() { local timestamp local container_staging_dir validate_environment + validate_backup_mount docker compose --profile backup config --quiet project_name="$(compose_project_name)" verify_snapshot "$snapshot_id" diff --git a/scripts/backup/run-backup.sh b/scripts/backup/run-backup.sh index 074248a..d0e7909 100755 --- a/scripts/backup/run-backup.sh +++ b/scripts/backup/run-backup.sh @@ -2,6 +2,7 @@ set -Eeuo pipefail readonly LOCK_FILE="/run/lock/dragons-stash-backup.lock" +readonly BACKUP_CONTAINER_ROOT="/backup" readonly -a MANAGED_SERVICES=(app worker bot) declare -a RUNNING_SERVICES=() @@ -22,6 +23,17 @@ validate_environment() { require_value BACKUP_RETENTION_DAYS } +validate_backup_repository() { + local repository="${BACKUP_REPOSITORY:-/backup/restic}" + local canonical_repository + + canonical_repository="$(realpath -ms -- "$repository")" + if [[ "$canonical_repository" == "$BACKUP_CONTAINER_ROOT" || "$canonical_repository" != "$BACKUP_CONTAINER_ROOT"/* ]]; then + printf 'BACKUP_REPOSITORY must be strictly below /backup; got %s.\n' "$repository" >&2 + return 1 + fi +} + validate_backup_mount() { local probe_file @@ -73,6 +85,7 @@ restart_running_services() { main() { validate_environment + validate_backup_repository exec 9>"$LOCK_FILE" if ! flock -n 9; then