mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-09-21 13:31:42 +00:00
425 lines
15 KiB
Bash
Executable File
425 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
readonly CONFIRM_REPLACE_LIVE_DATA="--confirm-replace-live-data"
|
|
readonly STAGING_CONTAINER_ROOT="/staging"
|
|
readonly -a LIVE_SERVICES=(app worker bot)
|
|
|
|
RESTORED_DUMP=""
|
|
RESTORED_UPLOADS=""
|
|
RESTORED_TDLIB_WORKER=""
|
|
RESTORED_TDLIB_BOT=""
|
|
LIVE_STAGING_DIR=""
|
|
SAFETY_DUMP=""
|
|
LIVE_RESTORE_ACTIVE=0
|
|
LIVE_REPLACEMENT_STARTED=0
|
|
TEMP_VERIFY_DATABASE=""
|
|
LIVE_UPLOADS_VOLUME=""
|
|
LIVE_WORKER_VOLUME=""
|
|
LIVE_BOT_VOLUME=""
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
./scripts/backup/restore.sh list
|
|
./scripts/backup/restore.sh verify SNAPSHOT_ID
|
|
./scripts/backup/restore.sh restore-to-staging SNAPSHOT_ID STAGING_DIR
|
|
./scripts/backup/restore.sh restore-live SNAPSHOT_ID --confirm-replace-live-data
|
|
EOF
|
|
}
|
|
|
|
require_value() {
|
|
local name="$1"
|
|
if [[ -z "${!name:-}" ]]; then
|
|
printf 'Required environment variable %s is not set.\n' "$name" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
validate_environment() {
|
|
require_value BACKUP_MOUNT_PATH
|
|
require_value BACKUP_STAGING_PATH
|
|
require_value BACKUP_RESTIC_PASSWORD_FILE
|
|
if [[ ! -r "$BACKUP_RESTIC_PASSWORD_FILE" ]]; then
|
|
printf 'Restic password file %s is not readable.\n' "$BACKUP_RESTIC_PASSWORD_FILE" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
backup_restic() {
|
|
docker compose --profile backup run --rm --no-deps backup "$@"
|
|
}
|
|
|
|
canonical_staging_root() {
|
|
realpath -m -- "$BACKUP_STAGING_PATH"
|
|
}
|
|
|
|
validate_staging_directory() {
|
|
local staging_dir="$1"
|
|
local staging_root
|
|
local canonical_staging_dir
|
|
|
|
staging_root="$(canonical_staging_root)"
|
|
canonical_staging_dir="$(realpath -m -- "$staging_dir")"
|
|
if [[ "$canonical_staging_dir" == "$staging_root" || "$canonical_staging_dir" != "$staging_root"/* ]]; then
|
|
printf 'Staging directory %s must be inside BACKUP_STAGING_PATH (%s).\n' "$staging_dir" "$staging_root" >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$canonical_staging_dir"
|
|
}
|
|
|
|
prepare_fresh_staging_directory() {
|
|
local staging_dir="$1"
|
|
|
|
if [[ -e "$staging_dir" ]]; then
|
|
if [[ ! -d "$staging_dir" ]]; then
|
|
printf 'Staging target %s exists but is not a directory.\n' "$staging_dir" >&2
|
|
return 1
|
|
fi
|
|
if [[ -n "$(find "$staging_dir" -mindepth 1 -print -quit)" ]]; then
|
|
printf 'Staging target %s must be fresh and empty; remove stale artifacts first.\n' "$staging_dir" >&2
|
|
return 1
|
|
fi
|
|
else
|
|
mkdir -p -- "$staging_dir"
|
|
fi
|
|
}
|
|
|
|
container_staging_directory() {
|
|
local host_staging_dir="$1"
|
|
local staging_root
|
|
|
|
staging_root="$(canonical_staging_root)"
|
|
printf '%s/%s\n' "$STAGING_CONTAINER_ROOT" "${host_staging_dir#"$staging_root"/}"
|
|
}
|
|
|
|
verify_custom_dump() {
|
|
local dump_path="$1"
|
|
docker compose --profile backup run --rm --no-deps \
|
|
--entrypoint pg_restore \
|
|
-v "$dump_path:/restore/database.dump:ro" \
|
|
backup --list /restore/database.dump >/dev/null
|
|
}
|
|
|
|
validate_restored_tree() {
|
|
local staging_dir="$1"
|
|
local -a backup_directories=("$staging_dir"/staging/backup-*)
|
|
local backup_directory
|
|
local manifest
|
|
|
|
if ((${#backup_directories[@]} != 1)) || [[ ! -d "${backup_directories[0]:-}" ]]; then
|
|
printf 'Restored tree %s must contain exactly one staging/backup-* directory.\n' "$staging_dir" >&2
|
|
return 1
|
|
fi
|
|
backup_directory="${backup_directories[0]}"
|
|
RESTORED_DUMP="$backup_directory/database.dump"
|
|
manifest="$backup_directory/manifest/backup-manifest.json"
|
|
RESTORED_UPLOADS="$staging_dir/data/uploads"
|
|
RESTORED_TDLIB_WORKER="$staging_dir/data/tdlib-worker"
|
|
RESTORED_TDLIB_BOT="$staging_dir/data/tdlib-bot"
|
|
if [[ ! -s "$RESTORED_DUMP" || ! -s "$manifest" || ! -d "$RESTORED_UPLOADS" || ! -d "$RESTORED_TDLIB_WORKER" || ! -d "$RESTORED_TDLIB_BOT" ]]; then
|
|
printf 'Restored tree %s is incomplete; require staging/backup-*/database.dump, its manifest, data/uploads, data/tdlib-worker, and data/tdlib-bot.\n' "$staging_dir" >&2
|
|
return 1
|
|
fi
|
|
verify_custom_dump "$RESTORED_DUMP"
|
|
}
|
|
|
|
verify_snapshot() {
|
|
local snapshot_id="$1"
|
|
backup_restic snapshots "$snapshot_id"
|
|
backup_restic check
|
|
}
|
|
|
|
restore_to_staging() {
|
|
local snapshot_id="$1"
|
|
local requested_staging_dir="$2"
|
|
local staging_dir
|
|
local container_staging_dir
|
|
validate_environment
|
|
staging_dir="$(validate_staging_directory "$requested_staging_dir")"
|
|
container_staging_dir="$(container_staging_directory "$staging_dir")"
|
|
prepare_fresh_staging_directory "$staging_dir"
|
|
verify_snapshot "$snapshot_id"
|
|
backup_restic restore "$snapshot_id" --target "$container_staging_dir"
|
|
validate_restored_tree "$staging_dir"
|
|
printf 'Staging restore verified: %s\n' "$staging_dir"
|
|
}
|
|
|
|
compose_project_name() {
|
|
local project_name
|
|
project_name="$(docker compose config --format json | awk -F'"' '/^[[:space:]]*"name"[[:space:]]*:/ { print $4; exit }')"
|
|
if [[ -z "$project_name" ]]; then
|
|
printf '%s\n' 'Unable to determine the Docker Compose project name.' >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$project_name"
|
|
}
|
|
|
|
compose_volume_name() {
|
|
local project_name="$1"
|
|
local logical_name="$2"
|
|
local -a matches=()
|
|
mapfile -t matches < <(docker volume ls \
|
|
--filter "label=com.docker.compose.project=$project_name" \
|
|
--filter "label=com.docker.compose.volume=$logical_name" \
|
|
--format '{{.Name}}')
|
|
if ((${#matches[@]} != 1)); then
|
|
printf 'Expected exactly one %s volume for Compose project %s; found %s.\n' "$logical_name" "$project_name" "${#matches[@]}" >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "${matches[0]}"
|
|
}
|
|
|
|
replace_volume() {
|
|
local source_dir="$1"
|
|
local volume_name="$2"
|
|
docker compose --profile backup run --rm --no-deps \
|
|
--entrypoint bash \
|
|
-v "$source_dir:/restore-source:ro" \
|
|
-v "$volume_name:/restore-target" \
|
|
backup -ceu '
|
|
find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
|
cp -a /restore-source/. /restore-target/
|
|
'
|
|
}
|
|
|
|
create_safety_dump() {
|
|
local database_user="${POSTGRES_USER:-dragons}"
|
|
local database_name="${POSTGRES_DB:-dragonsstash}"
|
|
SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump"
|
|
docker compose exec -T db pg_dump --format=custom --no-owner \
|
|
--username "$database_user" --dbname "$database_name" > "$SAFETY_DUMP"
|
|
if [[ ! -s "$SAFETY_DUMP" ]]; then
|
|
printf 'Safety database dump %s is missing or empty.\n' "$SAFETY_DUMP" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
restore_database_dump() {
|
|
local dump_path="$1"
|
|
local database_user="${POSTGRES_USER:-dragons}"
|
|
local database_name="${POSTGRES_DB:-dragonsstash}"
|
|
docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$database_name"
|
|
docker compose exec -T db createdb --username "$database_user" "$database_name"
|
|
docker compose exec -T db pg_restore --no-owner --exit-on-error \
|
|
--username "$database_user" --dbname "$database_name" < "$dump_path"
|
|
}
|
|
|
|
restore_database() {
|
|
restore_database_dump "$RESTORED_DUMP"
|
|
}
|
|
|
|
verify_file_references() {
|
|
local database_name="$1"
|
|
local uploads_source="$2"
|
|
local database_user="${POSTGRES_USER:-dragons}"
|
|
docker compose exec -T db psql --no-psqlrc --tuples-only --no-align --quiet \
|
|
--field-separator=$'\t' \
|
|
--username "$database_user" --dbname "$database_name" \
|
|
--command "SELECT 'legacy', \"filePath\" FROM \"manual_upload_files\" WHERE \"retainedAt\" IS NULL
|
|
UNION ALL
|
|
SELECT 'retained', \"filePath\" FROM \"manual_upload_files\" WHERE \"retainedAt\" IS NOT NULL
|
|
ORDER BY 2" |
|
|
docker compose --profile backup run --rm --no-deps -T --entrypoint bash \
|
|
-v "$uploads_source:/data/uploads:ro" backup -ceu '
|
|
missing=0
|
|
while IFS="$(printf "\t")" read -r retention file_path; do
|
|
if [[ "$retention" == "legacy" ]]; then
|
|
printf "Warning: legacy manual-upload file reference is not required because retainedAt is NULL: %s\\n" "$file_path" >&2
|
|
continue
|
|
fi
|
|
if [[ "$retention" != "retained" ]]; then
|
|
printf "Unexpected retention status for database reference: %s\\n" "$file_path" >&2
|
|
missing=1
|
|
continue
|
|
fi
|
|
case "$file_path" in
|
|
/data/uploads/*) relative_path="${file_path#/data/uploads/}" ;;
|
|
*)
|
|
printf "Database reference is outside /data/uploads: %s\\n" "$file_path" >&2
|
|
missing=1
|
|
continue
|
|
;;
|
|
esac
|
|
if [[ -z "$relative_path" || "$relative_path" == .. || "$relative_path" == ../* || "$relative_path" == */../* ]]; then
|
|
printf "Database reference has an invalid uploads path: %s\\n" "$file_path" >&2
|
|
missing=1
|
|
elif [[ ! -f "/data/uploads/$relative_path" ]]; then
|
|
printf "Missing restored upload for database reference: %s\\n" "$file_path" >&2
|
|
missing=1
|
|
fi
|
|
done
|
|
exit "$missing"
|
|
'
|
|
}
|
|
|
|
drop_temporary_verification_database() {
|
|
local database_user="${POSTGRES_USER:-dragons}"
|
|
|
|
if [[ -n "$TEMP_VERIFY_DATABASE" ]]; then
|
|
docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$TEMP_VERIFY_DATABASE"
|
|
TEMP_VERIFY_DATABASE=""
|
|
fi
|
|
}
|
|
|
|
verify_staged_snapshot_file_references() {
|
|
local database_user="${POSTGRES_USER:-dragons}"
|
|
|
|
TEMP_VERIFY_DATABASE="dragons_restore_verify_$$_$(date +%s)"
|
|
docker compose exec -T db dropdb --if-exists --force --username "$database_user" "$TEMP_VERIFY_DATABASE"
|
|
docker compose exec -T db createdb --username "$database_user" "$TEMP_VERIFY_DATABASE"
|
|
docker compose exec -T db pg_restore --no-owner --exit-on-error \
|
|
--username "$database_user" --dbname "$TEMP_VERIFY_DATABASE" < "$RESTORED_DUMP"
|
|
verify_file_references "$TEMP_VERIFY_DATABASE" "$RESTORED_UPLOADS"
|
|
drop_temporary_verification_database
|
|
}
|
|
|
|
archive_live_volume() {
|
|
local volume_name="$1"
|
|
local logical_name="$2"
|
|
local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar"
|
|
|
|
docker compose --profile backup run --rm --no-deps \
|
|
--entrypoint bash \
|
|
-v "$volume_name:/safety-source:ro" \
|
|
-v "$LIVE_STAGING_DIR:/safety-output" \
|
|
backup -ceu 'tar -C /safety-source -cf "$1" .' bash \
|
|
"/safety-output/pre-restore-$logical_name.tar"
|
|
if [[ ! -s "$archive_path" ]]; then
|
|
printf 'Safety archive %s is missing or empty.\n' "$archive_path" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
restore_live_volume_archive() {
|
|
local volume_name="$1"
|
|
local logical_name="$2"
|
|
local archive_path="$LIVE_STAGING_DIR/pre-restore-$logical_name.tar"
|
|
|
|
docker compose --profile backup run --rm --no-deps \
|
|
--entrypoint bash \
|
|
-v "$archive_path:/safety-archive:ro" \
|
|
-v "$volume_name:/restore-target" \
|
|
backup -ceu '
|
|
find /restore-target -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
|
tar -C /restore-target -xf /safety-archive
|
|
'
|
|
}
|
|
|
|
wait_for_health() {
|
|
local health_url="${RESTORE_HEALTH_URL:-http://localhost:${APP_PORT:-3000}/api/health}"
|
|
local attempt
|
|
for attempt in {1..30}; do
|
|
if curl --fail --silent --show-error "$health_url" >/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
printf 'Application health endpoint did not become ready: %s\n' "$health_url" >&2
|
|
return 1
|
|
}
|
|
|
|
live_restore_failure() {
|
|
local exit_code=$?
|
|
local rollback_ok=1
|
|
trap - EXIT
|
|
if ((LIVE_RESTORE_ACTIVE)); then
|
|
docker compose --profile full stop "${LIVE_SERVICES[@]}" || true
|
|
drop_temporary_verification_database || true
|
|
if ((LIVE_REPLACEMENT_STARTED)); then
|
|
restore_live_volume_archive "$LIVE_UPLOADS_VOLUME" manual_uploads || rollback_ok=0
|
|
restore_live_volume_archive "$LIVE_WORKER_VOLUME" tdlib_state || rollback_ok=0
|
|
restore_live_volume_archive "$LIVE_BOT_VOLUME" tdlib_bot_state || rollback_ok=0
|
|
restore_database_dump "$SAFETY_DUMP" || rollback_ok=0
|
|
if ((rollback_ok)); then
|
|
printf 'Rollback restored the pre-restore database and all three Docker volumes. Services remain stopped.\n' >&2
|
|
else
|
|
printf 'Rollback failed; services remain stopped. Restore the safety archives and database dump manually.\n' >&2
|
|
fi
|
|
fi
|
|
printf 'Live restore failed (exit %s). Services remain stopped. Staging directory: %s\n' "$exit_code" "$LIVE_STAGING_DIR" >&2
|
|
printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP" >&2
|
|
fi
|
|
exit "$exit_code"
|
|
}
|
|
|
|
restore_live() {
|
|
local snapshot_id="$1"
|
|
local project_name
|
|
local uploads_volume
|
|
local worker_volume
|
|
local bot_volume
|
|
local timestamp
|
|
local container_staging_dir
|
|
validate_environment
|
|
docker compose --profile backup config --quiet
|
|
project_name="$(compose_project_name)"
|
|
verify_snapshot "$snapshot_id"
|
|
printf 'Live restore target confirmed: Compose project %s, configured Restic repository, snapshot %s.\n' "$project_name" "$snapshot_id"
|
|
timestamp="$(date -u +'%Y-%m-%dT%H-%M-%SZ')"
|
|
LIVE_STAGING_DIR="$(validate_staging_directory "$BACKUP_STAGING_PATH/live-restore-$timestamp-$$")"
|
|
container_staging_dir="$(container_staging_directory "$LIVE_STAGING_DIR")"
|
|
prepare_fresh_staging_directory "$LIVE_STAGING_DIR"
|
|
SAFETY_DUMP="$LIVE_STAGING_DIR/pre-restore-database.dump"
|
|
trap live_restore_failure EXIT
|
|
LIVE_RESTORE_ACTIVE=1
|
|
docker compose --profile full stop "${LIVE_SERVICES[@]}"
|
|
create_safety_dump
|
|
backup_restic restore "$snapshot_id" --target "$container_staging_dir"
|
|
validate_restored_tree "$LIVE_STAGING_DIR"
|
|
uploads_volume="$(compose_volume_name "$project_name" manual_uploads)"
|
|
worker_volume="$(compose_volume_name "$project_name" tdlib_state)"
|
|
bot_volume="$(compose_volume_name "$project_name" tdlib_bot_state)"
|
|
LIVE_UPLOADS_VOLUME="$uploads_volume"
|
|
LIVE_WORKER_VOLUME="$worker_volume"
|
|
LIVE_BOT_VOLUME="$bot_volume"
|
|
archive_live_volume "$uploads_volume" manual_uploads
|
|
archive_live_volume "$worker_volume" tdlib_state
|
|
archive_live_volume "$bot_volume" tdlib_bot_state
|
|
verify_staged_snapshot_file_references
|
|
LIVE_REPLACEMENT_STARTED=1
|
|
replace_volume "$RESTORED_UPLOADS" "$uploads_volume"
|
|
replace_volume "$RESTORED_TDLIB_WORKER" "$worker_volume"
|
|
replace_volume "$RESTORED_TDLIB_BOT" "$bot_volume"
|
|
restore_database
|
|
docker compose --profile full up -d "${LIVE_SERVICES[@]}"
|
|
wait_for_health
|
|
LIVE_RESTORE_ACTIVE=0
|
|
trap - EXIT
|
|
printf 'Live restore completed. Staging directory retained at: %s\n' "$LIVE_STAGING_DIR"
|
|
printf 'Safety database dump retained at: %s\n' "$SAFETY_DUMP"
|
|
}
|
|
|
|
main() {
|
|
local command="${1:-}"
|
|
case "$command" in
|
|
list)
|
|
[[ $# -eq 1 ]] || { usage >&2; return 2; }
|
|
validate_environment
|
|
backup_restic snapshots
|
|
;;
|
|
verify)
|
|
[[ $# -eq 2 ]] || { usage >&2; return 2; }
|
|
validate_environment
|
|
verify_snapshot "$2"
|
|
;;
|
|
restore-to-staging)
|
|
[[ $# -eq 3 ]] || { usage >&2; return 2; }
|
|
restore_to_staging "$2" "$3"
|
|
;;
|
|
restore-live)
|
|
if [[ $# -ne 3 || "$3" != "$CONFIRM_REPLACE_LIVE_DATA" ]]; then
|
|
printf 'restore-live requires the exact confirmation flag: %s\n' "$CONFIRM_REPLACE_LIVE_DATA" >&2
|
|
return 2
|
|
fi
|
|
restore_live "$2"
|
|
;;
|
|
*)
|
|
usage >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|