Andrey
53769e6832
Add Healthchecks service details to architecture and backup documentation, including its role as a Dead man's switch for backups. Update backup scripts to utilize systemd timers instead of cron for improved scheduling. Enhance network topology documentation to reflect Healthchecks integration in the VPS Miran setup. This update clarifies backup processes and enhances overall system reliability.
148 lines
7.2 KiB
Bash
Executable File
148 lines
7.2 KiB
Bash
Executable File
#!/bin/bash
|
||
# Тест восстановления уровня 1: restic check и проверка дампа Nextcloud из restic.
|
||
# Запускать на хосте Proxmox под root.
|
||
# Режимы (аргумент): weekly | monthly-check | full-check | monthly-dump
|
||
# weekly — restic check (еженедельно)
|
||
# monthly-check — restic check --read-data-subset=10% (ежемесячно, 1-е число)
|
||
# full-check — restic check --read-data (раз в 6–12 мес, 1 янв и 1 июля)
|
||
# monthly-dump — restore дампа Nextcloud из restic, проверка целостности (ежемесячно)
|
||
# Секреты: из Vaultwarden (объект RESTIC), как в backup-restic-yandex.sh.
|
||
# Cron/Timer: отдельные таймеры для каждого режима.
|
||
set -e
|
||
|
||
export PATH="/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}"
|
||
|
||
MODE="${1:-weekly}"
|
||
NOTIFY_SCRIPT="${NOTIFY_SCRIPT:-/root/scripts/notify-telegram.sh}"
|
||
RESTORE_TARGET="/tmp/restore-test"
|
||
RESTIC_PATH_NEXTCLOUD="/mnt/backup/databases/ct101-nextcloud"
|
||
MIN_DUMP_SIZE_MB=1
|
||
|
||
if [ "$(id -u)" -ne 0 ]; then
|
||
echo "Запускайте под root."
|
||
exit 1
|
||
fi
|
||
|
||
# Загрузка кредов restic из Vaultwarden (как в backup-restic-yandex.sh)
|
||
setup_restic_env() {
|
||
BW_MASTER_PASSWORD_FILE="${BW_MASTER_PASSWORD_FILE:-/root/.bw-master}"
|
||
if [ ! -f "$BW_MASTER_PASSWORD_FILE" ]; then
|
||
echo "Нет файла с мастер-паролем Vaultwarden: $BW_MASTER_PASSWORD_FILE"
|
||
return 1
|
||
fi
|
||
if ! command -v bw >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then
|
||
echo "Установите bw (Bitwarden CLI) и jq."
|
||
return 1
|
||
fi
|
||
export BW_SESSION
|
||
BW_SESSION=$(bw unlock --passwordfile "$BW_MASTER_PASSWORD_FILE" --raw 2>/dev/null) || return 1
|
||
RESTIC_ITEM=$(bw get item "RESTIC" 2>/dev/null) || return 1
|
||
export RESTIC_REPOSITORY
|
||
RESTIC_REPOSITORY=$(echo "$RESTIC_ITEM" | jq -r '.fields[] | select(.name=="RESTIC_REPOSITORY") | .value')
|
||
export AWS_ACCESS_KEY_ID
|
||
AWS_ACCESS_KEY_ID=$(echo "$RESTIC_ITEM" | jq -r '.fields[] | select(.name=="AWS_ACCESS_KEY_ID") | .value')
|
||
export AWS_SECRET_ACCESS_KEY
|
||
AWS_SECRET_ACCESS_KEY=$(echo "$RESTIC_ITEM" | jq -r '.fields[] | select(.name=="AWS_SECRET_ACCESS_KEY") | .value')
|
||
export AWS_DEFAULT_REGION
|
||
AWS_DEFAULT_REGION=$(echo "$RESTIC_ITEM" | jq -r '.fields[] | select(.name=="AWS_DEFAULT_REGION") | .value')
|
||
[ -z "$AWS_DEFAULT_REGION" ] && AWS_DEFAULT_REGION="ru-central1"
|
||
RESTIC_PASS=$(echo "$RESTIC_ITEM" | jq -r '.fields[] | select(.name=="RESTIC_BACKUP_KEY") | .value')
|
||
for var in RESTIC_REPOSITORY AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||
[ -z "${!var}" ] && return 1
|
||
done
|
||
RESTIC_PASSWORD_FILE=$(mktemp -u)
|
||
echo -n "$RESTIC_PASS" > "$RESTIC_PASSWORD_FILE"
|
||
chmod 600 "$RESTIC_PASSWORD_FILE"
|
||
trap 'rm -f "$RESTIC_PASSWORD_FILE"' EXIT INT TERM
|
||
export RESTIC_PASSWORD_FILE
|
||
return 0
|
||
}
|
||
|
||
notify_ok() {
|
||
[ -x "$NOTIFY_SCRIPT" ] && "$NOTIFY_SCRIPT" "$1" "$2" || true
|
||
}
|
||
|
||
notify_err() {
|
||
[ -x "$NOTIFY_SCRIPT" ] && "$NOTIFY_SCRIPT" "⚠️ $1" "$2" || true
|
||
}
|
||
|
||
case "$MODE" in
|
||
weekly)
|
||
echo "[verify-restore-level1] Режим: weekly (restic check)"
|
||
setup_restic_env || { notify_err "Restic check" "Не удалось загрузить креды restic."; exit 1; }
|
||
if restic check 2>&1; then
|
||
echo "[verify-restore-level1] restic check OK"
|
||
# При еженедельном успехе — не спамим (только при ошибке)
|
||
else
|
||
notify_err "Restic check" "Ошибка проверки репозитория restic."
|
||
exit 1
|
||
fi
|
||
;;
|
||
monthly-check)
|
||
echo "[verify-restore-level1] Режим: monthly-check (restic check --read-data-subset=10%)"
|
||
setup_restic_env || { notify_err "Restic check (read-data-subset)" "Не удалось загрузить креды restic."; exit 1; }
|
||
if restic check --read-data-subset=10% 2>&1; then
|
||
echo "[verify-restore-level1] restic check --read-data-subset=10% OK"
|
||
notify_ok "Тест restic (read-data-subset)" "OK, 10% данных проверено."
|
||
else
|
||
notify_err "Restic check (read-data-subset)" "Ошибка проверки 10% данных репозитория."
|
||
exit 1
|
||
fi
|
||
;;
|
||
full-check)
|
||
echo "[verify-restore-level1] Режим: full-check (restic check --read-data)"
|
||
setup_restic_env || { notify_err "Restic check (read-data)" "Не удалось загрузить креды restic."; exit 1; }
|
||
if restic check --read-data 2>&1; then
|
||
echo "[verify-restore-level1] restic check --read-data OK"
|
||
notify_ok "Тест restic (full read-data)" "OK, полная проверка данных завершена."
|
||
else
|
||
notify_err "Restic check (read-data)" "Ошибка полной проверки данных репозитория."
|
||
exit 1
|
||
fi
|
||
;;
|
||
monthly-dump)
|
||
echo "[verify-restore-level1] Режим: monthly-dump (restore и проверка дампа Nextcloud)"
|
||
setup_restic_env || { notify_err "Тест дампа Nextcloud" "Не удалось загрузить креды restic."; exit 1; }
|
||
rm -rf "$RESTORE_TARGET"
|
||
mkdir -p "$RESTORE_TARGET"
|
||
trap 'rm -f "${RESTIC_PASSWORD_FILE:-}" 2>/dev/null; rm -rf "$RESTORE_TARGET"' EXIT INT TERM
|
||
if ! restic restore latest --target "$RESTORE_TARGET" --path "$RESTIC_PATH_NEXTCLOUD" 2>&1; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка restic restore: не удалось восстановить $RESTIC_PATH_NEXTCLOUD"
|
||
exit 1
|
||
fi
|
||
# Путь после restore: RESTORE_TARGET/mnt/backup/databases/ct101-nextcloud/
|
||
RESTORED_DIR="$RESTORE_TARGET/mnt/backup/databases/ct101-nextcloud"
|
||
if [ ! -d "$RESTORED_DIR" ]; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка: каталог $RESTORED_DIR не найден после restore."
|
||
exit 1
|
||
fi
|
||
LATEST_SQL=$(ls -t "$RESTORED_DIR"/nextcloud-db-*.sql.gz 2>/dev/null | head -1)
|
||
if [ -z "$LATEST_SQL" ] || [ ! -f "$LATEST_SQL" ]; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка: не найден .sql.gz в $RESTORED_DIR"
|
||
exit 1
|
||
fi
|
||
SIZE_BYTES=$(stat -c%s "$LATEST_SQL" 2>/dev/null || echo 0)
|
||
SIZE_MB=$(( SIZE_BYTES / 1024 / 1024 ))
|
||
if [ "$SIZE_MB" -lt "$MIN_DUMP_SIZE_MB" ]; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка: размер дампа ${SIZE_MB} MB < ${MIN_DUMP_SIZE_MB} MB (файл: $LATEST_SQL)"
|
||
exit 1
|
||
fi
|
||
if ! gunzip -t "$LATEST_SQL" 2>/dev/null; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка: gunzip -t не прошёл для $LATEST_SQL"
|
||
exit 1
|
||
fi
|
||
if ! gunzip -c "$LATEST_SQL" 2>/dev/null | grep -q 'CREATE TABLE'; then
|
||
notify_err "Тест дампа Nextcloud" "Ошибка: в распакованном дампе нет CREATE TABLE (возможно не SQL дамп)"
|
||
exit 1
|
||
fi
|
||
echo "[verify-restore-level1] Дамп Nextcloud OK: $LATEST_SQL, размер ${SIZE_MB} MB"
|
||
notify_ok "Тест дампа Nextcloud" "OK, размер ${SIZE_MB} MB."
|
||
;;
|
||
*)
|
||
echo "Использование: $0 {weekly|monthly-check|full-check|monthly-dump}"
|
||
exit 1
|
||
;;
|
||
esac
|
||
|
||
exit 0
|