#!/bin/sh # Managed by Ansible: HomeLab restic offsite backup audit (L0+L1+L2) set -eu if [ "$#" -ne 1 ]; then echo "usage: $0 " >&2 exit 64 fi PROFILE="$1" case "$PROFILE" in ''|*[!A-Za-z0-9_-]*) echo "invalid profile name" >&2 exit 64 ;; esac RESTIC_ENV="/etc/homelab-restic/${PROFILE}.env" AUDIT_ENV="/etc/homelab-backup-audit/${PROFILE}.env" LOG_FILE="{{ backup_audit_log_file }}" TMP="/var/lib/homelab-backup-audit/${PROFILE}" METRICS_DIR="{{ backup_audit_metrics_dir }}" METRICS_FILE="$METRICS_DIR/homelab_backup_audit_${PROFILE}.prom" METRICS_TMP=$(mktemp "$METRICS_FILE.XXXXXX") STATUS=OK trap 'rm -f "$METRICS_TMP"' EXIT publish_metrics() { chmod 0644 "$METRICS_TMP" mv "$METRICS_TMP" "$METRICS_FILE" } for f in "$RESTIC_ENV" "$AUDIT_ENV"; do if [ ! -f "$f" ]; then echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] [${PROFILE}] FAIL: missing env file $f" | tee -a "$LOG_FILE" printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP" printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP" publish_metrics exit 66 fi done set -a . "$RESTIC_ENV" . "$AUDIT_ENV" set +a export RESTIC_REPOSITORY RESTIC_PASSWORD_FILE RCLONE_CONFIG ts() { date '+%Y-%m-%dT%H:%M:%S%z'; } log() { echo "[$(ts)] [${PROFILE}] $*" | tee -a "$LOG_FILE"; } # ── L0: freshness ────────────────────────────────────────────────────────── latest_time=$(restic snapshots --latest 1 --json 2>/dev/null | jq -r '.[0].time // empty') if [ -z "$latest_time" ]; then log "FAIL L0: no snapshots found in repository" printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP" printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP" publish_metrics exit 1 fi now_epoch=$(date +%s) snap_epoch=$(date -d "$latest_time" +%s 2>/dev/null || echo 0) age_hours=$(( (now_epoch - snap_epoch) / 3600 )) max_age="${HOMELAB_AUDIT_MAX_AGE_HOURS:-36}" if [ "$age_hours" -gt "$max_age" ]; then log "FAIL L0: latest snapshot age ${age_hours}h > ${max_age}h (snapshot: $latest_time)" STATUS=FAIL else log "OK L0: latest snapshot age ${age_hours}h" fi printf 'homelab_backup_audit_snapshot_age_hours{profile="%s"} %s\n' "$PROFILE" "$age_hours" >> "$METRICS_TMP" # ── L1: repository integrity ─────────────────────────────────────────────── if restic check 2>&1 | tee -a "$LOG_FILE"; then log "OK L1: restic check passed" printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 1\n' "$PROFILE" >> "$METRICS_TMP" else log "FAIL L1: restic check failed" printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL fi # ── L2: SQLite restore + integrity ───────────────────────────────────────── if [ -n "${HOMELAB_AUDIT_SQLITE_NAME:-}" ]; then case "$TMP" in /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;; *) log "FAIL L2: unsafe temporary path" exit 64 ;; esac rm -rf "$TMP" mkdir -p "$TMP" if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_SQLITE_NAME}" 2>&1 | tee -a "$LOG_FILE"; then db=$(find "$TMP" -name "$HOMELAB_AUDIT_SQLITE_NAME" -type f | head -1) if [ -z "$db" ] || [ ! -f "$db" ]; then log "FAIL L2: $HOMELAB_AUDIT_SQLITE_NAME not found in restored data" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL else check=$(sqlite3 "$db" "PRAGMA integrity_check;" 2>&1) if [ "$check" = "ok" ]; then log "OK L2: SQLite integrity_check ok ($db)" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP" else log "FAIL L2: SQLite integrity_check: $check" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL fi fi else log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_SQLITE_NAME" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL fi rm -rf "$TMP" fi # ── L2: expected file restore ─────────────────────────────────────────────── if [ -n "${HOMELAB_AUDIT_EXPECTED_NAME:-}" ]; then case "$TMP" in /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;; *) log "FAIL L2: unsafe temporary path" exit 64 ;; esac rm -rf "$TMP" mkdir -p "$TMP" if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_EXPECTED_NAME}" 2>&1 | tee -a "$LOG_FILE"; then expected=$(find "$TMP" -name "$HOMELAB_AUDIT_EXPECTED_NAME" -type f | head -1) if [ -n "$expected" ] && [ -s "$expected" ]; then log "OK L2: restored non-empty expected file ($expected)" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP" else log "FAIL L2: $HOMELAB_AUDIT_EXPECTED_NAME not found or empty" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL fi else log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_EXPECTED_NAME" printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP" STATUS=FAIL fi rm -rf "$TMP" fi if [ "$STATUS" = "OK" ]; then log "AUDIT PASSED" printf 'homelab_backup_audit_success{profile="%s"} 1\n' "$PROFILE" >> "$METRICS_TMP" printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP" publish_metrics exit 0 else log "AUDIT FAILED" printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" >> "$METRICS_TMP" printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP" publish_metrics exit 1 fi