Capture current Ansible control plane state

Commit the accumulated infrastructure work that was living only in the working tree: monitoring stack, emergency access/bot, gyro allocator, grimmory, adguard, backup audit and the OpenCode agent definitions. Also ignore Python bytecode, local archives and Nix/direnv artifacts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
2026-08-26 21:39:28 +03:00
parent 4bafa7d09e
commit c676be81ec
126 changed files with 10583 additions and 44 deletions
@@ -0,0 +1,70 @@
+#!/bin/sh
+# Managed by Ansible: HomeLab PBS backup audit (L0 freshness)
+set -eu
+
+LOG_FILE="{{ backup_audit_log_file }}"
+METRICS_DIR="{{ backup_audit_metrics_dir }}"
+METRICS_FILE="$METRICS_DIR/homelab_backup_audit_pbs.prom"
+METRICS_TMP=$(mktemp "$METRICS_FILE.XXXXXX")
+STATUS=OK
+NOW_EPOCH=$(date +%s)
+
+trap 'rm -f "$METRICS_TMP"' EXIT
+
+publish_metrics() {
+  chmod 0644 "$METRICS_TMP"
+  mv "$METRICS_TMP" "$METRICS_FILE"
+}
+
+ts() { date '+%Y-%m-%dT%H:%M:%S%z'; }
+log() { echo "[$(ts)] [pbs] $*" | tee -a "$LOG_FILE"; }
+
+get_latest_snapshot() {
+  pvesm list pbs --vmid "$1" 2>/dev/null \
+    | tail -n +2 \
+    | awk '{print $1}' \
+    | sort -t/ -k4 \
+    | tail -1
+}
+
+{% for entry in backup_audit_pbs_vmids %}
+audit_vmid_{{ entry.vmid }}() {
+  vmid={{ entry.vmid }}
+  max_age={{ entry.max_age_hours }}
+  latest=$(get_latest_snapshot "$vmid")
+  if [ -z "$latest" ]; then
+    log "FAIL L0: vmid $vmid — no snapshots in PBS"
+    printf 'homelab_backup_audit_snapshot_age_hours{profile="pbs",vmid="%s"} -1\n' "$vmid" >> "$METRICS_TMP"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 0\n' "$vmid" >> "$METRICS_TMP"
+    STATUS=FAIL
+    return
+  fi
+  ts_str=$(printf '%s' "$latest" | sed -n 's#.*/\([0-9T:Z-]*\)$#\1#p')
+  snap_epoch=$(date -d "$ts_str" +%s 2>/dev/null || echo 0)
+  age_hours=$(( (NOW_EPOCH - snap_epoch) / 3600 ))
+  if [ "$age_hours" -gt "$max_age" ]; then
+    log "FAIL L0: vmid $vmid — latest snapshot age ${age_hours}h > ${max_age}h (snapshot: $ts_str)"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 0\n' "$vmid" >> "$METRICS_TMP"
+    STATUS=FAIL
+  else
+    log "OK L0: vmid $vmid — latest snapshot age ${age_hours}h"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 1\n' "$vmid" >> "$METRICS_TMP"
+  fi
+  printf 'homelab_backup_audit_snapshot_age_hours{profile="pbs",vmid="%s"} %s\n' "$vmid" "$age_hours" >> "$METRICS_TMP"
+}
+audit_vmid_{{ entry.vmid }}
+
+{% endfor %}
+if [ "$STATUS" = "OK" ]; then
+  log "AUDIT PASSED"
+  printf 'homelab_backup_audit_success{profile="pbs"} 1\n' >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="pbs"} %s\n' "$NOW_EPOCH" >> "$METRICS_TMP"
+  publish_metrics
+  exit 0
+else
+  log "AUDIT FAILED"
+  printf 'homelab_backup_audit_success{profile="pbs"} 0\n' >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="pbs"} %s\n' "$NOW_EPOCH" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi
@@ -0,0 +1,167 @@
+#!/bin/sh
+# Managed by Ansible: HomeLab restic offsite backup audit (L0+L1+L2)
+set -eu
+
+if [ "$#" -ne 1 ]; then
+  echo "usage: $0 <profile>" >&2
+  exit 64
+fi
+
+PROFILE="$1"
+case "$PROFILE" in
+  ''|*[!A-Za-z0-9_-]*)
+    echo "invalid profile name" >&2
+    exit 64
+    ;;
+esac
+RESTIC_ENV="/etc/homelab-restic/${PROFILE}.env"
+AUDIT_ENV="/etc/homelab-backup-audit/${PROFILE}.env"
+LOG_FILE="{{ backup_audit_log_file }}"
+TMP="/var/lib/homelab-backup-audit/${PROFILE}"
+METRICS_DIR="{{ backup_audit_metrics_dir }}"
+METRICS_FILE="$METRICS_DIR/homelab_backup_audit_${PROFILE}.prom"
+METRICS_TMP=$(mktemp "$METRICS_FILE.XXXXXX")
+STATUS=OK
+
+trap 'rm -f "$METRICS_TMP"' EXIT
+
+publish_metrics() {
+  chmod 0644 "$METRICS_TMP"
+  mv "$METRICS_TMP" "$METRICS_FILE"
+}
+
+for f in "$RESTIC_ENV" "$AUDIT_ENV"; do
+  if [ ! -f "$f" ]; then
+    echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] [${PROFILE}] FAIL: missing env file $f" | tee -a "$LOG_FILE"
+    printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP"
+    printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+    publish_metrics
+    exit 66
+  fi
+done
+
+set -a
+. "$RESTIC_ENV"
+. "$AUDIT_ENV"
+set +a
+
+export RESTIC_REPOSITORY RESTIC_PASSWORD_FILE RCLONE_CONFIG
+
+ts() { date '+%Y-%m-%dT%H:%M:%S%z'; }
+log() { echo "[$(ts)] [${PROFILE}] $*" | tee -a "$LOG_FILE"; }
+
+# ── L0: freshness ──────────────────────────────────────────────────────────
+latest_time=$(restic snapshots --latest 1 --json 2>/dev/null | jq -r '.[0].time // empty')
+if [ -z "$latest_time" ]; then
+  log "FAIL L0: no snapshots found in repository"
+  printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi
+
+now_epoch=$(date +%s)
+snap_epoch=$(date -d "$latest_time" +%s 2>/dev/null || echo 0)
+age_hours=$(( (now_epoch - snap_epoch) / 3600 ))
+max_age="${HOMELAB_AUDIT_MAX_AGE_HOURS:-36}"
+
+if [ "$age_hours" -gt "$max_age" ]; then
+  log "FAIL L0: latest snapshot age ${age_hours}h > ${max_age}h (snapshot: $latest_time)"
+  STATUS=FAIL
+else
+  log "OK L0: latest snapshot age ${age_hours}h"
+fi
+printf 'homelab_backup_audit_snapshot_age_hours{profile="%s"} %s\n' "$PROFILE" "$age_hours" >> "$METRICS_TMP"
+
+# ── L1: repository integrity ───────────────────────────────────────────────
+if restic check 2>&1 | tee -a "$LOG_FILE"; then
+  log "OK L1: restic check passed"
+  printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+else
+  log "FAIL L1: restic check failed"
+  printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+  STATUS=FAIL
+fi
+
+# ── L2: SQLite restore + integrity ─────────────────────────────────────────
+if [ -n "${HOMELAB_AUDIT_SQLITE_NAME:-}" ]; then
+  case "$TMP" in
+    /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;;
+    *)
+      log "FAIL L2: unsafe temporary path"
+      exit 64
+      ;;
+  esac
+  rm -rf "$TMP"
+  mkdir -p "$TMP"
+
+  if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_SQLITE_NAME}" 2>&1 | tee -a "$LOG_FILE"; then
+    db=$(find "$TMP" -name "$HOMELAB_AUDIT_SQLITE_NAME" -type f | head -1)
+    if [ -z "$db" ] || [ ! -f "$db" ]; then
+      log "FAIL L2: $HOMELAB_AUDIT_SQLITE_NAME not found in restored data"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+      STATUS=FAIL
+    else
+      check=$(sqlite3 "$db" "PRAGMA integrity_check;" 2>&1)
+      if [ "$check" = "ok" ]; then
+        log "OK L2: SQLite integrity_check ok ($db)"
+        printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+      else
+        log "FAIL L2: SQLite integrity_check: $check"
+        printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+        STATUS=FAIL
+      fi
+    fi
+  else
+    log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_SQLITE_NAME"
+    printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+    STATUS=FAIL
+  fi
+
+  rm -rf "$TMP"
+fi
+
+# ── L2: expected file restore ───────────────────────────────────────────────
+if [ -n "${HOMELAB_AUDIT_EXPECTED_NAME:-}" ]; then
+  case "$TMP" in
+    /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;;
+    *)
+      log "FAIL L2: unsafe temporary path"
+      exit 64
+      ;;
+  esac
+  rm -rf "$TMP"
+  mkdir -p "$TMP"
+
+  if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_EXPECTED_NAME}" 2>&1 | tee -a "$LOG_FILE"; then
+    expected=$(find "$TMP" -name "$HOMELAB_AUDIT_EXPECTED_NAME" -type f | head -1)
+    if [ -n "$expected" ] && [ -s "$expected" ]; then
+      log "OK L2: restored non-empty expected file ($expected)"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+    else
+      log "FAIL L2: $HOMELAB_AUDIT_EXPECTED_NAME not found or empty"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+      STATUS=FAIL
+    fi
+  else
+    log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_EXPECTED_NAME"
+    printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+    STATUS=FAIL
+  fi
+
+  rm -rf "$TMP"
+fi
+
+if [ "$STATUS" = "OK" ]; then
+  log "AUDIT PASSED"
+  printf 'homelab_backup_audit_success{profile="%s"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 0
+else
+  log "AUDIT FAILED"
+  printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi