Capture current Ansible control plane state

Commit the accumulated infrastructure work that was living only in the working tree: monitoring stack, emergency access/bot, gyro allocator, grimmory, adguard, backup audit and the OpenCode agent definitions. Also ignore Python bytecode, local archives and Nix/direnv artifacts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
2026-08-26 21:39:28 +03:00
parent 4bafa7d09e
commit c676be81ec
126 changed files with 10583 additions and 44 deletions
@@ -0,0 +1,31 @@
+---
+backup_audit_log_file: /var/log/homelab-backup-audit.log
+backup_audit_timer_oncalendar: "*-*-* 06:00:00"
+backup_audit_timer_randomized_delay: 10m
+
+backup_audit_pbs_vmids:
+  - vmid: 132
+    max_age_hours: 48
+  - vmid: 140
+    max_age_hours: 48
+  - vmid: 141
+    max_age_hours: 48
+  - vmid: 142
+    max_age_hours: 48
+  - vmid: 143
+    max_age_hours: 48
+  - vmid: 144
+    max_age_hours: 48
+  - vmid: 145
+    max_age_hours: 48
+  - vmid: 146
+    max_age_hours: 48
+  - vmid: 147
+    max_age_hours: 48
+  - vmid: 149
+    max_age_hours: 48
+  - vmid: 150
+    max_age_hours: 48
+
+backup_audit_restic_profiles: []
+backup_audit_metrics_dir: /var/lib/node_exporter/textfile_collector
@@ -0,0 +1,146 @@
+---
+- name: Install audit dependencies
+  ansible.builtin.apt:
+    name:
+      - jq
+      - sqlite3
+    state: present
+    update_cache: true
+
+- name: Ensure audit directories exist
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - /etc/homelab-backup-audit
+    - /var/lib/homelab-backup-audit
+    - "{{ backup_audit_metrics_dir }}"
+
+- name: Install PBS audit script
+  ansible.builtin.template:
+    src: audit-pbs.sh.j2
+    dest: /usr/local/sbin/homelab-backup-audit-pbs
+    owner: root
+    group: root
+    mode: "0755"
+  when: backup_audit_type | default('') == 'pbs'
+
+- name: Install restic audit script
+  ansible.builtin.template:
+    src: audit-restic.sh.j2
+    dest: /usr/local/sbin/homelab-backup-audit-restic
+    owner: root
+    group: root
+    mode: "0755"
+  when: backup_audit_type | default('') == 'restic'
+
+- name: Install restic audit env files
+  ansible.builtin.copy:
+    dest: "/etc/homelab-backup-audit/{{ item.name }}.env"
+    owner: root
+    group: root
+    mode: "0600"
+    content: |
+      HOMELAB_AUDIT_PROFILE={{ item.name }}
+      HOMELAB_AUDIT_MAX_AGE_HOURS={{ item.max_age_hours | default(36) }}
+      HOMELAB_AUDIT_SQLITE_NAME={{ item.sqlite_name | default('') }}
+      HOMELAB_AUDIT_EXPECTED_NAME={{ item.expected_name | default('') }}
+  loop: "{{ backup_audit_restic_profiles }}"
+  when: backup_audit_type | default('') == 'restic'
+  no_log: true
+
+- name: Install PBS audit systemd service
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/homelab-backup-audit-pbs.service
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Unit]
+      Description=HomeLab PBS backup audit
+      Wants=network-online.target
+      After=network-online.target
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/local/sbin/homelab-backup-audit-pbs
+  when: backup_audit_type | default('') == 'pbs'
+
+- name: Install restic audit systemd services
+  ansible.builtin.copy:
+    dest: "/etc/systemd/system/homelab-backup-audit-{{ item.name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Unit]
+      Description=HomeLab restic offsite backup audit ({{ item.name }})
+      Wants=network-online.target
+      After=network-online.target
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/bin/flock -w 1800 /var/lock/homelab-restic-{{ item.name }}.lock /usr/local/sbin/homelab-backup-audit-restic {{ item.name }}
+  loop: "{{ backup_audit_restic_profiles }}"
+  when: backup_audit_type | default('') == 'restic'
+
+- name: Install PBS audit systemd timer
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/homelab-backup-audit-pbs.timer
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Unit]
+      Description=Run HomeLab PBS backup audit
+
+      [Timer]
+      OnCalendar={{ backup_audit_timer_oncalendar }}
+      Persistent=true
+      RandomizedDelaySec={{ backup_audit_timer_randomized_delay }}
+
+      [Install]
+      WantedBy=timers.target
+  when: backup_audit_type | default('') == 'pbs'
+
+- name: Install restic audit systemd timers
+  ansible.builtin.copy:
+    dest: "/etc/systemd/system/homelab-backup-audit-{{ item.name }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Unit]
+      Description=Run HomeLab restic backup audit ({{ item.name }})
+
+      [Timer]
+      OnCalendar={{ backup_audit_timer_oncalendar }}
+      Persistent=true
+      RandomizedDelaySec={{ backup_audit_timer_randomized_delay }}
+
+      [Install]
+      WantedBy=timers.target
+  loop: "{{ backup_audit_restic_profiles }}"
+  when: backup_audit_type | default('') == 'restic'
+
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable PBS audit timer
+  ansible.builtin.systemd:
+    name: homelab-backup-audit-pbs.timer
+    enabled: true
+    state: started
+  when: backup_audit_type | default('') == 'pbs'
+
+- name: Enable restic audit timers
+  ansible.builtin.systemd:
+    name: "homelab-backup-audit-{{ item.name }}.timer"
+    enabled: true
+    state: started
+  loop: "{{ backup_audit_restic_profiles }}"
+  when: backup_audit_type | default('') == 'restic'
@@ -0,0 +1,70 @@
+#!/bin/sh
+# Managed by Ansible: HomeLab PBS backup audit (L0 freshness)
+set -eu
+
+LOG_FILE="{{ backup_audit_log_file }}"
+METRICS_DIR="{{ backup_audit_metrics_dir }}"
+METRICS_FILE="$METRICS_DIR/homelab_backup_audit_pbs.prom"
+METRICS_TMP=$(mktemp "$METRICS_FILE.XXXXXX")
+STATUS=OK
+NOW_EPOCH=$(date +%s)
+
+trap 'rm -f "$METRICS_TMP"' EXIT
+
+publish_metrics() {
+  chmod 0644 "$METRICS_TMP"
+  mv "$METRICS_TMP" "$METRICS_FILE"
+}
+
+ts() { date '+%Y-%m-%dT%H:%M:%S%z'; }
+log() { echo "[$(ts)] [pbs] $*" | tee -a "$LOG_FILE"; }
+
+get_latest_snapshot() {
+  pvesm list pbs --vmid "$1" 2>/dev/null \
+    | tail -n +2 \
+    | awk '{print $1}' \
+    | sort -t/ -k4 \
+    | tail -1
+}
+
+{% for entry in backup_audit_pbs_vmids %}
+audit_vmid_{{ entry.vmid }}() {
+  vmid={{ entry.vmid }}
+  max_age={{ entry.max_age_hours }}
+  latest=$(get_latest_snapshot "$vmid")
+  if [ -z "$latest" ]; then
+    log "FAIL L0: vmid $vmid — no snapshots in PBS"
+    printf 'homelab_backup_audit_snapshot_age_hours{profile="pbs",vmid="%s"} -1\n' "$vmid" >> "$METRICS_TMP"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 0\n' "$vmid" >> "$METRICS_TMP"
+    STATUS=FAIL
+    return
+  fi
+  ts_str=$(printf '%s' "$latest" | sed -n 's#.*/\([0-9T:Z-]*\)$#\1#p')
+  snap_epoch=$(date -d "$ts_str" +%s 2>/dev/null || echo 0)
+  age_hours=$(( (NOW_EPOCH - snap_epoch) / 3600 ))
+  if [ "$age_hours" -gt "$max_age" ]; then
+    log "FAIL L0: vmid $vmid — latest snapshot age ${age_hours}h > ${max_age}h (snapshot: $ts_str)"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 0\n' "$vmid" >> "$METRICS_TMP"
+    STATUS=FAIL
+  else
+    log "OK L0: vmid $vmid — latest snapshot age ${age_hours}h"
+    printf 'homelab_backup_audit_snapshot_success{profile="pbs",vmid="%s"} 1\n' "$vmid" >> "$METRICS_TMP"
+  fi
+  printf 'homelab_backup_audit_snapshot_age_hours{profile="pbs",vmid="%s"} %s\n' "$vmid" "$age_hours" >> "$METRICS_TMP"
+}
+audit_vmid_{{ entry.vmid }}
+
+{% endfor %}
+if [ "$STATUS" = "OK" ]; then
+  log "AUDIT PASSED"
+  printf 'homelab_backup_audit_success{profile="pbs"} 1\n' >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="pbs"} %s\n' "$NOW_EPOCH" >> "$METRICS_TMP"
+  publish_metrics
+  exit 0
+else
+  log "AUDIT FAILED"
+  printf 'homelab_backup_audit_success{profile="pbs"} 0\n' >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="pbs"} %s\n' "$NOW_EPOCH" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi
@@ -0,0 +1,167 @@
+#!/bin/sh
+# Managed by Ansible: HomeLab restic offsite backup audit (L0+L1+L2)
+set -eu
+
+if [ "$#" -ne 1 ]; then
+  echo "usage: $0 <profile>" >&2
+  exit 64
+fi
+
+PROFILE="$1"
+case "$PROFILE" in
+  ''|*[!A-Za-z0-9_-]*)
+    echo "invalid profile name" >&2
+    exit 64
+    ;;
+esac
+RESTIC_ENV="/etc/homelab-restic/${PROFILE}.env"
+AUDIT_ENV="/etc/homelab-backup-audit/${PROFILE}.env"
+LOG_FILE="{{ backup_audit_log_file }}"
+TMP="/var/lib/homelab-backup-audit/${PROFILE}"
+METRICS_DIR="{{ backup_audit_metrics_dir }}"
+METRICS_FILE="$METRICS_DIR/homelab_backup_audit_${PROFILE}.prom"
+METRICS_TMP=$(mktemp "$METRICS_FILE.XXXXXX")
+STATUS=OK
+
+trap 'rm -f "$METRICS_TMP"' EXIT
+
+publish_metrics() {
+  chmod 0644 "$METRICS_TMP"
+  mv "$METRICS_TMP" "$METRICS_FILE"
+}
+
+for f in "$RESTIC_ENV" "$AUDIT_ENV"; do
+  if [ ! -f "$f" ]; then
+    echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] [${PROFILE}] FAIL: missing env file $f" | tee -a "$LOG_FILE"
+    printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP"
+    printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+    publish_metrics
+    exit 66
+  fi
+done
+
+set -a
+. "$RESTIC_ENV"
+. "$AUDIT_ENV"
+set +a
+
+export RESTIC_REPOSITORY RESTIC_PASSWORD_FILE RCLONE_CONFIG
+
+ts() { date '+%Y-%m-%dT%H:%M:%S%z'; }
+log() { echo "[$(ts)] [${PROFILE}] $*" | tee -a "$LOG_FILE"; }
+
+# ── L0: freshness ──────────────────────────────────────────────────────────
+latest_time=$(restic snapshots --latest 1 --json 2>/dev/null | jq -r '.[0].time // empty')
+if [ -z "$latest_time" ]; then
+  log "FAIL L0: no snapshots found in repository"
+  printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" > "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi
+
+now_epoch=$(date +%s)
+snap_epoch=$(date -d "$latest_time" +%s 2>/dev/null || echo 0)
+age_hours=$(( (now_epoch - snap_epoch) / 3600 ))
+max_age="${HOMELAB_AUDIT_MAX_AGE_HOURS:-36}"
+
+if [ "$age_hours" -gt "$max_age" ]; then
+  log "FAIL L0: latest snapshot age ${age_hours}h > ${max_age}h (snapshot: $latest_time)"
+  STATUS=FAIL
+else
+  log "OK L0: latest snapshot age ${age_hours}h"
+fi
+printf 'homelab_backup_audit_snapshot_age_hours{profile="%s"} %s\n' "$PROFILE" "$age_hours" >> "$METRICS_TMP"
+
+# ── L1: repository integrity ───────────────────────────────────────────────
+if restic check 2>&1 | tee -a "$LOG_FILE"; then
+  log "OK L1: restic check passed"
+  printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+else
+  log "FAIL L1: restic check failed"
+  printf 'homelab_backup_audit_level_success{profile="%s",level="l1"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+  STATUS=FAIL
+fi
+
+# ── L2: SQLite restore + integrity ─────────────────────────────────────────
+if [ -n "${HOMELAB_AUDIT_SQLITE_NAME:-}" ]; then
+  case "$TMP" in
+    /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;;
+    *)
+      log "FAIL L2: unsafe temporary path"
+      exit 64
+      ;;
+  esac
+  rm -rf "$TMP"
+  mkdir -p "$TMP"
+
+  if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_SQLITE_NAME}" 2>&1 | tee -a "$LOG_FILE"; then
+    db=$(find "$TMP" -name "$HOMELAB_AUDIT_SQLITE_NAME" -type f | head -1)
+    if [ -z "$db" ] || [ ! -f "$db" ]; then
+      log "FAIL L2: $HOMELAB_AUDIT_SQLITE_NAME not found in restored data"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+      STATUS=FAIL
+    else
+      check=$(sqlite3 "$db" "PRAGMA integrity_check;" 2>&1)
+      if [ "$check" = "ok" ]; then
+        log "OK L2: SQLite integrity_check ok ($db)"
+        printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+      else
+        log "FAIL L2: SQLite integrity_check: $check"
+        printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+        STATUS=FAIL
+      fi
+    fi
+  else
+    log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_SQLITE_NAME"
+    printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+    STATUS=FAIL
+  fi
+
+  rm -rf "$TMP"
+fi
+
+# ── L2: expected file restore ───────────────────────────────────────────────
+if [ -n "${HOMELAB_AUDIT_EXPECTED_NAME:-}" ]; then
+  case "$TMP" in
+    /var/lib/homelab-backup-audit/[A-Za-z0-9_-]*) ;;
+    *)
+      log "FAIL L2: unsafe temporary path"
+      exit 64
+      ;;
+  esac
+  rm -rf "$TMP"
+  mkdir -p "$TMP"
+
+  if restic restore latest --target "$TMP" --include "**/${HOMELAB_AUDIT_EXPECTED_NAME}" 2>&1 | tee -a "$LOG_FILE"; then
+    expected=$(find "$TMP" -name "$HOMELAB_AUDIT_EXPECTED_NAME" -type f | head -1)
+    if [ -n "$expected" ] && [ -s "$expected" ]; then
+      log "OK L2: restored non-empty expected file ($expected)"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+    else
+      log "FAIL L2: $HOMELAB_AUDIT_EXPECTED_NAME not found or empty"
+      printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+      STATUS=FAIL
+    fi
+  else
+    log "FAIL L2: restic restore failed for $HOMELAB_AUDIT_EXPECTED_NAME"
+    printf 'homelab_backup_audit_level_success{profile="%s",level="l2"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+    STATUS=FAIL
+  fi
+
+  rm -rf "$TMP"
+fi
+
+if [ "$STATUS" = "OK" ]; then
+  log "AUDIT PASSED"
+  printf 'homelab_backup_audit_success{profile="%s"} 1\n' "$PROFILE" >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 0
+else
+  log "AUDIT FAILED"
+  printf 'homelab_backup_audit_success{profile="%s"} 0\n' "$PROFILE" >> "$METRICS_TMP"
+  printf 'homelab_backup_audit_timestamp_seconds{profile="%s"} %s\n' "$PROFILE" "$(date +%s)" >> "$METRICS_TMP"
+  publish_metrics
+  exit 1
+fi