Add read-only infrastructure status playbook

One command to see the state of everything: reachability, uptime, disk
usage, service unit states, failed units, pct list on the Proxmox nodes,
OpenVPN transport health, and the last run of each backup job.

An unreachable host is reported as data, not as a run failure, so a
single host being down still produces a full summary. Every command is
changed_when: false with check_mode: false, so the playbook is read-only
and works under --check. Backup freshness is read from what systemd
already recorded rather than by invoking the audit scripts, which would
hit PBS and Yandex Disk and take locks.

Service units are derived from inventory groups where possible; only
app-specific units need the per-host map, and each was taken from the
playbook or role that installs it.

Verified against live infrastructure: 15 hosts, changed=0.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71

ansible/playbooks/status.yml

@@ -0,0 +1,383 @@
+---
+# One-shot, human-readable "what is my HomeLab doing right now" report.
+#
+#   ansible-playbook playbooks/status.yml --ask-vault-pass
+#   ansible-playbook playbooks/status.yml --limit lxc_infra
+#   ansible-playbook playbooks/status.yml --check       # same output, changes nothing
+#
+# Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run
+# that includes the `gyro` host needs --ask-vault-pass / --vault-password-file
+# (same as playbooks/check.yml). Without the password use --limit '!gyro'.
+#
+# Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`,
+# nothing is started, installed or written. An unreachable host is a *result*
+# (reported as DOWN), not a playbook failure.
+
+- name: Collect HomeLab status
+  hosts: servers
+  gather_facts: false
+  ignore_unreachable: true
+  vars:
+    # Application units per host. Every name below is taken from the playbook or
+    # role that installs it:
+    #   adguard.service              playbooks/pve-adguard.yml
+    #   emergency-bot.service        roles/emergency_bot
+    #   gitea.service                playbooks/pve-gitea.yml
+    #   grimmory{,-docker-firewall}  playbooks/pve-grimmory.yml
+    #   gyro.timer                   roles/gyro (gyro.service is oneshot -> see jobs)
+    #   hermes-ai-tun-proxy.service  playbooks/pve-hermes-ai.yml
+    #   memoir-bot.service           playbooks/pve-memoir-bot.yml
+    #   mihomo{,-ui}.service         playbooks/pve-mihomo.yml
+    #   uptime-kuma.service          roles/uptime_kuma
+    #   vaultwarden.service          playbooks/pve-vaultwarden.yml
+    #   docker.service               installed by each of the compose-based playbooks
+    status_service_units:
+      adguard: [docker.service, adguard.service]
+      docker-test: [docker.service]
+      emergency-bot: [emergency-bot.service]
+      gitea: [docker.service, gitea.service]
+      grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service]
+      gyro: [gyro.timer]
+      hermes-ai: [docker.service, hermes-ai-tun-proxy.service]
+      memoir-bot: [docker.service, memoir-bot.service]
+      mihomo: [docker.service, mihomo.service, mihomo-ui.service]
+      monitoring: [docker.service, uptime-kuma.service]
+      ru-vps: [docker.service]
+      vaultwarden: [docker.service, vaultwarden.service]
+
+    # Oneshot / timer-driven jobs. They are normally "inactive", so instead of
+    # is-active we report the last run (ExecMainExitTimestamp + Result).
+    # Running the audit scripts themselves (roles/backup_audit) would hit PBS and
+    # Yandex Disk over the network, so we only read what systemd already knows.
+    status_job_units:
+      cloud-pc:
+        - homelab-restic-offsite-gitea.service
+        - homelab-backup-audit-gitea.service
+      grimmory:
+        - homelab-restic-offsite-grimmory.service
+        - homelab-backup-audit-grimmory.service
+      gyro: [gyro.service]
+      mini-pc: [homelab-backup-audit-pbs.service]
+      ru-vps: [homelab-monitoring-push.service]
+      vaultwarden:
+        - homelab-restic-offsite-vaultwarden.service
+        - homelab-backup-audit-vaultwarden.service
+
+    # Group-driven units: derived from inventory groups, not hardcoded per host.
+    status_units: >-
+      {{ (status_service_units[inventory_hostname] | default([]))
+         + (['prometheus-node-exporter.service']
+            if inventory_hostname in (groups['monitoring_exporters'] | default([])) else [])
+         + (['homelab-smartctl-metrics.timer']
+            if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else [])
+         + ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service']
+            if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }}
+
+    status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}"
+
+    # Root plus whatever extra storage the inventory declares for this host.
+    status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}"
+
+    # Same idea as playbooks/openvpn-check.yml, but built from the inventory:
+    # PVE web UI on every pve_nodes member, PBS web UI on pbs.
+    status_vpn_probe_targets: >-
+      {{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host')
+          | map('regex_replace', '^(.+)$', '\1:8006') | list)
+         + (groups['lxc_infra'] | default([]) | select('eq', 'pbs')
+            | map('extract', hostvars, 'ansible_host')
+            | map('regex_replace', '^(.+)$', '\1:8007') | list) }}
+
+  tasks:
+    - name: Probe host reachability
+      ansible.builtin.ping:
+      register: status_ping
+      ignore_unreachable: true
+      ignore_errors: true
+      check_mode: false
+
+    - name: Record reachability
+      ansible.builtin.set_fact:
+        status_reachable: >-
+          {{ (not (status_ping.unreachable | default(false)))
+             and (not (status_ping.failed | default(false))) }}
+
+    - name: Collect host state
+      when: status_reachable | bool
+      check_mode: false
+      ignore_unreachable: true
+      block:
+        - name: Gather hardware facts (uptime, memory)
+          ansible.builtin.setup:
+            gather_subset:
+              - hardware
+          failed_when: false
+
+        - name: Read root filesystem usage
+          ansible.builtin.shell:
+            cmd: >-
+              LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}'
+          register: status_root_df
+          changed_when: false
+          failed_when: false
+
+        - name: Read declared storage mount usage
+          ansible.builtin.command:
+            argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}"
+          register: status_disk_df
+          changed_when: false
+          failed_when: false
+          when: status_disk_paths | length > 1
+
+        - name: Check service unit states
+          ansible.builtin.command:
+            argv: "{{ ['systemctl', 'is-active'] + status_units }}"
+          register: status_units_state
+          changed_when: false
+          failed_when: false
+          when: status_units | length > 0
+
+        - name: List failed systemd units
+          ansible.builtin.shell:
+            cmd: >-
+              systemctl list-units --state=failed --no-legend --plain --no-pager
+              | awk '{print $1}' | head -n 5
+          register: status_failed_units
+          changed_when: false
+          failed_when: false
+
+        - name: Read last run of backup and oneshot jobs
+          ansible.builtin.shell:
+            cmd: |
+              set -u
+              now=$(date +%s)
+              for unit in {{ status_jobs | map('quote') | join(' ') }}; do
+                ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true)
+                res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true)
+                if [ -n "$ts" ]; then
+                  epoch=$(date -d "$ts" +%s 2>/dev/null || true)
+                  if [ -n "${epoch:-}" ]; then
+                    age="$(( (now - epoch) / 3600 ))h ago"
+                  else
+                    age="?"
+                  fi
+                else
+                  ts="never"
+                  age="-"
+                  res="-"
+                fi
+                printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}"
+              done
+          register: status_job_state
+          changed_when: false
+          failed_when: false
+          when: status_jobs | length > 0
+
+        - name: List Proxmox containers
+          ansible.builtin.shell:
+            cmd: >-
+              pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}'
+          register: status_pct
+          changed_when: false
+          failed_when: false
+          when: inventory_hostname in (groups['pve_nodes'] | default([]))
+
+        - name: Ping the OpenVPN peer
+          ansible.builtin.command:
+            argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"]
+          register: status_vpn_ping
+          changed_when: false
+          failed_when: false
+          when:
+            - inventory_hostname in (groups['vpn_openvpn'] | default([]))
+            - openvpn_peer_ip is defined
+
+        - name: Probe LAN services through the OpenVPN tunnel
+          ansible.builtin.shell:
+            cmd: |
+              for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do
+                host=${target%:*}
+                port=${target##*:}
+                if nc -z -w 3 "$host" "$port" 2>/dev/null; then
+                  printf '%-24s OK\n' "$target"
+                else
+                  printf '%-24s FAIL\n' "$target"
+                fi
+              done
+          register: status_vpn_probe
+          changed_when: false
+          failed_when: false
+          when:
+            - (openvpn_role | default('')) == 'server'
+            - status_vpn_probe_targets | length > 0
+
+    - name: Build status record
+      ansible.builtin.set_fact:
+        status_record:
+          state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}"
+          uptime: >-
+            {{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd '
+               ~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h'
+               if ansible_facts.uptime_seconds is defined else '-' }}
+          root_disk: "{{ status_root_df.stdout | default('-') | trim }}"
+          units: >-
+            {{ status_units
+               | zip(status_units_state.stdout_lines | default([]))
+               | map('join', '=') | join(' ') }}
+          failed_units: "{{ status_failed_units.stdout_lines | default([]) }}"
+          jobs: "{{ status_job_state.stdout_lines | default([]) }}"
+          disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}"
+          containers: "{{ status_pct.stdout_lines | default([]) }}"
+          vpn_peer: >-
+            {{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL')
+               if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }}
+          vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}"
+          vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}"
+
+
+- name: Print HomeLab status summary
+  # Runs once, on the controller, so it also works with --limit.
+  hosts: servers
+  gather_facts: false
+  become: false
+  ignore_unreachable: true
+  tasks:
+    - name: Render the summary
+      run_once: true
+      delegate_to: localhost
+      block:
+        - name: Select hosts that produced a record
+          ansible.builtin.set_fact:
+            status_hosts: >-
+              {{ ansible_play_hosts_all | sort
+                 | map('extract', hostvars)
+                 | selectattr('status_record', 'defined')
+                 | map(attribute='inventory_hostname') | list }}
+            status_states: >-
+              {{ ansible_play_hosts_all | sort
+                 | map('extract', hostvars)
+                 | selectattr('status_record', 'defined')
+                 | map(attribute='status_record.state') | list }}
+
+        - name: Start the report
+          ansible.builtin.set_fact:
+            status_report:
+              - ""
+              - "================ HOMELAB STATUS ================================================"
+              - >-
+                hosts: {{ status_hosts | length }}
+                up: {{ status_states | select('equalto', 'UP') | list | length }}
+                down: {{ status_states | select('equalto', 'DOWN') | list | length }}
+              - ""
+              - "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}"
+              - "{{ '-' * 80 }}"
+
+        - name: Add one line per host
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + [line] }}"
+          vars:
+            rec: "{{ hostvars[item].status_record }}"
+            line: >-
+              {{ '%-13s %-5s %-9s %-24s %s' | format(
+                   item,
+                   rec.state,
+                   rec.uptime | trim | default('-', true),
+                   rec.root_disk | trim | default('-', true),
+                   rec.units | trim | default('-', true)) }}
+          loop: "{{ status_hosts }}"
+
+        - name: Add Proxmox container section header
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}"
+
+        - name: Add containers per Proxmox node
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report
+                 + ['  ' ~ item ~ ':  running=' ~ (running | length)
+                    ~ '  other=' ~ (stopped | length)]
+                 + (stopped | map('regex_replace', '^\s*', '    not running: ') | list) }}
+          vars:
+            lines: "{{ hostvars[item].status_record.containers }}"
+            running: "{{ lines | select('search', ' running ') | list }}"
+            stopped: "{{ lines | reject('search', ' running ') | list }}"
+          loop: "{{ status_hosts }}"
+          when: hostvars[item].status_record.containers | length > 0
+
+        - name: Add storage section header
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}"
+
+        - name: Add storage lines per host
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report + ['  ' ~ item ~ ':']
+                 + (hostvars[item].status_record.disk_lines
+                    | map('regex_replace', '^', '    ') | list) }}
+          loop: "{{ status_hosts }}"
+          when: hostvars[item].status_record.disk_lines | length > 0
+
+        - name: Add OpenVPN transport section header
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}"
+
+        - name: Add OpenVPN peer results
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report
+                 + ['  ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip
+                    ~ ' : ' ~ hostvars[item].status_record.vpn_peer]
+                 + (hostvars[item].status_record.vpn_probes
+                    | map('regex_replace', '^', '    via tunnel: ') | list) }}
+          loop: "{{ status_hosts }}"
+          when: hostvars[item].status_record.vpn_peer != '-'
+
+        - name: Add backup and job section header
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }}
+
+        - name: Add job lines per host
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report + ['  ' ~ item ~ ':']
+                 + (hostvars[item].status_record.jobs
+                    | map('regex_replace', '^', '    ') | list) }}
+          loop: "{{ status_hosts }}"
+          when: hostvars[item].status_record.jobs | length > 0
+
+        - name: Add failed units section header
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}"
+
+        - name: Add failed units per host
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report + ['  ' ~ item ~ ': '
+                 ~ (hostvars[item].status_record.failed_units | join(', '))] }}
+          loop: "{{ status_hosts }}"
+          when: hostvars[item].status_record.failed_units | length > 0
+
+        - name: Note when nothing failed
+          ansible.builtin.set_fact:
+            status_report: "{{ status_report + ['  none'] }}"
+          when: >-
+            (status_hosts | map('extract', hostvars)
+             | map(attribute='status_record.failed_units', default=[])
+             | flatten | length) == 0
+
+        - name: Add unreachable section
+          ansible.builtin.set_fact:
+            status_report: >-
+              {{ status_report + ['', 'UNREACHABLE', '-' * 80]
+                 + (down_hosts | map('regex_replace', '^', '  ') | list
+                    if (down_hosts | length) > 0 else ['  none'])
+                 + ['', '=' * 80, ''] }}
+          vars:
+            down_hosts: >-
+              {{ status_hosts | map('extract', hostvars)
+                 | selectattr('status_record.state', 'equalto', 'DOWN')
+                 | map(attribute='inventory_hostname') | list }}
+
+        - name: Print HomeLab status
+          ansible.builtin.debug:
+            msg: "{{ status_report | join('\n') }}"