ada/infra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
646f2bbc8f1081ae7286fbc9150dc0620b691195
infra/ansible/playbooks/status.yml
T
DmitryandClaude Opus 5 d535ef2d32 Add read-only infrastructure status playbook 
One command to see the state of everything: reachability, uptime, disk
usage, service unit states, failed units, pct list on the Proxmox nodes,
OpenVPN transport health, and the last run of each backup job.

An unreachable host is reported as data, not as a run failure, so a
single host being down still produces a full summary. Every command is
changed_when: false with check_mode: false, so the playbook is read-only
and works under --check. Backup freshness is read from what systemd
already recorded rather than by invoking the audit scripts, which would
hit PBS and Yandex Disk and take locks.

Service units are derived from inventory groups where possible; only
app-specific units need the per-host map, and each was taken from the
playbook or role that installs it.

Verified against live infrastructure: 15 hosts, changed=0.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
2026-08-26 22:10:29 +03:00

384 lines
16 KiB
YAML
Raw Blame History

---
# One-shot, human-readable "what is my HomeLab doing right now" report.
#
# ansible-playbook playbooks/status.yml --ask-vault-pass
# ansible-playbook playbooks/status.yml --limit lxc_infra
# ansible-playbook playbooks/status.yml --check # same output, changes nothing
#
# Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run
# that includes the `gyro` host needs --ask-vault-pass / --vault-password-file
# (same as playbooks/check.yml). Without the password use --limit '!gyro'.
#
# Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`,
# nothing is started, installed or written. An unreachable host is a *result*
# (reported as DOWN), not a playbook failure.
- name: Collect HomeLab status
hosts: servers
gather_facts: false
ignore_unreachable: true
vars:
# Application units per host. Every name below is taken from the playbook or
# role that installs it:
# adguard.service playbooks/pve-adguard.yml
# emergency-bot.service roles/emergency_bot
# gitea.service playbooks/pve-gitea.yml
# grimmory{,-docker-firewall} playbooks/pve-grimmory.yml
# gyro.timer roles/gyro (gyro.service is oneshot -> see jobs)
# hermes-ai-tun-proxy.service playbooks/pve-hermes-ai.yml
# memoir-bot.service playbooks/pve-memoir-bot.yml
# mihomo{,-ui}.service playbooks/pve-mihomo.yml
# uptime-kuma.service roles/uptime_kuma
# vaultwarden.service playbooks/pve-vaultwarden.yml
# docker.service installed by each of the compose-based playbooks
status_service_units:
adguard: [docker.service, adguard.service]
docker-test: [docker.service]
emergency-bot: [emergency-bot.service]
gitea: [docker.service, gitea.service]
grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service]
gyro: [gyro.timer]
hermes-ai: [docker.service, hermes-ai-tun-proxy.service]
memoir-bot: [docker.service, memoir-bot.service]
mihomo: [docker.service, mihomo.service, mihomo-ui.service]
monitoring: [docker.service, uptime-kuma.service]
ru-vps: [docker.service]
vaultwarden: [docker.service, vaultwarden.service]
# Oneshot / timer-driven jobs. They are normally "inactive", so instead of
# is-active we report the last run (ExecMainExitTimestamp + Result).
# Running the audit scripts themselves (roles/backup_audit) would hit PBS and
# Yandex Disk over the network, so we only read what systemd already knows.
status_job_units:
cloud-pc:
- homelab-restic-offsite-gitea.service
- homelab-backup-audit-gitea.service
grimmory:
- homelab-restic-offsite-grimmory.service
- homelab-backup-audit-grimmory.service
gyro: [gyro.service]
mini-pc: [homelab-backup-audit-pbs.service]
ru-vps: [homelab-monitoring-push.service]
vaultwarden:
- homelab-restic-offsite-vaultwarden.service
- homelab-backup-audit-vaultwarden.service
# Group-driven units: derived from inventory groups, not hardcoded per host.
status_units: >-
{{ (status_service_units[inventory_hostname] | default([]))
+ (['prometheus-node-exporter.service']
if inventory_hostname in (groups['monitoring_exporters'] | default([])) else [])
+ (['homelab-smartctl-metrics.timer']
if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else [])
+ ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service']
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }}
status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}"
# Root plus whatever extra storage the inventory declares for this host.
status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}"
# Same idea as playbooks/openvpn-check.yml, but built from the inventory:
# PVE web UI on every pve_nodes member, PBS web UI on pbs.
status_vpn_probe_targets: >-
{{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host')
| map('regex_replace', '^(.+)$', '\1:8006') | list)
+ (groups['lxc_infra'] | default([]) | select('eq', 'pbs')
| map('extract', hostvars, 'ansible_host')
| map('regex_replace', '^(.+)$', '\1:8007') | list) }}
tasks:
- name: Probe host reachability
ansible.builtin.ping:
register: status_ping
ignore_unreachable: true
ignore_errors: true
check_mode: false
- name: Record reachability
ansible.builtin.set_fact:
status_reachable: >-
{{ (not (status_ping.unreachable | default(false)))
and (not (status_ping.failed | default(false))) }}
- name: Collect host state
when: status_reachable | bool
check_mode: false
ignore_unreachable: true
block:
- name: Gather hardware facts (uptime, memory)
ansible.builtin.setup:
gather_subset:
- hardware
failed_when: false
- name: Read root filesystem usage
ansible.builtin.shell:
cmd: >-
LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}'
register: status_root_df
changed_when: false
failed_when: false
- name: Read declared storage mount usage
ansible.builtin.command:
argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}"
register: status_disk_df
changed_when: false
failed_when: false
when: status_disk_paths | length > 1
- name: Check service unit states
ansible.builtin.command:
argv: "{{ ['systemctl', 'is-active'] + status_units }}"
register: status_units_state
changed_when: false
failed_when: false
when: status_units | length > 0
- name: List failed systemd units
ansible.builtin.shell:
cmd: >-
systemctl list-units --state=failed --no-legend --plain --no-pager
| awk '{print $1}' | head -n 5
register: status_failed_units
changed_when: false
failed_when: false
- name: Read last run of backup and oneshot jobs
ansible.builtin.shell:
cmd: |
set -u
now=$(date +%s)
for unit in {{ status_jobs | map('quote') | join(' ') }}; do
ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true)
res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true)
if [ -n "$ts" ]; then
epoch=$(date -d "$ts" +%s 2>/dev/null || true)
if [ -n "${epoch:-}" ]; then
age="$(( (now - epoch) / 3600 ))h ago"
else
age="?"
fi
else
ts="never"
age="-"
res="-"
fi
printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}"
done
register: status_job_state
changed_when: false
failed_when: false
when: status_jobs | length > 0
- name: List Proxmox containers
ansible.builtin.shell:
cmd: >-
pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}'
register: status_pct
changed_when: false
failed_when: false
when: inventory_hostname in (groups['pve_nodes'] | default([]))
- name: Ping the OpenVPN peer
ansible.builtin.command:
argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"]
register: status_vpn_ping
changed_when: false
failed_when: false
when:
- inventory_hostname in (groups['vpn_openvpn'] | default([]))
- openvpn_peer_ip is defined
- name: Probe LAN services through the OpenVPN tunnel
ansible.builtin.shell:
cmd: |
for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do
host=${target%:*}
port=${target##*:}
if nc -z -w 3 "$host" "$port" 2>/dev/null; then
printf '%-24s OK\n' "$target"
else
printf '%-24s FAIL\n' "$target"
fi
done
register: status_vpn_probe
changed_when: false
failed_when: false
when:
- (openvpn_role | default('')) == 'server'
- status_vpn_probe_targets | length > 0
- name: Build status record
ansible.builtin.set_fact:
status_record:
state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}"
uptime: >-
{{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd '
~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h'
if ansible_facts.uptime_seconds is defined else '-' }}
root_disk: "{{ status_root_df.stdout | default('-') | trim }}"
units: >-
{{ status_units
| zip(status_units_state.stdout_lines | default([]))
| map('join', '=') | join(' ') }}
failed_units: "{{ status_failed_units.stdout_lines | default([]) }}"
jobs: "{{ status_job_state.stdout_lines | default([]) }}"
disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}"
containers: "{{ status_pct.stdout_lines | default([]) }}"
vpn_peer: >-
{{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL')
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }}
vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}"
vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}"
- name: Print HomeLab status summary
# Runs once, on the controller, so it also works with --limit.
hosts: servers
gather_facts: false
become: false
ignore_unreachable: true
tasks:
- name: Render the summary
run_once: true
delegate_to: localhost
block:
- name: Select hosts that produced a record
ansible.builtin.set_fact:
status_hosts: >-
{{ ansible_play_hosts_all | sort
| map('extract', hostvars)
| selectattr('status_record', 'defined')
| map(attribute='inventory_hostname') | list }}
status_states: >-
{{ ansible_play_hosts_all | sort
| map('extract', hostvars)
| selectattr('status_record', 'defined')
| map(attribute='status_record.state') | list }}
- name: Start the report
ansible.builtin.set_fact:
status_report:
- ""
- "================ HOMELAB STATUS ================================================"
- >-
hosts: {{ status_hosts | length }}
up: {{ status_states | select('equalto', 'UP') | list | length }}
down: {{ status_states | select('equalto', 'DOWN') | list | length }}
- ""
- "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}"
- "{{ '-' * 80 }}"
- name: Add one line per host
ansible.builtin.set_fact:
status_report: "{{ status_report + [line] }}"
vars:
rec: "{{ hostvars[item].status_record }}"
line: >-
{{ '%-13s %-5s %-9s %-24s %s' | format(
item,
rec.state,
rec.uptime | trim | default('-', true),
rec.root_disk | trim | default('-', true),
rec.units | trim | default('-', true)) }}
loop: "{{ status_hosts }}"
- name: Add Proxmox container section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}"
- name: Add containers per Proxmox node
ansible.builtin.set_fact:
status_report: >-
{{ status_report
+ [' ' ~ item ~ ': running=' ~ (running | length)
~ ' other=' ~ (stopped | length)]
+ (stopped | map('regex_replace', '^\s*', ' not running: ') | list) }}
vars:
lines: "{{ hostvars[item].status_record.containers }}"
running: "{{ lines | select('search', ' running ') | list }}"
stopped: "{{ lines | reject('search', ' running ') | list }}"
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.containers | length > 0
- name: Add storage section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}"
- name: Add storage lines per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ':']
+ (hostvars[item].status_record.disk_lines
| map('regex_replace', '^', ' ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.disk_lines | length > 0
- name: Add OpenVPN transport section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}"
- name: Add OpenVPN peer results
ansible.builtin.set_fact:
status_report: >-
{{ status_report
+ [' ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip
~ ' : ' ~ hostvars[item].status_record.vpn_peer]
+ (hostvars[item].status_record.vpn_probes
| map('regex_replace', '^', ' via tunnel: ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.vpn_peer != '-'
- name: Add backup and job section header
ansible.builtin.set_fact:
status_report: >-
{{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }}
- name: Add job lines per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ':']
+ (hostvars[item].status_record.jobs
| map('regex_replace', '^', ' ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.jobs | length > 0
- name: Add failed units section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}"
- name: Add failed units per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ': '
~ (hostvars[item].status_record.failed_units | join(', '))] }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.failed_units | length > 0
- name: Note when nothing failed
ansible.builtin.set_fact:
status_report: "{{ status_report + [' none'] }}"
when: >-
(status_hosts | map('extract', hostvars)
| map(attribute='status_record.failed_units', default=[])
| flatten | length) == 0
- name: Add unreachable section
ansible.builtin.set_fact:
status_report: >-
{{ status_report + ['', 'UNREACHABLE', '-' * 80]
+ (down_hosts | map('regex_replace', '^', ' ') | list
if (down_hosts | length) > 0 else [' none'])
+ ['', '=' * 80, ''] }}
vars:
down_hosts: >-
{{ status_hosts | map('extract', hostvars)
| selectattr('status_record.state', 'equalto', 'DOWN')
| map(attribute='inventory_hostname') | list }}
- name: Print HomeLab status
ansible.builtin.debug:
msg: "{{ status_report | join('\n') }}"
Reference in New Issue View Git Blame Copy Permalink