diff --git a/ansible/playbooks/status.yml b/ansible/playbooks/status.yml new file mode 100644 index 0000000..34a60ae --- /dev/null +++ b/ansible/playbooks/status.yml @@ -0,0 +1,383 @@ +--- +# One-shot, human-readable "what is my HomeLab doing right now" report. +# +# ansible-playbook playbooks/status.yml --ask-vault-pass +# ansible-playbook playbooks/status.yml --limit lxc_infra +# ansible-playbook playbooks/status.yml --check # same output, changes nothing +# +# Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run +# that includes the `gyro` host needs --ask-vault-pass / --vault-password-file +# (same as playbooks/check.yml). Without the password use --limit '!gyro'. +# +# Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`, +# nothing is started, installed or written. An unreachable host is a *result* +# (reported as DOWN), not a playbook failure. + +- name: Collect HomeLab status + hosts: servers + gather_facts: false + ignore_unreachable: true + vars: + # Application units per host. Every name below is taken from the playbook or + # role that installs it: + # adguard.service playbooks/pve-adguard.yml + # emergency-bot.service roles/emergency_bot + # gitea.service playbooks/pve-gitea.yml + # grimmory{,-docker-firewall} playbooks/pve-grimmory.yml + # gyro.timer roles/gyro (gyro.service is oneshot -> see jobs) + # hermes-ai-tun-proxy.service playbooks/pve-hermes-ai.yml + # memoir-bot.service playbooks/pve-memoir-bot.yml + # mihomo{,-ui}.service playbooks/pve-mihomo.yml + # uptime-kuma.service roles/uptime_kuma + # vaultwarden.service playbooks/pve-vaultwarden.yml + # docker.service installed by each of the compose-based playbooks + status_service_units: + adguard: [docker.service, adguard.service] + docker-test: [docker.service] + emergency-bot: [emergency-bot.service] + gitea: [docker.service, gitea.service] + grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service] + gyro: [gyro.timer] + hermes-ai: [docker.service, hermes-ai-tun-proxy.service] + memoir-bot: [docker.service, memoir-bot.service] + mihomo: [docker.service, mihomo.service, mihomo-ui.service] + monitoring: [docker.service, uptime-kuma.service] + ru-vps: [docker.service] + vaultwarden: [docker.service, vaultwarden.service] + + # Oneshot / timer-driven jobs. They are normally "inactive", so instead of + # is-active we report the last run (ExecMainExitTimestamp + Result). + # Running the audit scripts themselves (roles/backup_audit) would hit PBS and + # Yandex Disk over the network, so we only read what systemd already knows. + status_job_units: + cloud-pc: + - homelab-restic-offsite-gitea.service + - homelab-backup-audit-gitea.service + grimmory: + - homelab-restic-offsite-grimmory.service + - homelab-backup-audit-grimmory.service + gyro: [gyro.service] + mini-pc: [homelab-backup-audit-pbs.service] + ru-vps: [homelab-monitoring-push.service] + vaultwarden: + - homelab-restic-offsite-vaultwarden.service + - homelab-backup-audit-vaultwarden.service + + # Group-driven units: derived from inventory groups, not hardcoded per host. + status_units: >- + {{ (status_service_units[inventory_hostname] | default([])) + + (['prometheus-node-exporter.service'] + if inventory_hostname in (groups['monitoring_exporters'] | default([])) else []) + + (['homelab-smartctl-metrics.timer'] + if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else []) + + ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service'] + if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }} + + status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}" + + # Root plus whatever extra storage the inventory declares for this host. + status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}" + + # Same idea as playbooks/openvpn-check.yml, but built from the inventory: + # PVE web UI on every pve_nodes member, PBS web UI on pbs. + status_vpn_probe_targets: >- + {{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host') + | map('regex_replace', '^(.+)$', '\1:8006') | list) + + (groups['lxc_infra'] | default([]) | select('eq', 'pbs') + | map('extract', hostvars, 'ansible_host') + | map('regex_replace', '^(.+)$', '\1:8007') | list) }} + + tasks: + - name: Probe host reachability + ansible.builtin.ping: + register: status_ping + ignore_unreachable: true + ignore_errors: true + check_mode: false + + - name: Record reachability + ansible.builtin.set_fact: + status_reachable: >- + {{ (not (status_ping.unreachable | default(false))) + and (not (status_ping.failed | default(false))) }} + + - name: Collect host state + when: status_reachable | bool + check_mode: false + ignore_unreachable: true + block: + - name: Gather hardware facts (uptime, memory) + ansible.builtin.setup: + gather_subset: + - hardware + failed_when: false + + - name: Read root filesystem usage + ansible.builtin.shell: + cmd: >- + LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}' + register: status_root_df + changed_when: false + failed_when: false + + - name: Read declared storage mount usage + ansible.builtin.command: + argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}" + register: status_disk_df + changed_when: false + failed_when: false + when: status_disk_paths | length > 1 + + - name: Check service unit states + ansible.builtin.command: + argv: "{{ ['systemctl', 'is-active'] + status_units }}" + register: status_units_state + changed_when: false + failed_when: false + when: status_units | length > 0 + + - name: List failed systemd units + ansible.builtin.shell: + cmd: >- + systemctl list-units --state=failed --no-legend --plain --no-pager + | awk '{print $1}' | head -n 5 + register: status_failed_units + changed_when: false + failed_when: false + + - name: Read last run of backup and oneshot jobs + ansible.builtin.shell: + cmd: | + set -u + now=$(date +%s) + for unit in {{ status_jobs | map('quote') | join(' ') }}; do + ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true) + res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true) + if [ -n "$ts" ]; then + epoch=$(date -d "$ts" +%s 2>/dev/null || true) + if [ -n "${epoch:-}" ]; then + age="$(( (now - epoch) / 3600 ))h ago" + else + age="?" + fi + else + ts="never" + age="-" + res="-" + fi + printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}" + done + register: status_job_state + changed_when: false + failed_when: false + when: status_jobs | length > 0 + + - name: List Proxmox containers + ansible.builtin.shell: + cmd: >- + pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}' + register: status_pct + changed_when: false + failed_when: false + when: inventory_hostname in (groups['pve_nodes'] | default([])) + + - name: Ping the OpenVPN peer + ansible.builtin.command: + argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"] + register: status_vpn_ping + changed_when: false + failed_when: false + when: + - inventory_hostname in (groups['vpn_openvpn'] | default([])) + - openvpn_peer_ip is defined + + - name: Probe LAN services through the OpenVPN tunnel + ansible.builtin.shell: + cmd: | + for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do + host=${target%:*} + port=${target##*:} + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + printf '%-24s OK\n' "$target" + else + printf '%-24s FAIL\n' "$target" + fi + done + register: status_vpn_probe + changed_when: false + failed_when: false + when: + - (openvpn_role | default('')) == 'server' + - status_vpn_probe_targets | length > 0 + + - name: Build status record + ansible.builtin.set_fact: + status_record: + state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}" + uptime: >- + {{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd ' + ~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h' + if ansible_facts.uptime_seconds is defined else '-' }} + root_disk: "{{ status_root_df.stdout | default('-') | trim }}" + units: >- + {{ status_units + | zip(status_units_state.stdout_lines | default([])) + | map('join', '=') | join(' ') }} + failed_units: "{{ status_failed_units.stdout_lines | default([]) }}" + jobs: "{{ status_job_state.stdout_lines | default([]) }}" + disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}" + containers: "{{ status_pct.stdout_lines | default([]) }}" + vpn_peer: >- + {{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL') + if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }} + vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}" + vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}" + + +- name: Print HomeLab status summary + # Runs once, on the controller, so it also works with --limit. + hosts: servers + gather_facts: false + become: false + ignore_unreachable: true + tasks: + - name: Render the summary + run_once: true + delegate_to: localhost + block: + - name: Select hosts that produced a record + ansible.builtin.set_fact: + status_hosts: >- + {{ ansible_play_hosts_all | sort + | map('extract', hostvars) + | selectattr('status_record', 'defined') + | map(attribute='inventory_hostname') | list }} + status_states: >- + {{ ansible_play_hosts_all | sort + | map('extract', hostvars) + | selectattr('status_record', 'defined') + | map(attribute='status_record.state') | list }} + + - name: Start the report + ansible.builtin.set_fact: + status_report: + - "" + - "================ HOMELAB STATUS ================================================" + - >- + hosts: {{ status_hosts | length }} + up: {{ status_states | select('equalto', 'UP') | list | length }} + down: {{ status_states | select('equalto', 'DOWN') | list | length }} + - "" + - "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}" + - "{{ '-' * 80 }}" + + - name: Add one line per host + ansible.builtin.set_fact: + status_report: "{{ status_report + [line] }}" + vars: + rec: "{{ hostvars[item].status_record }}" + line: >- + {{ '%-13s %-5s %-9s %-24s %s' | format( + item, + rec.state, + rec.uptime | trim | default('-', true), + rec.root_disk | trim | default('-', true), + rec.units | trim | default('-', true)) }} + loop: "{{ status_hosts }}" + + - name: Add Proxmox container section header + ansible.builtin.set_fact: + status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}" + + - name: Add containers per Proxmox node + ansible.builtin.set_fact: + status_report: >- + {{ status_report + + [' ' ~ item ~ ': running=' ~ (running | length) + ~ ' other=' ~ (stopped | length)] + + (stopped | map('regex_replace', '^\s*', ' not running: ') | list) }} + vars: + lines: "{{ hostvars[item].status_record.containers }}" + running: "{{ lines | select('search', ' running ') | list }}" + stopped: "{{ lines | reject('search', ' running ') | list }}" + loop: "{{ status_hosts }}" + when: hostvars[item].status_record.containers | length > 0 + + - name: Add storage section header + ansible.builtin.set_fact: + status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}" + + - name: Add storage lines per host + ansible.builtin.set_fact: + status_report: >- + {{ status_report + [' ' ~ item ~ ':'] + + (hostvars[item].status_record.disk_lines + | map('regex_replace', '^', ' ') | list) }} + loop: "{{ status_hosts }}" + when: hostvars[item].status_record.disk_lines | length > 0 + + - name: Add OpenVPN transport section header + ansible.builtin.set_fact: + status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}" + + - name: Add OpenVPN peer results + ansible.builtin.set_fact: + status_report: >- + {{ status_report + + [' ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip + ~ ' : ' ~ hostvars[item].status_record.vpn_peer] + + (hostvars[item].status_record.vpn_probes + | map('regex_replace', '^', ' via tunnel: ') | list) }} + loop: "{{ status_hosts }}" + when: hostvars[item].status_record.vpn_peer != '-' + + - name: Add backup and job section header + ansible.builtin.set_fact: + status_report: >- + {{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }} + + - name: Add job lines per host + ansible.builtin.set_fact: + status_report: >- + {{ status_report + [' ' ~ item ~ ':'] + + (hostvars[item].status_record.jobs + | map('regex_replace', '^', ' ') | list) }} + loop: "{{ status_hosts }}" + when: hostvars[item].status_record.jobs | length > 0 + + - name: Add failed units section header + ansible.builtin.set_fact: + status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}" + + - name: Add failed units per host + ansible.builtin.set_fact: + status_report: >- + {{ status_report + [' ' ~ item ~ ': ' + ~ (hostvars[item].status_record.failed_units | join(', '))] }} + loop: "{{ status_hosts }}" + when: hostvars[item].status_record.failed_units | length > 0 + + - name: Note when nothing failed + ansible.builtin.set_fact: + status_report: "{{ status_report + [' none'] }}" + when: >- + (status_hosts | map('extract', hostvars) + | map(attribute='status_record.failed_units', default=[]) + | flatten | length) == 0 + + - name: Add unreachable section + ansible.builtin.set_fact: + status_report: >- + {{ status_report + ['', 'UNREACHABLE', '-' * 80] + + (down_hosts | map('regex_replace', '^', ' ') | list + if (down_hosts | length) > 0 else [' none']) + + ['', '=' * 80, ''] }} + vars: + down_hosts: >- + {{ status_hosts | map('extract', hostvars) + | selectattr('status_record.state', 'equalto', 'DOWN') + | map(attribute='inventory_hostname') | list }} + + - name: Print HomeLab status + ansible.builtin.debug: + msg: "{{ status_report | join('\n') }}"