--- # One-shot, human-readable "what is my HomeLab doing right now" report. # # ansible-playbook playbooks/status.yml --ask-vault-pass # ansible-playbook playbooks/status.yml --limit lxc_infra # ansible-playbook playbooks/status.yml --check # same output, changes nothing # # Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run # that includes the `gyro` host needs --ask-vault-pass / --vault-password-file # (same as playbooks/check.yml). Without the password use --limit '!gyro'. # # Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`, # nothing is started, installed or written. An unreachable host is a *result* # (reported as DOWN), not a playbook failure. - name: Collect HomeLab status hosts: servers gather_facts: false ignore_unreachable: true vars: # Application units per host. Every name below is taken from the playbook or # role that installs it: # adguard.service playbooks/pve-adguard.yml # emergency-bot.service roles/emergency_bot # gitea.service playbooks/pve-gitea.yml # grimmory{,-docker-firewall} playbooks/pve-grimmory.yml # gyro.timer roles/gyro (gyro.service is oneshot -> see jobs) # hermes-ai-tun-proxy.service playbooks/pve-hermes-ai.yml # memoir-bot.service playbooks/pve-memoir-bot.yml # mihomo{,-ui}.service playbooks/pve-mihomo.yml # uptime-kuma.service roles/uptime_kuma # vaultwarden.service playbooks/pve-vaultwarden.yml # docker.service installed by each of the compose-based playbooks status_service_units: adguard: [docker.service, adguard.service] docker-test: [docker.service] emergency-bot: [emergency-bot.service] gitea: [docker.service, gitea.service] grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service] gyro: [gyro.timer] hermes-ai: [docker.service, hermes-ai-tun-proxy.service] memoir-bot: [docker.service, memoir-bot.service] mihomo: [docker.service, mihomo.service, mihomo-ui.service] monitoring: [docker.service, uptime-kuma.service] ru-vps: [docker.service] vaultwarden: [docker.service, vaultwarden.service] # Oneshot / timer-driven jobs. They are normally "inactive", so instead of # is-active we report the last run (ExecMainExitTimestamp + Result). # Running the audit scripts themselves (roles/backup_audit) would hit PBS and # Yandex Disk over the network, so we only read what systemd already knows. status_job_units: cloud-pc: - homelab-restic-offsite-gitea.service - homelab-backup-audit-gitea.service grimmory: - homelab-restic-offsite-grimmory.service - homelab-backup-audit-grimmory.service gyro: [gyro.service] mini-pc: [homelab-backup-audit-pbs.service] ru-vps: [homelab-monitoring-push.service] vaultwarden: - homelab-restic-offsite-vaultwarden.service - homelab-backup-audit-vaultwarden.service # Group-driven units: derived from inventory groups, not hardcoded per host. status_units: >- {{ (status_service_units[inventory_hostname] | default([])) + (['prometheus-node-exporter.service'] if inventory_hostname in (groups['monitoring_exporters'] | default([])) else []) + (['homelab-smartctl-metrics.timer'] if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else []) + ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service'] if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }} status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}" # Root plus whatever extra storage the inventory declares for this host. status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}" # Same idea as playbooks/openvpn-check.yml, but built from the inventory: # PVE web UI on every pve_nodes member, PBS web UI on pbs. status_vpn_probe_targets: >- {{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.+)$', '\1:8006') | list) + (groups['lxc_infra'] | default([]) | select('eq', 'pbs') | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.+)$', '\1:8007') | list) }} tasks: - name: Probe host reachability ansible.builtin.ping: register: status_ping ignore_unreachable: true ignore_errors: true check_mode: false - name: Record reachability ansible.builtin.set_fact: status_reachable: >- {{ (not (status_ping.unreachable | default(false))) and (not (status_ping.failed | default(false))) }} - name: Collect host state when: status_reachable | bool check_mode: false ignore_unreachable: true block: - name: Gather hardware facts (uptime, memory) ansible.builtin.setup: gather_subset: - hardware failed_when: false - name: Read root filesystem usage ansible.builtin.shell: cmd: >- LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}' register: status_root_df changed_when: false failed_when: false - name: Read declared storage mount usage ansible.builtin.command: argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}" register: status_disk_df changed_when: false failed_when: false when: status_disk_paths | length > 1 - name: Check service unit states ansible.builtin.command: argv: "{{ ['systemctl', 'is-active'] + status_units }}" register: status_units_state changed_when: false failed_when: false when: status_units | length > 0 - name: List failed systemd units ansible.builtin.shell: cmd: >- systemctl list-units --state=failed --no-legend --plain --no-pager | awk '{print $1}' | head -n 5 register: status_failed_units changed_when: false failed_when: false - name: Read last run of backup and oneshot jobs ansible.builtin.shell: cmd: | set -u now=$(date +%s) for unit in {{ status_jobs | map('quote') | join(' ') }}; do ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true) res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true) if [ -n "$ts" ]; then epoch=$(date -d "$ts" +%s 2>/dev/null || true) if [ -n "${epoch:-}" ]; then age="$(( (now - epoch) / 3600 ))h ago" else age="?" fi else ts="never" age="-" res="-" fi printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}" done register: status_job_state changed_when: false failed_when: false when: status_jobs | length > 0 - name: List Proxmox containers ansible.builtin.shell: cmd: >- pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}' register: status_pct changed_when: false failed_when: false when: inventory_hostname in (groups['pve_nodes'] | default([])) - name: Ping the OpenVPN peer ansible.builtin.command: argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"] register: status_vpn_ping changed_when: false failed_when: false when: - inventory_hostname in (groups['vpn_openvpn'] | default([])) - openvpn_peer_ip is defined - name: Probe LAN services through the OpenVPN tunnel ansible.builtin.shell: cmd: | for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do host=${target%:*} port=${target##*:} if nc -z -w 3 "$host" "$port" 2>/dev/null; then printf '%-24s OK\n' "$target" else printf '%-24s FAIL\n' "$target" fi done register: status_vpn_probe changed_when: false failed_when: false when: - (openvpn_role | default('')) == 'server' - status_vpn_probe_targets | length > 0 - name: Build status record ansible.builtin.set_fact: status_record: state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}" uptime: >- {{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd ' ~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h' if ansible_facts.uptime_seconds is defined else '-' }} root_disk: "{{ status_root_df.stdout | default('-') | trim }}" units: >- {{ status_units | zip(status_units_state.stdout_lines | default([])) | map('join', '=') | join(' ') }} failed_units: "{{ status_failed_units.stdout_lines | default([]) }}" jobs: "{{ status_job_state.stdout_lines | default([]) }}" disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}" containers: "{{ status_pct.stdout_lines | default([]) }}" vpn_peer: >- {{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL') if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }} vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}" vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}" - name: Print HomeLab status summary # Runs once, on the controller, so it also works with --limit. hosts: servers gather_facts: false become: false ignore_unreachable: true tasks: - name: Render the summary run_once: true delegate_to: localhost block: - name: Select hosts that produced a record ansible.builtin.set_fact: status_hosts: >- {{ ansible_play_hosts_all | sort | map('extract', hostvars) | selectattr('status_record', 'defined') | map(attribute='inventory_hostname') | list }} status_states: >- {{ ansible_play_hosts_all | sort | map('extract', hostvars) | selectattr('status_record', 'defined') | map(attribute='status_record.state') | list }} - name: Start the report ansible.builtin.set_fact: status_report: - "" - "================ HOMELAB STATUS ================================================" - >- hosts: {{ status_hosts | length }} up: {{ status_states | select('equalto', 'UP') | list | length }} down: {{ status_states | select('equalto', 'DOWN') | list | length }} - "" - "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}" - "{{ '-' * 80 }}" - name: Add one line per host ansible.builtin.set_fact: status_report: "{{ status_report + [line] }}" vars: rec: "{{ hostvars[item].status_record }}" line: >- {{ '%-13s %-5s %-9s %-24s %s' | format( item, rec.state, rec.uptime | trim | default('-', true), rec.root_disk | trim | default('-', true), rec.units | trim | default('-', true)) }} loop: "{{ status_hosts }}" - name: Add Proxmox container section header ansible.builtin.set_fact: status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}" - name: Add containers per Proxmox node ansible.builtin.set_fact: status_report: >- {{ status_report + [' ' ~ item ~ ': running=' ~ (running | length) ~ ' other=' ~ (stopped | length)] + (stopped | map('regex_replace', '^\s*', ' not running: ') | list) }} vars: lines: "{{ hostvars[item].status_record.containers }}" running: "{{ lines | select('search', ' running ') | list }}" stopped: "{{ lines | reject('search', ' running ') | list }}" loop: "{{ status_hosts }}" when: hostvars[item].status_record.containers | length > 0 - name: Add storage section header ansible.builtin.set_fact: status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}" - name: Add storage lines per host ansible.builtin.set_fact: status_report: >- {{ status_report + [' ' ~ item ~ ':'] + (hostvars[item].status_record.disk_lines | map('regex_replace', '^', ' ') | list) }} loop: "{{ status_hosts }}" when: hostvars[item].status_record.disk_lines | length > 0 - name: Add OpenVPN transport section header ansible.builtin.set_fact: status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}" - name: Add OpenVPN peer results ansible.builtin.set_fact: status_report: >- {{ status_report + [' ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip ~ ' : ' ~ hostvars[item].status_record.vpn_peer] + (hostvars[item].status_record.vpn_probes | map('regex_replace', '^', ' via tunnel: ') | list) }} loop: "{{ status_hosts }}" when: hostvars[item].status_record.vpn_peer != '-' - name: Add backup and job section header ansible.builtin.set_fact: status_report: >- {{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }} - name: Add job lines per host ansible.builtin.set_fact: status_report: >- {{ status_report + [' ' ~ item ~ ':'] + (hostvars[item].status_record.jobs | map('regex_replace', '^', ' ') | list) }} loop: "{{ status_hosts }}" when: hostvars[item].status_record.jobs | length > 0 - name: Add failed units section header ansible.builtin.set_fact: status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}" - name: Add failed units per host ansible.builtin.set_fact: status_report: >- {{ status_report + [' ' ~ item ~ ': ' ~ (hostvars[item].status_record.failed_units | join(', '))] }} loop: "{{ status_hosts }}" when: hostvars[item].status_record.failed_units | length > 0 - name: Note when nothing failed ansible.builtin.set_fact: status_report: "{{ status_report + [' none'] }}" when: >- (status_hosts | map('extract', hostvars) | map(attribute='status_record.failed_units', default=[]) | flatten | length) == 0 - name: Add unreachable section ansible.builtin.set_fact: status_report: >- {{ status_report + ['', 'UNREACHABLE', '-' * 80] + (down_hosts | map('regex_replace', '^', ' ') | list if (down_hosts | length) > 0 else [' none']) + ['', '=' * 80, ''] }} vars: down_hosts: >- {{ status_hosts | map('extract', hostvars) | selectattr('status_record.state', 'equalto', 'DOWN') | map(attribute='inventory_hostname') | list }} - name: Print HomeLab status ansible.builtin.debug: msg: "{{ status_report | join('\n') }}"