DmitryandClaude Sonnet 5
79878e36f9
One ru-vps housekeeping sweep (2026-09-02/03): - playbooks/ru-vps-base.yml (new): adopt the Caddy compose stack into Ansible (pinned image by digest, homelab-caddy.service), and manage the corosync-qnetd UFW rule - allow 5403/tcp from homelab_pve_egress_ip, drop the stale rule for the retired ZeroTier 10.122.62.0/24. The qdevice had gone silent because its only allowed path was the decommissioned ZeroTier network. - group_vars/all/main.yml: homelab_pve_egress_ip (the NATed home egress the PVE nodes reach corosync-qnetd from - a direct path that does not depend on the OpenVPN tunnel). Marked dynamic: a change silently re-breaks the qdevice. - playbooks/status.yml: CLUSTER QUORUM section (pvecm status per PVE node) so a repeat failure is visible. Also drops the memoir-bot unit list and moves the gitea offsite-restic unit to the gitea host (see the OpenTofu-migration commit). - playbooks/ru-vps-zerotier-decommission.yml (new): stop the zerotier container, disable ssh-zt22.service, remove the interface/9993/9001/10.122.62.0/24 UFW rules. Node identity and data are kept; removal is a separate step. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012uoq5AVK8mkBgg83Mq6o5V
409 lines
18 KiB
YAML
409 lines
18 KiB
YAML
---
|
|
# One-shot, human-readable "what is my HomeLab doing right now" report.
|
|
#
|
|
# ansible-playbook playbooks/status.yml --ask-vault-pass
|
|
# ansible-playbook playbooks/status.yml --limit lxc_infra
|
|
# ansible-playbook playbooks/status.yml --check # same output, changes nothing
|
|
#
|
|
# Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run
|
|
# that includes the `gyro` host needs --ask-vault-pass / --vault-password-file
|
|
# (same as playbooks/check.yml). Without the password use --limit '!gyro'.
|
|
#
|
|
# Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`,
|
|
# nothing is started, installed or written. An unreachable host is a *result*
|
|
# (reported as DOWN), not a playbook failure.
|
|
|
|
- name: Collect HomeLab status
|
|
hosts: servers
|
|
gather_facts: false
|
|
ignore_unreachable: true
|
|
vars:
|
|
# Application units per host. Every name below is taken from the playbook or
|
|
# role that installs it:
|
|
# adguard.service playbooks/pve-adguard.yml
|
|
# emergency-bot.service roles/emergency_bot
|
|
# gitea.service playbooks/pve-gitea.yml
|
|
# grimmory{,-docker-firewall} playbooks/pve-grimmory.yml
|
|
# gyro.timer roles/gyro (gyro.service is oneshot -> see jobs)
|
|
# hermes-ai-tun-proxy.service playbooks/pve-hermes-ai.yml
|
|
# mihomo{,-ui}.service playbooks/pve-mihomo.yml
|
|
# uptime-kuma.service roles/uptime_kuma
|
|
# vaultwarden.service playbooks/pve-vaultwarden.yml
|
|
# docker.service installed by each of the compose-based playbooks
|
|
status_service_units:
|
|
adguard: [docker.service, adguard.service]
|
|
docker-test: [docker.service]
|
|
emergency-bot: [emergency-bot.service]
|
|
gitea: [docker.service, gitea.service]
|
|
grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service]
|
|
gyro: [gyro.timer]
|
|
hermes-ai: [docker.service, hermes-ai-tun-proxy.service]
|
|
mihomo: [docker.service, mihomo.service, mihomo-ui.service]
|
|
monitoring: [docker.service, uptime-kuma.service]
|
|
ru-vps: [docker.service]
|
|
vaultwarden: [docker.service, vaultwarden.service]
|
|
|
|
# Oneshot / timer-driven jobs. They are normally "inactive", so instead of
|
|
# is-active we report the last run (ExecMainExitTimestamp + Result).
|
|
# Running the audit scripts themselves (roles/backup_audit) would hit PBS and
|
|
# Yandex Disk over the network, so we only read what systemd already knows.
|
|
status_job_units:
|
|
cloud-pc:
|
|
- homelab-backup-audit-gitea.service
|
|
gitea:
|
|
- homelab-restic-offsite-gitea.service
|
|
grimmory:
|
|
- homelab-restic-offsite-grimmory.service
|
|
- homelab-backup-audit-grimmory.service
|
|
gyro: [gyro.service]
|
|
mini-pc: [homelab-backup-audit-pbs.service]
|
|
ru-vps: [homelab-monitoring-push.service]
|
|
vaultwarden:
|
|
- homelab-restic-offsite-vaultwarden.service
|
|
- homelab-backup-audit-vaultwarden.service
|
|
|
|
# Group-driven units: derived from inventory groups, not hardcoded per host.
|
|
status_units: >-
|
|
{{ (status_service_units[inventory_hostname] | default([]))
|
|
+ (['prometheus-node-exporter.service']
|
|
if inventory_hostname in (groups['monitoring_exporters'] | default([])) else [])
|
|
+ (['homelab-smartctl-metrics.timer']
|
|
if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else [])
|
|
+ ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service']
|
|
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }}
|
|
|
|
status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}"
|
|
|
|
# Root plus whatever extra storage the inventory declares for this host.
|
|
status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}"
|
|
|
|
# Same idea as playbooks/openvpn-check.yml, but built from the inventory:
|
|
# PVE web UI on every pve_nodes member, PBS web UI on pbs.
|
|
status_vpn_probe_targets: >-
|
|
{{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host')
|
|
| map('regex_replace', '^(.+)$', '\1:8006') | list)
|
|
+ (groups['lxc_infra'] | default([]) | select('eq', 'pbs')
|
|
| map('extract', hostvars, 'ansible_host')
|
|
| map('regex_replace', '^(.+)$', '\1:8007') | list) }}
|
|
|
|
tasks:
|
|
- name: Probe host reachability
|
|
ansible.builtin.ping:
|
|
register: status_ping
|
|
ignore_unreachable: true
|
|
ignore_errors: true
|
|
check_mode: false
|
|
|
|
- name: Record reachability
|
|
ansible.builtin.set_fact:
|
|
status_reachable: >-
|
|
{{ (not (status_ping.unreachable | default(false)))
|
|
and (not (status_ping.failed | default(false))) }}
|
|
|
|
- name: Collect host state
|
|
when: status_reachable | bool
|
|
check_mode: false
|
|
ignore_unreachable: true
|
|
block:
|
|
- name: Gather hardware facts (uptime, memory)
|
|
ansible.builtin.setup:
|
|
gather_subset:
|
|
- hardware
|
|
failed_when: false
|
|
|
|
- name: Read root filesystem usage
|
|
ansible.builtin.shell:
|
|
cmd: >-
|
|
LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}'
|
|
register: status_root_df
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: Read declared storage mount usage
|
|
ansible.builtin.command:
|
|
argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}"
|
|
register: status_disk_df
|
|
changed_when: false
|
|
failed_when: false
|
|
when: status_disk_paths | length > 1
|
|
|
|
- name: Check service unit states
|
|
ansible.builtin.command:
|
|
argv: "{{ ['systemctl', 'is-active'] + status_units }}"
|
|
register: status_units_state
|
|
changed_when: false
|
|
failed_when: false
|
|
when: status_units | length > 0
|
|
|
|
# Двухнодовый кластер держится на арбитре corosync-qnetd (ru-vps:5403).
|
|
# Когда арбитр молчит, Total votes < Expected votes и падение ЛЮБОЙ ноды
|
|
# оставляет выжившую без кворума. Один раз это уже сломалось молча —
|
|
# после вывода ZeroTier правило UFW для 5403 осталось на мёртвой сети.
|
|
- name: Read cluster quorum state
|
|
ansible.builtin.shell:
|
|
cmd: LC_ALL=C pvecm status 2>/dev/null | grep -E 'Quorate:|Expected votes|Total votes' | tr -s ' ' | tr '\n' ' '
|
|
register: status_quorum
|
|
changed_when: false
|
|
failed_when: false
|
|
when: inventory_hostname in (groups['pve_nodes'] | default([]))
|
|
|
|
- name: List failed systemd units
|
|
ansible.builtin.shell:
|
|
cmd: >-
|
|
systemctl list-units --state=failed --no-legend --plain --no-pager
|
|
| awk '{print $1}' | head -n 5
|
|
register: status_failed_units
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: Read last run of backup and oneshot jobs
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
set -u
|
|
now=$(date +%s)
|
|
for unit in {{ status_jobs | map('quote') | join(' ') }}; do
|
|
ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true)
|
|
res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true)
|
|
if [ -n "$ts" ]; then
|
|
epoch=$(date -d "$ts" +%s 2>/dev/null || true)
|
|
if [ -n "${epoch:-}" ]; then
|
|
age="$(( (now - epoch) / 3600 ))h ago"
|
|
else
|
|
age="?"
|
|
fi
|
|
else
|
|
ts="never"
|
|
age="-"
|
|
res="-"
|
|
fi
|
|
printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}"
|
|
done
|
|
register: status_job_state
|
|
changed_when: false
|
|
failed_when: false
|
|
when: status_jobs | length > 0
|
|
|
|
- name: List Proxmox containers
|
|
ansible.builtin.shell:
|
|
cmd: >-
|
|
pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}'
|
|
register: status_pct
|
|
changed_when: false
|
|
failed_when: false
|
|
when: inventory_hostname in (groups['pve_nodes'] | default([]))
|
|
|
|
- name: Ping the OpenVPN peer
|
|
ansible.builtin.command:
|
|
argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"]
|
|
register: status_vpn_ping
|
|
changed_when: false
|
|
failed_when: false
|
|
when:
|
|
- inventory_hostname in (groups['vpn_openvpn'] | default([]))
|
|
- openvpn_peer_ip is defined
|
|
|
|
- name: Probe LAN services through the OpenVPN tunnel
|
|
ansible.builtin.shell:
|
|
cmd: |
|
|
for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do
|
|
host=${target%:*}
|
|
port=${target##*:}
|
|
if nc -z -w 3 "$host" "$port" 2>/dev/null; then
|
|
printf '%-24s OK\n' "$target"
|
|
else
|
|
printf '%-24s FAIL\n' "$target"
|
|
fi
|
|
done
|
|
register: status_vpn_probe
|
|
changed_when: false
|
|
failed_when: false
|
|
when:
|
|
- (openvpn_role | default('')) == 'server'
|
|
- status_vpn_probe_targets | length > 0
|
|
|
|
- name: Build status record
|
|
ansible.builtin.set_fact:
|
|
status_record:
|
|
state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}"
|
|
uptime: >-
|
|
{{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd '
|
|
~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h'
|
|
if ansible_facts.uptime_seconds is defined else '-' }}
|
|
root_disk: "{{ status_root_df.stdout | default('-') | trim }}"
|
|
units: >-
|
|
{{ status_units
|
|
| zip(status_units_state.stdout_lines | default([]))
|
|
| map('join', '=') | join(' ') }}
|
|
failed_units: "{{ status_failed_units.stdout_lines | default([]) }}"
|
|
jobs: "{{ status_job_state.stdout_lines | default([]) }}"
|
|
disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}"
|
|
containers: "{{ status_pct.stdout_lines | default([]) }}"
|
|
vpn_peer: >-
|
|
{{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL')
|
|
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }}
|
|
vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}"
|
|
vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}"
|
|
quorum: "{{ status_quorum.stdout | default('') | trim }}"
|
|
|
|
|
|
- name: Print HomeLab status summary
|
|
# Runs once, on the controller, so it also works with --limit.
|
|
hosts: servers
|
|
gather_facts: false
|
|
become: false
|
|
ignore_unreachable: true
|
|
tasks:
|
|
- name: Render the summary
|
|
run_once: true
|
|
delegate_to: localhost
|
|
block:
|
|
- name: Select hosts that produced a record
|
|
ansible.builtin.set_fact:
|
|
status_hosts: >-
|
|
{{ ansible_play_hosts_all | sort
|
|
| map('extract', hostvars)
|
|
| selectattr('status_record', 'defined')
|
|
| map(attribute='inventory_hostname') | list }}
|
|
status_states: >-
|
|
{{ ansible_play_hosts_all | sort
|
|
| map('extract', hostvars)
|
|
| selectattr('status_record', 'defined')
|
|
| map(attribute='status_record.state') | list }}
|
|
|
|
- name: Start the report
|
|
ansible.builtin.set_fact:
|
|
status_report:
|
|
- ""
|
|
- "================ HOMELAB STATUS ================================================"
|
|
- >-
|
|
hosts: {{ status_hosts | length }}
|
|
up: {{ status_states | select('equalto', 'UP') | list | length }}
|
|
down: {{ status_states | select('equalto', 'DOWN') | list | length }}
|
|
- ""
|
|
- "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}"
|
|
- "{{ '-' * 80 }}"
|
|
|
|
- name: Add one line per host
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + [line] }}"
|
|
vars:
|
|
rec: "{{ hostvars[item].status_record }}"
|
|
line: >-
|
|
{{ '%-13s %-5s %-9s %-24s %s' | format(
|
|
item,
|
|
rec.state,
|
|
rec.uptime | trim | default('-', true),
|
|
rec.root_disk | trim | default('-', true),
|
|
rec.units | trim | default('-', true)) }}
|
|
loop: "{{ status_hosts }}"
|
|
|
|
- name: Add Proxmox container section header
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}"
|
|
|
|
- name: Add containers per Proxmox node
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report
|
|
+ [' ' ~ item ~ ': running=' ~ (running | length)
|
|
~ ' other=' ~ (stopped | length)]
|
|
+ (stopped | map('regex_replace', '^\s*', ' not running: ') | list) }}
|
|
vars:
|
|
lines: "{{ hostvars[item].status_record.containers }}"
|
|
running: "{{ lines | select('search', ' running ') | list }}"
|
|
stopped: "{{ lines | reject('search', ' running ') | list }}"
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.containers | length > 0
|
|
|
|
- name: Add storage section header
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}"
|
|
|
|
- name: Add storage lines per host
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + [' ' ~ item ~ ':']
|
|
+ (hostvars[item].status_record.disk_lines
|
|
| map('regex_replace', '^', ' ') | list) }}
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.disk_lines | length > 0
|
|
|
|
- name: Add OpenVPN transport section header
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}"
|
|
|
|
- name: Add OpenVPN peer results
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report
|
|
+ [' ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip
|
|
~ ' : ' ~ hostvars[item].status_record.vpn_peer]
|
|
+ (hostvars[item].status_record.vpn_probes
|
|
| map('regex_replace', '^', ' via tunnel: ') | list) }}
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.vpn_peer != '-'
|
|
|
|
- name: Add cluster quorum section header
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + ['', 'CLUSTER QUORUM (qdevice = третий голос)', '-' * 80] }}
|
|
|
|
- name: Add quorum line per PVE node
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + [' ' ~ item ~ ': '
|
|
~ (hostvars[item].status_record.quorum | default('') | trim | default('нет данных', true))] }}
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.quorum | default('') | length > 0
|
|
|
|
- name: Add backup and job section header
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }}
|
|
|
|
- name: Add job lines per host
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + [' ' ~ item ~ ':']
|
|
+ (hostvars[item].status_record.jobs
|
|
| map('regex_replace', '^', ' ') | list) }}
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.jobs | length > 0
|
|
|
|
- name: Add failed units section header
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}"
|
|
|
|
- name: Add failed units per host
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + [' ' ~ item ~ ': '
|
|
~ (hostvars[item].status_record.failed_units | join(', '))] }}
|
|
loop: "{{ status_hosts }}"
|
|
when: hostvars[item].status_record.failed_units | length > 0
|
|
|
|
- name: Note when nothing failed
|
|
ansible.builtin.set_fact:
|
|
status_report: "{{ status_report + [' none'] }}"
|
|
when: >-
|
|
(status_hosts | map('extract', hostvars)
|
|
| map(attribute='status_record.failed_units', default=[])
|
|
| flatten | length) == 0
|
|
|
|
- name: Add unreachable section
|
|
ansible.builtin.set_fact:
|
|
status_report: >-
|
|
{{ status_report + ['', 'UNREACHABLE', '-' * 80]
|
|
+ (down_hosts | map('regex_replace', '^', ' ') | list
|
|
if (down_hosts | length) > 0 else [' none'])
|
|
+ ['', '=' * 80, ''] }}
|
|
vars:
|
|
down_hosts: >-
|
|
{{ status_hosts | map('extract', hostvars)
|
|
| selectattr('status_record.state', 'equalto', 'DOWN')
|
|
| map(attribute='inventory_hostname') | list }}
|
|
|
|
- name: Print HomeLab status
|
|
ansible.builtin.debug:
|
|
msg: "{{ status_report | join('\n') }}"
|