ada/infra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
5e27ba251344ac5515cf2578769f00cdc95ece0d
infra/ansible/playbooks/status.yml
T
DmitryandClaude Sonnet 5 79878e36f9 feat: adopt the ru-vps Caddy stack, fix cluster quorum, decommission ZeroTier 
One ru-vps housekeeping sweep (2026-09-02/03):

- playbooks/ru-vps-base.yml (new): adopt the Caddy compose stack into Ansible
  (pinned image by digest, homelab-caddy.service), and manage the corosync-qnetd
  UFW rule - allow 5403/tcp from homelab_pve_egress_ip, drop the stale rule for
  the retired ZeroTier 10.122.62.0/24. The qdevice had gone silent because its
  only allowed path was the decommissioned ZeroTier network.
- group_vars/all/main.yml: homelab_pve_egress_ip (the NATed home egress the PVE
  nodes reach corosync-qnetd from - a direct path that does not depend on the
  OpenVPN tunnel). Marked dynamic: a change silently re-breaks the qdevice.
- playbooks/status.yml: CLUSTER QUORUM section (pvecm status per PVE node) so a
  repeat failure is visible. Also drops the memoir-bot unit list and moves the
  gitea offsite-restic unit to the gitea host (see the OpenTofu-migration commit).
- playbooks/ru-vps-zerotier-decommission.yml (new): stop the zerotier container,
  disable ssh-zt22.service, remove the interface/9993/9001/10.122.62.0/24 UFW
  rules. Node identity and data are kept; removal is a separate step.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012uoq5AVK8mkBgg83Mq6o5V
2026-09-03 07:05:11 +03:00

409 lines
18 KiB
YAML
Raw Blame History

---
# One-shot, human-readable "what is my HomeLab doing right now" report.
#
# ansible-playbook playbooks/status.yml --ask-vault-pass
# ansible-playbook playbooks/status.yml --limit lxc_infra
# ansible-playbook playbooks/status.yml --check # same output, changes nothing
#
# Note: inventory/host_vars/gyro/vault.yml is Ansible Vault encrypted, so any run
# that includes the `gyro` host needs --ask-vault-pass / --vault-password-file
# (same as playbooks/check.yml). Without the password use --limit '!gyro'.
#
# Strictly READ-ONLY: every command is `changed_when: false` + `check_mode: false`,
# nothing is started, installed or written. An unreachable host is a *result*
# (reported as DOWN), not a playbook failure.
- name: Collect HomeLab status
hosts: servers
gather_facts: false
ignore_unreachable: true
vars:
# Application units per host. Every name below is taken from the playbook or
# role that installs it:
# adguard.service playbooks/pve-adguard.yml
# emergency-bot.service roles/emergency_bot
# gitea.service playbooks/pve-gitea.yml
# grimmory{,-docker-firewall} playbooks/pve-grimmory.yml
# gyro.timer roles/gyro (gyro.service is oneshot -> see jobs)
# hermes-ai-tun-proxy.service playbooks/pve-hermes-ai.yml
# mihomo{,-ui}.service playbooks/pve-mihomo.yml
# uptime-kuma.service roles/uptime_kuma
# vaultwarden.service playbooks/pve-vaultwarden.yml
# docker.service installed by each of the compose-based playbooks
status_service_units:
adguard: [docker.service, adguard.service]
docker-test: [docker.service]
emergency-bot: [emergency-bot.service]
gitea: [docker.service, gitea.service]
grimmory: [docker.service, grimmory.service, grimmory-docker-firewall.service]
gyro: [gyro.timer]
hermes-ai: [docker.service, hermes-ai-tun-proxy.service]
mihomo: [docker.service, mihomo.service, mihomo-ui.service]
monitoring: [docker.service, uptime-kuma.service]
ru-vps: [docker.service]
vaultwarden: [docker.service, vaultwarden.service]
# Oneshot / timer-driven jobs. They are normally "inactive", so instead of
# is-active we report the last run (ExecMainExitTimestamp + Result).
# Running the audit scripts themselves (roles/backup_audit) would hit PBS and
# Yandex Disk over the network, so we only read what systemd already knows.
status_job_units:
cloud-pc:
- homelab-backup-audit-gitea.service
gitea:
- homelab-restic-offsite-gitea.service
grimmory:
- homelab-restic-offsite-grimmory.service
- homelab-backup-audit-grimmory.service
gyro: [gyro.service]
mini-pc: [homelab-backup-audit-pbs.service]
ru-vps: [homelab-monitoring-push.service]
vaultwarden:
- homelab-restic-offsite-vaultwarden.service
- homelab-backup-audit-vaultwarden.service
# Group-driven units: derived from inventory groups, not hardcoded per host.
status_units: >-
{{ (status_service_units[inventory_hostname] | default([]))
+ (['prometheus-node-exporter.service']
if inventory_hostname in (groups['monitoring_exporters'] | default([])) else [])
+ (['homelab-smartctl-metrics.timer']
if inventory_hostname in (groups['monitoring_smart_exporters'] | default([])) else [])
+ ([(openvpn_service_name | default('homelab-openvpn')) ~ '.service']
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else []) }}
status_jobs: "{{ status_job_units[inventory_hostname] | default([]) }}"
# Root plus whatever extra storage the inventory declares for this host.
status_disk_paths: "{{ ['/'] + (storage_mounts | default([]) | map(attribute='path') | list) }}"
# Same idea as playbooks/openvpn-check.yml, but built from the inventory:
# PVE web UI on every pve_nodes member, PBS web UI on pbs.
status_vpn_probe_targets: >-
{{ (groups['pve_nodes'] | default([]) | map('extract', hostvars, 'ansible_host')
| map('regex_replace', '^(.+)$', '\1:8006') | list)
+ (groups['lxc_infra'] | default([]) | select('eq', 'pbs')
| map('extract', hostvars, 'ansible_host')
| map('regex_replace', '^(.+)$', '\1:8007') | list) }}
tasks:
- name: Probe host reachability
ansible.builtin.ping:
register: status_ping
ignore_unreachable: true
ignore_errors: true
check_mode: false
- name: Record reachability
ansible.builtin.set_fact:
status_reachable: >-
{{ (not (status_ping.unreachable | default(false)))
and (not (status_ping.failed | default(false))) }}
- name: Collect host state
when: status_reachable | bool
check_mode: false
ignore_unreachable: true
block:
- name: Gather hardware facts (uptime, memory)
ansible.builtin.setup:
gather_subset:
- hardware
failed_when: false
- name: Read root filesystem usage
ansible.builtin.shell:
cmd: >-
LC_ALL=C df -hP / | awk 'NR == 2 {print $5 " used, " $4 " free"}'
register: status_root_df
changed_when: false
failed_when: false
- name: Read declared storage mount usage
ansible.builtin.command:
argv: "{{ ['df', '-h', '--output=target,size,used,avail,pcent'] + status_disk_paths }}"
register: status_disk_df
changed_when: false
failed_when: false
when: status_disk_paths | length > 1
- name: Check service unit states
ansible.builtin.command:
argv: "{{ ['systemctl', 'is-active'] + status_units }}"
register: status_units_state
changed_when: false
failed_when: false
when: status_units | length > 0
# Двухнодовый кластер держится на арбитре corosync-qnetd (ru-vps:5403).
# Когда арбитр молчит, Total votes < Expected votes и падение ЛЮБОЙ ноды
# оставляет выжившую без кворума. Один раз это уже сломалось молча —
# после вывода ZeroTier правило UFW для 5403 осталось на мёртвой сети.
- name: Read cluster quorum state
ansible.builtin.shell:
cmd: LC_ALL=C pvecm status 2>/dev/null | grep -E 'Quorate:|Expected votes|Total votes' | tr -s ' ' | tr '\n' ' '
register: status_quorum
changed_when: false
failed_when: false
when: inventory_hostname in (groups['pve_nodes'] | default([]))
- name: List failed systemd units
ansible.builtin.shell:
cmd: >-
systemctl list-units --state=failed --no-legend --plain --no-pager
| awk '{print $1}' | head -n 5
register: status_failed_units
changed_when: false
failed_when: false
- name: Read last run of backup and oneshot jobs
ansible.builtin.shell:
cmd: |
set -u
now=$(date +%s)
for unit in {{ status_jobs | map('quote') | join(' ') }}; do
ts=$(systemctl show -p ExecMainExitTimestamp --value "$unit" 2>/dev/null || true)
res=$(systemctl show -p Result --value "$unit" 2>/dev/null || true)
if [ -n "$ts" ]; then
epoch=$(date -d "$ts" +%s 2>/dev/null || true)
if [ -n "${epoch:-}" ]; then
age="$(( (now - epoch) / 3600 ))h ago"
else
age="?"
fi
else
ts="never"
age="-"
res="-"
fi
printf '%-44s %-32s %-10s %s\n' "$unit" "$ts" "$age" "${res:-unknown}"
done
register: status_job_state
changed_when: false
failed_when: false
when: status_jobs | length > 0
- name: List Proxmox containers
ansible.builtin.shell:
cmd: >-
pct list | awk 'NR > 1 {printf "%-6s %-9s %s\n", $1, $2, $NF}'
register: status_pct
changed_when: false
failed_when: false
when: inventory_hostname in (groups['pve_nodes'] | default([]))
- name: Ping the OpenVPN peer
ansible.builtin.command:
argv: [ping, -c, '2', -W, '2', "{{ openvpn_peer_ip | default('') }}"]
register: status_vpn_ping
changed_when: false
failed_when: false
when:
- inventory_hostname in (groups['vpn_openvpn'] | default([]))
- openvpn_peer_ip is defined
- name: Probe LAN services through the OpenVPN tunnel
ansible.builtin.shell:
cmd: |
for target in {{ status_vpn_probe_targets | map('quote') | join(' ') }}; do
host=${target%:*}
port=${target##*:}
if nc -z -w 3 "$host" "$port" 2>/dev/null; then
printf '%-24s OK\n' "$target"
else
printf '%-24s FAIL\n' "$target"
fi
done
register: status_vpn_probe
changed_when: false
failed_when: false
when:
- (openvpn_role | default('')) == 'server'
- status_vpn_probe_targets | length > 0
- name: Build status record
ansible.builtin.set_fact:
status_record:
state: "{{ 'UP' if (status_reachable | bool) else 'DOWN' }}"
uptime: >-
{{ ((ansible_facts.uptime_seconds | int) // 86400) ~ 'd '
~ (((ansible_facts.uptime_seconds | int) % 86400) // 3600) ~ 'h'
if ansible_facts.uptime_seconds is defined else '-' }}
root_disk: "{{ status_root_df.stdout | default('-') | trim }}"
units: >-
{{ status_units
| zip(status_units_state.stdout_lines | default([]))
| map('join', '=') | join(' ') }}
failed_units: "{{ status_failed_units.stdout_lines | default([]) }}"
jobs: "{{ status_job_state.stdout_lines | default([]) }}"
disk_lines: "{{ status_disk_df.stdout_lines | default([]) }}"
containers: "{{ status_pct.stdout_lines | default([]) }}"
vpn_peer: >-
{{ ('OK' if (status_vpn_ping.rc | default(1)) == 0 else 'FAIL')
if inventory_hostname in (groups['vpn_openvpn'] | default([])) else '-' }}
vpn_peer_ip: "{{ openvpn_peer_ip | default('-') }}"
vpn_probes: "{{ status_vpn_probe.stdout_lines | default([]) }}"
quorum: "{{ status_quorum.stdout | default('') | trim }}"
- name: Print HomeLab status summary
# Runs once, on the controller, so it also works with --limit.
hosts: servers
gather_facts: false
become: false
ignore_unreachable: true
tasks:
- name: Render the summary
run_once: true
delegate_to: localhost
block:
- name: Select hosts that produced a record
ansible.builtin.set_fact:
status_hosts: >-
{{ ansible_play_hosts_all | sort
| map('extract', hostvars)
| selectattr('status_record', 'defined')
| map(attribute='inventory_hostname') | list }}
status_states: >-
{{ ansible_play_hosts_all | sort
| map('extract', hostvars)
| selectattr('status_record', 'defined')
| map(attribute='status_record.state') | list }}
- name: Start the report
ansible.builtin.set_fact:
status_report:
- ""
- "================ HOMELAB STATUS ================================================"
- >-
hosts: {{ status_hosts | length }}
up: {{ status_states | select('equalto', 'UP') | list | length }}
down: {{ status_states | select('equalto', 'DOWN') | list | length }}
- ""
- "{{ '%-13s %-5s %-9s %-24s %s' | format('HOST', 'STATE', 'UPTIME', 'ROOT FS', 'UNITS') }}"
- "{{ '-' * 80 }}"
- name: Add one line per host
ansible.builtin.set_fact:
status_report: "{{ status_report + [line] }}"
vars:
rec: "{{ hostvars[item].status_record }}"
line: >-
{{ '%-13s %-5s %-9s %-24s %s' | format(
item,
rec.state,
rec.uptime | trim | default('-', true),
rec.root_disk | trim | default('-', true),
rec.units | trim | default('-', true)) }}
loop: "{{ status_hosts }}"
- name: Add Proxmox container section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'PROXMOX CONTAINERS (pct list)', '-' * 80] }}"
- name: Add containers per Proxmox node
ansible.builtin.set_fact:
status_report: >-
{{ status_report
+ [' ' ~ item ~ ': running=' ~ (running | length)
~ ' other=' ~ (stopped | length)]
+ (stopped | map('regex_replace', '^\s*', ' not running: ') | list) }}
vars:
lines: "{{ hostvars[item].status_record.containers }}"
running: "{{ lines | select('search', ' running ') | list }}"
stopped: "{{ lines | reject('search', ' running ') | list }}"
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.containers | length > 0
- name: Add storage section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'STORAGE (root + declared mounts)', '-' * 80] }}"
- name: Add storage lines per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ':']
+ (hostvars[item].status_record.disk_lines
| map('regex_replace', '^', ' ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.disk_lines | length > 0
- name: Add OpenVPN transport section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'OPENVPN TRANSPORT', '-' * 80] }}"
- name: Add OpenVPN peer results
ansible.builtin.set_fact:
status_report: >-
{{ status_report
+ [' ' ~ item ~ ' -> peer ' ~ hostvars[item].status_record.vpn_peer_ip
~ ' : ' ~ hostvars[item].status_record.vpn_peer]
+ (hostvars[item].status_record.vpn_probes
| map('regex_replace', '^', ' via tunnel: ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.vpn_peer != '-'
- name: Add cluster quorum section header
ansible.builtin.set_fact:
status_report: >-
{{ status_report + ['', 'CLUSTER QUORUM (qdevice = третий голос)', '-' * 80] }}
- name: Add quorum line per PVE node
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ': '
~ (hostvars[item].status_record.quorum | default('') | trim | default('нет данных', true))] }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.quorum | default('') | length > 0
- name: Add backup and job section header
ansible.builtin.set_fact:
status_report: >-
{{ status_report + ['', 'BACKUP / SCHEDULED JOBS (last run, from systemd)', '-' * 80] }}
- name: Add job lines per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ':']
+ (hostvars[item].status_record.jobs
| map('regex_replace', '^', ' ') | list) }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.jobs | length > 0
- name: Add failed units section header
ansible.builtin.set_fact:
status_report: "{{ status_report + ['', 'FAILED SYSTEMD UNITS', '-' * 80] }}"
- name: Add failed units per host
ansible.builtin.set_fact:
status_report: >-
{{ status_report + [' ' ~ item ~ ': '
~ (hostvars[item].status_record.failed_units | join(', '))] }}
loop: "{{ status_hosts }}"
when: hostvars[item].status_record.failed_units | length > 0
- name: Note when nothing failed
ansible.builtin.set_fact:
status_report: "{{ status_report + [' none'] }}"
when: >-
(status_hosts | map('extract', hostvars)
| map(attribute='status_record.failed_units', default=[])
| flatten | length) == 0
- name: Add unreachable section
ansible.builtin.set_fact:
status_report: >-
{{ status_report + ['', 'UNREACHABLE', '-' * 80]
+ (down_hosts | map('regex_replace', '^', ' ') | list
if (down_hosts | length) > 0 else [' none'])
+ ['', '=' * 80, ''] }}
vars:
down_hosts: >-
{{ status_hosts | map('extract', hostvars)
| selectattr('status_record.state', 'equalto', 'DOWN')
| map(attribute='inventory_hostname') | list }}
- name: Print HomeLab status
ansible.builtin.debug:
msg: "{{ status_report | join('\n') }}"
Reference in New Issue View Git Blame Copy Permalink