DmitryandClaude Opus 5
c676be81ec
Commit the accumulated infrastructure work that was living only in the working tree: monitoring stack, emergency access/bot, gyro allocator, grimmory, adguard, backup audit and the OpenCode agent definitions. Also ignore Python bytecode, local archives and Nix/direnv artifacts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
135 lines
5.0 KiB
Django/Jinja
135 lines
5.0 KiB
Django/Jinja
groups:
|
|
- name: homelab-availability
|
|
rules:
|
|
- alert: ExporterDown
|
|
expr: up{job=~"node|smartctl|pve"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Exporter unavailable: {{ '{{' }} $labels.job {{ '}}' }} {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- alert: CriticalServiceUnavailable
|
|
expr: probe_success{job="blackbox-vps"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "External or VPN check failed: {{ '{{' }} $labels.probe_name {{ '}}' }}"
|
|
|
|
- alert: RuVpsMonitoringStale
|
|
expr: time() - homelab_vps_probe_timestamp_seconds > 5 * 60
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "ru-vps has not pushed monitoring metrics through OpenVPN"
|
|
|
|
- alert: RuVpsMonitoringMissing
|
|
expr: absent(homelab_vps_probe_timestamp_seconds{job="blackbox-vps"})
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "ru-vps has no monitoring metrics in Pushgateway"
|
|
|
|
- alert: LanServiceUnavailable
|
|
expr: probe_success{job=~"blackbox-lan-http|blackbox-lan-tcp"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "LAN service unavailable: {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- alert: TLSCertificateExpiringSoon
|
|
expr: (probe_ssl_earliest_cert_expiry - time()) < 14 * 24 * 3600
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "TLS certificate expires within 14 days: {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- name: homelab-capacity
|
|
rules:
|
|
- alert: FilesystemAlmostFull
|
|
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.15
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Filesystem is more than 85% full: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
|
|
|
- alert: FilesystemWillFillSoon
|
|
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 7 * 24 * 3600) < 0
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Filesystem may fill within seven days: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
|
|
|
- alert: HostMemoryPressure
|
|
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.1
|
|
for: 20m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Memory pressure on {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- alert: SMARTDeviceFailed
|
|
expr: homelab_smart_device_healthy == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SMART health check failed: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
|
|
|
- alert: NVMECriticalWarning
|
|
expr: homelab_smart_nvme_critical_warning > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "NVMe critical warning: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
|
|
|
- name: homelab-backups
|
|
rules:
|
|
- alert: BackupAuditFailed
|
|
expr: homelab_backup_audit_success == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Backup audit failed: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- alert: BackupAuditStale
|
|
expr: time() - homelab_backup_audit_timestamp_seconds > 48 * 3600
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Backup audit result is stale: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
|
|
- alert: BackupAuditMissing
|
|
expr: absent(homelab_backup_audit_timestamp_seconds{profile="pbs"}) or absent(homelab_backup_audit_timestamp_seconds{profile="gitea"}) or absent(homelab_backup_audit_timestamp_seconds{profile="vaultwarden"}) or absent(homelab_backup_audit_timestamp_seconds{profile="grimmory"})
|
|
for: 12h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Expected backup audit metric has not been published"
|
|
|
|
- alert: ResticBackupTooOld
|
|
expr: homelab_backup_audit_snapshot_age_hours{profile=~"gitea|vaultwarden|grimmory"} > 36
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Restic backup is older than 36 hours: {{ '{{' }} $labels.profile {{ '}}' }}"
|
|
|
|
- alert: PBSBackupTooOld
|
|
expr: homelab_backup_audit_snapshot_age_hours{profile="pbs"} > 48
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "PBS backup is older than 48 hours: VMID {{ '{{' }} $labels.vmid {{ '}}' }}"
|