Capture current Ansible control plane state
Commit the accumulated infrastructure work that was living only in the working tree: monitoring stack, emergency access/bot, gyro allocator, grimmory, adguard, backup audit and the OpenCode agent definitions. Also ignore Python bytecode, local archives and Nix/direnv artifacts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
This commit is contained in:
Dmitry
@@ -0,0 +1,40 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: telegram-warning
|
||||
group_by: [alertname, instance]
|
||||
group_wait: 30s
|
||||
group_interval: 10m
|
||||
repeat_interval: 6h
|
||||
routes:
|
||||
- matchers:
|
||||
- severity="critical"
|
||||
receiver: telegram-critical
|
||||
repeat_interval: 2h
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- alertname="ExporterDown"
|
||||
target_matchers:
|
||||
- alertname="LanServiceUnavailable"
|
||||
equal: [instance]
|
||||
|
||||
receivers:
|
||||
- name: telegram-warning
|
||||
telegram_configs:
|
||||
- bot_token: "{{ monitoring_telegram_bot_token }}"
|
||||
chat_id: {{ monitoring_telegram_chat_id }}
|
||||
send_resolved: true
|
||||
message: |-
|
||||
[{{ "{{" }} .Status | toUpper {{ "}}" }}] {{ "{{" }} .CommonLabels.alertname {{ "}}" }}
|
||||
{{ "{{" }} .CommonAnnotations.summary {{ "}}" }}
|
||||
|
||||
- name: telegram-critical
|
||||
telegram_configs:
|
||||
- bot_token: "{{ monitoring_telegram_bot_token }}"
|
||||
chat_id: {{ monitoring_telegram_chat_id }}
|
||||
send_resolved: true
|
||||
message: |-
|
||||
[{{ "{{" }} .Status | toUpper {{ "}}" }}] CRITICAL: {{ "{{" }} .CommonLabels.alertname {{ "}}" }}
|
||||
{{ "{{" }} .CommonAnnotations.summary {{ "}}" }}
|
||||
@@ -0,0 +1,134 @@
|
||||
groups:
|
||||
- name: homelab-availability
|
||||
rules:
|
||||
- alert: ExporterDown
|
||||
expr: up{job=~"node|smartctl|pve"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Exporter unavailable: {{ '{{' }} $labels.job {{ '}}' }} {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: CriticalServiceUnavailable
|
||||
expr: probe_success{job="blackbox-vps"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "External or VPN check failed: {{ '{{' }} $labels.probe_name {{ '}}' }}"
|
||||
|
||||
- alert: RuVpsMonitoringStale
|
||||
expr: time() - homelab_vps_probe_timestamp_seconds > 5 * 60
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ru-vps has not pushed monitoring metrics through OpenVPN"
|
||||
|
||||
- alert: RuVpsMonitoringMissing
|
||||
expr: absent(homelab_vps_probe_timestamp_seconds{job="blackbox-vps"})
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ru-vps has no monitoring metrics in Pushgateway"
|
||||
|
||||
- alert: LanServiceUnavailable
|
||||
expr: probe_success{job=~"blackbox-lan-http|blackbox-lan-tcp"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "LAN service unavailable: {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: TLSCertificateExpiringSoon
|
||||
expr: (probe_ssl_earliest_cert_expiry - time()) < 14 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "TLS certificate expires within 14 days: {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- name: homelab-capacity
|
||||
rules:
|
||||
- alert: FilesystemAlmostFull
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem is more than 85% full: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
||||
|
||||
- alert: FilesystemWillFillSoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 7 * 24 * 3600) < 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem may fill within seven days: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
||||
|
||||
- alert: HostMemoryPressure
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.1
|
||||
for: 20m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Memory pressure on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: SMARTDeviceFailed
|
||||
expr: homelab_smart_device_healthy == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART health check failed: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
||||
|
||||
- alert: NVMECriticalWarning
|
||||
expr: homelab_smart_nvme_critical_warning > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe critical warning: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
||||
|
||||
- name: homelab-backups
|
||||
rules:
|
||||
- alert: BackupAuditFailed
|
||||
expr: homelab_backup_audit_success == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Backup audit failed: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: BackupAuditStale
|
||||
expr: time() - homelab_backup_audit_timestamp_seconds > 48 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Backup audit result is stale: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: BackupAuditMissing
|
||||
expr: absent(homelab_backup_audit_timestamp_seconds{profile="pbs"}) or absent(homelab_backup_audit_timestamp_seconds{profile="gitea"}) or absent(homelab_backup_audit_timestamp_seconds{profile="vaultwarden"}) or absent(homelab_backup_audit_timestamp_seconds{profile="grimmory"})
|
||||
for: 12h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Expected backup audit metric has not been published"
|
||||
|
||||
- alert: ResticBackupTooOld
|
||||
expr: homelab_backup_audit_snapshot_age_hours{profile=~"gitea|vaultwarden|grimmory"} > 36
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Restic backup is older than 36 hours: {{ '{{' }} $labels.profile {{ '}}' }}"
|
||||
|
||||
- alert: PBSBackupTooOld
|
||||
expr: homelab_backup_audit_snapshot_age_hours{profile="pbs"} > 48
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PBS backup is older than 48 hours: VMID {{ '{{' }} $labels.vmid {{ '}}' }}"
|
||||
@@ -0,0 +1,10 @@
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
valid_status_codes: [200, 301, 302]
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
@@ -0,0 +1,63 @@
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ monitoring_prometheus_retention }}
|
||||
- --storage.tsdb.retention.size={{ monitoring_prometheus_retention_size }}
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./prometheus:/etc/prometheus:ro
|
||||
- ./prometheus/data:/prometheus
|
||||
restart: unless-stopped
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.0
|
||||
user: "0:0"
|
||||
command:
|
||||
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||
- --storage.path=/alertmanager
|
||||
volumes:
|
||||
- ./alertmanager:/etc/alertmanager:ro
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.1
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: {{ monitoring_grafana_admin_user | to_json }}
|
||||
GF_SECURITY_ADMIN_PASSWORD: {{ monitoring_grafana_admin_password | to_json }}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
ports:
|
||||
- "192.168.1.30:3000:3000"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/etc/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
restart: unless-stopped
|
||||
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.25.0
|
||||
command:
|
||||
- --config.file=/config/blackbox.yml
|
||||
volumes:
|
||||
- ./blackbox.yml:/config/blackbox.yml:ro
|
||||
restart: unless-stopped
|
||||
|
||||
pve-exporter:
|
||||
image: prompve/prometheus-pve-exporter:3.5.5
|
||||
user: "0:0"
|
||||
command:
|
||||
- --config.file=/etc/pve-exporter/pve.yml
|
||||
volumes:
|
||||
- ./pve-exporter/pve.yml:/etc/pve-exporter/pve.yml:ro
|
||||
restart: unless-stopped
|
||||
|
||||
pushgateway:
|
||||
image: prom/pushgateway:v1.11.0
|
||||
ports:
|
||||
- "192.168.1.30:9091:9091"
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,94 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ["alertmanager:9093"]
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5:9100"
|
||||
- "192.168.1.10:9100"
|
||||
- "192.168.1.20:9100"
|
||||
- "192.168.1.23:9100"
|
||||
- "192.168.1.24:9100"
|
||||
- "192.168.1.25:9100"
|
||||
- "192.168.1.26:9100"
|
||||
- "192.168.1.27:9100"
|
||||
- "192.168.1.28:9100"
|
||||
- "192.168.1.30:9100"
|
||||
- "192.168.1.34:9100"
|
||||
|
||||
- job_name: pve
|
||||
metrics_path: /pve
|
||||
params:
|
||||
module: [default]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5"
|
||||
- "192.168.1.10"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: pve-exporter:9221
|
||||
|
||||
- job_name: blackbox-lan-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "http://192.168.1.24"
|
||||
- "http://192.168.1.25:3000"
|
||||
- "http://192.168.1.27:8080"
|
||||
- "http://192.168.1.28"
|
||||
- "http://192.168.1.34:6060/api/v1/healthcheck"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- target_label: probe_location
|
||||
replacement: lan
|
||||
|
||||
- job_name: blackbox-lan-tcp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_connect]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5:8006"
|
||||
- "192.168.1.10:8006"
|
||||
- "192.168.1.20:8007"
|
||||
- "192.168.1.25:2222"
|
||||
- "192.168.1.27:9090"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- target_label: probe_location
|
||||
replacement: lan
|
||||
|
||||
- job_name: pushgateway
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets: ["pushgateway:9091"]
|
||||
@@ -0,0 +1,5 @@
|
||||
default:
|
||||
user: "{{ monitoring_pve_api_user }}"
|
||||
token_name: "{{ monitoring_pve_api_token_id }}"
|
||||
token_value: "{{ monitoring_pve_api_token_secret }}"
|
||||
verify_ssl: false
|
||||
Reference in New Issue
Block a user