Capture current Ansible control plane state
Commit the accumulated infrastructure work that was living only in the working tree: monitoring stack, emergency access/bot, gyro allocator, grimmory, adguard, backup audit and the OpenCode agent definitions. Also ignore Python bytecode, local archives and Nix/direnv artifacts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GTocXkGUUazHdKKd3r9k71
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
---
|
||||
monitoring_root: /opt/monitoring
|
||||
monitoring_prometheus_retention: 30d
|
||||
monitoring_prometheus_retention_size: 8GB
|
||||
monitoring_grafana_admin_user: admin
|
||||
monitoring_grafana_admin_password: "{{ lookup('env', 'MONITORING_GRAFANA_ADMIN_PASSWORD') }}"
|
||||
monitoring_telegram_bot_token: "{{ lookup('env', 'MONITORING_TELEGRAM_BOT_TOKEN') }}"
|
||||
monitoring_telegram_chat_id: "{{ lookup('env', 'MONITORING_TELEGRAM_CHAT_ID') }}"
|
||||
monitoring_pve_api_user: "{{ lookup('env', 'MONITORING_PVE_API_USER') }}"
|
||||
monitoring_pve_api_token_id: "{{ lookup('env', 'MONITORING_PVE_API_TOKEN_ID') }}"
|
||||
monitoring_pve_api_token_secret: "{{ lookup('env', 'MONITORING_PVE_API_TOKEN_SECRET') }}"
|
||||
monitoring_lan_cidr: 192.168.1.0/24
|
||||
monitoring_openvpn_gateway_ip: 192.168.1.23
|
||||
monitoring_docker_cidr: 172.16.0.0/12
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"title": "Backups",
|
||||
"uid": "homelab-backups",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"panels": [
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Audit Result",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 0, "y": 0},
|
||||
"targets": [{"expr": "homelab_backup_audit_success", "format": "table", "instant": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Backup Age (Hours)",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 12, "y": 0},
|
||||
"targets": [{"expr": "homelab_backup_audit_snapshot_age_hours", "legendFormat": "{{profile}} {{vmid}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Audit Levels",
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 10},
|
||||
"targets": [{"expr": "homelab_backup_audit_level_success", "legendFormat": "{{profile}} {{level}}", "refId": "A"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"title": "HomeLab Overview",
|
||||
"uid": "homelab-overview",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Active Alerts",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0},
|
||||
"targets": [{"expr": "count(ALERTS{alertstate=\"firing\"})", "refId": "A"}],
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Successful Probes",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0},
|
||||
"targets": [{"expr": "sum(probe_success)", "refId": "A"}],
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Filesystem Free Space",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 0, "y": 8},
|
||||
"targets": [{"expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\"}", "legendFormat": "{{instance}} {{mountpoint}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Backup Audit Success",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 12, "y": 8},
|
||||
"targets": [{"expr": "homelab_backup_audit_success", "legendFormat": "{{profile}} {{instance}}", "refId": "A"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"title": "Proxmox and Storage",
|
||||
"uid": "proxmox-storage",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Node CPU Usage",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 0},
|
||||
"targets": [{"expr": "1 - avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Node Memory Available",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 0},
|
||||
"targets": [{"expr": "node_memory_MemAvailable_bytes", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Disk Read and Write",
|
||||
"gridPos": {"h": 9, "w": 24, "x": 0, "y": 9},
|
||||
"targets": [{"expr": "rate(node_disk_read_bytes_total[5m])", "legendFormat": "read {{instance}} {{device}}", "refId": "A"}, {"expr": "rate(node_disk_written_bytes_total[5m])", "legendFormat": "write {{instance}} {{device}}", "refId": "B"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"title": "Services and Network",
|
||||
"uid": "services-network",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"panels": [
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Probe State",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 0, "y": 0},
|
||||
"targets": [{"expr": "probe_success", "format": "table", "instant": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Probe Duration",
|
||||
"gridPos": {"h": 10, "w": 12, "x": 12, "y": 0},
|
||||
"targets": [{"expr": "probe_duration_seconds", "legendFormat": "{{job}} {{instance}}", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "TLS Certificate Remaining",
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 10},
|
||||
"targets": [{"expr": "(probe_ssl_earliest_cert_expiry - time()) / 86400", "legendFormat": "{{instance}}", "refId": "A"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: HomeLab
|
||||
orgId: 1
|
||||
folder: HomeLab
|
||||
type: file
|
||||
disableDeletion: true
|
||||
editable: false
|
||||
options:
|
||||
path: /etc/grafana/dashboards
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: restart monitoring stack
|
||||
ansible.builtin.systemd:
|
||||
name: homelab-monitoring
|
||||
state: restarted
|
||||
@@ -0,0 +1,233 @@
|
||||
---
|
||||
- name: Validate monitoring secrets
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- monitoring_telegram_bot_token | length > 0
|
||||
- monitoring_telegram_bot_token != 'replace-me'
|
||||
- monitoring_telegram_chat_id | length > 0
|
||||
- monitoring_telegram_chat_id != 'replace-me'
|
||||
- monitoring_pve_api_user | length > 0
|
||||
- monitoring_pve_api_token_id | length > 0
|
||||
- monitoring_pve_api_token_secret | length > 0
|
||||
- monitoring_pve_api_token_secret != 'replace-me'
|
||||
- monitoring_grafana_admin_password | length > 0
|
||||
- monitoring_grafana_admin_password != 'change-me'
|
||||
fail_msg: Load monitoring secrets from ignored ansible/.env or Ansible Vault before running this playbook.
|
||||
no_log: true
|
||||
|
||||
- name: Install monitoring runtime packages
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- ca-certificates
|
||||
- docker.io
|
||||
- docker-compose
|
||||
- ufw
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Ensure Docker is enabled and running
|
||||
ansible.builtin.systemd:
|
||||
name: docker
|
||||
enabled: true
|
||||
state: started
|
||||
|
||||
- name: Allow SSH and Grafana access from the LAN
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "{{ item }}"
|
||||
proto: tcp
|
||||
src: "{{ monitoring_lan_cidr }}"
|
||||
loop:
|
||||
- "22"
|
||||
- "3000"
|
||||
|
||||
- name: Allow Pushgateway only from the OpenVPN gateway
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "9091"
|
||||
proto: tcp
|
||||
src: "{{ monitoring_openvpn_gateway_ip }}"
|
||||
|
||||
- name: Allow local Prometheus container to scrape Node Exporter
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
port: "9100"
|
||||
proto: tcp
|
||||
src: "{{ monitoring_docker_cidr }}"
|
||||
|
||||
- name: Enable monitoring LXC firewall
|
||||
community.general.ufw:
|
||||
state: enabled
|
||||
policy: deny
|
||||
direction: incoming
|
||||
|
||||
- name: Create monitoring directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ monitoring_root }}"
|
||||
- "{{ monitoring_root }}/prometheus"
|
||||
- "{{ monitoring_root }}/alertmanager"
|
||||
- "{{ monitoring_root }}/grafana"
|
||||
- "{{ monitoring_root }}/grafana/provisioning"
|
||||
- "{{ monitoring_root }}/grafana/provisioning/dashboards"
|
||||
- "{{ monitoring_root }}/grafana/provisioning/datasources"
|
||||
- "{{ monitoring_root }}/grafana/dashboards"
|
||||
- "{{ monitoring_root }}/pve-exporter"
|
||||
|
||||
- name: Grant Prometheus access to its data directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ monitoring_root }}/prometheus/data"
|
||||
state: directory
|
||||
owner: "65534"
|
||||
group: "65534"
|
||||
mode: "0755"
|
||||
|
||||
- name: Install monitoring compose definition
|
||||
ansible.builtin.template:
|
||||
src: compose.yml.j2
|
||||
dest: "{{ monitoring_root }}/compose.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
no_log: true
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install Prometheus configuration
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ monitoring_root }}/prometheus/prometheus.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install Prometheus alert rules
|
||||
ansible.builtin.template:
|
||||
src: alerts.yml.j2
|
||||
dest: "{{ monitoring_root }}/prometheus/alerts.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install central Blackbox configuration
|
||||
ansible.builtin.template:
|
||||
src: blackbox.yml.j2
|
||||
dest: "{{ monitoring_root }}/blackbox.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install Alertmanager configuration
|
||||
ansible.builtin.template:
|
||||
src: alertmanager.yml.j2
|
||||
dest: "{{ monitoring_root }}/alertmanager/alertmanager.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0640"
|
||||
no_log: true
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install PVE exporter configuration
|
||||
ansible.builtin.template:
|
||||
src: pve.yml.j2
|
||||
dest: "{{ monitoring_root }}/pve-exporter/pve.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
no_log: true
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Provision Grafana datasource
|
||||
ansible.builtin.copy:
|
||||
src: grafana-datasource.yml
|
||||
dest: "{{ monitoring_root }}/grafana/provisioning/datasources/prometheus.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Provision Grafana dashboard provider
|
||||
ansible.builtin.copy:
|
||||
src: grafana-dashboard-provider.yml
|
||||
dest: "{{ monitoring_root }}/grafana/provisioning/dashboards/homelab.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Install Grafana dashboards
|
||||
ansible.builtin.copy:
|
||||
src: "dashboards/{{ item }}"
|
||||
dest: "{{ monitoring_root }}/grafana/dashboards/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- homelab-overview.json
|
||||
- proxmox-storage.json
|
||||
- services-network.json
|
||||
- backups.json
|
||||
notify: restart monitoring stack
|
||||
|
||||
- name: Validate Docker Compose configuration
|
||||
ansible.builtin.command: >-
|
||||
docker-compose -f {{ monitoring_root }}/compose.yml config --quiet
|
||||
changed_when: false
|
||||
no_log: true
|
||||
|
||||
- name: Validate Prometheus configuration
|
||||
ansible.builtin.command: >-
|
||||
docker run --rm --network none
|
||||
-v {{ monitoring_root }}/prometheus:/etc/prometheus:ro
|
||||
--entrypoint promtool prom/prometheus:v3.2.1
|
||||
check config /etc/prometheus/prometheus.yml
|
||||
changed_when: false
|
||||
|
||||
- name: Validate Alertmanager configuration
|
||||
ansible.builtin.command: >-
|
||||
docker run --rm --network none
|
||||
-v {{ monitoring_root }}/alertmanager:/etc/alertmanager:ro
|
||||
--user 0:0
|
||||
--entrypoint amtool prom/alertmanager:v0.28.0
|
||||
check-config /etc/alertmanager/alertmanager.yml
|
||||
changed_when: false
|
||||
no_log: true
|
||||
|
||||
- name: Install monitoring systemd unit
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-monitoring.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=HomeLab monitoring stack
|
||||
Requires=docker.service
|
||||
After=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
WorkingDirectory={{ monitoring_root }}
|
||||
ExecStart=/usr/bin/docker-compose -f {{ monitoring_root }}/compose.yml up -d --remove-orphans
|
||||
ExecStop=/usr/bin/docker-compose -f {{ monitoring_root }}/compose.yml down
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart monitoring stack
|
||||
|
||||
- name: Enable monitoring stack
|
||||
ansible.builtin.systemd:
|
||||
name: homelab-monitoring
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,40 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: telegram-warning
|
||||
group_by: [alertname, instance]
|
||||
group_wait: 30s
|
||||
group_interval: 10m
|
||||
repeat_interval: 6h
|
||||
routes:
|
||||
- matchers:
|
||||
- severity="critical"
|
||||
receiver: telegram-critical
|
||||
repeat_interval: 2h
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- alertname="ExporterDown"
|
||||
target_matchers:
|
||||
- alertname="LanServiceUnavailable"
|
||||
equal: [instance]
|
||||
|
||||
receivers:
|
||||
- name: telegram-warning
|
||||
telegram_configs:
|
||||
- bot_token: "{{ monitoring_telegram_bot_token }}"
|
||||
chat_id: {{ monitoring_telegram_chat_id }}
|
||||
send_resolved: true
|
||||
message: |-
|
||||
[{{ "{{" }} .Status | toUpper {{ "}}" }}] {{ "{{" }} .CommonLabels.alertname {{ "}}" }}
|
||||
{{ "{{" }} .CommonAnnotations.summary {{ "}}" }}
|
||||
|
||||
- name: telegram-critical
|
||||
telegram_configs:
|
||||
- bot_token: "{{ monitoring_telegram_bot_token }}"
|
||||
chat_id: {{ monitoring_telegram_chat_id }}
|
||||
send_resolved: true
|
||||
message: |-
|
||||
[{{ "{{" }} .Status | toUpper {{ "}}" }}] CRITICAL: {{ "{{" }} .CommonLabels.alertname {{ "}}" }}
|
||||
{{ "{{" }} .CommonAnnotations.summary {{ "}}" }}
|
||||
@@ -0,0 +1,134 @@
|
||||
groups:
|
||||
- name: homelab-availability
|
||||
rules:
|
||||
- alert: ExporterDown
|
||||
expr: up{job=~"node|smartctl|pve"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Exporter unavailable: {{ '{{' }} $labels.job {{ '}}' }} {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: CriticalServiceUnavailable
|
||||
expr: probe_success{job="blackbox-vps"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "External or VPN check failed: {{ '{{' }} $labels.probe_name {{ '}}' }}"
|
||||
|
||||
- alert: RuVpsMonitoringStale
|
||||
expr: time() - homelab_vps_probe_timestamp_seconds > 5 * 60
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ru-vps has not pushed monitoring metrics through OpenVPN"
|
||||
|
||||
- alert: RuVpsMonitoringMissing
|
||||
expr: absent(homelab_vps_probe_timestamp_seconds{job="blackbox-vps"})
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ru-vps has no monitoring metrics in Pushgateway"
|
||||
|
||||
- alert: LanServiceUnavailable
|
||||
expr: probe_success{job=~"blackbox-lan-http|blackbox-lan-tcp"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "LAN service unavailable: {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: TLSCertificateExpiringSoon
|
||||
expr: (probe_ssl_earliest_cert_expiry - time()) < 14 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "TLS certificate expires within 14 days: {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- name: homelab-capacity
|
||||
rules:
|
||||
- alert: FilesystemAlmostFull
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem is more than 85% full: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
||||
|
||||
- alert: FilesystemWillFillSoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 7 * 24 * 3600) < 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem may fill within seven days: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.mountpoint {{ '}}' }}"
|
||||
|
||||
- alert: HostMemoryPressure
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.1
|
||||
for: 20m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Memory pressure on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: SMARTDeviceFailed
|
||||
expr: homelab_smart_device_healthy == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SMART health check failed: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
||||
|
||||
- alert: NVMECriticalWarning
|
||||
expr: homelab_smart_nvme_critical_warning > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe critical warning: {{ '{{' }} $labels.instance {{ '}}' }} {{ '{{' }} $labels.device {{ '}}' }}"
|
||||
|
||||
- name: homelab-backups
|
||||
rules:
|
||||
- alert: BackupAuditFailed
|
||||
expr: homelab_backup_audit_success == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Backup audit failed: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: BackupAuditStale
|
||||
expr: time() - homelab_backup_audit_timestamp_seconds > 48 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Backup audit result is stale: {{ '{{' }} $labels.profile {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
|
||||
- alert: BackupAuditMissing
|
||||
expr: absent(homelab_backup_audit_timestamp_seconds{profile="pbs"}) or absent(homelab_backup_audit_timestamp_seconds{profile="gitea"}) or absent(homelab_backup_audit_timestamp_seconds{profile="vaultwarden"}) or absent(homelab_backup_audit_timestamp_seconds{profile="grimmory"})
|
||||
for: 12h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Expected backup audit metric has not been published"
|
||||
|
||||
- alert: ResticBackupTooOld
|
||||
expr: homelab_backup_audit_snapshot_age_hours{profile=~"gitea|vaultwarden|grimmory"} > 36
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Restic backup is older than 36 hours: {{ '{{' }} $labels.profile {{ '}}' }}"
|
||||
|
||||
- alert: PBSBackupTooOld
|
||||
expr: homelab_backup_audit_snapshot_age_hours{profile="pbs"} > 48
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PBS backup is older than 48 hours: VMID {{ '{{' }} $labels.vmid {{ '}}' }}"
|
||||
@@ -0,0 +1,10 @@
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
valid_status_codes: [200, 301, 302]
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
@@ -0,0 +1,63 @@
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ monitoring_prometheus_retention }}
|
||||
- --storage.tsdb.retention.size={{ monitoring_prometheus_retention_size }}
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./prometheus:/etc/prometheus:ro
|
||||
- ./prometheus/data:/prometheus
|
||||
restart: unless-stopped
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.0
|
||||
user: "0:0"
|
||||
command:
|
||||
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||
- --storage.path=/alertmanager
|
||||
volumes:
|
||||
- ./alertmanager:/etc/alertmanager:ro
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.1
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: {{ monitoring_grafana_admin_user | to_json }}
|
||||
GF_SECURITY_ADMIN_PASSWORD: {{ monitoring_grafana_admin_password | to_json }}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
ports:
|
||||
- "192.168.1.30:3000:3000"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/etc/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
restart: unless-stopped
|
||||
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.25.0
|
||||
command:
|
||||
- --config.file=/config/blackbox.yml
|
||||
volumes:
|
||||
- ./blackbox.yml:/config/blackbox.yml:ro
|
||||
restart: unless-stopped
|
||||
|
||||
pve-exporter:
|
||||
image: prompve/prometheus-pve-exporter:3.5.5
|
||||
user: "0:0"
|
||||
command:
|
||||
- --config.file=/etc/pve-exporter/pve.yml
|
||||
volumes:
|
||||
- ./pve-exporter/pve.yml:/etc/pve-exporter/pve.yml:ro
|
||||
restart: unless-stopped
|
||||
|
||||
pushgateway:
|
||||
image: prom/pushgateway:v1.11.0
|
||||
ports:
|
||||
- "192.168.1.30:9091:9091"
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,94 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ["alertmanager:9093"]
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5:9100"
|
||||
- "192.168.1.10:9100"
|
||||
- "192.168.1.20:9100"
|
||||
- "192.168.1.23:9100"
|
||||
- "192.168.1.24:9100"
|
||||
- "192.168.1.25:9100"
|
||||
- "192.168.1.26:9100"
|
||||
- "192.168.1.27:9100"
|
||||
- "192.168.1.28:9100"
|
||||
- "192.168.1.30:9100"
|
||||
- "192.168.1.34:9100"
|
||||
|
||||
- job_name: pve
|
||||
metrics_path: /pve
|
||||
params:
|
||||
module: [default]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5"
|
||||
- "192.168.1.10"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: pve-exporter:9221
|
||||
|
||||
- job_name: blackbox-lan-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "http://192.168.1.24"
|
||||
- "http://192.168.1.25:3000"
|
||||
- "http://192.168.1.27:8080"
|
||||
- "http://192.168.1.28"
|
||||
- "http://192.168.1.34:6060/api/v1/healthcheck"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- target_label: probe_location
|
||||
replacement: lan
|
||||
|
||||
- job_name: blackbox-lan-tcp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_connect]
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.5:8006"
|
||||
- "192.168.1.10:8006"
|
||||
- "192.168.1.20:8007"
|
||||
- "192.168.1.25:2222"
|
||||
- "192.168.1.27:9090"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- target_label: probe_location
|
||||
replacement: lan
|
||||
|
||||
- job_name: pushgateway
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets: ["pushgateway:9091"]
|
||||
@@ -0,0 +1,5 @@
|
||||
default:
|
||||
user: "{{ monitoring_pve_api_user }}"
|
||||
token_name: "{{ monitoring_pve_api_token_id }}"
|
||||
token_value: "{{ monitoring_pve_api_token_secret }}"
|
||||
verify_ssl: false
|
||||
Reference in New Issue
Block a user