infra/ansible/roles/monitoring_server/tasks/main.yml

---
- name: Validate monitoring secrets
  ansible.builtin.assert:
    that:
      - monitoring_telegram_bot_token | length > 0
      - monitoring_telegram_bot_token != 'replace-me'
      - monitoring_telegram_chat_id | length > 0
      - monitoring_telegram_chat_id != 'replace-me'
      - monitoring_pve_api_user | length > 0
      - monitoring_pve_api_token_id | length > 0
      - monitoring_pve_api_token_secret | length > 0
      - monitoring_pve_api_token_secret != 'replace-me'
      - monitoring_grafana_admin_password | length > 0
      - monitoring_grafana_admin_password != 'change-me'
    fail_msg: Load monitoring secrets from the ignored repository-root .env or Ansible Vault before running this playbook.
  no_log: true

- name: Install monitoring runtime packages
  ansible.builtin.apt:
    name:
      - ca-certificates
      - docker.io
      - docker-compose
      - ufw
    state: present
    update_cache: true

- name: Ensure Docker is enabled and running
  ansible.builtin.systemd:
    name: docker
    enabled: true
    state: started

- name: Allow SSH and Grafana access from the LAN
  community.general.ufw:
    rule: allow
    port: "{{ item }}"
    proto: tcp
    src: "{{ monitoring_lan_cidr }}"
  loop:
    - "22"
    - "3000"

- name: Allow Pushgateway only from the OpenVPN gateway
  community.general.ufw:
    rule: allow
    port: "9091"
    proto: tcp
    src: "{{ monitoring_openvpn_gateway_ip }}"

- name: Allow local Prometheus container to scrape Node Exporter
  community.general.ufw:
    rule: allow
    port: "9100"
    proto: tcp
    src: "{{ monitoring_docker_cidr }}"

- name: Enable monitoring LXC firewall
  community.general.ufw:
    state: enabled
    policy: deny
    direction: incoming

- name: Create monitoring directories
  ansible.builtin.file:
    path: "{{ item }}"
    state: directory
    owner: root
    group: root
    mode: "0755"
  loop:
    - "{{ monitoring_root }}"
    - "{{ monitoring_root }}/prometheus"
    - "{{ monitoring_root }}/alertmanager"
    - "{{ monitoring_root }}/grafana"
    - "{{ monitoring_root }}/grafana/provisioning"
    - "{{ monitoring_root }}/grafana/provisioning/dashboards"
    - "{{ monitoring_root }}/grafana/provisioning/datasources"
    - "{{ monitoring_root }}/grafana/dashboards"
    - "{{ monitoring_root }}/pve-exporter"

- name: Grant Prometheus access to its data directory
  ansible.builtin.file:
    path: "{{ monitoring_root }}/prometheus/data"
    state: directory
    owner: "65534"
    group: "65534"
    mode: "0755"

- name: Install monitoring compose definition
  ansible.builtin.template:
    src: compose.yml.j2
    dest: "{{ monitoring_root }}/compose.yml"
    owner: root
    group: root
    mode: "0600"
  no_log: true
  notify: restart monitoring stack

- name: Install Prometheus configuration
  ansible.builtin.template:
    src: prometheus.yml.j2
    dest: "{{ monitoring_root }}/prometheus/prometheus.yml"
    owner: root
    group: root
    mode: "0644"
  notify: restart monitoring stack

- name: Install Prometheus alert rules
  ansible.builtin.template:
    src: alerts.yml.j2
    dest: "{{ monitoring_root }}/prometheus/alerts.yml"
    owner: root
    group: root
    mode: "0644"
  notify: restart monitoring stack

- name: Install central Blackbox configuration
  ansible.builtin.template:
    src: blackbox.yml.j2
    dest: "{{ monitoring_root }}/blackbox.yml"
    owner: root
    group: root
    mode: "0644"
  notify: restart monitoring stack

- name: Install Alertmanager configuration
  ansible.builtin.template:
    src: alertmanager.yml.j2
    dest: "{{ monitoring_root }}/alertmanager/alertmanager.yml"
    owner: root
    group: root
    mode: "0640"
  no_log: true
  notify: restart monitoring stack

- name: Install PVE exporter configuration
  ansible.builtin.template:
    src: pve.yml.j2
    dest: "{{ monitoring_root }}/pve-exporter/pve.yml"
    owner: root
    group: root
    mode: "0600"
  no_log: true
  notify: restart monitoring stack

- name: Provision Grafana datasource
  ansible.builtin.copy:
    src: grafana-datasource.yml
    dest: "{{ monitoring_root }}/grafana/provisioning/datasources/prometheus.yml"
    owner: root
    group: root
    mode: "0644"
  notify: restart monitoring stack

- name: Provision Grafana dashboard provider
  ansible.builtin.copy:
    src: grafana-dashboard-provider.yml
    dest: "{{ monitoring_root }}/grafana/provisioning/dashboards/homelab.yml"
    owner: root
    group: root
    mode: "0644"
  notify: restart monitoring stack

- name: Install Grafana dashboards
  ansible.builtin.copy:
    src: "dashboards/{{ item }}"
    dest: "{{ monitoring_root }}/grafana/dashboards/{{ item }}"
    owner: root
    group: root
    mode: "0644"
  loop:
    - homelab-overview.json
    - proxmox-storage.json
    - services-network.json
    - backups.json
  notify: restart monitoring stack

- name: Validate Docker Compose configuration
  ansible.builtin.command: >-
    docker-compose -f {{ monitoring_root }}/compose.yml config --quiet
  changed_when: false
  no_log: true

- name: Validate Prometheus configuration
  ansible.builtin.command: >-
    docker run --rm --network none
    -v {{ monitoring_root }}/prometheus:/etc/prometheus:ro
    --entrypoint promtool prom/prometheus:v3.2.1
    check config /etc/prometheus/prometheus.yml
  changed_when: false

- name: Validate Alertmanager configuration
  ansible.builtin.command: >-
    docker run --rm --network none
    -v {{ monitoring_root }}/alertmanager:/etc/alertmanager:ro
    --user 0:0
    --entrypoint amtool prom/alertmanager:v0.28.0
    check-config /etc/alertmanager/alertmanager.yml
  changed_when: false
  no_log: true

- name: Install monitoring systemd unit
  ansible.builtin.copy:
    dest: /etc/systemd/system/homelab-monitoring.service
    owner: root
    group: root
    mode: "0644"
    content: |
      [Unit]
      Description=HomeLab monitoring stack
      Requires=docker.service
      After=docker.service

      [Service]
      Type=oneshot
      RemainAfterExit=yes
      WorkingDirectory={{ monitoring_root }}
      ExecStart=/usr/bin/docker-compose -f {{ monitoring_root }}/compose.yml up -d --remove-orphans
      ExecStop=/usr/bin/docker-compose -f {{ monitoring_root }}/compose.yml down

      [Install]
      WantedBy=multi-user.target
  notify:
    - reload systemd
    - restart monitoring stack

- name: Enable monitoring stack
  ansible.builtin.systemd:
    name: homelab-monitoring
    daemon_reload: true
    enabled: true
    state: started