Implement observable dashboard via Grafana and Prometheus

ansible/playbooks/07_observability.yml

@@ -0,0 +1,118 @@
+---
+- name: Deploy Node Exporter on All Hosts
+  hosts: all
+  gather_facts: true
+
+  tasks:
+    - name: Ensure tar and gzip are installed
+      become: true
+      ansible.builtin.package:
+        name:
+          - tar
+          - gzip
+        state: present
+      failed_when: false # Ignore if a host doesn't use standard repositories
+
+    - name: Download Node Exporter binary
+      ansible.builtin.get_url:
+        url: "https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz"
+        dest: "/tmp/node_exporter.tar.gz"
+        mode: "0644"
+
+    - name: Extract Node Exporter package
+      ansible.builtin.unarchive:
+        src: "/tmp/node_exporter.tar.gz"
+        dest: "/tmp"
+        remote_src: true
+
+    - name: Copy Node Exporter binary to destination
+      become: true
+      ansible.builtin.copy:
+        src: "/tmp/node_exporter-1.8.2.linux-amd64/node_exporter"
+        dest: "/usr/local/bin/node_exporter"
+        mode: "0755"
+        remote_src: true
+
+    - name: Deploy Node Exporter Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "templates/node_exporter.service.j2"
+        dest: "/etc/systemd/system/node_exporter.service"
+        mode: "0644"
+
+    - name: Restore SELinux contexts (RedHat/AlmaLinux)
+      become: true
+      ansible.builtin.command: restorecon -v /usr/local/bin/node_exporter /etc/systemd/system/node_exporter.service
+      when: ansible_os_family == "RedHat"
+
+    - name: Open port 9100 in firewalld
+      become: true
+      ansible.builtin.command: firewall-cmd --permanent --add-port=9100/tcp && firewall-cmd --reload
+      when: ansible_os_family == "RedHat"
+      failed_when: false
+
+    - name: Start and enable Node Exporter
+      become: true
+      ansible.builtin.systemd_service:
+        name: node_exporter.service
+        state: restarted
+        enabled: true
+        daemon_reload: true
+
+- name: Deploy Prometheus and Grafana on Portfolio VM
+  hosts: portfolio.lab.local
+  gather_facts: true
+
+  tasks:
+    - name: Create Prometheus and Grafana directories
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: "{{ admin_user }}"
+        group: "{{ admin_user }}"
+        mode: "0755"
+      loop:
+        - "/home/{{ admin_user }}/containers/prometheus"
+        - "/home/{{ admin_user }}/containers/prometheus/config"
+        - "/home/{{ admin_user }}/containers/prometheus/data"
+        - "/home/{{ admin_user }}/containers/grafana"
+        - "/home/{{ admin_user }}/containers/grafana/data"
+
+    - name: Deploy Prometheus configuration
+      become: true
+      ansible.builtin.template:
+        src: "templates/prometheus.yml.j2"
+        dest: "/home/{{ admin_user }}/containers/prometheus/config/prometheus.yml"
+        owner: "{{ admin_user }}"
+        group: "{{ admin_user }}"
+        mode: "0644"
+
+    - name: Deploy Prometheus Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "templates/prometheus.service.j2"
+        dest: "/etc/systemd/system/prometheus.service"
+        mode: "0644"
+
+    - name: Deploy Grafana Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "templates/grafana.service.j2"
+        dest: "/etc/systemd/system/grafana.service"
+        mode: "0644"
+
+    - name: Start and enable Prometheus service
+      become: true
+      ansible.builtin.systemd_service:
+        name: prometheus.service
+        state: restarted
+        enabled: true
+        daemon_reload: true
+
+    - name: Start and enable Grafana service
+      become: true
+      ansible.builtin.systemd_service:
+        name: grafana.service
+        state: restarted
+        enabled: true
+        daemon_reload: true

ansible/playbooks/templates/grafana.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=Grafana Dashboard Container
+After=network-online.target prometheus.service
+Wants=network-online.target prometheus.service
+
+[Service]
+Type=simple
+Restart=always
+ExecStartPre=-/usr/bin/podman rm -f grafana
+ExecStart=/usr/bin/podman run --name grafana -p 3000:3000 -v /home/{{ admin_user }}/containers/grafana/data:/var/lib/grafana:z,U docker.io/grafana/grafana-oss:latest
+ExecStop=/usr/bin/podman stop -t 10 grafana
+ExecStopPost=-/usr/bin/podman rm -f grafana
+
+[Install]
+WantedBy=multi-user.target

ansible/playbooks/templates/node_exporter.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Prometheus Node Exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+ExecStart=/usr/local/bin/node_exporter
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

ansible/playbooks/templates/prometheus.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Prometheus Metrics Database Container
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+Restart=always
+ExecStartPre=-/usr/bin/podman rm -f Prometheus
+ExecStart=/usr/bin/podman run --name prometheus \
+    --net=host \
+    -v /home/{{ admin_user }}/containers/prometheus/config/prometheus.yml:/etc/prometheus/prometheus.yml:z \
+    -v /home/{{ admin_user }}/containers/prometheus/data:/prometheus:z,U \
+    docker.io/prom/prometheus:latest \
+    --config.file=/etc/prometheus/prometheus.yml \
+    --storage.tsdb.path=/prometheus
+ExecStop=/usr/bin/podman stop -t 10 prometheus
+ExecStopPost=-/usr/bin/podman rm -f prometheus
+
+[Install]
+WantedBy=multi-user.target

ansible/playbooks/templates/prometheus.yml.j2

@@ -0,0 +1,13 @@
+global:
+    scrape_interval: 15s
+    evaluation_interval: 15s
+
+scrape_configs:
+    - job_name: 'homelab-nodes'
+      static_configs:
+        - targets:
+            - '172.30.1.200:9100'
+            - '172.30.1.85:9100'
+            - '172.30.1.93:9100'
+            - '172.30.1.91:9100'
+            - '172.30.1.92:9100'

ansible/site.yml

@@ -22,3 +22,7 @@
 - name: Cloudflared Tunnel Setup
   import_playbook: ./playbooks/06_cloudflared_setup.yml
   tags: cloudflared
+
+- name: Grafana/Prometheus Setup
+  import_playbook: ./playbooks/07_observability.yml
+  tags: observability