From 4ffd19d92a417239658737335512f61470ba7ed3 Mon Sep 17 00:00:00 2001 From: Sho Ishida Date: Thu, 16 Jul 2026 09:55:12 +0000 Subject: [PATCH] Implement observable dashboard via Grafana and Prometheus --- ansible/playbooks/07_observability.yml | 118 ++++++++++++++++++ .../playbooks/templates/grafana.service.j2 | 15 +++ .../templates/node_exporter.service.j2 | 13 ++ .../playbooks/templates/prometheus.service.j2 | 21 ++++ ansible/playbooks/templates/prometheus.yml.j2 | 13 ++ ansible/site.yml | 4 + 6 files changed, 184 insertions(+) create mode 100644 ansible/playbooks/07_observability.yml create mode 100644 ansible/playbooks/templates/grafana.service.j2 create mode 100644 ansible/playbooks/templates/node_exporter.service.j2 create mode 100644 ansible/playbooks/templates/prometheus.service.j2 create mode 100644 ansible/playbooks/templates/prometheus.yml.j2 diff --git a/ansible/playbooks/07_observability.yml b/ansible/playbooks/07_observability.yml new file mode 100644 index 0000000..ae1dc8f --- /dev/null +++ b/ansible/playbooks/07_observability.yml @@ -0,0 +1,118 @@ +--- +- name: Deploy Node Exporter on All Hosts + hosts: all + gather_facts: true + + tasks: + - name: Ensure tar and gzip are installed + become: true + ansible.builtin.package: + name: + - tar + - gzip + state: present + failed_when: false # Ignore if a host doesn't use standard repositories + + - name: Download Node Exporter binary + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz" + dest: "/tmp/node_exporter.tar.gz" + mode: "0644" + + - name: Extract Node Exporter package + ansible.builtin.unarchive: + src: "/tmp/node_exporter.tar.gz" + dest: "/tmp" + remote_src: true + + - name: Copy Node Exporter binary to destination + become: true + ansible.builtin.copy: + src: "/tmp/node_exporter-1.8.2.linux-amd64/node_exporter" + dest: "/usr/local/bin/node_exporter" + mode: "0755" + remote_src: true + + - name: Deploy Node Exporter Systemd Service + become: true + ansible.builtin.template: + src: "templates/node_exporter.service.j2" + dest: "/etc/systemd/system/node_exporter.service" + mode: "0644" + + - name: Restore SELinux contexts (RedHat/AlmaLinux) + become: true + ansible.builtin.command: restorecon -v /usr/local/bin/node_exporter /etc/systemd/system/node_exporter.service + when: ansible_os_family == "RedHat" + + - name: Open port 9100 in firewalld + become: true + ansible.builtin.command: firewall-cmd --permanent --add-port=9100/tcp && firewall-cmd --reload + when: ansible_os_family == "RedHat" + failed_when: false + + - name: Start and enable Node Exporter + become: true + ansible.builtin.systemd_service: + name: node_exporter.service + state: restarted + enabled: true + daemon_reload: true + +- name: Deploy Prometheus and Grafana on Portfolio VM + hosts: portfolio.lab.local + gather_facts: true + + tasks: + - name: Create Prometheus and Grafana directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0755" + loop: + - "/home/{{ admin_user }}/containers/prometheus" + - "/home/{{ admin_user }}/containers/prometheus/config" + - "/home/{{ admin_user }}/containers/prometheus/data" + - "/home/{{ admin_user }}/containers/grafana" + - "/home/{{ admin_user }}/containers/grafana/data" + + - name: Deploy Prometheus configuration + become: true + ansible.builtin.template: + src: "templates/prometheus.yml.j2" + dest: "/home/{{ admin_user }}/containers/prometheus/config/prometheus.yml" + owner: "{{ admin_user }}" + group: "{{ admin_user }}" + mode: "0644" + + - name: Deploy Prometheus Systemd Service + become: true + ansible.builtin.template: + src: "templates/prometheus.service.j2" + dest: "/etc/systemd/system/prometheus.service" + mode: "0644" + + - name: Deploy Grafana Systemd Service + become: true + ansible.builtin.template: + src: "templates/grafana.service.j2" + dest: "/etc/systemd/system/grafana.service" + mode: "0644" + + - name: Start and enable Prometheus service + become: true + ansible.builtin.systemd_service: + name: prometheus.service + state: restarted + enabled: true + daemon_reload: true + + - name: Start and enable Grafana service + become: true + ansible.builtin.systemd_service: + name: grafana.service + state: restarted + enabled: true + daemon_reload: true diff --git a/ansible/playbooks/templates/grafana.service.j2 b/ansible/playbooks/templates/grafana.service.j2 new file mode 100644 index 0000000..35919cb --- /dev/null +++ b/ansible/playbooks/templates/grafana.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Grafana Dashboard Container +After=network-online.target prometheus.service +Wants=network-online.target prometheus.service + +[Service] +Type=simple +Restart=always +ExecStartPre=-/usr/bin/podman rm -f grafana +ExecStart=/usr/bin/podman run --name grafana -p 3000:3000 -v /home/{{ admin_user }}/containers/grafana/data:/var/lib/grafana:z,U docker.io/grafana/grafana-oss:latest +ExecStop=/usr/bin/podman stop -t 10 grafana +ExecStopPost=-/usr/bin/podman rm -f grafana + +[Install] +WantedBy=multi-user.target diff --git a/ansible/playbooks/templates/node_exporter.service.j2 b/ansible/playbooks/templates/node_exporter.service.j2 new file mode 100644 index 0000000..4db9c73 --- /dev/null +++ b/ansible/playbooks/templates/node_exporter.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus Node Exporter +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +ExecStart=/usr/local/bin/node_exporter +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/ansible/playbooks/templates/prometheus.service.j2 b/ansible/playbooks/templates/prometheus.service.j2 new file mode 100644 index 0000000..511a8b5 --- /dev/null +++ b/ansible/playbooks/templates/prometheus.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Prometheus Metrics Database Container +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +ExecStartPre=-/usr/bin/podman rm -f Prometheus +ExecStart=/usr/bin/podman run --name prometheus \ + --net=host \ + -v /home/{{ admin_user }}/containers/prometheus/config/prometheus.yml:/etc/prometheus/prometheus.yml:z \ + -v /home/{{ admin_user }}/containers/prometheus/data:/prometheus:z,U \ + docker.io/prom/prometheus:latest \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus +ExecStop=/usr/bin/podman stop -t 10 prometheus +ExecStopPost=-/usr/bin/podman rm -f prometheus + +[Install] +WantedBy=multi-user.target diff --git a/ansible/playbooks/templates/prometheus.yml.j2 b/ansible/playbooks/templates/prometheus.yml.j2 new file mode 100644 index 0000000..624870e --- /dev/null +++ b/ansible/playbooks/templates/prometheus.yml.j2 @@ -0,0 +1,13 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'homelab-nodes' + static_configs: + - targets: + - '172.30.1.200:9100' + - '172.30.1.85:9100' + - '172.30.1.93:9100' + - '172.30.1.91:9100' + - '172.30.1.92:9100' diff --git a/ansible/site.yml b/ansible/site.yml index 266ddda..1a94cf3 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -22,3 +22,7 @@ - name: Cloudflared Tunnel Setup import_playbook: ./playbooks/06_cloudflared_setup.yml tags: cloudflared + +- name: Grafana/Prometheus Setup + import_playbook: ./playbooks/07_observability.yml + tags: observability