Deploy Gitea and configure GitOps CI/CD pipeline

.gitignore

@@ -5,17 +5,21 @@ ansible/playbooks/*.retry
 *.retry
 
 # === Terraform ===
-terraform/.terraform/
+.terraform/
-terraform/.terraform.lock.*
+.terraform.lock.hcl
+*.terraform.lock.*
+crash.log
+*.crash.log
 
 # === State Files ===
-terraform/terraform.tfstate
+*.tfstate
-terraform/terraform.tfstate.backup
+*.tfstate.*
-terraform/terraform.tfstate.d/
+terraform.tfstate
+terraform.tfstate.*
 
 # === Variables ===
-terraform/*.tfvars
+*.tfvars
-terraform/*.tfvars.json
+*.tfvars.json
 
 # === OS & IDE ===
 .DS_Store

README.md

@@ -102,3 +102,7 @@ graph TD
 *   [**Local Development Setup**](./docs/01-dev-env.md)
 *   [**Bare-Metal Hypervisor Preparation**](./docs/02-hypervisor.md)
 *   [**Infrastructure Provisioning with Terraform**](./docs/03-terraform.md)
+*   [**Centralized Identity & DNS Management**](./docs/04-identity.md)
+*   [**Application Container Deployments**](./docs/05-applications.md)
+*   [**Ingress & Secure Tunnels**](./docs/06-tunnels.md)
+*   [**Observability & Telemetry**](./docs/07-observability.md)

ansible/my_inventory/group_vars/all.yml

@@ -7,6 +7,7 @@ lan_subnet: "172.30.1.0/24"
 host_ip: "172.30.1.200"
 gateway_ip: "172.30.1.254"
 
+utility_ip: "172.30.1.80"
 freeipa_ip: "172.30.1.85"
 portfolio_ip: "172.30.1.93"
 minecraft_ip: "172.30.1.91"
@@ -20,3 +21,9 @@ admin_user: "sho"
 
 ipa_admin_password: "ChangeMeIPAAdmin123!"
 ipa_directory_manager_password: "ChangeMeIPAAdmin123!"
+
+minio_root_user: "admin"
+minio_root_password: "ChangeThisPasswordSecurely"
+minio_port: "9000"
+minio_console_port: "9001"
+minio_data_dir: "/home/{{ admin_user }}/containers/minio/data"

ansible/my_inventory/hosts

@@ -2,6 +2,7 @@
 hypervisor.lab.local ansible_host=172.30.1.200 ansible_user=sho
 
 [vms]
+utility.lab.local ansible_host=172.30.1.80 ansible_user=sho
 freeipa.lab.local ansible_host=172.30.1.85 ansible_user=sho
 portfolio.lab.local ansible_host=172.30.1.93 ansible_user=sho
 minecraft.lab.local ansible_host=172.30.1.91 ansible_user=sho

ansible/playbooks/08_utility_services.yml

@@ -0,0 +1,65 @@
+---
+- name: Deploy MinIO Infrastructure on Utility VM
+  hosts: utility.lab.local
+  gather_facts: true
+  become: false
+
+  tasks:
+    - name: Install Podman and rootless container dependencies
+      become: true
+      ansible.builtin.package:
+        name:
+          - podman
+          - uidmap
+          - dbus-user-session
+        state: present
+
+    - name: Create MinIO directory structures
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: "{{ admin_user }}"
+        group: "{{ admin_user }}"
+        mode: "0755"
+      loop:
+        - "/home/{{ admin_user }}/containers/minio"
+        - "{{ minio_data_dir }}"
+
+    - name: Deploy MinIO Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "./templates/minio.service.j2"
+        dest: "/etc/systemd/system/minio.service"
+        mode: "0644"
+
+    - name: Start and enable MinIO service
+      become: true
+      ansible.builtin.systemd_service:
+        name: minio.service
+        state: restarted
+        enabled: true
+        daemon_reload: true
+
+    - name: Wait for MinIO API to be ready
+      ansible.builtin.wait_for:
+        port: 9000
+        delay: 2
+        timeout: 30
+
+    - name: Install MinIO client (mc)
+      become: true
+      ansible.builtin.get_url:
+        url: "https://dl.min.io/client/mc/release/linux-amd64/mc"
+        dest: "/usr/local/bin/mc"
+        mode: "0755"
+
+    - name: Configure mc local alias
+      ansible.builtin.command:
+        cmd: "/usr/local/bin/mc alias set localhttp http://localhost:9000 {{ minio_root_user }} {{ minio_root_password }}"
+      changed_when: false
+
+    - name: Create 'terraform-state' bucket if it doesn't exist
+      ansible.builtin.command:
+        cmd: "/usr/local/bin/mc mb --ignore-existing localhttp/terraform-state"
+      register: mb_result
+      changed_when: "'Bucket created successfully' in mb_result.stdout"

ansible/playbooks/09_gitea_setup.yml

@@ -0,0 +1,152 @@
+---
+- name: Deploy Gitea and Act Runner on Utility VM
+  hosts: utility.lab.local
+  gather_facts: true
+  become: false
+
+  vars:
+    gitea_container_dir: "/home/{{ admin_user }}/containers/gitea"
+    runner_dir: "/home/{{ admin_user }}/containers/act_runner"
+
+  tasks:
+    - name: Create Gitea persistent volume directories
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+      loop:
+        - "{{ gitea_container_dir }}/data"
+        - "{{ gitea_container_dir }}/config"
+
+    - name: Deploy pre-seeded Gitea Configuration
+      become: true
+      ansible.builtin.template:
+        src: "./templates/app.ini.j2"
+        dest: "{{ gitea_container_dir }}/config/app.ini"
+        owner: 100999
+        group: 100999
+        mode: "0644"
+
+    - name: Enable systemd lingering for the administrator user
+      become: true
+      ansible.builtin.command:
+        cmd: "loginctl enable-linger {{ admin_user }}"
+      changed_when: false
+
+    - name: Deploy Gitea Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "./templates/gitea.service.j2"
+        dest: "/etc/systemd/system/gitea.service"
+        mode: "0644"
+
+    - name: Enable and start Gitea system service
+      become: true
+      ansible.builtin.systemd_service:
+        name: gitea.service
+        state: restarted
+        enabled: true
+        daemon_reload: true
+
+    - name: Wait for Gitea to become responsive
+      ansible.builtin.wait_for:
+        port: 3000
+        delay: 2
+        timeout: 60
+
+
+    - name: Install build tool dependencies
+      become: true
+      ansible.builtin.package:
+        name:
+          - unzip
+          - git
+          - python3-pip
+        state: present
+
+    - name: Install Ansible-Lint via pip
+      become: true
+      ansible.builtin.pip:
+        name: ansible-lint
+        state: present
+        extra_args: --break-system-packages
+
+    - name: Download and install Terraform CLI
+      become: true
+      ansible.builtin.unarchive:
+        src: "https://releases.hashicorp.com/terraform/1.8.5/terraform_1.8.5_linux_amd64.zip"
+        dest: "/usr/local/bin"
+        remote_src: true
+        mode: "0755"
+
+    - name: Download and install TFLint
+      become: true
+      ansible.builtin.unarchive:
+        src: "https://github.com/terraform-linters/tflint/releases/download/v0.50.1/tflint_linux_amd64.zip"
+        dest: "/usr/local/bin"
+        remote_src: true
+        mode: "0755"
+
+
+    - name: Create Gitea Runner directories
+      become: true
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: directory
+        owner: "{{ item.owner }}"
+        group: "{{ item.owner }}"
+        mode: "0755"
+      loop:
+        - { path: "{{ runner_dir }}", owner: "{{ admin_user }}" }
+        - { path: "/etc/act_runner", owner: "root" }
+
+    - name: Deploy Gitea Runner config file
+      become: true
+      ansible.builtin.template:
+        src: "./templates/act_runner_config.yaml.j2"
+        dest: "/etc/act_runner/config.yaml"
+        mode: "0644"
+
+    - name: Download Gitea Runner binary
+      become: true
+      ansible.builtin.get_url:
+        url: "https://dl.gitea.com/gitea-runner/2.1.0/gitea-runner-2.1.0-linux-amd64"
+        dest: "/usr/local/bin/gitea-runner"
+        mode: "0755"
+
+    - name: Generate Gitea runner registration token
+      ansible.builtin.command:
+        cmd: "podman exec -w / -u git gitea gitea --config /etc/gitea/app.ini actions generate-runner-token"
+      register: gitea_runner_token_cmd
+      changed_when: false
+
+    - name: Check if Gitea Runner is already registered
+      ansible.builtin.stat:
+        path: "{{ runner_dir }}/.runner"
+      register: runner_state_file
+
+    - name: Register Gitea Runner with Gitea
+      ansible.builtin.command:
+        cmd: >
+          /usr/local/bin/gitea-runner register
+          --instance http://localhost:3000
+          --token {{ gitea_runner_token_cmd.stdout | trim }}
+          --name utility-runner
+          --labels "linux_amd64:host,self-hosted:host"
+          --no-interactive
+        chdir: "{{ runner_dir }}"
+      when: not runner_state_file.stat.exists
+
+    - name: Deploy Gitea Runner Systemd Service
+      become: true
+      ansible.builtin.template:
+        src: "./templates/act_runner.service.j2"
+        dest: "/etc/systemd/system/act_runner.service"
+        mode: "0644"
+
+    - name: Enable and start Gitea Runner service
+      become: true
+      ansible.builtin.systemd_service:
+        name: act_runner.service
+        state: restarted
+        enabled: true
+        daemon_reload: true

ansible/playbooks/templates/act_runner.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Gitea Act Runner
+After=network-online.target gitea.service
+Wants=gitea.service
+
+[Service]
+Type=simple
+User={{ admin_user }}
+Group={{ admin_user }}
+WorkingDirectory=/home/{{ admin_user }}/containers/act_runner
+ExecStart=/usr/local/bin/gitea-runner daemon --config /etc/act_runner/config.yaml
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

ansible/playbooks/templates/act_runner_config.yaml.j2

@@ -0,0 +1,14 @@
+log:
+    level: info
+
+runner:
+    file: .runner
+    capacity: 2
+    envs: {}
+    timeout: 3h
+
+container:
+    network: ""
+
+host:
+    enable: true

ansible/playbooks/templates/app.ini.j2

@@ -0,0 +1,25 @@
+APP_NAME = Gitea: Git with a cup of tea
+RUN_USER = git
+RUN_MODE = prod
+
+[database]
+DB_TYPE = sqlite3
+PATH = /var/lib/gitea/data/gitea.db
+
+[repository]
+ROOT = /var/lib/gitea/git/repositories
+
+[server]
+SSH_DOMAIN = 172.30.1.80
+HTTP_PORT = 3000
+ROOT_URL = http://172.30.1.80:3000/
+DISABLE_SSH = false
+SSH_PORT = 2222
+LFS_START_SERVER = true
+
+[security]
+INSTALL_LOCK = true
+SECRET_KEY = {{ gitea_secret_key | default('GiteaSecretKeyChangeMeDefault123') }}
+
+[service]
+DISABLE_REGISTRATION = false

ansible/playbooks/templates/gitea.service.j2

@@ -0,0 +1,27 @@
+[Unit]
+Description=Gitea Git Server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ admin_user }}
+Group={{ admin_user }}
+WorkingDirectory=/home/{{ admin_user }}/containers/gitea
+ExecStartPre=-/usr/bin/podman rm -f gitea
+ExecStart=/usr/bin/podman run --name gitea --rm \
+    -p 3000:3000 \
+    -p 2222:22 \
+    -v /home/{{ admin_user }}/containers/gitea/data:/var/lib/gitea:z,U \
+    -v /home/{{ admin_user }}/containers/gitea/config:/etc/gitea:z,U \
+    -e GITEA__security__INSTALL_LOCK=true \
+    -e GITEA__database__DB_TYPE=sqlite3 \
+    -e GITEA__database__PATH=/var/lib/gitea/data/gitea.db \
+    docker.io/gitea/gitea:1.22-rootless
+ExecStop=/usr/bin/podman -t 10 gitea
+ExecStopPost=-/usr/bin/podman rm -f gitea
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

ansible/playbooks/templates/minio.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=MinIO Object Storage Container
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+Restart=always
+ExecStartPre=-/usr/bin/podman rm -f minio
+ExecStart=/usr/bin/podman run --name minio \
+    -p 9000:9000 \
+    -p 9001:9001 \
+    -v {{ minio_data_dir }}:/data:z,U \
+    -e "MINIO_ROOT_USER={{ minio_root_user }}" \
+    -e "MINIO_ROOT_PASSWORD={{ minio_root_password }}" \
+    quay.io/minio/minio:latest server /data --console-address ":9001"
+ExecStop=/usr/bin/podman stop -t 10 minio
+ExecStopPost=-/usr/bin/podman rm -f minio
+
+[Install]
+WantedBy=multi-user.target

ansible/site.yml

@@ -26,3 +26,7 @@
 - name: Grafana/Prometheus Setup
   import_playbook: ./playbooks/07_observability.yml
   tags: observability
+
+- name: Gitea & Act Runner Setup
+  import_playbook: ./playbooks/09_gitea_setup.yml
+  tags: gitea

docs/03-terraform.md

@@ -19,7 +19,7 @@ graph TD
     subgraph Configs ["Declarative Configuration Files"]
         MainTF["📄 main.tf<br>(Resource Specifications)"]:::inputNode
         CloudInit["📄 cloud_init.cfg<br>(User Provisioning Template)"]:::inputNode
-        NetConfig["📄 network_config.cfg.tpl<br>(Static IP Mappings)"]:::inputNOde
+        NetConfig["📄 network_config.cfg.tpl<br>(Static IP Mappings)"]:::inputNode
     end
 
     Terraform["🚀 Terraform Provider<br>(dmacvicar/libvirt)"]:::tfNode
@@ -108,7 +108,7 @@ ethernets:
                 - 1.1.1.1
 ```
 
-*Note: The primary nameserver points to `172.30.1.85`(the FreeIPA DNS server), with a fallback to `1.1.1.1`.
+*Note: The primary nameserver points to `172.30.1.85`(the FreeIPA DNS server), with a fallback to `1.1.1.1`.*
 
 ---
 

docs/04-identity.md

@@ -0,0 +1,130 @@
+# 🔑 Centralized Identity & DNS Management (FreeIPA)
+
+This document details the configuration, deployment, and operation of the centralized identity domain and directory services managed by **FreeIPA** running on `freeipa.lab.local` (`172.30.1.85`).
+
+---
+
+## 🏛️ Directory & DNS Resolution Architecture
+
+The FreeIPA virtual machine functions as the central authority for authentication, authorization, domain-name resolution, and internal certificates:
+
+```mermaid
+graph LR
+    %% My Color Palette
+    classDef clientNode fill:#212c2a,stroke:#70a99f,color:#f8f8f2,stroke-width:1.5px;
+    classDef ipaNode fill:#2b3b38,stroke:#ff9580,color:#ff9580,stroke-width:1.5px;
+    classDef extNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:2px;
+
+    subgraph Clients ["Homelab Clients (*.lab.local)"]
+        VMs["🖥️ Guest VMs<br>(portfolio, minecraft, etc.)"]:::clientNode
+    end
+
+    subgraph FreeIPA ["FreeIPA Server (freeipa.lab.local)"]
+        DNS["🌐 BIND DNS Server<br>(Port 53)"]:::ipaNode
+        KDC["🎟️ Kerberos KDC<br>(Ports 88/464)"]:::ipaNode
+        LDAP["🗄️ 389 Directory Server<br>(Ports 389/636)"]:::ipaNode
+        CA["🛡️ Dogtag PKI CA<br>(Certificate Authority)"]:::ipaNode
+    end
+
+    Upstream["🌐 Cloudflare DNS<br>(1.1.1.1)"]:::extNode
+
+    VMs -->|1. Resolves local queries| DNS
+    DNS -->|2. Forwards external queries| Upstream
+    VMs -->|3. Requests Kerberos ticket| KDC
+    VMs -->|4. Authenticates user query| LDAP
+    VMs -->|5. Trusts issued SSL certs| CA
+
+    %% Subgraph Colors
+    style Clients fill:#161d1c,stroke:#70a99f,stroke-width:1px;
+    style FreeIPA fill:#161d1c,stroke:#ff9580,stroke-width:2px;
+```
+
+---
+
+## 📄 Ansible Configuration (`ansible/playbooks/03_freeipa_install.yml`)
+
+The deployment of FreeIPA is automated using Ansible:
+
+### 1.  FQDN and Hostname Setup
+
+Updates the virtual machine kernel hostname to `freeipa.lab.local` to satisfy FreeIPA's strict fully-qualified domain name (FQDN) verification requirements.
+
+### 2.  Package Management
+
+Installs the following packages:
+
+*   `freeipa-server` & `freeipa-server-dns`: The directory server components.
+*   `bind` & `bind-utils`: DNS server utilities.
+*   `firewalld` & `python3-firewall`: Local system firewall management tools.
+
+### 3.  Unattended Server Installation
+
+The playbook runs the installer in non-interactive mode using variables declared in the inventory group variables:
+
+```bash
+ipa-server-install \
+    --unattended \
+    --realm=LAB.LOCAL \
+    --domain=lab.local \
+    --ds-password=xxxxx \
+    --admin-password=xxxxx \
+    --setup-dns \
+    --forwarder=1.1.1.1 \
+    --no-host-dns
+```
+
+*   **Integrated DNS**: DNS is configured to resolve local hosts(`*.lab.local`) and forward unresolved queries to the upstream server (`1.1.1.1`).
+*   **Timeout Handling**: Due to the time-intensive generation of cryptographic PKI keys during installation, the Ansible command timeout is increased to 1200 seconds.
+
+### 4.  Port Protection & Firewall Rules
+
+Ensures the local system firewall allows traffic across the directory network services:
+
+*   `freeipa-ldaps`(Port 636) and `freeipa-ldap`(Port 389)
+*   `dns`(Port 53 TCP/UDP)
+*   `kerberos`(Ports 88/464 TCP/UDP)
+
+---
+
+## 🚀 Execution & Verification
+
+Run the playbook using the following command:
+
+```bash
+ansible-playbook site.yml --tags "freeipa" --ask-vault-pass
+```
+
+### Verification Checks
+
+1.  **Access the GUI Console**: Open a web browser on a workstation connected to the bridge network and navigate to `https://freeipa.lab.local`. Log in using the `admin` username.
+2.  **Kerberos Authentication Check**: SSH into the FreeIPA VM and verify ticket validation:
+
+```bash
+# Request a Kerberos ticket
+kinit admin
+
+# View active ticket credentials
+klist
+```
+
+3.  **DNS Verification**: Query the DNS server directly to verify lookups:
+
+```bash
+dig @172.30.1.85 freeipa.lab.local +short
+# 172.30.1.85
+```
+
+---
+
+## 🔑 Client Enrollment
+
+To enroll a client virtual machine(such as `portfolio` or `minecraft`) into the `LAB.LOCAL` identity realm, execute the client installer on the target node:
+
+```bash
+sudo ipa-client-install \
+    --mkhomedir \
+    --no-ntp \
+    --unattended
+```
+
+*   `--mkhomedir`: Configures PAM(`pam_oddjob_mkhomedir` or `pam_mkhomedir`) to automatically create a local `/home/` directory upon a user's first login via SSH.

docs/05-applications.md

@@ -0,0 +1,142 @@
+# 📦 Application Container Deployments
+
+This document details the containerized application architecture, storage mounts, and Systemd service mappings deployed across the virtual machine nodes using rootless **Podman**.
+
+---
+
+## 🏗️ Application & Systemd Service Architecture
+
+Every application workload runs inside an isolated rootless container. The host's local **Systemd** daemon manages the startup dependencies and lifecycle of the containers:
+
+### 1. Portfolio Web Server (`portfolio.lab.local`)
+Exposes the portfolio static webpage assets using an Nginx Alpine container containerized by a simple Systemd unit wrapper.
+
+```mermaid
+graph TD
+    %% My Color Palette
+    classDef vmNode fill:#161d1c,stroke:#70a99f,color:#f8f8f2,stroke-width:1.5px;
+    classDef svcNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:1px;
+    classDef appNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+
+    subgraph VM_Portfolio ["portfolio.lab.local VM"]
+        Svc_Portal["⚙️ portal.service<br>(Systemd)"]:::svcNode
+        App_Nginx["🌐 Nginx Web Server<br>(Port 80)"]:::appNode
+        SiteFiles["📁 /home/sho/containers/portal<br>(Static HTML/CSS/JS)"]:::vmNode
+    end
+
+    Svc_Portal ===>|Launches & restarts| App_Nginx
+    App_Nginx -->|Mounts assets read-only| SiteFiles
+
+    %% Subgraph Colors
+    style VM_Portfolio fill:#111615,stroke:#70a99f,stroke-width:1px;
+```
+
+---
+
+### 2. Fabric Minecraft Server (`minecraft.lab.local`)
+Runs a Java-based Minecraft game server node with Fabric mod loaders and maps a persistent directory for user/world preservation.
+
+```mermaid
+graph TD
+    %% My Color Palette
+    classDef vmNode fill:#161d1c,stroke:#70a99f,color:#f8f8f2,stroke-width:1.5px;
+    classDef svcNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:1px;
+    classDef appNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+
+    subgraph VM_Minecraft ["minecraft.lab.local VM"]
+        Svc_Minecraft["⚙️ minecraft.service<br>(Systemd)"]:::svcNode
+        App_MC["⚔️ Fabric Server<br>(Port 25565)"]:::appNode
+        MCDatabase["📁 /home/sho/containers/minecraft<br>(Persistent /data)"]:::vmNode
+    end
+
+    Svc_Minecraft ===>|Launches & restarts| App_MC
+    App_MC -->|Mounts database read-write| MCDatabase
+
+    %% Subgraph Colors
+    style VM_Minecraft fill:#111615,stroke:#70a99f,stroke-width:1px;
+```
+
+---
+
+### 3. Navidrome Music Server (`navidrome.lab.local`)
+Traces the startup dependencies where Google Drive is FUSE-mounted via `rclone` before the Navidrome audio engine mounts and indexes the path.
+
+```mermaid
+graph TD
+    %% My Color Palette
+    classDef vmNode fill:#161d1c,stroke:#70a99f,color:#f8f8f2,stroke-width:1.5px;
+    classDef svcNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:1px;
+    classDef appNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+    classDef mountNode fill:#212c2a,stroke:#ffca80,color:#ffca80,stroke-width:1px;
+
+    subgraph VM_Navidrome ["navidrome.lab.local VM"]
+        Svc_Rclone["⚙️ rclone-mount.service<br>(Systemd)"]:::svcNode
+        Svc_Navidrome["⚙️ navidrome.service<br>(Systemd)"]:::svcNode
+        App_ND["🎵 Navidrome Streamer<br>(Port 4533)"]:::appNode
+        GDrive["☁️ Google Drive<br>(FUSE Mount: /mnt/gdrive)"]:::mountNode
+    end
+
+    Svc_Rclone ===>|Mounts remote GDrive via| GDrive
+    Svc_Navidrome ===>|Launches streamer| App_ND
+    Svc_Navidrome -.->|Requires active mount| Svc_Rclone
+    App_ND -->|Scrapes MP3 audio files from| GDrive
+
+    %% Subgraph Colors
+    style VM_Navidrome fill:#111615,stroke:#70a99f,stroke-width:1px;
+```
+
+---
+
+## 📄 Application Specifications
+
+The container lifecycle is automated via `ansible/playbooks/04_services_deploy.yml`:
+
+### 1.  Portfolio Hub (`portfolio.lab.local`)
+
+*   **Engine**: Launches `docker.io/library/nginx:alpine` using rootless Podman.
+*   **Statis Assets**: Clones and mounts the HTML/CSS website files into `/usr/share/nginx/html` in read-only(`ro`) mode.
+*   **Port Mapping**: Maps container port to 80 to target port 80 of the virtual machine.
+
+### 2.  Fabric Minecraft Server (`minecraft.lab.local`)
+
+*   **Engine**: Launches `docker.io/itzg/minecraft-server:java17`(java-based wrapper).
+*   **Configurations**:
+    *   `EULA=TRUE`: Accepts user agreements.
+    *   `TYPE=FABRIC`: Deploys Fabric mod loader.
+    *   `VERSION=1.20.1`: Deploys game runtime version.
+    *   `MEMORY=4G`: Allocates memory bounds(configured dynamically via variables).
+    *   `MODRINTH_PROJECTS`: Deploys specific mod files(Fabric API, Architectury API, Geckolib, Kotlin runtime, hamster pets, etc.).
+*   **Persistent Storage**: Mounts `/home/sho/containers/minecraft` to `/data` in read-write mode to preserve world data, player profiles, and server properties.
+
+### 3.  Music Streaming Node (`navidrome.lab.local`)
+
+*   **rclone Google Drive FUSE Mount**:
+    *   Installed packages `fuse3` and `rclone`.
+    *   Modifies `/etc/fuse.conf` to enable `user_allow_other`(allowing rootless containers to read paths mounted by host users).
+    *   Configures `rclone` with Google Drive credentials and spawns a background mount service daemon(`rclone-mount.service`) directing drive data to `/mnt/gdrive` using cache mode `full`(cached locally for 24h).
+*   **Navidrome Engine**:
+    *   Launches `docker.io/deluan/navidrome:latest`.
+    *   Mounts the FUSE path `/mnt/gdrive` into `/music:ro`.
+    *   Systemd configurations map a `Requires=rclone-mount.service` dependency, preventing the Navidrome container from running if the Google Drive FUSE mount fails.
+
+---
+
+## 🚀 Execution & Management
+
+Deploy the application containers using the Ansible tags target:
+
+```bash
+ansible-playbook site.yml --tags "services" --ask-vault-pass
+```
+
+### Container Status Verification
+
+Log in to any service virtual machine and query Podman:
+
+```bash
+# View running container status
+podman ps
+
+# Inspect container startup logs
+podman logs navidrome
+```

docs/06-tunnels.md

@@ -0,0 +1,121 @@
+# 🌩️ Ingress & Secure Tunnels
+
+This document outlines the zero-trust remote access architecture of the homelab. By utilizing outbound-only secure tunnels (**Cloudflared** and **Playit.gg**), external services are exposed without configuring port forwarding or opening inbound router ports.
+
+---
+
+## 🏗️ Tunnel Ingress Architectures
+
+Traffic ingress is divided into two separate pathways: HTTP web traffic via Cloudflare, and TCP/UDP game server traffic via Playit.gg.
+
+### 1.  Cloudflare HTTPS Ingress (Web Portals)
+
+Handles secure HTTPS traffic routing for the Nginx portfolio website and the Grafana analytics dashboard.
+
+```mermaid
+graph LR
+    %% My Color Palette
+    classDef extNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:1.5px;
+    classDef agentNode fill:#212c2a,stroke:#ffca80,color:#ffca80,stroke-width:1px,stroke-dasharray: 5 5;
+    classDef appNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+
+    subgraph WAN ["Public WAN"]
+        User["🌐 Web Browser"]:::extNode
+        CFEdge["☁️ Cloudflare Edge Network"]:::extNode
+    end
+
+    subgraph VM_Portfolio ["portfolio.lab.local VM"]
+        Agent_CF["🐳 cloudflared container<br>(--net=host)"]:::agentNode
+        App_Nginx["🌐 Nginx Port 80"]:::appNode
+        App_Grafana["📊 Grafana Port 3000"]:::appNode
+    end
+
+    User -->|HTTPS Query| CFEdge
+    CFEdge ===>|Encrypted Tunnel Stream| Agent_CF
+    Agent_CF -->|Proxies HTTP traffic| App_Nginx
+    Agent_CF -->|Proxies HTTP traffic| App_Grafana
+
+    %% Subgraph Colors
+    style VM_Portfolio fill:#111615,stroke:#70a99f,stroke-width:1px;
+    style WAN fill:#161d1c,stroke:#9580ff,stroke-width:1px;
+```
+
+### 2. Playit.gg Game Ingress (Minecraft Server)
+
+Routes game packets for external players into the local Fabric server instance using a secure UDP/TCP agent connection.
+
+```mermaid
+graph LR
+    %% My Color Palette
+    classDef extNode fill:#212c2a,stroke:#9580ff,color:#f8f8f2,stroke-width:1.5px;
+    classDef agentNode fill:#212c2a,stroke:#ffca80,color:#ffca80,stroke-width:1px,stroke-dasharray: 5 5;
+    classDef appNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+
+    subgraph WAN_Game ["Public WAN"]
+        Player["🎮 Game Client"]:::extNode
+        PlayitWAN["🔌 Playit.gg Global Proxy"]:::extNode
+    end
+
+    subgraph VM_Minecraft ["minecraft.lab.local VM"]
+        Agent_Playit["🐳 playit-agent container<br>(--net=host)"]:::agentNode
+        App_MC["⚔️ Fabric Server Port 25565"]:::appNode
+    end
+
+    Player -->|TCP/UDP Game Packets| PlayitWAN
+    PlayitWAN ===>|Encrypted Tunnel Stream| Agent_Playit
+    Agent_Playit -->|Proxies raw TCP/UDP| App_MC
+
+    %% Subgraph Colors
+    style VM_Minecraft fill:#111615,stroke:#70a99f,stroke-width:1px;
+    style WAN_Game fill:#161d1c,stroke:#9580ff,stroke-width:1px;
+```
+
+---
+
+## 📄 Service Configurations
+
+The outbound tunnel agents are launched as rootless container daemons:
+
+### 1.  Cloudflare Tunnel Agent (`06_cloudflared_setup.yml`)
+
+*   **Engine**: Spawns `docker.io/cloudflare/cloudflared:latest` using `--net=host`.
+*   **Credentials**: Authenticates with Cloudflare using a secure credential token(`cloudflare_token`) managed inside the encrypted Ansible Vault file(`vms/vault.yml`).
+*   **Systemd Integration(`cloudflared.service`)**: Configures startup sequencing to ensure the local web application services(`portal.service`) are active before starting the tunnel agent:
+
+```ini
+[Unit]
+After=network-online.target portal.service
+Wants=network-online.target portal.service
+```
+
+### 2.  Playit.gg Tunnel Agent (`05_playit_setup.yml`)
+
+*   **Engine**: Spawns `ghcr.io/playit-cloud/playit-agent:1.0` using `--net=host` to bridge internal traffic.
+*   **Credentials**: Authenticates using a cryptographically generated static secret key(`playit_secret_key`) mapped to the agent container's environment variables(`SECRET_KEY`).
+*   **Systemd Integration(`playit.service`)**: Chains the agent startup process to load immediately after Minecraft service completes:
+
+```ini
+[Unit]
+After=network-online.target minecraft.service
+Wants=network-online.target minecraft.service
+```
+
+---
+
+## 🚀 Execution & Administration
+
+To deploy or refresh the ingress tunnels, run the following playbooks:
+
+```bash
+ansible-playbook site.yml --tags "playit,cloudflared" --ask-vault-pass
+```
+
+### Checking Tunnel Status
+
+Inspect active connections on the respective VMs by viewing Systemd status outputs:
+
+```bash
+# Verify connection logs and exit statuses
+systemctl status cloudflared.service
+systemctl status playit.service
+```

docs/07-observability.md

@@ -0,0 +1,96 @@
+# 📊 Observability & Telemetry
+
+This document details the configuation and design of the cluster-wide telemetry scraping infrastructure utilizing **Prometheus**, **Grafana**, and native **Node Exporter**.
+
+---
+
+## 📈 Observability Architecture
+
+Prometheus scrapes node metrics periodically via Node Exporter agents listening on port `9100` across the bridge network:
+
+```mermaid
+graph TD
+    %% My Color Palette
+    classDef hostNode fill:#161d1c,stroke:#415854,color:#f8f8f2,stroke-width:2px;
+    classDef ipdNode fill:#2b3b38,stroke:#ff9580,color:#ff9580,stroke-width:1.5px;
+    classDef vmNode fill:#2b3b38,stroke:#70a99f,color:#f8f8f2,stroke-width:1.5px;
+    classDef obsNode fill:#2b3b38,stroke:#8aff80,color:#8aff80,stroke-width:1.5px;
+
+    VM1["🔑 freeipa.lab.local (172.30.1.85)<br>LDAP / Kerberos / BIND DNS"]:::ipdNode
+
+    subgraph Nodes ["Monitored Nodes (Port: 9100)"]
+        HostOS["🖥️ Hypervisor Host (172.30.1.200)"]:::hostNode
+        VM2["📄 portfolio VM (172.30.1.93)"]:::vmNode
+        VM3["⚔️ minecraft VM (172.30.1.91)"]:::vmNode
+        VM4["🎵 navidrome VM (172.30.1.92)"]:::vmNode
+    end
+
+    subgraph PortfolioServices ["portfolio VM Services"]
+        Prom["📈 Prometheus TSDB"]:::obsNode
+        Grafana["📊 Grafana Dashboard"]:::obsNode
+    end
+
+    HostOS & VM1 & VM2 & VM3 & VM4 -.->|Node Exporter Scrape| Prom
+    Prom -->|Data Source Query| Grafana
+
+    Nodes --->|DNS Lookups| VM1
+
+    %% Subgraph Colors
+    style Nodes fill:#212c2a,stroke:#70a99f,stroke-width:1px;
+    style PortfolioServices fill:#161d1c,stroke:#415854,stroke-width:2px;
+```
+
+---
+
+## 📄 Service Configurations
+
+Telemetry collectors are automated via `ansible/playbooks/07_observability.yml`:
+
+### 1.  Prometheus Node Exporter (Daemon Node)
+
+*   **Engine**: Downloads the native `node_exporter-1.8.2.linux-amd64` release binary and places it under `/usr/local/bin/node_exporter`.
+*   **Systemd Integration (`node_exporter.service`)**: Deploys a background daemon unit configured to run the exporter immediately after network interfaces load.
+*   **SELinux Contexts (AlmaLinux)**: Automatically runs `restorecon` to preserve SELinux contexts for the binary and service files.
+*   **Security**: Opens port `9100/tcp` on firewalld on RedHat-family systems to allow scrapers to read metric inputs.
+
+### 2. Prometheus Engine (`prometheus.yml.j2`)
+
+*   **Engine**: Containerized using `docker.io/prom/prometheus:latest` running with `--net=host` on the `portfolio` VM.
+*   **Storage Mounts**: Maps the host folder `/home/sho/containers/prometheus/data` to `/prometheus` with tag properties `:z,U`(ensuring rootless Podman SELinux permissions map correctly to local storage directories).
+*   **Scrape Loop Specifications**: Sets a scrape and evaluation interval of 15 seconds:
+
+```yaml
+global:
+    scrape_interval: 15s
+    evaluation_interval: 15s
+
+scrape_configs:
+    - job_name: 'homelab-nodes'
+      static_configs:
+        - targets:
+            - '172.30.1.200:9100' # Host
+            - '172.30.1.85:9100' # freeipa
+            - '172.30.1.93:9100' # portfolio
+            - '172.30.1.91:9100' # minecraft
+            - '172.30.1.92:9100' # navidrome
+```
+
+### 3.  Grafana Dashboard
+
+*   **Engine**: Runs `docker.io/grafana/grafana-oss:latest` in a container mapping port `3000:3000`.
+*   **Data Persistence**: Mounts `/home/sho/containers/grafana/data` to preserve custom dashboards, datasources, and user configurations between restarts.
+
+---
+
+## 🚀 Execution & Monitoring
+
+To deploy the observability stack across the cluster nodes, run the following:
+
+```bash
+ansible-playbook site.yml --tags "observability" --ask-vault-pass
+```
+
+### Checking Scraping Sinks
+
+1.  **Prometheus Targets Console**: Access the Prometheus TUI interface by opening `http://172.30.1.93:9000/targets` and verify that all 5 target hosts report `UP`.
+2   **Grafana Portal**: Navigate to `http://172.30.1.93:3000` to create custom query dashboards. *(Default port is mapped externally via Cloudflared Tunnel).*

terraform/bootstrap/main.tf

@@ -0,0 +1,67 @@
+terraform {
+  required_version = ">=1.5.0"
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.7.6"
+    }
+  }
+}
+
+provider "libvirt" {
+  uri = "qemu+ssh://sho@172.30.1.200/system"
+}
+
+resource "libvirt_volume" "debian12_base" {
+  name   = "debian12-base-bootstrap.qcow2"
+  pool   = "vm_pool"
+  source = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+  format = "qcow2"
+}
+
+resource "libvirt_volume" "utility_disk" {
+  name           = "utility-disk.qcow2"
+  pool           = "vm_pool"
+  base_volume_id = libvirt_volume.debian12_base.id
+  size           = 21474836480
+  format         = "qcow2"
+}
+
+data "template_file" "user_data" {
+  template = file("${path.module}/templates/cloud_init.cfg")
+  vars = {
+    admin_user = "sho"
+    ssh_key    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILM/84tpkx+yYsA8Zr5or1xuELOGMl0JEP576SyUc9eC sho@bazzite"
+  }
+}
+
+resource "libvirt_cloudinit_disk" "utility_init" {
+  name      = "utility-init.iso"
+  pool      = "vm_pool"
+  user_data = data.template_file.user_data.rendered
+  network_config = templatefile("${path.module}/templates/network_config.cfg.tpl", {
+    interface_name = "ens3"
+    ip_address     = "172.30.1.80"
+    gateway_ip     = "172.30.1.254"
+    dns_ip         = "172.30.1.85"
+  })
+}
+
+resource "libvirt_domain" "utility_vm" {
+  name   = "utility"
+  memory = "2048"
+  vcpu   = 2
+  cpu { mode = "host-passthrough" }
+  cloudinit = libvirt_cloudinit_disk.utility_init.id
+  network_interface {
+    bridge = "br0"
+    mac    = "52:54:00:ee:ef:60"
+  }
+  disk { volume_id = libvirt_volume.utility_disk.id }
+  
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+}

terraform/bootstrap/templates/cloud_init.cfg

@@ -0,0 +1,15 @@
+#cloud-config
+package_update: true
+package_upgrade: false
+
+users:
+    - name: ${admin_user}
+      groups: wheel, systemd-journal
+      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+      shell: /bin/bash
+      ssh_authorized_keys:
+        - ${ssh_key}
+
+runcmd:
+    - sed -i 's/^#PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config
+    - systemctl restart sshd

terraform/bootstrap/templates/network_config.cfg.tpl

@@ -0,0 +1,14 @@
+#cloud-config
+version: 2
+ethernets:
+    ${interface_name}:
+        dhcp4: no
+        addresses:
+            - ${ip_address}/24
+        routes:
+            - to: default
+              via: ${gateway_ip}
+        nameservers:
+            addresses:
+                - ${dns_ip}
+                - 1.1.1.1

terraform/workloads/main.tf

@@ -0,0 +1,220 @@
+terraform {
+  required_version = ">=1.5.0"
+
+  backend "s3" {
+    bucket                      = "terraform-state"
+    key                         = "workloads/terraform.tfstate"
+    region                      = "main"
+    endpoints                   = { s3 = "http://172.30.1.80:9000" }
+    skip_credentials_validation = true
+    skip_metadata_api_check     = true
+    skip_region_validation      = true
+    skip_requesting_account_id  = true
+    use_path_style              = true
+  }
+
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.7.6"
+    }
+  }
+}
+
+provider "libvirt" {
+  uri = "qemu+ssh://sho@172.30.1.200/system"
+}
+
+resource "libvirt_volume" "almalinux10_image" {
+  name   = "almalinux10-base.qcow2"
+  pool   = "vm_pool"
+  source = "https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
+  format = "qcow2"
+}
+
+resource "libvirt_volume" "debian12_image" {
+  name   = "debian12-base.qcow2"
+  pool   = "vm_pool"
+  source = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+  format = "qcow2"
+}
+
+resource "libvirt_volume" "freeipa_disk" {
+  name           = "freeipa-disk.qcow2"
+  pool           = "vm_pool"
+  base_volume_id = libvirt_volume.almalinux10_image.id
+  size           = 42949672960
+  format         = "qcow2"
+}
+
+resource "libvirt_volume" "portfolio_disk" {
+  name           = "portfolio-disk.qcow2"
+  pool           = "vm_pool"
+  base_volume_id = libvirt_volume.debian12_image.id
+  size           = 10737418240
+  format         = "qcow2"
+}
+
+resource "libvirt_volume" "minecraft_disk" {
+  name           = "minecraft-disk.qcow2"
+  pool           = "vm_pool"
+  base_volume_id = libvirt_volume.debian12_image.id
+  size           = 21474836480
+  format         = "qcow2"
+}
+
+resource "libvirt_volume" "navidrome_disk" {
+  name           = "navidrome-disk.qcow2"
+  pool           = "vm_pool"
+  base_volume_id = libvirt_volume.debian12_image.id
+  size           = 16106127360
+  format         = "qcow2"
+}
+
+data "template_file" "user_data" {
+  template = file("${path.module}/templates/cloud_init.cfg")
+  vars = {
+    admin_user = "sho"
+    ssh_key    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILM/84tpkx+yYsA8Zr5or1xuELOGMl0JEP576SyUc9eC sho@bazzite"
+  }
+}
+
+resource "libvirt_cloudinit_disk" "freeipa_init" {
+  name      = "freeipa-init.iso"
+  pool      = "vm_pool"
+  user_data = data.template_file.user_data.rendered
+  network_config = templatefile("${path.module}/templates/network_config.cfg.tpl", {
+    interface_name = "etho0"
+    ip_address     = "172.30.1.85"
+    gateway_ip     = "172.30.1.254"
+    dns_ip         = "172.30.1.85"
+  })
+}
+
+resource "libvirt_cloudinit_disk" "portfolio_init" {
+  name      = "portfolio-init.iso"
+  pool      = "vm_pool"
+  user_data = data.template_file.user_data.rendered
+  network_config = templatefile("${path.module}/templates/network_config.cfg.tpl", {
+    interface_name = "ens3"
+    ip_address     = "172.30.1.93"
+    gateway_ip     = "172.30.1.254"
+    dns_ip         = "172.30.1.85"
+  })
+}
+
+resource "libvirt_cloudinit_disk" "minecraft_init" {
+  name      = "minecraft-init.iso"
+  pool      = "vm_pool"
+  user_data = data.template_file.user_data.rendered
+  network_config = templatefile("${path.module}/templates/network_config.cfg.tpl", {
+    interface_name = "ens3"
+    ip_address     = "172.30.1.91"
+    gateway_ip     = "172.30.1.254"
+    dns_ip         = "172.30.1.85"
+  })
+}
+
+resource "libvirt_cloudinit_disk" "navidrome_init" {
+  name      = "navidrome-init.iso"
+  pool      = "vm_pool"
+  user_data = data.template_file.user_data.rendered
+  network_config = templatefile("${path.module}/templates/network_config.cfg.tpl", {
+    interface_name = "ens3"
+    ip_address     = "172.30.1.92"
+    gateway_ip     = "172.30.1.254"
+    dns_ip         = "172.30.1.85"
+  })
+}
+
+resource "libvirt_domain" "freeipa_vm" {
+  name   = "freeipa"
+  memory = "3072"
+  vcpu   = 2
+  cpu { mode = "host-passthrough" }
+  cloudinit = libvirt_cloudinit_disk.freeipa_init.id
+  network_interface {
+    bridge = "br0"
+    mac    = "52:54:00:ee:ef:61"
+  }
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+  disk { volume_id = libvirt_volume.freeipa_disk.id }
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}
+
+resource "libvirt_domain" "portfolio_vm" {
+  name   = "portfolio"
+  memory = "1024"
+  vcpu   = 1
+  cpu { mode = "host-passthrough" }
+  cloudinit = libvirt_cloudinit_disk.portfolio_init.id
+  network_interface {
+    bridge = "br0"
+    mac    = "52:54:00:ee:ef:62"
+  }
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+  disk { volume_id = libvirt_volume.portfolio_disk.id }
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}
+
+resource "libvirt_domain" "minecraft_vm" {
+  name   = "minecraft"
+  memory = "6144"
+  vcpu   = 2
+  cpu { mode = "host-passthrough" }
+  cloudinit = libvirt_cloudinit_disk.minecraft_init.id
+  network_interface {
+    bridge = "br0"
+    mac    = "52:54:00:ee:ef:63"
+  }
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+  disk { volume_id = libvirt_volume.minecraft_disk.id }
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}
+
+resource "libvirt_domain" "navidrome_vm" {
+  name   = "navidrome"
+  memory = "1024"
+  vcpu   = 1
+  cpu { mode = "host-passthrough" }
+  cloudinit = libvirt_cloudinit_disk.navidrome_init.id
+  network_interface {
+    bridge = "br0"
+    mac    = "52:54:00:ee:ef:65"
+  }
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+  disk { volume_id = libvirt_volume.navidrome_disk.id }
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}

terraform/workloads/templates/cloud_init.cfg

@@ -0,0 +1,15 @@
+#cloud-config
+package_update: true
+package_upgrade: false
+
+users:
+    - name: ${admin_user}
+      groups: wheel, systemd-journal
+      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+      shell: /bin/bash
+      ssh_authorized_keys:
+        - ${ssh_key}
+
+runcmd:
+    - sed -i 's/^#PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config
+    - systemctl restart sshd

terraform/workloads/templates/network_config.cfg.tpl

@@ -0,0 +1,14 @@
+#cloud-config
+version: 2
+ethernets:
+    ${interface_name}:
+        dhcp4: no
+        addresses:
+            - ${ip_address}/24
+        routes:
+            - to: default
+              via: ${gateway_ip}
+        nameservers:
+            addresses:
+                - ${dns_ip}
+                - 1.1.1.1