From 5169aecc6ddbcbe43058e8a9f6c7efd5452f2a9e Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Tue, 9 May 2023 15:56:20 -0700
Subject: [PATCH] Add pushgateway to prometheus

---
 core/metrics/prometheus.nomad | 54 ++++++++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 13 deletions(-)

diff --git a/core/metrics/prometheus.nomad b/core/metrics/prometheus.nomad
index 76da3fe..f065d3c 100644
--- a/core/metrics/prometheus.nomad
+++ b/core/metrics/prometheus.nomad
@@ -11,12 +11,17 @@ job "prometheus" {
         host_network = "wesher"
         to = 9090
       }
+
+      port "pushgateway" {
+        host_network = "wesher"
+        static = 9091
+      }
     }
 
-    # ephemeral_disk {
-    #   migrate = true
-    #   sticky = true
-    # }
+    ephemeral_disk {
+      migrate = true
+      sticky = true
+    }
 
     service {
       name = "prometheus"
@@ -30,6 +35,12 @@ job "prometheus" {
       ]
     }
 
+    service {
+      name = "pushgateway"
+      provider = "nomad"
+      port = "pushgateway"
+    }
+
     task "prometheus" {
       driver = "docker"
 
@@ -37,18 +48,12 @@ job "prometheus" {
         image = "prom/prometheus:v2.43.0"
         ports = ["web"]
         args = [
-          "--config.file=/etc/prometheus/config/prometheus.yml",
+          "--config.file=${NOMAD_TASK_DIR}/prometheus.yml",
           "--storage.tsdb.path=${NOMAD_ALLOC_DIR}/data/tsdb",
           "--web.listen-address=0.0.0.0:9090",
           "--web.console.libraries=/usr/share/prometheus/console_libraries",
           "--web.console.templates=/usr/share/prometheus/consoles",
         ]
-
-        mount {
-          type = "bind"
-          target = "/etc/prometheus/config"
-          source = "local/config"
-        }
       }
 
       template {
@@ -62,7 +67,13 @@ scrape_configs:
   - job_name: prometheus
     static_configs:
       - targets:
-        - 0.0.0.0:9090
+        - 127.0.0.1:9090
+
+  - job_name: "pushgateway"
+    honor_labels: true
+    static_configs:
+      - targets:
+        - 127.0.0.1:9091
 
   - job_name: "nomad_client"
     metrics_path: "/v1/metrics"
@@ -101,7 +112,7 @@ scrape_configs:
         EOF
         change_mode = "signal"
         change_signal = "SIGHUP"
-        destination = "local/config/prometheus.yml"
+        destination = "${NOMAD_TASK_DIR}/prometheus.yml"
       }
 
       resources {
@@ -109,5 +120,22 @@ scrape_configs:
         memory = 300
       }
     }
+
+    task "pushgateway" {
+      driver = "docker"
+
+      config {
+        image = "prom/pushgateway"
+        ports = ["pushgateway"]
+        args = [
+          "--persistence.file=${NOMAD_ALLOC_DIR}/pushgateway-persistence",
+        ]
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
   }
 }