Get nomad client scraping working

ansible_playbooks/setup-cluster.yml

@@ -189,7 +189,7 @@
             interface: lo
             reserved_ports: "22"
           - name: wesher
-            interface: wesher
+            interface: wgoverlay
             reserved_ports: "22"
 
         # Enable ACLs

core/main.tf

@@ -18,6 +18,10 @@ module "metrics" {
   # depends_on = [module.databases]
 }
 
+resource "nomad_job" "nomad-client-stalker" {
+  jobspec = file("${path.module}/nomad-client-stalker.nomad")
+}
+
 module "loki" {
   source  = "IamTheFij/levant/nomad"
   version = "0.1.0"

core/metrics/prometheus.nomad

@@ -13,10 +13,10 @@ job "prometheus" {
       }
     }
 
-    ephemeral_disk {
+    # ephemeral_disk {
-      migrate = true
+    #   migrate = true
-      sticky = true
+    #   sticky = true
-    }
+    # }
 
     service {
       name = "prometheus"
@@ -69,23 +69,14 @@ scrape_configs:
     params:
       format:
         - "prometheus"
-    consul_sd_configs:
+    nomad_sd_configs:
-      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
+      - server: "http://{{env "attr.unique.network.ip-address"}}:4646"
-        services:
-          - "nomad-client"
-
-  - job_name: "consul"
-    metrics_path: "/v1/agent/metrics"
-    params:
-      format:
-        - "prometheus"
-    consul_sd_configs:
-      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
-        services:
-          - "consul"
     relabel_configs:
-      - source_labels: [__meta_consul_address]
+      - source_labels: [__meta_nomad_service]
-        replacement: $1:8500
+        regex: nomad-client-stalker
+        action: keep
+      - source_labels: [__meta_nomad_address]
+        replacement: "$1:4646"
         target_label: __address__
 
   - job_name: "nomad_services"
@@ -105,50 +96,6 @@ scrape_configs:
         target_label: nomad_dc
       - source_labels: [__meta_nomad_node_id]
         target_label: nomad_node_id
-
-  - job_name: "exporters"
-    metrics_path: "/metrics"
-    consul_sd_configs:
-      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
-    relabel_configs:
-      - source_labels: [__meta_consul_service]
-        action: drop
-        regex: (.+)-sidecar-proxy
-      - source_labels: [__meta_consul_service_metadata_metrics_addr]
-        action: keep
-        regex: (.+)
-      - source_labels: [__meta_consul_service_metadata_metrics_addr]
-        target_label: __address__
-      - source_labels: [__meta_consul_service]
-        target_label: consul_service
-      - source_labels: [__meta_consul_node]
-        target_label: consul_node
-      - source_labels: [__meta_consul_service_nomad_dc]
-        target_label: nomad_dc
-      - source_labels: [__meta_consul_service_nomad_node_name]
-        target_label: nomad_node_name
-
-  - job_name: "envoy"
-    metrics_path: "/metrics"
-    consul_sd_configs:
-      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
-    relabel_configs:
-      - source_labels: [__meta_consul_service]
-        action: keep
-        regex: (.+)-sidecar-proxy
-      - source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
-        action: keep
-        regex: (.+)
-      - source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
-        target_label: __address__
-      - source_labels: [__meta_consul_service]
-        target_label: consul_service
-      - source_labels: [__meta_consul_node]
-        target_label: consul_node
-      - source_labels: [__meta_consul_service_nomad_dc]
-        target_label: nomad_dc
-      - source_labels: [__meta_consul_service_nomad_node_name]
-        target_label: nomad_node_name
         EOF
         change_mode = "signal"
         change_signal = "SIGHUP"

core/nomad-client-stalker.nomad

@@ -0,0 +1,31 @@
+job "nomad-client-stalker" {
+  type = "system"
+
+  group "main" {
+
+    network {
+      mode = "host"
+      port "main" {}
+    }
+
+    service {
+      name = "nomad-client-stalker"
+      provider = "nomad"
+      port = "main"
+    }
+
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "busybox"
+        args = ["tail", "-f", "/dev/null"]
+      }
+
+      resources {
+        cpu = 10
+        memory = 10
+      }
+    }
+  }
+}