homelab-nomad/core/metrics/prometheus.nomad

job "prometheus" {
  datacenters = ["dc1"]

  group "prometheus" {
    count = 1

    network {
      mode = "bridge"

      port "web" {
        %{~ if use_wesher ~}
        host_network = "wesher"
        %{~ endif ~}
        to = 9090
      }

      port "pushgateway" {
        %{~ if use_wesher ~}
        host_network = "wesher"
        %{~ endif ~}
        static = 9091
      }
    }

    ephemeral_disk {
      migrate = true
      sticky = true
    }

    service {
      name = "prometheus"
      provider = "nomad"
      port = "web"

      // TODO: Remove traefik tags
      tags = [
        "traefik.enable=true",
        "traefik.http.routers.prometheus.entryPoints=websecure",
      ]
    }

    service {
      name = "pushgateway"
      provider = "nomad"
      port = "pushgateway"
    }

    task "prometheus" {
      driver = "docker"

      config {
        image = "prom/prometheus:v2.43.0"
        ports = ["web"]
        args = [
          "--config.file=$${NOMAD_TASK_DIR}/prometheus.yml",
          "--storage.tsdb.path=$${NOMAD_ALLOC_DIR}/data/tsdb",
          "--web.listen-address=0.0.0.0:9090",
          "--web.console.libraries=/usr/share/prometheus/console_libraries",
          "--web.console.templates=/usr/share/prometheus/consoles",
        ]
      }

      template {
        data = <<EOF
---
global:
  scrape_interval: 30s
  evaluation_interval: 3s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets:
        - 127.0.0.1:9090

  - job_name: "pushgateway"
    honor_labels: true
    static_configs:
      - targets:
        - 127.0.0.1:9091

  - job_name: "nomad_client"
    metrics_path: "/v1/metrics"
    params:
      format:
        - "prometheus"
    nomad_sd_configs:
        # TODO: Use NOMAD_SECRETS_DIR/api.sock and workload idenity when
        # workload acls can be set using terraform
      - server: "http://{{env "attr.unique.network.ip-address"}}:4646"
    relabel_configs:
      - source_labels: [__meta_nomad_service]
        regex: nomad-client-stalker
        action: keep
      - source_labels: [__meta_nomad_address]
        replacement: "$1:4646"
        target_label: __address__

  - job_name: "nomad_services"
    metrics_path: "/metrics"
    nomad_sd_configs:
      - server: "http://{{env "attr.unique.network.ip-address"}}:4646"
    relabel_configs:
      - source_labels: [__meta_nomad_tags]
        regex: .*(prometheus.scrape).*
        action: keep
      - source_labels: [__meta_nomad_service_address,__meta_nomad_service_port]
        separator: ":"
        target_label: __address__
      - source_labels: [__meta_nomad_service]
        target_label: nomad_service
      - source_labels: [__meta_nomad_dc]
        target_label: nomad_dc
      - source_labels: [__meta_nomad_node_id]
        target_label: nomad_node_id
        EOF
        change_mode = "signal"
        change_signal = "SIGHUP"
        destination = "$${NOMAD_TASK_DIR}/prometheus.yml"
      }

      resources {
        cpu = 100
        memory = 300
      }
    }

    task "pushgateway" {
      driver = "docker"

      config {
        image = "prom/pushgateway"
        ports = ["pushgateway"]
        args = [
          "--persistence.file=$${NOMAD_ALLOC_DIR}/pushgateway-persistence",
        ]
      }

      resources {
        cpu = 50
        memory = 50
      }
    }
  }
}