homelab-nomad/metrics/prometheus.nomad

variable "consul_address" {
  type = string
  description = "Full address of Consul instance to get catalog from"
  default = "http://127.0.0.1:5400"
}

job "prometheus" {
  datacenters = ["dc1"]

  group "prometheus" {
    count = 1

    network {
      mode = "bridge"

      port "web" {
        host_network = "loopback"
        to = 9090
      }
    }

    ephemeral_disk {
      migrate = true
      sticky = true
    }

    service {
      name = "prometheus"
      port = "web"

      connect {
        sidecar_service {
          proxy {
            local_service_port = 9090
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }

      check {
        type = "http"
        path = "/"
        port = "web"
        interval = "10s"
        timeout = "10s"
      }

      // TODO: Remove traefik tags
      tags = [
        "traefik.enable=true",
      ]
    }

    task "prometheus" {
      driver = "docker"

      config {
        image = "prom/prometheus:v2.30.2"
        ports = ["web"]
        args = [
          "--config.file=/etc/prometheus/config/prometheus.yml",
          "--storage.tsdb.path=/prometheus",
          "--web.listen-address=0.0.0.0:9090",
          "--web.console.libraries=/usr/share/prometheus/console_libraries",
          "--web.console.templates=/usr/share/prometheus/consoles",
        ]

        mount {
          type = "bind"
          target = "/etc/prometheus/config"
          source = "local/config"
        }
      }

      template {
        data = <<EOF
---
global:
  scrape_interval: 30s
  evaluation_interval: 3s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets:
        - 0.0.0.0:9090

  - job_name: "nomad_server"
    metrics_path: "/v1/metrics"
    params:
      format:
        - "prometheus"
    consul_sd_configs:
      - server: "${var.consul_address}"
      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
        services:
          - "nomad"
        tags:
          - "http"

  - job_name: "nomad_client"
    metrics_path: "/v1/metrics"
    params:
      format:
        - "prometheus"
    consul_sd_configs:
      - server: "${var.consul_address}"
      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
        services:
          - "nomad-client"

  - job_name: "consul"
    metrics_path: "/v1/agent/metrics"
    params:
      format:
        - "prometheus"
    consul_sd_configs:
      - server: "${var.consul_address}"
      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
        services:
          - "consul"
    relabel_configs:
      - source_labels: [__meta_consul_address]
        replacement: $1:8500
        target_label: __address__

  - job_name: "exporters"
    metrics_path: "/metrics"
    consul_sd_configs:
      - server: "${var.consul_address}"
      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: drop
        regex: (.+)-sidecar-proxy
      - source_labels: [__meta_consul_service_metadata_metrics_addr]
        action: keep
        regex: (.+)
      - source_labels: [__meta_consul_service_metadata_metrics_addr]
        target_label: __address__
      - source_labels: [__meta_consul_service]
        target_label: consul_service
      - source_labels: [__meta_consul_node]
        target_label: consul_node
      - source_labels: [__meta_consul_service_nomad_dc]
        target_label: nomad_dc
      - source_labels: [__meta_consul_service_nomad_node_name]
        target_label: nomad_node_name

  - job_name: "envoy"
    metrics_path: "/metrics"
    consul_sd_configs:
      - server: "${var.consul_address}"
      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: keep
        regex: (.+)-sidecar-proxy
      - source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
        action: keep
        regex: (.+)
      - source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
        target_label: __address__
      - source_labels: [__meta_consul_service]
        target_label: consul_service
      - source_labels: [__meta_consul_node]
        target_label: consul_node
      - source_labels: [__meta_consul_service_nomad_dc]
        target_label: nomad_dc
      - source_labels: [__meta_consul_service_nomad_node_name]
        target_label: nomad_node_name
        EOF
        change_mode = "signal"
        change_signal = "SIGHUP"
        destination = "local/config/prometheus.yml"
      }

      resources {
        cpu = 100
        memory = 200
      }
    }
  }
}