homelab-nomad/metrics/prometheus.nomad

183 lines
4.4 KiB
HCL

variable "consul_address" {
type = string
description = "Full address of Consul instance to get catalog from"
default = "http://127.0.0.1:5400"
}
job "prometheus" {
datacenters = ["dc1"]
group "prometheus" {
count = 1
network {
mode = "bridge"
port "web" {
host_network = "nomad-bridge"
to = 9090
}
}
service {
port = "web"
connect {
sidecar_service {
proxy {
local_service_port = 9090
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "http"
path = "/"
port = "web"
interval = "10s"
timeout = "10s"
}
// TODO: Remove traefik tags
tags = [
"traefik.enable=true",
"traefik.http.routers.prometheus.entrypoints=web,websecure",
"traefik.http.routers.prometheus.rule=Host(`prometheus.dev.homelab`)",
"traefik.http.routers.prometheus.tls=true",
]
}
task "prometheus" {
driver = "docker"
config {
image = "prom/prometheus:v2.30.2"
ports = ["web"]
args = [
"--config.file=/etc/prometheus/config/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--web.listen-address=0.0.0.0:9090",
"--web.console.libraries=/usr/share/prometheus/console_libraries",
"--web.console.templates=/usr/share/prometheus/consoles",
]
mount {
type = "bind"
target = "/etc/prometheus/config"
source = "local/config"
}
}
template {
data = <<EOF
---
global:
scrape_interval: 30s
evaluation_interval: 3s
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- 0.0.0.0:9090
- job_name: "nomad_server"
metrics_path: "/v1/metrics"
params:
format:
- "prometheus"
consul_sd_configs:
- server: "${var.consul_address}"
services:
- "nomad"
tags:
- "http"
- job_name: "nomad_client"
metrics_path: "/v1/metrics"
params:
format:
- "prometheus"
consul_sd_configs:
- server: "${var.consul_address}"
services:
- "nomad-client"
- job_name: "consul"
metrics_path: "/v1/agent/metrics"
params:
format:
- "prometheus"
consul_sd_configs:
- server: "${var.consul_address}"
services:
- "consul"
relabel_configs:
- source_labels: [__meta_consul_address]
replacement: $1:8500
target_label: __address__
- job_name: "exporters"
metrics_path: "/metrics"
consul_sd_configs:
- server: "${var.consul_address}"
services:
- "cadvisor"
- "nodeexporter"
relabel_configs:
- source_labels: [__meta_consul_service_metadata_metrics_addr]
action: keep
regex: (.+)
- source_labels: [__meta_consul_service_metadata_metrics_addr]
target_label: __address__
- source_labels: [__meta_consul_service]
target_label: consul_service
- source_labels: [__meta_consul_node]
target_label: consul_node
- source_labels: [__meta_consul_service_nomad_dc]
target_label: nomad_dc
- source_labels: [__meta_consul_service_nomad_node_name]
target_label: nomad_node_name
- job_name: "envoy"
metrics_path: "/metrics"
consul_sd_configs:
- server: "${var.consul_address}"
relabel_configs:
- source_labels: [__meta_consul_service]
action: drop
regex: (.+)-sidecar-proxy
- source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
action: keep
regex: (.+)
- source_labels: [__meta_consul_service_metadata_envoy_metrics_addr]
target_label: __address__
- source_labels: [__meta_consul_service]
target_label: consul_service
- source_labels: [__meta_consul_node]
target_label: consul_node
- source_labels: [__meta_consul_service_nomad_dc]
target_label: nomad_dc
- source_labels: [__meta_consul_service_nomad_node_name]
target_label: nomad_node_name
EOF
change_mode = "signal"
change_signal = "SIGHUP"
destination = "local/config/prometheus.yml"
}
resources {
cpu = 100
memory = 200
}
}
}
}