homelab-nomad/metrics/exporters.nomad

344 lines
7.0 KiB
HCL

job "metrics" {
datacenters = ["dc1"]
type = "system"
group "exporters" {
network {
mode = "bridge"
port "cadvisor" {
host_network = "nomad-bridge"
to = 8080
}
port "node_exporter" {
host_network = "nomad-bridge"
to = 9100
}
port "promtail" {
host_network = "nomad-bridge"
to = 9080
}
port "expose" {
host_network = "nomad-bridge"
}
port "cadvisor_envoy_metrics" {
host_network = "nomad-bridge"
to = 9102
}
}
service {
name = "cadvisor"
port = "cadvisor"
meta {
metrics_addr = "${NOMAD_ADDR_expose}"
envoy_metrics_addr = "${NOMAD_ADDR_cadvisor_envoy_metrics}"
nomad_dc = "${NOMAD_DC}"
nomad_node_name = "${node.unique.name}"
}
connect {
sidecar_service {
proxy {
local_service_port = 8080
expose {
path {
path = "/metrics"
protocol = "http"
local_path_port = 8080
listener_port = "expose"
}
}
config {
envoy_prometheus_bind_addr = "0.0.0.0:9102"
}
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "http"
path = "/metrics"
port = "cadvisor"
interval = "10s"
timeout = "10s"
}
}
task "cadvisor" {
driver = "docker"
config {
# image = "iamthefij/cadvisor:0.37.5"
image = "gcr.io/cadvisor/cadvisor:v0.39.3"
args = ["--docker_only=true"]
ports = ["cadvisor"]
# volumes = [
# "/:/rootfs:ro",
# "/var/run:/var/run:rw",
# "/sys:/sys:ro",
# "/var/lib/docker/:/var/lib/docker:ro",
# "/cgroup:/cgroup:ro",
# "/etc/machine-id:/etc/machine-id:ro",
# ]
mount {
type = "bind"
source = "/"
target = "/rootfs"
readonly = true
}
mount {
type = "bind"
source = "/var/run"
target = "/var/run"
readonly = false
}
mount {
type = "bind"
source = "/sys"
target = "/sys"
readonly = true
}
mount {
type = "bind"
source = "/var/lib/docker"
target = "/var/lib/docker"
readonly = true
}
# mount {
# type = "bind"
# source = "/cgroup"
# target = "/cgroup"
# readonly = true
# }
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
}
resources {
cpu = 50
memory = 100
}
}
service {
name = "nodeexporter"
port = "node_exporter"
meta {
metrics_addr = "${NOMAD_ADDR_node_exporter}"
nomad_dc = "${NOMAD_DC}"
nomad_node_name = "${node.unique.name}"
}
connect {
sidecar_service {
proxy {
local_service_port = 9100
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "http"
path = "/metrics"
port = "node_exporter"
interval = "10s"
timeout = "10s"
}
}
task "node_exporter" {
driver = "docker"
config {
image = "prom/node-exporter:v1.0.1"
args = ["--path.rootfs", "/host"]
ports = ["node_exporter"]
mount {
type = "bind"
source = "/"
target = "/host"
readonly = true
}
}
resources {
cpu = 50
memory = 50
}
}
service {
name = "promtail"
port = "promtail"
meta {
metrics_addr = "${NOMAD_ADDR_promtail}"
nomad_dc = "${NOMAD_DC}"
nomad_node_name = "${node.unique.name}"
}
connect {
sidecar_service {
proxy {
local_service_port = 9080
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "http"
path = "/metrics"
port = "promtail"
interval = "10s"
timeout = "10s"
}
}
task "promtail" {
driver = "docker"
config {
image = "grafana/promtail:2.2.1"
args = ["-config.file=/etc/promtail/promtail.yml"]
ports = ["promtail"]
# Mount config
mount {
type = "bind"
target = "/etc/promtail/promtail.yml"
source = "local/promtail.yml"
}
# Bind mount host machine-id and log directories
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
mount {
type = "bind"
source = "/var/log/journal/"
target = "/var/log/journal/"
readonly = true
}
mount {
type = "bind"
source = "/run/log/journal/"
target = "/run/log/journal/"
readonly = true
}
mount {
type = "bind"
source = "/var/log/audit"
target = "/var/log/audit"
readonly = true
}
}
template {
data = <<EOF
---
server:
http_listen_address: 0.0.0.0
http_listen_port: 9080
clients:
- url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push
scrape_configs:
- job_name: journal
journal:
json: false
max_age: 12h
path: /var/log/journal
labels:
job: systemd-journal
relabel_configs:
- source_labels: ['__journal__systemd_unit']
target_label: unit
- source_labels: ['__journal__hostname']
target_label: hostname
- source_labels: ['__journal__transport']
target_label: journal_transport
# Docker log labels
- source_labels: ['__journal_syslog_identifier']
target_label: syslog_identifier
- source_labels: ['__journal_image_name']
target_label: docker_image_name
- source_labels: ['__journal_container_name']
target_label: docker_container_name
- source_labels: ['__journal_container_id']
target_label: docker_container_id
- source_labels: ['__journal_com_docker_compose_project']
target_label: docker_compose_project
- source_labels: ['__journal_com_docker_compose_service']
target_label: docker_compose_service
EOF
destination = "local/promtail.yml"
}
resources {
cpu = 50
memory = 50
}
}
}
}