homelab-nomad/core/exporters.nomad
2024-11-21 11:11:57 -08:00

150 lines
3.8 KiB
HCL

job "exporters" {
datacenters = ["dc1"]
type = "service"
priority = 55
constraint {
distinct_hosts = true
}
group "promtail" {
# TODO: This must be updated to match the nubmer of servers (possibly grabbed from TF)
# I am moving away from `system` jobs because of https://github.com/hashicorp/nomad/issues/1202
count = 2
network {
mode = "bridge"
port "promtail" {
%{~ if use_wesher ~}
host_network = "wesher"
%{~ endif ~}
to = 9080
}
}
service {
name = "promtail"
provider = "nomad"
port = "promtail"
meta {
nomad_dc = "$${NOMAD_DC}"
nomad_node_name = "$${node.unique.name}"
}
tags = [
"prometheus.scrape",
]
}
task "promtail" {
driver = "docker"
meta = {
"diun.sort_tags" = "semver"
"diun.watch_repo" = true
"diun.include_tags" = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
}
config {
image = "grafana/promtail:3.3.0"
args = ["-config.file=$${NOMAD_TASK_DIR}/promtail.yml"]
ports = ["promtail"]
# Bind mount host machine-id and log directories
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
mount {
type = "bind"
source = "/var/log/journal/"
target = "/var/log/journal/"
readonly = true
}
mount {
type = "bind"
source = "/run/log/journal/"
target = "/run/log/journal/"
readonly = true
}
# mount {
# type = "bind"
# source = "/var/log/audit"
# target = "/var/log/audit"
# readonly = true
# }
}
template {
data = <<EOF
---
server:
http_listen_address: 0.0.0.0
http_listen_port: 9080
clients:
{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "loki" -}}
- url: http://{{ .Address }}:{{ .Port }}/loki/api/v1/push
{{- end }}
scrape_configs:
- job_name: journal
journal:
json: false
max_age: 12h
path: /var/log/journal
labels:
job: systemd-journal
relabel_configs:
- source_labels: ['__journal__systemd_unit']
target_label: unit
- source_labels: ['__journal__hostname']
target_label: hostname
- source_labels: ['__journal__transport']
target_label: journal_transport
# Docker log labels
- source_labels: ['__journal_syslog_identifier']
target_label: syslog_identifier
- source_labels: ['__journal_image_name']
target_label: docker_image_name
- source_labels: ['__journal_container_name']
target_label: docker_container_name
- source_labels: ['__journal_container_id']
target_label: docker_container_id
- source_labels: ['__journal_com_docker_compose_project']
target_label: docker_compose_project
- source_labels: ['__journal_com_docker_compose_service']
target_label: docker_compose_service
- source_labels: ['__journal_com_hashicorp_nomad_alloc_id']
target_label: nomad_alloc_id
- source_labels: ['__journal_com_hashicorp_nomad_job_id']
target_label: nomad_job_id
- source_labels: ['__journal_com_hashicorp_nomad_job_name']
target_label: nomad_job_name
- source_labels: ['__journal_com_hashicorp_nomad_node_name']
target_label: nomad_node_name
- source_labels: ['__journal_com_hashicorp_nomad_group_name']
target_label: nomad_group_name
- source_labels: ['__journal_com_hashicorp_nomad_task_name']
target_label: nomad_task_name
EOF
destination = "$${NOMAD_TASK_DIR}/promtail.yml"
}
resources {
cpu = 50
memory = 100
}
}
}
}