homelab-nomad/core/metrics/exporters.nomad

job "metrics" {
  datacenters = ["dc1"]
  type = "system"

  group "promtail" {

    network {
      mode = "bridge"

      port "promtail" {
        to = 9080
      }
    }

    service {
      name = "promtail"
      port = "promtail"

      meta {
        metrics_addr = "${NOMAD_ADDR_promtail}"
        nomad_dc = "${NOMAD_DC}"
        nomad_node_name = "${node.unique.name}"
      }

      connect {
        sidecar_service {
          proxy {
            local_service_port = 9080

            upstreams {
              destination_name = "loki"
              local_bind_port = 1000
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 20
          }
        }
      }

      check {
        type = "http"
        path = "/metrics"
        port = "promtail"
        interval = "10s"
        timeout = "10s"
      }
    }

    task "promtail" {
      driver = "docker"

      meta = {
        "diun.sort_tags" = "semver"
        "diun.watch_repo" = true
        "diun.include_tags" = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
      }

      config {
        image = "grafana/promtail:2.7.1"
        args = ["-config.file=/etc/promtail/promtail.yml"]
        ports = ["promtail"]

        # Mount config
        mount {
          type = "bind"
          target = "/etc/promtail/promtail.yml"
          source = "local/promtail.yml"
        }

        # Bind mount host machine-id and log directories

        mount {
          type = "bind"
          source = "/etc/machine-id"
          target = "/etc/machine-id"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/var/log/journal/"
          target = "/var/log/journal/"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/run/log/journal/"
          target = "/run/log/journal/"
          readonly = true
        }

        # mount {
        #   type = "bind"
        #   source = "/var/log/audit"
        #   target = "/var/log/audit"
        #   readonly = true
        # }
      }

      template {
        data = <<EOF
---
server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

clients:
  - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push

scrape_configs:

  - job_name: journal
    journal:
      json: false
      max_age: 12h
      path: /var/log/journal
      labels:
        job: systemd-journal
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit
      - source_labels: ['__journal__hostname']
        target_label: hostname
      - source_labels: ['__journal__transport']
        target_label: journal_transport
      # Docker log labels
      - source_labels: ['__journal_syslog_identifier']
        target_label: syslog_identifier
      - source_labels: ['__journal_image_name']
        target_label: docker_image_name
      - source_labels: ['__journal_container_name']
        target_label: docker_container_name
      - source_labels: ['__journal_container_id']
        target_label: docker_container_id
      - source_labels: ['__journal_com_docker_compose_project']
        target_label: docker_compose_project
      - source_labels: ['__journal_com_docker_compose_service']
        target_label: docker_compose_service
      - source_labels: ['__journal_com_hashicorp_nomad_alloc_id']
        target_label: nomad_alloc_id
      - source_labels: ['__journal_com_hashicorp_nomad_job_id']
        target_label: nomad_job_id
      - source_labels: ['__journal_com_hashicorp_nomad_job_name']
        target_label: nomad_job_name
      - source_labels: ['__journal_com_hashicorp_nomad_node_name']
        target_label: nomad_node_name
      - source_labels: ['__journal_com_hashicorp_nomad_group_name']
        target_label: nomad_group_name
      - source_labels: ['__journal_com_hashicorp_nomad_task_name']
        target_label: nomad_task_name
EOF
        destination = "local/promtail.yml"
      }

      resources {
        cpu = 50
        memory = 100
      }
    }
  }
}