homelab-nomad/metrics/exporters.nomad

job "metrics" {
  datacenters = ["dc1"]
  type = "system"

  group "cadvisor" {

    network {
      mode = "bridge"

      port "cadvisor" {
        to = 8080
      }

      port "expose" {
      }

      port "cadvisor_envoy_metrics" {
        to = 9102
      }
    }

    service {
      name = "cadvisor"
      port = "cadvisor"

      meta {
        metrics_addr = "${NOMAD_ADDR_expose}"
        envoy_metrics_addr = "${NOMAD_ADDR_cadvisor_envoy_metrics}"
        nomad_dc = "${NOMAD_DC}"
        nomad_node_name = "${node.unique.name}"
      }

      connect {
        sidecar_service {
          proxy {
            local_service_port = 8080

            expose {
              path {
                path = "/metrics"
                protocol = "http"
                local_path_port = 8080
                listener_port = "expose"
              }
            }

            config {
              envoy_prometheus_bind_addr = "0.0.0.0:9102"
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }

      check {
        type = "http"
        path = "/metrics"
        port = "cadvisor"
        interval = "10s"
        timeout = "10s"
      }
    }

    task "cadvisor" {
      driver = "docker"

      config {
        # image = "iamthefij/cadvisor:0.37.5"
        image = "gcr.io/cadvisor/cadvisor:v0.39.3"
        args = ["--docker_only=true"]

        ports = ["cadvisor"]

        # volumes = [
        #   "/:/rootfs:ro",
        #   "/var/run:/var/run:rw",
        #   "/sys:/sys:ro",
        #   "/var/lib/docker/:/var/lib/docker:ro",
        #   "/cgroup:/cgroup:ro",
        #   "/etc/machine-id:/etc/machine-id:ro",
        # ]

        mount {
          type = "bind"
          source = "/"
          target = "/rootfs"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/var/run"
          target = "/var/run"
          readonly = false
        }

        mount {
          type = "bind"
          source = "/sys"
          target = "/sys"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/var/lib/docker"
          target = "/var/lib/docker"
          readonly = true
        }

        # mount {
        #   type = "bind"
        #   source = "/cgroup"
        #   target = "/cgroup"
        #   readonly = true
        # }

        mount {
          type = "bind"
          source = "/etc/machine-id"
          target = "/etc/machine-id"
          readonly = true
        }
      }

      resources {
        cpu = 50
        memory = 100
      }
    }

  }

  group "node_exporter" {

    network {
      mode = "bridge"

      port "node_exporter" {
        to = 9100
      }
    }

    service {
      name = "nodeexporter"
      port = "node_exporter"

      meta {
        metrics_addr = "${NOMAD_ADDR_node_exporter}"
        nomad_dc = "${NOMAD_DC}"
        nomad_node_name = "${node.unique.name}"
      }

      connect {
        sidecar_service {
          proxy {
            local_service_port = 9100

            expose {
              path {
                path = "/metrics"
                protocol = "http"
                local_path_port = 9100
                listener_port = "node_exporter"
              }
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }

      check {
        type = "http"
        path = "/metrics"
        port = "node_exporter"
        interval = "10s"
        timeout = "10s"
      }
    }

    task "node_exporter" {
      driver = "docker"

      config {
        image = "prom/node-exporter:v1.0.1"
        args = ["--path.rootfs", "/host"]

        ports = ["node_exporter"]

        mount {
          type = "bind"
          source = "/"
          target = "/host"
          readonly = true
        }
      }

      resources {
        cpu = 50
        memory = 50
      }
    }

  }

  group "promtail" {

    network {
      mode = "bridge"

      port "promtail" {
        to = 9080
      }
    }

    service {
      name = "promtail"
      port = "promtail"

      meta {
        metrics_addr = "${NOMAD_ADDR_promtail}"
        nomad_dc = "${NOMAD_DC}"
        nomad_node_name = "${node.unique.name}"
      }

      connect {
        sidecar_service {
          proxy {
            local_service_port = 9080

            upstreams {
              destination_name = "loki"
              local_bind_port = 1000
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }

      check {
        type = "http"
        path = "/metrics"
        port = "promtail"
        interval = "10s"
        timeout = "10s"
      }
    }

    task "promtail" {
      driver = "docker"

      config {
        image = "grafana/promtail:2.2.1"
        args = ["-config.file=/etc/promtail/promtail.yml"]
        ports = ["promtail"]

        # Mount config
        mount {
          type = "bind"
          target = "/etc/promtail/promtail.yml"
          source = "local/promtail.yml"
        }

        # Bind mount host machine-id and log directories

        mount {
          type = "bind"
          source = "/etc/machine-id"
          target = "/etc/machine-id"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/var/log/journal/"
          target = "/var/log/journal/"
          readonly = true
        }

        mount {
          type = "bind"
          source = "/run/log/journal/"
          target = "/run/log/journal/"
          readonly = true
        }

        # mount {
        #   type = "bind"
        #   source = "/var/log/audit"
        #   target = "/var/log/audit"
        #   readonly = true
        # }
      }

      template {
        data = <<EOF
---
server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

clients:
  # loki upstream: {{ env "NOMAD_UPSTREAM_ADDR_loki" }}
  - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push

scrape_configs:

  - job_name: journal
    journal:
      json: false
      max_age: 12h
      path: /var/log/journal
      labels:
        job: systemd-journal
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit
      - source_labels: ['__journal__hostname']
        target_label: hostname
      - source_labels: ['__journal__transport']
        target_label: journal_transport
      # Docker log labels
      - source_labels: ['__journal_syslog_identifier']
        target_label: syslog_identifier
      - source_labels: ['__journal_image_name']
        target_label: docker_image_name
      - source_labels: ['__journal_container_name']
        target_label: docker_container_name
      - source_labels: ['__journal_container_id']
        target_label: docker_container_id
      - source_labels: ['__journal_com_docker_compose_project']
        target_label: docker_compose_project
      - source_labels: ['__journal_com_docker_compose_service']
        target_label: docker_compose_service
EOF
        destination = "local/promtail.yml"
      }

      resources {
        cpu = 50
        memory = 50
      }
    }
  }
}