job "metrics" { datacenters = ["dc1"] type = "system" group "cadvisor" { network { mode = "bridge" port "cadvisor" { to = 8080 } port "expose" { } port "cadvisor_envoy_metrics" { to = 9102 } } service { name = "cadvisor" port = "cadvisor" meta { metrics_addr = "${NOMAD_ADDR_expose}" envoy_metrics_addr = "${NOMAD_ADDR_cadvisor_envoy_metrics}" nomad_dc = "${NOMAD_DC}" nomad_node_name = "${node.unique.name}" } connect { sidecar_service { proxy { local_service_port = 8080 expose { path { path = "/metrics" protocol = "http" local_path_port = 8080 listener_port = "expose" } } config { envoy_prometheus_bind_addr = "0.0.0.0:9102" } } } sidecar_task { resources { cpu = 50 memory = 50 } } } check { type = "http" path = "/metrics" port = "cadvisor" interval = "10s" timeout = "10s" } } task "cadvisor" { driver = "docker" config { # image = "iamthefij/cadvisor:0.37.5" image = "gcr.io/cadvisor/cadvisor:v0.39.3" args = ["--docker_only=true"] ports = ["cadvisor"] # volumes = [ # "/:/rootfs:ro", # "/var/run:/var/run:rw", # "/sys:/sys:ro", # "/var/lib/docker/:/var/lib/docker:ro", # "/cgroup:/cgroup:ro", # "/etc/machine-id:/etc/machine-id:ro", # ] mount { type = "bind" source = "/" target = "/rootfs" readonly = true } mount { type = "bind" source = "/var/run" target = "/var/run" readonly = false } mount { type = "bind" source = "/sys" target = "/sys" readonly = true } mount { type = "bind" source = "/var/lib/docker" target = "/var/lib/docker" readonly = true } # mount { # type = "bind" # source = "/cgroup" # target = "/cgroup" # readonly = true # } mount { type = "bind" source = "/etc/machine-id" target = "/etc/machine-id" readonly = true } } resources { cpu = 50 memory = 100 } } } group "node_exporter" { network { mode = "bridge" port "node_exporter" { to = 9100 } } service { name = "nodeexporter" port = "node_exporter" meta { metrics_addr = "${NOMAD_ADDR_node_exporter}" nomad_dc = "${NOMAD_DC}" nomad_node_name = "${node.unique.name}" } connect { sidecar_service { proxy { local_service_port = 9100 expose { path { path = "/metrics" protocol = "http" local_path_port = 9100 listener_port = "node_exporter" } } } } sidecar_task { resources { cpu = 50 memory = 50 } } } check { type = "http" path = "/metrics" port = "node_exporter" interval = "10s" timeout = "10s" } } task "node_exporter" { driver = "docker" config { image = "prom/node-exporter:v1.0.1" args = ["--path.rootfs", "/host"] ports = ["node_exporter"] mount { type = "bind" source = "/" target = "/host" readonly = true } } resources { cpu = 50 memory = 50 } } } group "promtail" { network { mode = "bridge" port "promtail" { to = 9080 } } service { name = "promtail" port = "promtail" meta { metrics_addr = "${NOMAD_ADDR_promtail}" nomad_dc = "${NOMAD_DC}" nomad_node_name = "${node.unique.name}" } connect { sidecar_service { proxy { local_service_port = 9080 upstreams { destination_name = "loki" local_bind_port = 1000 } } } sidecar_task { resources { cpu = 50 memory = 50 } } } check { type = "http" path = "/metrics" port = "promtail" interval = "10s" timeout = "10s" } } task "promtail" { driver = "docker" config { image = "grafana/promtail:2.2.1" args = ["-config.file=/etc/promtail/promtail.yml"] ports = ["promtail"] # Mount config mount { type = "bind" target = "/etc/promtail/promtail.yml" source = "local/promtail.yml" } # Bind mount host machine-id and log directories mount { type = "bind" source = "/etc/machine-id" target = "/etc/machine-id" readonly = true } mount { type = "bind" source = "/var/log/journal/" target = "/var/log/journal/" readonly = true } mount { type = "bind" source = "/run/log/journal/" target = "/run/log/journal/" readonly = true } # mount { # type = "bind" # source = "/var/log/audit" # target = "/var/log/audit" # readonly = true # } } template { data = <