From 18807de608f19208b91fbfbefdc653d557b51e33 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Mon, 25 Jul 2022 15:49:29 -0700 Subject: [PATCH] Clean up Grafana and Loki bootstraps --- nomad/core.tf | 45 +++++++++++ nomad/metrics/.terraform.lock.hcl | 2 + nomad/metrics/exporters.nomad | 74 ++++++++++++------- nomad/metrics/grafana.nomad | 35 ++++++++- nomad/metrics/grafana/grafana.ini | 9 ++- .../grafana/provisioning/datasources/loki.yml | 2 +- .../provisioning/datasources/prometheus.yml | 2 +- .../grafana/provisioning/notifiers/main.yml | 11 +-- nomad/metrics/metrics.tf | 14 ++-- nomad/metrics/prometheus.nomad | 20 ++--- nomad/syslogng.nomad | 19 ++--- 11 files changed, 161 insertions(+), 72 deletions(-) diff --git a/nomad/core.tf b/nomad/core.tf index 4db9369..961140b 100644 --- a/nomad/core.tf +++ b/nomad/core.tf @@ -32,6 +32,7 @@ module "loki" { service_port = 3100 ingress = true sticky_disk = true + healthcheck = "/ready" templates = jsonencode([ { data = file("./loki-config.yml") @@ -41,6 +42,34 @@ module "loki" { } } +resource "consul_config_entry" "loki_intent" { + name = "loki" + kind = "service-intentions" + + config_json = jsonencode({ + Sources = [ + { + Action = "allow" + Name = "grafana" + Precedence = 9 + Type = "consul" + }, + { + Action = "allow" + Name = "promtail" + Precedence = 9 + Type = "consul" + }, + { + Action = "allow" + Name = "syslogng-promtail" + Precedence = 9 + Type = "consul" + }, + ] + }) +} + resource "nomad_job" "syslog-ng" { hcl2 { enabled = true @@ -48,3 +77,19 @@ resource "nomad_job" "syslog-ng" { jobspec = file("${path.module}/syslogng.nomad") } + +resource "consul_config_entry" "syslogng_promtail_intent" { + name = "syslogng-promtail" + kind = "service-intentions" + + config_json = jsonencode({ + Sources = [ + { + Action = "allow" + Name = "syslogng" + Precedence = 9 + Type = "consul" + }, + ] + }) +} diff --git a/nomad/metrics/.terraform.lock.hcl b/nomad/metrics/.terraform.lock.hcl index 027dbaa..b5be304 100644 --- a/nomad/metrics/.terraform.lock.hcl +++ b/nomad/metrics/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/consul" { version = "2.15.0" hashes = [ "h1:o+Su3YqeOkHgf86GEArIVDZfaZQphYFjAOwpi/b0bzs=", + "h1:tAb2gwW+oZ8/t2j7lExdqpNrxmaWsHbyA2crFWClPb0=", "zh:0bd2a9873099d89bd52e9eee623dd20ccb275d1e2f750da229a53a4d5b23450c", "zh:1c9f87d4d97b2c61d006c0bef159d61d2a661a103025f8276ebbeb000129f931", "zh:25b73a34115255c464be10a53f2510c4a1db958a71be31974d30654d5472e624", @@ -22,6 +23,7 @@ provider "registry.terraform.io/hashicorp/consul" { provider "registry.terraform.io/hashicorp/nomad" { version = "1.4.16" hashes = [ + "h1:PQxNPNmMVOErxryTWIJwr22k95DTSODmgRylqjc2TjI=", "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=", "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e", "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572", diff --git a/nomad/metrics/exporters.nomad b/nomad/metrics/exporters.nomad index 936ae2b..0a49007 100644 --- a/nomad/metrics/exporters.nomad +++ b/nomad/metrics/exporters.nomad @@ -2,32 +2,19 @@ job "metrics" { datacenters = ["dc1"] type = "system" - group "exporters" { + group "cadvisor" { network { mode = "bridge" port "cadvisor" { - host_network = "nomad-bridge" to = 8080 } - port "node_exporter" { - host_network = "nomad-bridge" - to = 9100 - } - - port "promtail" { - host_network = "nomad-bridge" - to = 9080 - } - port "expose" { - host_network = "nomad-bridge" } port "cadvisor_envoy_metrics" { - host_network = "nomad-bridge" to = 9102 } } @@ -148,6 +135,18 @@ job "metrics" { } } + } + + group "node_exporter" { + + network { + mode = "bridge" + + port "node_exporter" { + to = 9100 + } + } + service { name = "nodeexporter" port = "node_exporter" @@ -162,6 +161,15 @@ job "metrics" { sidecar_service { proxy { local_service_port = 9100 + + expose { + path { + path = "/metrics" + protocol = "http" + local_path_port = 9100 + listener_port = "node_exporter" + } + } } } @@ -205,6 +213,18 @@ job "metrics" { } } + } + + group "promtail" { + + network { + mode = "bridge" + + port "promtail" { + to = 9080 + } + } + service { name = "promtail" port = "promtail" @@ -219,6 +239,11 @@ job "metrics" { sidecar_service { proxy { local_service_port = 9080 + + upstreams { + destination_name = "loki" + local_bind_port = 1000 + } } } @@ -255,12 +280,6 @@ job "metrics" { } # Bind mount host machine-id and log directories - mount { - type = "bind" - source = "/etc/machine-id" - target = "/etc/machine-id" - readonly = true - } mount { type = "bind" @@ -283,12 +302,12 @@ job "metrics" { readonly = true } - mount { - type = "bind" - source = "/var/log/audit" - target = "/var/log/audit" - readonly = true - } + # mount { + # type = "bind" + # source = "/var/log/audit" + # target = "/var/log/audit" + # readonly = true + # } } template { @@ -299,7 +318,8 @@ server: http_listen_port: 9080 clients: - - url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push + # loki upstream: {{ env "NOMAD_UPSTREAM_ADDR_loki" }} + - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push scrape_configs: diff --git a/nomad/metrics/grafana.nomad b/nomad/metrics/grafana.nomad index deb2609..c2f6a9b 100644 --- a/nomad/metrics/grafana.nomad +++ b/nomad/metrics/grafana.nomad @@ -33,6 +33,11 @@ job "grafana" { destination_name = "prometheus" local_bind_port = 9090 } + + upstreams { + destination_name = "loki" + local_bind_port = 3100 + } } } @@ -72,10 +77,36 @@ job "grafana" { } env = { - "GF_SECURITY_ADMIN_PASSWORD" = "password", "GF_INSTALL_PLUGINS" = "grafana-clock-panel,grafana-piechart-panel,grafana-polystat-panel", } + vault { + policies = [ + "access-tables", + "nomad-task", + ] + } + + template { + data = < 0 ~} + %{ if length(regexall("dashboard", config_file)) > 0 ~} left_delimiter = "<<<<" right_delimiter = ">>>>" %{ endif ~} diff --git a/nomad/metrics/grafana/grafana.ini b/nomad/metrics/grafana/grafana.ini index f04184f..7a45c67 100644 --- a/nomad/metrics/grafana/grafana.ini +++ b/nomad/metrics/grafana/grafana.ini @@ -281,11 +281,18 @@ log_queries = #################################### Auth Proxy ########################## [auth.proxy] +{{ with service "traefik" -}} enabled = true header_name = X-WEBAUTH-USER header_property = username auto_sign_up = true -whitelist = 192.168.2.20 +{{- $last := len . | subtract 1 -}} +{{- $services := . -}} +whitelist = {{ range $i := loop $last -}} + {{- with index $services $i }}{{ .Address }},{{ end -}} +{{- end -}} + {{- with index . $last }}{{ .Address }}{{ end -}} +{{- end }} #################################### Basic Auth ########################## [auth.basic] diff --git a/nomad/metrics/grafana/provisioning/datasources/loki.yml b/nomad/metrics/grafana/provisioning/datasources/loki.yml index ef78fb7..7ba5958 100644 --- a/nomad/metrics/grafana/provisioning/datasources/loki.yml +++ b/nomad/metrics/grafana/provisioning/datasources/loki.yml @@ -3,7 +3,7 @@ apiVersion: 1 datasources: - name: Loki - url: http://loki:3100 + url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }} type: loki access: proxy isDefault: false diff --git a/nomad/metrics/grafana/provisioning/datasources/prometheus.yml b/nomad/metrics/grafana/provisioning/datasources/prometheus.yml index abf4fee..a971b93 100644 --- a/nomad/metrics/grafana/provisioning/datasources/prometheus.yml +++ b/nomad/metrics/grafana/provisioning/datasources/prometheus.yml @@ -3,7 +3,7 @@ apiVersion: 1 datasources: - name: Prometheus - url: http://prom:9090 + url: http://{{ env "NOMAD_UPSTREAM_ADDR_prometheus" }} type: prometheus access: proxy isDefault: true diff --git a/nomad/metrics/grafana/provisioning/notifiers/main.yml b/nomad/metrics/grafana/provisioning/notifiers/main.yml index a895c21..6bb0c53 100644 --- a/nomad/metrics/grafana/provisioning/notifiers/main.yml +++ b/nomad/metrics/grafana/provisioning/notifiers/main.yml @@ -1,15 +1,6 @@ --- -notifiers: - # - name: Telegram - # type: telegram - # uid: telegram-1 - # org_id: 1 - # is_default: false - # settings: - # chatid: ${TELEGRAM_CHATID} - # bottoken: ${TELEGRAM_BOTTOKEN} - # uploadImage: true {{ with secret "kv/data/grafana" -}} +notifiers: - name: Personal email type: email uid: email-1 diff --git a/nomad/metrics/metrics.tf b/nomad/metrics/metrics.tf index ff198c0..dc5ae19 100644 --- a/nomad/metrics/metrics.tf +++ b/nomad/metrics/metrics.tf @@ -3,13 +3,13 @@ variable "consul_address" { description = "address of consul server for dynamic scraping" } -# resource "nomad_job" "exporters" { -# hcl2 { -# enabled = true -# } -# -# jobspec = file("${path.module}/exporters.nomad") -# } +resource "nomad_job" "exporters" { + hcl2 { + enabled = true + } + + jobspec = file("${path.module}/exporters.nomad") +} data "consul_nodes" "all-nodes" { query_options { diff --git a/nomad/metrics/prometheus.nomad b/nomad/metrics/prometheus.nomad index ea86af7..150a64d 100644 --- a/nomad/metrics/prometheus.nomad +++ b/nomad/metrics/prometheus.nomad @@ -97,8 +97,8 @@ scrape_configs: format: - "prometheus" consul_sd_configs: - # - server: "${var.consul_address}" - - server: "${CONSUL_HTTP_ADDR}" + - server: "${var.consul_address}" + # - server: "{{ env "CONSUL_HTTP_ADDR" }}" services: - "nomad" tags: @@ -110,8 +110,8 @@ scrape_configs: format: - "prometheus" consul_sd_configs: - # - server: "${var.consul_address}" - - server: "${CONSUL_HTTP_ADDR}" + - server: "${var.consul_address}" + # - server: "{{ env "CONSUL_HTTP_ADDR" }}" services: - "nomad-client" @@ -121,8 +121,8 @@ scrape_configs: format: - "prometheus" consul_sd_configs: - # - server: "${var.consul_address}" - - server: "${CONSUL_HTTP_ADDR}" + - server: "${var.consul_address}" + # - server: "{{ env "CONSUL_HTTP_ADDR" }}" services: - "consul" relabel_configs: @@ -133,8 +133,8 @@ scrape_configs: - job_name: "exporters" metrics_path: "/metrics" consul_sd_configs: - # - server: "${var.consul_address}" - - server: "${CONSUL_HTTP_ADDR}" + - server: "${var.consul_address}" + # - server: "{{ env "CONSUL_HTTP_ADDR" }}" relabel_configs: - source_labels: [__meta_consul_service] action: drop @@ -156,8 +156,8 @@ scrape_configs: - job_name: "envoy" metrics_path: "/metrics" consul_sd_configs: - # - server: "${var.consul_address}" - - server: "${CONSUL_HTTP_ADDR}" + - server: "${var.consul_address}" + # - server: "{{ env "CONSUL_HTTP_ADDR" }}" relabel_configs: - source_labels: [__meta_consul_service] action: keep diff --git a/nomad/syslogng.nomad b/nomad/syslogng.nomad index 9819269..a6a87c5 100644 --- a/nomad/syslogng.nomad +++ b/nomad/syslogng.nomad @@ -23,8 +23,9 @@ job "syslogng" { connect { sidecar_service { - local_service_port = 1514 proxy { + local_service_port = 1514 + upstreams { destination_name = "loki" local_bind_port = 1000 @@ -64,13 +65,13 @@ server: http_listen_port: 9080 clients: - - url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push + - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push scrape_configs: # TCP syslog receiver - job_name: syslog syslog: - listen_address: 0.0.0.0:${NOMAD_PORT_main} + listen_address: 0.0.0.0:{{ env "NOMAD_PORT_main" }} labels: job: syslog relabel_configs: @@ -125,21 +126,13 @@ EOF } } } - - check { - type = "tcp" - port = "main" - interval = "10s" - timeout = "10s" - } - } task "syslogng" { driver = "docker" config { - image = "balbit/syslog-ng:latest" + image = "balabit/syslog-ng:3.37.1" ports = ["main"] args = ["--no-caps"] @@ -165,7 +158,7 @@ source s_internal { destination d_loki { # Forward to Connect proxy to Promtail - syslog("${NOMAD_UPSTREAM_IP_syslogngpromtail}" transport("tcp") port(${NOMAD_UPSTREAM_PORT_syslogngpromtail})); + syslog("{{ env "NOMAD_UPSTREAM_IP_syslogng-promtail" }}" transport("tcp") port({{ env "NOMAD_UPSTREAM_PORT_syslogng-promtail" }})); }; log { source(s_internal); destination(d_loki); };