homelab-nomad/core/metrics/grafana.nomad

job "grafana" {
  datacenters = ["dc1"]

  group "grafana" {
    count = 1

    network {
      mode = "bridge"

      port "web" {
        host_network = "loopback"
        to = 3000
      }

      port "renderer" {
        host_network = "loopback"
        to = 8081
      }
    }

    ephemeral_disk {
      migrate = true
      sticky = true
    }

    service {
      name = "grafana"
      port = "web"

      connect {
        sidecar_service {
          proxy {
            local_service_port = 3000

            upstreams {
              destination_name = "prometheus"
              local_bind_port = 9090
            }

            upstreams {
              destination_name = "loki"
              local_bind_port = 3100
            }

            upstreams {
              destination_name = "mysql-server"
              local_bind_port = 6060
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }

      check {
        type = "http"
        path = "/"
        port = "web"
        interval = "10s"
        timeout = "10s"
      }

      tags = [
        "traefik.enable=true",
        "traefik.http.routers.grafana.entryPoints=websecure",
      ]
    }

    task "grafana-bootstrap" {
      driver = "docker"

      lifecycle {
        hook = "prestart"
        sidecar = false
      }

      config {
        image = "mariadb:10"
        args = [
          "/bin/bash",
          "-c",
          "/usr/bin/mysql --defaults-extra-file=$${NOMAD_SECRETS_DIR}/my.cnf < $${NOMAD_SECRETS_DIR}/bootstrap.sql",
        ]
      }

      vault {
        policies = [
          "access-tables",
          "nomad-task",
        ]
      }

      template {
        data = <<EOF
[client]
host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
user=root
{{ with secret "kv/data/mysql" }}
password={{ .Data.data.root_password }}
{{ end }}
        EOF
        destination = "$${NOMAD_SECRETS_DIR}/my.cnf"
      }

      template {
        data = <<EOF
{{ with secret "kv/data/grafana" -}}
{{ if .Data.data.db_name -}}
CREATE DATABASE IF NOT EXISTS `{{ .Data.data.db_name }}`;
CREATE USER IF NOT EXISTS '{{ .Data.data.db_user }}'@'%' IDENTIFIED BY '{{ .Data.data.db_pass }}';
GRANT ALL ON `{{ .Data.data.db_name }}`.* to '{{ .Data.data.db_user }}'@'%';
-- Add grafana read_only user
CREATE USER IF NOT EXISTS '{{ .Data.data.db_user_ro }}'@'%' IDENTIFIED BY '{{ .Data.data.db_pass_ro }}';
{{ else -}}
SELECT 'NOOP';
{{ end -}}
{{ end -}}
        EOF
        destination = "$${NOMAD_SECRETS_DIR}/bootstrap.sql"
      }

      resources {
        cpu = 50
        memory = 50
      }
    }

    task "grafana" {
      driver = "docker"

      config {
        image = "grafana/grafana:9.2.4"
        ports = ["web"]

        mount {
          type = "bind"
          target = "/etc/grafana"
          source = "local/config"
        }
      }

      env = {
        "GF_INSTALL_PLUGINS" = "grafana-clock-panel,grafana-piechart-panel,grafana-polystat-panel,natel-discrete-panel",
      }

      vault {
        policies = [
          "access-tables",
          "nomad-task",
        ]
      }

      template {
        data = <<EOF
{{ with secret "kv/data/grafana" -}}
GF_SECURITY_ADMIN_PASSWORD={{ .Data.data.admin_pw }}
GF_SMTP_USER={{ .Data.data.smtp_user }}
GF_SMTP_PASSWORD={{ .Data.data.smtp_password }}
GF_EXTERNAL_IMAGE_STORAGE_S3_ACCESS_KEY={{ .Data.data.minio_access_key }}
GF_EXTERNAL_IMAGE_STORAGE_S3_SECRET_KEY={{ .Data.data.minio_secret_key }}
GRAFANA_ALERT_EMAIL_ADDRESSES={{ .Data.data.alert_email_addresses }}
{{ if .Data.data.db_name -}}
# Database storage
GF_DATABASE_TYPE=mysql
GF_DATABASE_HOST={{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}
GF_DATABASE_NAME={{ .Data.data.db_name }}
GF_DATABASE_USER={{ .Data.data.db_user }}
GF_DATABASE_PASSWORD={{ .Data.data.db_pass }}
{{ end -}}
{{ end -}}
{{ with secret "kv/data/slack" -}}
SLACK_BOT_URL={{ .Data.data.bot_url }}
SLACK_BOT_TOKEN={{ .Data.data.bot_token }}
SLACK_HOOK_URL={{ .Data.data.hook_url }}
{{ end -}}
        EOF
        env = true
        destination = "secrets/conf.env"
      }

      %{ for config_file in fileset(join("/", [module_path, "grafana"]), "**") ~}
      template {
        data = <<EOF
${file(join("/", [module_path, "grafana", config_file]))}
        EOF
        change_mode = "signal"
        change_signal = "SIGHUP"
        destination = "local/config/${config_file}"
        perms = 777
        # Set owner to grafana uid
        # uid = 472
        # Change template delimeter for dashboard files that use json and have double curly braces and square braces
        %{ if length(regexall("dashboard", config_file)) > 0 ~}
        left_delimiter = "<<<<"
        right_delimiter = ">>>>"
        %{ endif }
      }
      %{ endfor }

      resources {
        cpu = 100
        memory = 200
      }
    }

    task "grafana-image-renderer" {
      driver = "docker"

      config {
        image = "grafana/grafana-image-renderer:3.6.1"
        ports = ["renderer"]
      }

      env = {
        "RENDERING_MODE" = "clustered"
        "RENDERING_CLUSTERING_MODE" = "browser"
        "RENDERING_CLUSTERING_MAX_CONCURRENCY" = 5
        "RENDERING_CLUSTERING_TIMEOUT" = 30
      }
    }
  }
}