Automatically re-provision grafana when data source addresses change

core/metrics/grafana.nomad

@@ -11,12 +11,6 @@ job "grafana" {
         host_network = "wesher"
         to = 3000
       }
-
-      # TODO: Not sure if this needs to be mapped here since it's within the group
-      port "renderer" {
-        host_network = "loopback"
-        to = 8081
-      }
     }
 
     ephemeral_disk {
@@ -96,16 +90,12 @@ SELECT 'NOOP';
       config {
         image = "grafana/grafana:9.4.2"
         ports = ["web"]
-
-        mount {
-          type = "bind"
-          target = "/etc/grafana"
-          source = "local/config"
-        }
       }
 
       env = {
         "GF_INSTALL_PLUGINS" = "grafana-clock-panel,grafana-piechart-panel,grafana-polystat-panel,natel-discrete-panel",
+        "GF_PATHS_CONFIG" = "$${NOMAD_ALLOC_DIR}/config/grafana.ini"
+        "GF_PATHS_PROVISIONING" = "$${NOMAD_ALLOC_DIR}/config/provisioning"
       }
 
       template {
@@ -136,14 +126,79 @@ SLACK_HOOK_URL={{ .slack_hook_url }}
         destination = "secrets/conf.env"
       }
 
+      resources {
+        cpu = 100
+        memory = 200
+      }
+    }
+
+    task "grafana-reprovisioner" {
+      driver = "docker"
+
+      lifecycle {
+        hook = "prestart"
+        sidecar = true
+      }
+
+      config {
+        image = "alpine"
+        args = ["$${NOMAD_TASK_DIR}/startup.sh"]
+      }
+
+      resources {
+        cpu = 100
+        memory = 100
+      }
+
+      env = {
+        LOG_FILE = "/var/log/grafana_reloader.log"
+      }
+
+      template {
+        data = <<EOF
+#! /bin/sh
+apk add curl
+touch "$LOG_FILE"
+exec tail -f "$LOG_FILE"
+        EOF
+        perms = "777"
+        destination = "$${NOMAD_TASK_DIR}/startup.sh"
+      }
+
+      template {
+        data = <<EOF
+{{ with nomadVar "nomad/jobs/grafana" -}}
+GF_SECURITY_ADMIN_PASSWORD={{ .admin_pw }}
+{{ end -}}
+        EOF
+        env = true
+        destination = "$${NOMAD_SECRETS_DIR}/conf.env"
+      }
+
+      template {
+        data = <<EOF
+#! /bin/sh
+exec > "$LOG_FILE"
+exec 2>&1
+GRAFANA_URL=http://127.0.0.1:3000
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/dashboards/reload
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/datasources/reload
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/plugins/reload
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/notifications/reload
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/access-control/reload
+curl -s -S --user admin:$GF_SECURITY_ADMIN_PASSWORD --request POST $GRAFANA_URL/api/admin/provisioning/alerting/reload
+        EOF
+        change_mode = "noop"
+        perms = "777"
+        destination = "$${NOMAD_TASK_DIR}/reload_config.sh"
+      }
+
       %{ for config_file in fileset(join("/", [module_path, "grafana"]), "**") ~}
       template {
         data = <<EOF
 ${file(join("/", [module_path, "grafana", config_file]))}
         EOF
-        change_mode = "signal"
+        destination = "$${NOMAD_ALLOC_DIR}/config/${config_file}"
-        change_signal = "SIGHUP"
-        destination = "local/config/${config_file}"
         perms = 777
         # Set owner to grafana uid
         # uid = 472
@@ -152,13 +207,13 @@ ${file(join("/", [module_path, "grafana", config_file]))}
         left_delimiter = "<<<<"
         right_delimiter = ">>>>"
         %{ endif }
+        change_mode = "script"
+        change_script {
+          command = "$${NOMAD_TASK_DIR}/reload_config.sh"
+        }
       }
       %{ endfor }
 
-      resources {
-        cpu = 100
-        memory = 200
-      }
     }
 
     task "grafana-image-renderer" {

core/metrics/grafana/grafana.ini

@@ -21,7 +21,7 @@ data = /var/lib/grafana
 ;plugins = /var/lib/grafana/plugins
 
 # folder that contains provisioning config files that grafana will apply on startup and while running.
-provisioning = /etc/grafana/provisioning
+; provisioning = /etc/grafana/provisioning
 
 #################################### Server ####################################
 [server]
@@ -442,5 +442,5 @@ path_style_access = true
 
 [rendering]
 # Since they are inside the same group, we can reference their bound ports
-server_url = http://localhost:8081/render
+server_url = http://127.0.0.1:8081/render
-callback_url = http://localhost:3000/
+callback_url = http://127.0.0.1:3000/