Add mysql database storage to Grafana

databases/mysql.tf

@@ -42,6 +42,12 @@ resource "consul_config_entry" "mysql_intents" {
         Precedence = 9
         Type       = "consul"
       },
+      {
+        Action     = "allow"
+        Name       = "grafana"
+        Precedence = 9
+        Type       = "consul"
+      },
     ]
   })
 }

metrics/grafana.nomad

@@ -38,6 +38,11 @@ job "grafana" {
               destination_name = "loki"
               local_bind_port = 3100
             }
+
+            upstreams {
+              destination_name = "mysql-server"
+              local_bind_port = 6060
+            }
           }
         }
 
@@ -62,6 +67,70 @@ job "grafana" {
       ]
     }
 
+    task "grafana-bootstrap" {
+      driver = "docker"
+
+      lifecycle {
+        hook = "prestart"
+        sidecar = false
+      }
+
+      config {
+        image = "mysql:8"
+        args = [
+          "/bin/bash",
+          "-c",
+          "/usr/bin/mysql --defaults-extra-file=/task/my.cnf < /task/bootstrap.sql",
+        ]
+
+        mount {
+          type = "bind"
+          source = "local/"
+          target = "/task/"
+        }
+      }
+
+      vault {
+        policies = [
+          "access-tables",
+          "nomad-task",
+        ]
+      }
+
+      template {
+        data = <<EOF
+[client]
+host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
+port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
+user=root
+{{ with secret "kv/data/mysql" }}
+password={{ .Data.data.root_password }}
+{{ end }}
+        EOF
+        destination = "local/my.cnf"
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/grafana" -}}
+{{ if .Data.data.db_name -}}
+CREATE DATABASE IF NOT EXISTS `{{ .Data.data.db_name }}`;
+CREATE USER IF NOT EXISTS '{{ .Data.data.db_user }}'@'%' IDENTIFIED BY '{{ .Data.data.db_pass }}';
+GRANT ALL ON `{{ .Data.data.db_name }}`.* to '{{ .Data.data.db_user }}'@'%';
+{{ else -}}
+SELECT 'NOOP';
+{{ end -}}
+{{ end -}}
+        EOF
+        destination = "local/bootstrap.sql"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
+
     task "grafana" {
       driver = "docker"
 
@@ -89,19 +158,27 @@ job "grafana" {
 
       template {
         data = <<EOF
-{{ with secret "kv/data/grafana" }}
+{{ with secret "kv/data/grafana" -}}
 GF_SECURITY_ADMIN_PASSWORD={{ .Data.data.admin_pw }}
 GF_SMTP_USER={{ .Data.data.smtp_user }}
 GF_SMTP_PASSWORD={{ .Data.data.smtp_password }}
 GF_EXTERNAL_IMAGE_STORAGE_S3_ACCESS_KEY={{ .Data.data.minio_access_key }}
 GF_EXTERNAL_IMAGE_STORAGE_S3_SECRET_KEY={{ .Data.data.minio_secret_key }}
 GRAFANA_ALERT_EMAIL_ADDRESSES={{ .Data.data.alert_email_addresses }}
-{{ end }}
-{{ with secret "kv/data/slack" }}
+{{ if .Data.data.db_name -}}
+# Database storage
+GF_DATABASE_TYPE=mysql
+GF_DATABASE_HOST={{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}
+GF_DATABASE_NAME={{ .Data.data.db_name }}
+GF_DATABASE_USER={{ .Data.data.db_user }}
+GF_DATABASE_PASSWORD={{ .Data.data.db_pass }}
+{{ end -}}
+{{ end -}}
+{{ with secret "kv/data/slack" -}}
 SLACK_BOT_URL={{ .Data.data.bot_url }}
 SLACK_BOT_TOKEN={{ .Data.data.bot_token }}
 SLACK_HOOK_URL={{ .Data.data.hook_url }}
-{{ end }}
+{{ end -}}
         EOF
         env = true
         destination = "secrets/conf.env"