From d8307935f50359e0b7ffab0d8b151050ca35aaed Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Fri, 24 Mar 2023 11:24:36 -0700
Subject: [PATCH] Refactor everything for nomad vars

---
 .secrets-baseline                             |  20 +--
 core/blocky/blocky.nomad                      |   5 +-
 core/metrics/exporters.nomad                  |  34 +-----
 core/metrics/grafana.nomad                    | 107 +++++-----------
 core/metrics/grafana/grafana.ini              |   2 +-
 .../grafana/provisioning/datasources/loki.yml |   4 +-
 .../provisioning/datasources/mysql.yml        |  17 +--
 .../provisioning/datasources/prometheus.yml   |   4 +-
 .../grafana/provisioning/notifiers/main.yml   |   4 +-
 .../grafana/provisioning/notifiers/slack.yml  |   8 +-
 core/metrics/metrics.tf                       |  32 ++---
 core/metrics/prometheus.nomad                 |  43 +++----
 core/syslogng.nomad                           |  22 +---
 databases/redis.nomad                         |  14 ---
 service.nomad                                 |  21 ++--
 services/backups/backup.nomad                 |  78 +++---------
 services/ip-dvr.nomad                         | 115 +-----------------
 services/main.tf                              |  99 ++++++++-------
 services/media/caddy.nomad                    |  18 +--
 services/nextcloud/nextcloud.nomad            |   4 +-
 services/service/service_template.nomad       | 110 +++--------------
 services/whoami.nomad                         |  26 +---
 22 files changed, 208 insertions(+), 579 deletions(-)

diff --git a/.secrets-baseline b/.secrets-baseline
index af31148..1db1eab 100644
--- a/.secrets-baseline
+++ b/.secrets-baseline
@@ -193,25 +193,7 @@
         "line_number": 252,
         "is_secret": false
       }
-    ],
-    "core/syslogng.nomad": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "core/syslogng.nomad",
-        "hashed_secret": "298b5925fe7c7458cb8a12a74621fdedafea5ad6",
-        "is_verified": false,
-        "line_number": 165,
-        "is_secret": false
-      },
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "core/syslogng.nomad",
-        "hashed_secret": "3a1cec2d3c3de7e4da4d99c6731ca696c24b72b4",
-        "is_verified": false,
-        "line_number": 165,
-        "is_secret": false
-      }
     ]
   },
-  "generated_at": "2023-01-13T23:47:42Z"
+  "generated_at": "2023-03-24T18:23:24Z"
 }
diff --git a/core/blocky/blocky.nomad b/core/blocky/blocky.nomad
index ecf078f..a22aac2 100644
--- a/core/blocky/blocky.nomad
+++ b/core/blocky/blocky.nomad
@@ -45,11 +45,8 @@ job "blocky" {
       provider = "nomad"
       port = "api"
 
-      meta {
-        metrics_addr = "${NOMAD_ADDR_api}"
-      }
-
       tags = [
+        "prometheus.scrape",
         "traefik.enable=true",
         "traefik.http.routers.blocky-api.entryPoints=websecure",
       ]
diff --git a/core/metrics/exporters.nomad b/core/metrics/exporters.nomad
index 5f57852..c2cafc2 100644
--- a/core/metrics/exporters.nomad
+++ b/core/metrics/exporters.nomad
@@ -14,40 +14,16 @@ job "metrics" {
 
     service {
       name = "promtail"
+      provider = "nomad"
       port = "promtail"
 
       meta {
-        metrics_addr = "${NOMAD_ADDR_promtail}"
         nomad_dc = "${NOMAD_DC}"
         nomad_node_name = "${node.unique.name}"
       }
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 9080
-
-            upstreams {
-              destination_name = "loki"
-              local_bind_port = 1000
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 20
-          }
-        }
-      }
-
-      check {
-        type = "http"
-        path = "/metrics"
-        port = "promtail"
-        interval = "10s"
-        timeout = "10s"
+      tags = [
+        "prometheus.scrape",
       }
     }
 
@@ -111,7 +87,9 @@ server:
   http_listen_port: 9080
 
 clients:
-  - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "loki" -}}
+  - url: http://{{ .Address }}:{{ .Port }}/loki/api/v1/push
+{{- end }}
 
 scrape_configs:
 
diff --git a/core/metrics/grafana.nomad b/core/metrics/grafana.nomad
index e6619a5..d856827 100644
--- a/core/metrics/grafana.nomad
+++ b/core/metrics/grafana.nomad
@@ -8,10 +8,11 @@ job "grafana" {
       mode = "bridge"
 
       port "web" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 3000
       }
 
+      # TODO: Not sure if this needs to be mapped here since it's within the group
       port "renderer" {
         host_network = "loopback"
         to = 8081
@@ -25,46 +26,9 @@ job "grafana" {
 
     service {
       name = "grafana"
+      provider = "nomad"
       port = "web"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 3000
-
-            upstreams {
-              destination_name = "prometheus"
-              local_bind_port = 9090
-            }
-
-            upstreams {
-              destination_name = "loki"
-              local_bind_port = 3100
-            }
-
-            upstreams {
-              destination_name = "mysql-server"
-              local_bind_port = 6060
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-          }
-        }
-      }
-
-      check {
-        type = "http"
-        path = "/"
-        port = "web"
-        interval = "10s"
-        timeout = "10s"
-      }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.grafana.entryPoints=websecure",
@@ -98,25 +62,27 @@ job "grafana" {
       template {
         data = <<EOF
 [client]
-host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
-port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+host={{ .Address }}
+port={{ .Port }}
+{{ end -}}
 user=root
-{{ with secret "kv/data/mysql" }}
-password={{ .Data.data.root_password }}
-{{ end }}
+{{ with nomadVar "nomad/jobs" -}}
+password={{ .mysql_root_password }}
+{{ end -}}
         EOF
         destination = "$${NOMAD_SECRETS_DIR}/my.cnf"
       }
 
       template {
         data = <<EOF
-{{ with secret "kv/data/grafana" -}}
-{{ if .Data.data.db_name -}}
-CREATE DATABASE IF NOT EXISTS `{{ .Data.data.db_name }}`;
-CREATE USER IF NOT EXISTS '{{ .Data.data.db_user }}'@'%' IDENTIFIED BY '{{ .Data.data.db_pass }}';
-GRANT ALL ON `{{ .Data.data.db_name }}`.* to '{{ .Data.data.db_user }}'@'%';
--- Add grafana read_only user
-CREATE USER IF NOT EXISTS '{{ .Data.data.db_user_ro }}'@'%' IDENTIFIED BY '{{ .Data.data.db_pass_ro }}';
+{{ with nomadVar "nomad/jobs/grafana" -}}
+{{ if .db_name -}}
+CREATE DATABASE IF NOT EXISTS `{{ .db_name }}`;
+CREATE USER IF NOT EXISTS '{{ .db_user }}'@'%' IDENTIFIED BY '{{ .db_pass }}';
+GRANT ALL ON `{{ .db_name }}`.* to '{{ .db_user }}'@'%';
+-- Create Read Only user
+CREATE USER IF NOT EXISTS '{{ .db_user_ro }}'@'%' IDENTIFIED BY '{{ .db_pass_ro }}';
 {{ else -}}
 SELECT 'NOOP';
 {{ end -}}
@@ -149,35 +115,28 @@ SELECT 'NOOP';
         "GF_INSTALL_PLUGINS" = "grafana-clock-panel,grafana-piechart-panel,grafana-polystat-panel,natel-discrete-panel",
       }
 
-      vault {
-        policies = [
-          "access-tables",
-          "nomad-task",
-        ]
-      }
-
       template {
         data = <<EOF
-{{ with secret "kv/data/grafana" -}}
-GF_SECURITY_ADMIN_PASSWORD={{ .Data.data.admin_pw }}
-GF_SMTP_USER={{ .Data.data.smtp_user }}
-GF_SMTP_PASSWORD={{ .Data.data.smtp_password }}
-GF_EXTERNAL_IMAGE_STORAGE_S3_ACCESS_KEY={{ .Data.data.minio_access_key }}
-GF_EXTERNAL_IMAGE_STORAGE_S3_SECRET_KEY={{ .Data.data.minio_secret_key }}
+{{ with nomadVar "nomad/jobs/grafana" -}}
+GF_SECURITY_ADMIN_PASSWORD={{ .admin_pw }}
+GF_SMTP_USER={{ .smtp_user }}
+GF_SMTP_PASSWORD={{ .smtp_password }}
+GF_EXTERNAL_IMAGE_STORAGE_S3_ACCESS_KEY={{ .minio_access_key }}
+GF_EXTERNAL_IMAGE_STORAGE_S3_SECRET_KEY={{ .minio_secret_key }}
 GRAFANA_ALERT_EMAIL_ADDRESSES={{ .Data.data.alert_email_addresses }}
 {{ if .Data.data.db_name -}}
 # Database storage
 GF_DATABASE_TYPE=mysql
-GF_DATABASE_HOST={{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}
-GF_DATABASE_NAME={{ .Data.data.db_name }}
-GF_DATABASE_USER={{ .Data.data.db_user }}
-GF_DATABASE_PASSWORD={{ .Data.data.db_pass }}
-{{ end -}}
-{{ end -}}
-{{ with secret "kv/data/slack" -}}
-SLACK_BOT_URL={{ .Data.data.bot_url }}
-SLACK_BOT_TOKEN={{ .Data.data.bot_token }}
-SLACK_HOOK_URL={{ .Data.data.hook_url }}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+GF_DATABASE_HOST={{ .Address }}:{{ .Port }}
+{{- end }}
+GF_DATABASE_NAME={{ .db_name }}
+GF_DATABASE_USER={{ .db_user }}
+GF_DATABASE_PASSWORD={{ .db_pass }}
+{{- end }}
+SLACK_BOT_URL={{ .slack_bot_url }}
+SLACK_BOT_TOKEN={{ .slack_bot_token }}
+SLACK_HOOK_URL={{ .slack_hook_url }}
 {{ end -}}
         EOF
         env = true
diff --git a/core/metrics/grafana/grafana.ini b/core/metrics/grafana/grafana.ini
index 5126333..148869a 100644
--- a/core/metrics/grafana/grafana.ini
+++ b/core/metrics/grafana/grafana.ini
@@ -281,7 +281,7 @@ log_queries =
 
 #################################### Auth Proxy ##########################
 [auth.proxy]
-{{ with service "traefik" -}}
+{{ with nomadService "traefik" -}}
 enabled = true
 header_name = X-WEBAUTH-USER
 header_property = username
diff --git a/core/metrics/grafana/provisioning/datasources/loki.yml b/core/metrics/grafana/provisioning/datasources/loki.yml
index 7ba5958..71fd830 100644
--- a/core/metrics/grafana/provisioning/datasources/loki.yml
+++ b/core/metrics/grafana/provisioning/datasources/loki.yml
@@ -2,9 +2,11 @@
 apiVersion: 1
 
 datasources:
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "loki" -}}
   - name: Loki
-    url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}
+    url: http://{{ .Address }}:{{ .Port }}
     type: loki
     access: proxy
     isDefault: false
     version: 1
+{{- end }}
diff --git a/core/metrics/grafana/provisioning/datasources/mysql.yml b/core/metrics/grafana/provisioning/datasources/mysql.yml
index e56d51c..5f7f433 100644
--- a/core/metrics/grafana/provisioning/datasources/mysql.yml
+++ b/core/metrics/grafana/provisioning/datasources/mysql.yml
@@ -2,16 +2,17 @@
 apiVersion: 1
 
 datasources:
-  {{ with secret "kv/data/blocky" }}
   - name: Blocky logs
-    url: {{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}
-    database: {{ .Data.data.db_name }}
+    {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+    url: {{ .Address }}:{{ .Port }}
+    {{- end }}
+    # TODO: Looking for an acl friendly way to expose this since it's a variable in blocky setup
+    database: blocky
     type: mysql
     isDefault: false
     version: 1
-    {{ with secret "kv/data/grafana" }}
-    user: {{ .Data.data.db_user_ro }}
+    {{ with nomadVar "nomad/jobs/grafana" -}}
+    user: {{ .db_user_ro }}
     secureJsonData:
-      password: {{ .Data.data.db_pass_ro }}
-    {{ end }}
-  {{ end -}}
+      password: {{ .db_pass_ro }}
+    {{- end }}
diff --git a/core/metrics/grafana/provisioning/datasources/prometheus.yml b/core/metrics/grafana/provisioning/datasources/prometheus.yml
index a971b93..81f4f49 100644
--- a/core/metrics/grafana/provisioning/datasources/prometheus.yml
+++ b/core/metrics/grafana/provisioning/datasources/prometheus.yml
@@ -2,9 +2,11 @@
 apiVersion: 1
 
 datasources:
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "prometheus" -}}
   - name: Prometheus
-    url: http://{{ env "NOMAD_UPSTREAM_ADDR_prometheus" }}
+    url: http://{{ .Address }}:{{ .Port }}
     type: prometheus
     access: proxy
     isDefault: true
     version: 1
+{{- end }}
diff --git a/core/metrics/grafana/provisioning/notifiers/main.yml b/core/metrics/grafana/provisioning/notifiers/main.yml
index 6bb0c53..5b6079a 100644
--- a/core/metrics/grafana/provisioning/notifiers/main.yml
+++ b/core/metrics/grafana/provisioning/notifiers/main.yml
@@ -1,5 +1,5 @@
 ---
-{{ with secret "kv/data/grafana" -}}
+{{ with nomadVar "nomad/jobs/grafana" -}}
 notifiers:
   - name: Personal email
     type: email
@@ -7,5 +7,5 @@ notifiers:
     org_id: 1
     is_default: false
     settings:
-      addresses: "{{ .Data.data.alert_email_addresses }}"
+      addresses: "{{ .alert_email_addresses }}"
 {{ end -}}
diff --git a/core/metrics/grafana/provisioning/notifiers/slack.yml b/core/metrics/grafana/provisioning/notifiers/slack.yml
index c4e52b7..1d7edb3 100644
--- a/core/metrics/grafana/provisioning/notifiers/slack.yml
+++ b/core/metrics/grafana/provisioning/notifiers/slack.yml
@@ -1,5 +1,5 @@
 ---
-{{ with secret "kv/data/slack" -}}
+{{ with nomadVar "nomad/jobs/grafana" -}}
 notifiers:
   - name: Slack Bot
     type: slack
@@ -7,11 +7,11 @@ notifiers:
     org_id: 1
     is_default: false
     settings:
-      url: "{{ .Data.data.bot_url }}"
+      url: "{{ .slack_bot_url }}"
       recipient: "#site-notifications"
       username: Grafana Alerts
       icon_url: https://grafana.iamthefij.com/public/img/grafana_icon.svg
-      token: "{{ .Data.data.bot_token }}"
+      token: "{{ .slack_bot_token }}"
       uploadImage: true
       mentionChannel: channel
   - name: Slack Hook
@@ -20,7 +20,7 @@ notifiers:
     org_id: 1
     is_default: true
     settings:
-      url: "{{ .Data.data.hook_url }}"
+      url: "{{ .slack_hook_url }}"
       icon_url: https://grafana.iamthefij.com/public/img/grafana_icon.svg
       uploadImage: true
       mentionChannel: channel
diff --git a/core/metrics/metrics.tf b/core/metrics/metrics.tf
index 72429bb..7fd0b1b 100644
--- a/core/metrics/metrics.tf
+++ b/core/metrics/metrics.tf
@@ -22,25 +22,25 @@ resource "nomad_job" "grafana" {
   jobspec = templatefile("${path.module}/grafana.nomad", {
     module_path = path.module
   })
-  
+
   depends_on = [nomad_job.prometheus]
 }
 
-resource "consul_config_entry" "prometheus_intent" {
-  name = "prometheus"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "grafana"
-        Precedence = 9
-        Type       = "consul"
-      },
-    ]
-  })
-}
+# resource "consul_config_entry" "prometheus_intent" {
+#   name = "prometheus"
+#   kind = "service-intentions"
+#
+#   config_json = jsonencode({
+#     Sources = [
+#       {
+#         Action     = "allow"
+#         Name       = "grafana"
+#         Precedence = 9
+#         Type       = "consul"
+#       },
+#     ]
+#   })
+# }
 
 # resource "consul_config_entry" "envoy_prometheus_bind" {
 #   name = "global"
diff --git a/core/metrics/prometheus.nomad b/core/metrics/prometheus.nomad
index a879984..b2677f0 100644
--- a/core/metrics/prometheus.nomad
+++ b/core/metrics/prometheus.nomad
@@ -8,7 +8,7 @@ job "prometheus" {
       mode = "bridge"
 
       port "web" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 9090
       }
     }
@@ -20,31 +20,9 @@ job "prometheus" {
 
     service {
       name = "prometheus"
+      provider = "nomad"
       port = "web"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 9090
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-          }
-        }
-      }
-
-      check {
-        type = "http"
-        path = "/"
-        port = "web"
-        interval = "10s"
-        timeout = "10s"
-      }
-
       // TODO: Remove traefik tags
       tags = [
         "traefik.enable=true",
@@ -110,6 +88,23 @@ scrape_configs:
         replacement: $1:8500
         target_label: __address__
 
+  - job_name: "nomad_services"
+    metrics_path: "/metrics"
+    nomad_sd_configs:
+      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
+    relabel_configs:
+      - source_labels: [__meta_nomad_tags]
+        regex: .*(prometheus.scrape).*
+        action: keep
+      - source_labels: [__meta_nomad_address]
+        target_label: __address__
+      - source_labels: [__meta_nomad_service]
+        target_label: nomad_service
+      - source_labels: [__meta_nomad_dc]
+        target_label: nomad_dc
+      - source_labels: [__meta_nomad_node_id]
+        target_label: nomad_node_id
+
   - job_name: "exporters"
     metrics_path: "/metrics"
     consul_sd_configs:
diff --git a/core/syslogng.nomad b/core/syslogng.nomad
index e60988d..da5f1aa 100644
--- a/core/syslogng.nomad
+++ b/core/syslogng.nomad
@@ -89,28 +89,8 @@ EOF
 
     service {
       name = "syslogng"
+      provider = "nomad"
       port = "main"
-
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 514
-
-            upstreams {
-              destination_name = "syslogng-promtail"
-              local_bind_port = 1000
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 20
-            memory_max = 50
-          }
-        }
-      }
     }
 
     task "syslogng" {
diff --git a/databases/redis.nomad b/databases/redis.nomad
index ff8ab3b..40e0088 100644
--- a/databases/redis.nomad
+++ b/databases/redis.nomad
@@ -18,26 +18,12 @@ job "redis" {
       port "main" {
         to = 6379
       }
-      port "envoy_metrics" {
-        to = 9123
-      }
     }
 
     service {
       name = "redis"
       provider = "nomad"
       port = "main"
-
-      # check {
-      #   name     = "alive"
-      #   type     = "tcp"
-      #   interval = "10s"
-      #   timeout  = "2s"
-      # }
-
-      meta {
-        envoy_metrics_addr = "${NOMAD_ADDR_envoy_metrics}"
-      }
     }
 
     task "redis" {
diff --git a/service.nomad b/service.nomad
index e6d6912..54a1fd7 100644
--- a/service.nomad
+++ b/service.nomad
@@ -198,11 +198,14 @@ EOF
       template {
         data = <<EOF
 [client]
-host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
-port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+host={{ .Address }}
+port={{ .Port }}
+{{ end -}}
 user=root
-{{ with secret "kv/data/mysql" -}}
-password={{ .Data.data.root_password }}
+# TODO: Use via lesser scoped access
+{{ with nomadVar "nomad/jobs" -}}
+password={{ .mysql_root_password }}
 {{ end -}}
         EOF
         destination = "${NOMAD_SECRETS_DIR}/my.cnf"
@@ -210,13 +213,13 @@ password={{ .Data.data.root_password }}
 
       template {
         data = <<EOF
-{{ with secret "[[.vault_key]]" -}}
-CREATE DATABASE IF NOT EXISTS `{{ .Data.data.[[ default "db_name" .db_name_key ]] }}`
+{{ with nomadVar "[[.vault_key]]" -}}
+CREATE DATABASE IF NOT EXISTS `{{ .[[ default "db_name" .db_name_key ]] }}`
   CHARACTER SET = 'utf8mb4'
   COLLATE = 'utf8mb4_unicode_ci';
-CREATE USER IF NOT EXISTS '{{ .Data.data.[[ default "db_user" .db_user_key ]] }}'@'%'
-  IDENTIFIED BY '{{ .Data.data.[[ default "db_pass" .db_pass_key ]] }}';
-GRANT ALL ON `{{ .Data.data.[[ default "db_name" .db_name_key ]] }}`.* to '{{ .Data.data.[[ default "db_user" .db_user_key ]] }}'@'%';
+CREATE USER IF NOT EXISTS '{{ .[[ default "db_user" .db_user_key ]] }}'@'%'
+  IDENTIFIED BY '{{ .[[ default "db_pass" .db_pass_key ]] }}';
+GRANT ALL ON `{{ .[[ default "db_name" .db_name_key ]] }}`.* to '{{ .[[ default "db_user" .db_user_key ]] }}'@'%';
 {{ end -}}
 {{ end -}}
         EOF
diff --git a/services/backups/backup.nomad b/services/backups/backup.nomad
index e8747c1..25af9aa 100644
--- a/services/backups/backup.nomad
+++ b/services/backups/backup.nomad
@@ -50,54 +50,12 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
 
     service {
       name = "backups"
+      provider = "nomad"
       port = "metrics"
 
-      # Add connect to mysql
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 8080
-
-            # TODO: Do I need this?
-            expose {
-              path {
-                path = "/metrics"
-                protocol = "http"
-                local_path_port = 8080
-                listener_port = "metrics"
-              }
-            }
-
-            upstreams {
-              destination_name = "mysql-server"
-              local_bind_port = 6060
-            }
-
-            config {
-              protocol = "tcp"
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-          }
-        }
-      }
-
-      check {
-        port = "metrics"
-        type = "http"
-        path = "/health"
-        interval = "10s"
-        timeout = "3s"
-      }
-
-      meta {
-        metrics_addr = "$${NOMAD_ADDR_metrics}"
-      }
+      tags = [
+        "prometheus.scrape"
+      ]
     }
 
     task "backup" {
@@ -122,16 +80,7 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
         ]
       }
 
-      vault {
-        policies = [
-          "access-tables",
-          "nomad-task",
-        ]
-      }
-
       env = {
-        "MYSQL_HOST" = "$${NOMAD_UPSTREAM_IP_mysql_server}"
-        "MYSQL_PORT" = "$${NOMAD_UPSTREAM_PORT_mysql_server}"
         "RCLONE_CHECKERS" = "2"
         "RCLONE_TRANSFERS" = "2"
         "RCLONE_FTP_CONCURRENCY" = "5"
@@ -139,15 +88,20 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
 
       template {
         data = <<EOF
-{{ with secret "kv/data/mysql" }}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+MYSQL_HOST={{ .Address }}
+MYSQL_PORT={{ .Port }}
+{{- end }}
+# TODO: Move this to new mysql root pass path
+{{ with nomadVar "nomad/jobs" }}
 MYSQL_USER=root
-MYSQL_PASSWORD={{ .Data.data.root_password }}
+MYSQL_PASSWORD={{ .mysql_root_password }}
 {{ end -}}
-{{ with secret "kv/data/backups" -}}
-BACKUP_PASSPHRASE={{ .Data.data.backup_passphrase }}
-RCLONE_FTP_HOST={{ .Data.data.nas_ftp_host }}
-RCLONE_FTP_USER={{ .Data.data.nas_ftp_user }}
-RCLONE_FTP_PASS={{ .Data.data.nas_ftp_pass | toJSON }}
+{{ with nomadVar "nomad/jobs/backups" -}}
+BACKUP_PASSPHRASE={{ .backup_passphrase }}
+RCLONE_FTP_HOST={{ .nas_ftp_host }}
+RCLONE_FTP_USER={{ .nas_ftp_user }}
+RCLONE_FTP_PASS={{ .nas_ftp_pass.Value | toJSON }}
 RCLONE_FTP_EXPLICIT_TLS=true
 RCLONE_FTP_NO_CHECK_CERTIFICATE=true
 {{ end -}}
diff --git a/services/ip-dvr.nomad b/services/ip-dvr.nomad
index 5776ded..adadacb 100644
--- a/services/ip-dvr.nomad
+++ b/services/ip-dvr.nomad
@@ -9,12 +9,9 @@ job "ipdvr" {
       mode = "bridge"
 
       port "main" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 8080
       }
-      port "envoy_metrics" {
-        to = 9123
-      }
     }
 
     volume "sabnzbd-config" {
@@ -31,36 +28,13 @@ job "ipdvr" {
 
     service {
       name = "sabnzbd"
+      provider = "nomad"
       port = "main"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 8080
-
-            config {
-              envoy_prometheus_bind_addr = "0.0.0.0:9123"
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-            memory_max = 100
-          }
-        }
-      }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.sabnzbd.entryPoints=websecure",
       ]
-
-      meta {
-        envoy_metrics_addr = "${NOMAD_ADDR_envoy_metrics}"
-      }
     }
 
     task "sabnzbd" {
@@ -101,12 +75,9 @@ job "ipdvr" {
     network {
       mode = "bridge"
       port "main" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 6789
       }
-      port "envoy_metrics" {
-        to = 9123
-      }
     }
 
     volume "nzbget-config" {
@@ -123,44 +94,13 @@ job "ipdvr" {
 
     service {
       name = "nzbget"
+      provider = "nomad"
       port = "main"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 6789
-
-            config {
-              envoy_prometheus_bind_addr = "0.0.0.0:9123"
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-            memory_max = 100  
-          }
-        }
-      }
-
-      # check {
-      #   type = "http"
-      #   path = "/"
-      #   port = "main"
-      #   interval = "10s"
-      #   timeout = "10s"
-      # }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.nzbget.entryPoints=websecure",
       ]
-
-      meta {
-        envoy_metrics_addr = "${NOMAD_ADDR_envoy_metrics}"
-      }
     }
 
     task "nzbget" {
@@ -202,12 +142,9 @@ job "ipdvr" {
     network {
       mode = "bridge"
       port "main" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 8989
       }
-      port "envoy_metrics" {
-        to = 9123
-      }
     }
 
     volume "sonarr-data" {
@@ -224,53 +161,13 @@ job "ipdvr" {
 
     service {
       name = "sonarr"
+      provider = "nomad"
       port = "main"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 8989
-            upstreams {
-              destination_name = "nzbget"
-              local_bind_port = 6789
-            }
-
-            upstreams {
-              destination_name = "sabnzbd"
-              local_bind_port = 8080
-            }
-
-            config {
-              envoy_prometheus_bind_addr = "0.0.0.0:9123"
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-            memory_max = 100
-          }
-        }
-      }
-
-      # check {
-      #   type = "http"
-      #   path = "/"
-      #   port = "main"
-      #   interval = "10s"
-      #   timeout = "10s"
-      # }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.sonarr.entryPoints=websecure",
       ]
-
-      meta {
-        envoy_metrics_addr = "${NOMAD_ADDR_envoy_metrics}"
-      }
     }
 
     task "sonarr" {
diff --git a/services/main.tf b/services/main.tf
index 0ea798d..d0570f6 100644
--- a/services/main.tf
+++ b/services/main.tf
@@ -27,41 +27,41 @@ resource "nomad_job" "ipdvr" {
   jobspec = file("${path.module}/ip-dvr.nomad")
 }
 
-resource "consul_config_entry" "sabnzbd_intents" {
-  depends_on = [nomad_job.ipdvr]
-
-  name = "sabnzbd"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "sonarr"
-        Precedence = 9
-        Type       = "consul"
-      },
-    ]
-  })
-}
-
-resource "consul_config_entry" "nzbget_intents" {
-  depends_on = [nomad_job.ipdvr]
-
-  name = "nzbget"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "sonarr"
-        Precedence = 9
-        Type       = "consul"
-      },
-    ]
-  })
-}
+# resource "consul_config_entry" "sabnzbd_intents" {
+#   depends_on = [nomad_job.ipdvr]
+#
+#   name = "sabnzbd"
+#   kind = "service-intentions"
+#
+#   config_json = jsonencode({
+#     Sources = [
+#       {
+#         Action     = "allow"
+#         Name       = "sonarr"
+#         Precedence = 9
+#         Type       = "consul"
+#       },
+#     ]
+#   })
+# }
+#
+# resource "consul_config_entry" "nzbget_intents" {
+#   depends_on = [nomad_job.ipdvr]
+#
+#   name = "nzbget"
+#   kind = "service-intentions"
+#
+#   config_json = jsonencode({
+#     Sources = [
+#       {
+#         Action     = "allow"
+#         Name       = "sonarr"
+#         Precedence = 9
+#         Type       = "consul"
+#       },
+#     ]
+#   })
+# }
 
 module "media-library" {
   source = "./service"
@@ -88,14 +88,12 @@ module "minitor" {
   args              = ["-metrics", "-config=$${NOMAD_TASK_DIR}/config.yml"]
   service_port      = 8080
   metrics_port_name = "main"
-  healthcheck_path  = "/metrics"
-  use_vault         = true
 
   templates = [
     {
       data        = <<EOF
-      {{ with secret "kv/data/mailgun_api" -}}
-      MAILGUN_API_KEY={{ .Data.data.key }}
+      {{ with nomadVar "nomad/jobs/minitor" -}}
+      MAILGUN_API_KEY={{ .mailgun_api_key }}
       {{ end -}}
       EOF
       dest        = "env"
@@ -161,19 +159,20 @@ module "photoprism_module" {
     },
   ]
   mysql_bootstrap = {
-    vault_key = "kv/data/photoprism"
+    vault_key = "nomad/jobs/photoprism"
   }
   templates = [
     {
       data        = <<EOF
-      {{ with secret "kv/data/photoprism" -}}
-      PHOTOPRISM_ADMIN_USER={{ .Data.data.admin_user }}
-      PHOTOPRISM_ADMIN_PASSWORD={{ .Data.data.admin_password }}
+      {{ with nomadVar "nomad/jobs/photoprism" -}}
+      PHOTOPRISM_ADMIN_USER={{ .admin_user }}
+      PHOTOPRISM_ADMIN_PASSWORD={{ .admin_password }}
       PHOTOPRISM_DATABASE_DRIVER=mysql
-      PHOTOPRISM_DATABASE_NAME={{ .Data.data.db_name }}
-      PHOTOPRISM_DATABASE_USER={{ .Data.data.db_user }}
-      PHOTOPRISM_DATABASE_PASSWORD={{ .Data.data.db_pass }}
-      PHOTOPRISM_DATABASE_SERVER="{{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}"
+      PHOTOPRISM_DATABASE_NAME={{ .db_name }}
+      PHOTOPRISM_DATABASE_USER={{ .db_user }}
+      PHOTOPRISM_DATABASE_PASSWORD={{ .db_pass }}
+      {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+      PHOTOPRISM_DATABASE_SERVER="{{ .Address" }}:{{ .Port }}"
       {{ end -}}
       EOF
       dest_prefix = "$${NOMAD_SECRETS_DIR}/"
@@ -206,9 +205,9 @@ module "diun" {
   templates = [
     {
       data        = <<EOF
-      {{ with secret "kv/data/slack" -}}
-      DIUN_NOTIF_SLACK_WEBHOOKURL={{ .Data.data.hook_url }}
-      {{ end -}}
+      {{ with nomadVar "nomad/jobs/diun" -}}
+      DIUN_NOTIF_SLACK_WEBHOOKURL={{ .slack_hook_url }}
+      {{- end }}
       EOF
       dest_prefix = "$${NOMAD_SECRETS_DIR}"
       dest        = "env"
diff --git a/services/media/caddy.nomad b/services/media/caddy.nomad
index 019108d..9991ad6 100644
--- a/services/media/caddy.nomad
+++ b/services/media/caddy.nomad
@@ -8,7 +8,7 @@ job "multimedia" {
     network {
       mode = "bridge"
       port "web" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 80
       }
     }
@@ -21,23 +21,9 @@ job "multimedia" {
 
     service {
       name = "library"
+      provider = "nomad"
       port = "web"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 80
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-          }
-        }
-      }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.library.entryPoints=websecure",
diff --git a/services/nextcloud/nextcloud.nomad b/services/nextcloud/nextcloud.nomad
index 916165d..16153e1 100644
--- a/services/nextcloud/nextcloud.nomad
+++ b/services/nextcloud/nextcloud.nomad
@@ -184,8 +184,8 @@ MYSQL_PASSWORD={{ .Data.data.db_pass }}
       name = "nextcloud-backups"
       port = "backup"
 
-      meta {
-        metrics_addr = "${NOMAD_ADDR_backup}"
+      tags = [
+        "prometheus.scrape",
       }
     }
 
diff --git a/services/service/service_template.nomad b/services/service/service_template.nomad
index 85d2551..3d70982 100644
--- a/services/service/service_template.nomad
+++ b/services/service/service_template.nomad
@@ -10,16 +10,11 @@ job "${name}" {
       %{ if service_port != null ~}
       port "main" {
         %{ if ingress }
-        host_network = "loopback"
+        host_network = "wgoverlay"
         %{~ endif }
         to = ${service_port}
       }
       %{ endif }
-      %{ if connect }
-      port "envoy_metrics" {
-        to = 9123
-      }
-      %{~ endif }
     }
 
     %{ if length(group_meta) > 0 }
@@ -45,74 +40,14 @@ job "${name}" {
     %{ if service_port != null ~}
     service {
       name = "${replace(name, "_", "-")}"
+      provider = "nomad"
       port = "main"
 
-      %{ if connect }
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = ${service_port}
-            %{ if use_mysql }
-            upstreams {
-              destination_name = "mysql-server"
-              local_bind_port = 4040
-            }
-            %{ endif ~}
-            %{ if use_redis }
-            upstreams {
-              destination_name = "redis"
-              local_bind_port = 6379
-            }
-            %{ endif ~}
-            %{ if use_ldap }
-            upstreams {
-              destination_name = "lldap"
-              local_bind_port = 3890
-            }
-            %{ endif ~}
-            %{ for upstream in upstreams ~}
-            upstreams {
-              destination_name = "${upstream.destination_name}"
-              local_bind_port = ${upstream.local_bind_port}
-            }
-            %{ endfor }
-
-            config {
-              envoy_prometheus_bind_addr = "0.0.0.0:9123"
-            }
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 50
-            memory_max = 100
-          }
-        }
-      }
-
-      %{~ endif }
-      %{ if healthcheck_path != null }
-      check {
-        type = "http"
-        path = "${healthcheck_path}"
-        port = "main"
-        interval = "10s"
-        timeout = "10s"
-      }
-
-      %{~ endif }
-      meta {
-        %{ if metrics_port_name != null }
-        metrics_addr = "$${NOMAD_ADDR_${metrics_port_name}}"
-        %{ endif }
-        %{ if connect }
-        envoy_metrics_addr = "$${NOMAD_ADDR_envoy_metrics}"
-        %{~ endif }
-      }
-
       tags = [
+        # TODO: Rename metrics_port_name to something like "prometheus_scrape"
+        %{ if metrics_port_name != null }
+        "prometheus.scrape",
+        %{ endif }
         %{ if ingress }
         "traefik.enable=true",
         "traefik.http.routers.${name}.entryPoints=websecure",
@@ -160,15 +95,6 @@ job "${name}" {
         %{ endif ~}
         %{ endfor ~}
       }
-      %{ if use_vault ~}
-
-      vault {
-        policies = [
-          "access-tables",
-          "nomad-task",
-        ]
-      }
-      %{ endif ~}
       %{ if length(env) > 0 ~}
 
       env = {
@@ -236,11 +162,14 @@ EOF
       template {
         data = <<EOF
 [client]
-host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
-port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-server" -}}
+host={{ .Address }}
+port={{ .Port }}
+{{ end -}}
 user=root
-{{ with secret "kv/data/mysql" -}}
-password={{ .Data.data.root_password }}
+# TODO: Use via lesser scoped access
+{{ with nomadVar "nomad/jobs" -}}
+password={{ .mysql_root_password }}
 {{ end -}}
         EOF
         destination = "$${NOMAD_SECRETS_DIR}/my.cnf"
@@ -248,14 +177,15 @@ password={{ .Data.data.root_password }}
 
       template {
         data = <<EOF
-{{ with secret "${mysql_bootstrap.vault_key}" -}}
-CREATE DATABASE IF NOT EXISTS `{{ .Data.data.${mysql_bootstrap.db_name_key} }}`
+{{ with nomadVar "${mysql_bootstrap.vault_key}" -}}
+CREATE DATABASE IF NOT EXISTS `{{ .${mysql_bootstrap.db_name_key} }}`
   CHARACTER SET = 'utf8mb4'
   COLLATE = 'utf8mb4_unicode_ci';
-CREATE USER IF NOT EXISTS '{{ .Data.data.${mysql_bootstrap.db_user_key} }}'@'%'
-  IDENTIFIED BY '{{ .Data.data.${mysql_bootstrap.db_pass_key} }}';
-GRANT ALL ON `{{ .Data.data.${mysql_bootstrap.db_name_key} }}`.*
-  TO '{{ .Data.data.${mysql_bootstrap.db_user_key} }}'@'%';
+CREATE USER IF NOT EXISTS '{{ .${mysql_bootstrap.db_user_key} }}'@'%'
+  IDENTIFIED BY '{{ .${mysql_bootstrap.db_pass_key} }}';
+GRANT ALL ON `{{ .${mysql_bootstrap.db_name_key} }}`.*
+  TO '{{ .${mysql_bootstrap.db_user_key} }}'@'%';
+# TODO: Optionally grant ro access to ro user
 {{ else -}}
 SELECT 'NOOP';
 {{ end -}}
diff --git a/services/whoami.nomad b/services/whoami.nomad
index ac5a2a7..e0648ad 100644
--- a/services/whoami.nomad
+++ b/services/whoami.nomad
@@ -58,38 +58,16 @@ job "whoami" {
     network {
       mode = "bridge"
       port "web" {
-        host_network = "loopback"
+        host_network = "wgoverlay"
         to = 80
       }
     }
 
     service {
       name = "whoami"
+      provider = "nomad"
       port = "web"
 
-      connect {
-        sidecar_service {
-          proxy {
-            local_service_port = 80
-          }
-        }
-
-        sidecar_task {
-          resources {
-            cpu    = 50
-            memory = 20
-          }
-        }
-      }
-
-      check {
-        type = "http"
-        path = "/health"
-        port = "web"
-        interval = "10s"
-        timeout = "10s"
-      }
-
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.whoami.entryPoints=websecure",