From 0bd995ec2b3c0f4ea85b05eaf7051fdbd3b5d17f Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Wed, 3 Jan 2024 13:56:43 -0800
Subject: [PATCH] Traefik: Use nomad vars for dynamic certs

Rather than having Traefik handle cert fetching, instead
it is delegated to a separate job so that multiple Traefik
instances can share certs
---
 ansible_playbooks/vars/nomad_vars.sample.yml |  2 -
 core/traefik/.terraform.lock.hcl             | 26 ++++----
 core/traefik/traefik.nomad                   | 67 +++++++++++---------
 core/traefik/traefik.tf                      | 20 ++++++
 4 files changed, 70 insertions(+), 45 deletions(-)

diff --git a/ansible_playbooks/vars/nomad_vars.sample.yml b/ansible_playbooks/vars/nomad_vars.sample.yml
index 1b21a3c..207a197 100644
--- a/ansible_playbooks/vars/nomad_vars.sample.yml
+++ b/ansible_playbooks/vars/nomad_vars.sample.yml
@@ -134,8 +134,6 @@ nomad/jobs/redis-blocky:
 nomad/jobs/rediscommander:
   redis_stunnel_psk: VALUE
 nomad/jobs/traefik:
-  acme_email: VALUE
-  domain_lego_dns: VALUE
   usersfile: VALUE
 nomad/jobs/unifi-traffic-route-ips:
   unifi_password: VALUE
diff --git a/core/traefik/.terraform.lock.hcl b/core/traefik/.terraform.lock.hcl
index d216884..0b1316c 100644
--- a/core/traefik/.terraform.lock.hcl
+++ b/core/traefik/.terraform.lock.hcl
@@ -2,20 +2,20 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/nomad" {
-  version = "1.4.17"
+  version = "2.1.0"
   hashes = [
-    "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
-    "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
-    "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
-    "zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
-    "zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
-    "zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
-    "zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
+    "h1:ek0L7fA+4R1/BXhbutSRqlQPzSZ5aY/I2YfVehuYeEU=",
+    "zh:39ba4d4fc9557d4d2c1e4bf866cf63973359b73e908cce237c54384512bdb454",
+    "zh:40d2b66e3f3675e6b88000c145977c1d5288510c76b702c6c131d9168546c605",
+    "zh:40fbe575d85a083f96d4703c6b7334e9fc3e08e4f1d441de2b9513215184ebcc",
+    "zh:42ce6db79e2f94557fae516ee3f22e5271f0b556638eb45d5fbad02c99fc7af3",
+    "zh:4acf63dfb92f879b3767529e75764fef68886521b7effa13dd0323c38133ce88",
+    "zh:72cf35a13c2fb542cd3c8528826e2390db9b8f6f79ccb41532e009ad140a3269",
     "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
-    "zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
-    "zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
-    "zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
-    "zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
+    "zh:8b8bcc136c05916234cb0c3bcc3d48fda7ca551a091ad8461ea4ab16fb6960a3",
+    "zh:8e1c2f924eae88afe7ac83775f000ae8fd71a04e06228edf7eddce4df2421169",
+    "zh:abc6e725531fc06a8e02e84946aaabc3453ecafbc1b7a442ea175db14fd9c86a",
+    "zh:b735fcd1fb20971df3e92f81bb6d73eef845dcc9d3d98e908faa3f40013f0f69",
+    "zh:ce59797282505d872903789db8f092861036da6ec3e73f6507dac725458a5ec9",
   ]
 }
diff --git a/core/traefik/traefik.nomad b/core/traefik/traefik.nomad
index 8d2746a..6218d3a 100644
--- a/core/traefik/traefik.nomad
+++ b/core/traefik/traefik.nomad
@@ -100,6 +100,12 @@ job "traefik" {
           target = "/etc/traefik/usersfile"
           source = "secrets/usersfile"
         }
+
+        mount {
+          type = "bind"
+          target = "/etc/traefik/certs"
+          source = "secrets/certs"
+        }
       }
 
       template {
@@ -122,9 +128,6 @@ job "traefik" {
   [entryPoints.websecure]
     address = ":443"
     [entryPoints.websecure.http.tls]
-      certResolver = "letsEncrypt"
-      [[entryPoints.websecure.http.tls.domains]]
-        main = "*.<< with nomadVar "nomad/jobs" >><< .base_hostname >><< end >>"
 
   [entryPoints.metrics]
     address = ":8989"
@@ -158,30 +161,8 @@ job "traefik" {
   defaultRule = "Host(`{{normalize .Name}}.<< with nomadVar "nomad/jobs" >><< .base_hostname >><< end >>`)"
   [providers.nomad.endpoint]
   address = "http://<< env "attr.unique.network.ip-address" >>:4646"
-
-<< if nomadVarExists "nomad/jobs/traefik" ->>
-[certificatesResolvers.letsEncrypt.acme]
-  email = "<< with nomadVar "nomad/jobs/traefik" >><< .acme_email >><< end >>"
-  # Store in /local because /secrets doesn't persist with ephemeral disk
-  storage = "/local/acme.json"
-  [certificatesResolvers.letsEncrypt.acme.dnsChallenge]
-    provider = "cloudflare"
-    resolvers = ["1.1.1.1:53", "8.8.8.8:53"]
-    delayBeforeCheck = 0
-<<- end >>
         EOH
-        destination = "local/config/traefik.toml"
-      }
-
-      template {
-        data = <<EOH
-{{ with nomadVar "nomad/jobs/traefik" -}}
-CF_DNS_API_TOKEN={{ .domain_lego_dns }}
-CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
-{{- end }}
-        EOH
-        destination = "secrets/cloudflare.env"
-        env = true
+        destination = "${NOMAD_TASK_DIR}/config/traefik.toml"
       }
 
       template {
@@ -207,7 +188,7 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
         [[http.services.hass.loadBalancer.servers]]
           url = "http://192.168.3.65:8123"
         EOH
-        destination = "local/config/conf/route-hashi.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/route-hashi.toml"
         change_mode = "noop"
       }
 
@@ -244,7 +225,33 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
       {{ end -}}
 {{- end }}
         EOH
-        destination = "local/config/conf/route-syslog-ng.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/route-syslog-ng.toml"
+        change_mode = "noop"
+      }
+
+      template {
+        data = <<EOF
+{{- with nomadVar "secrets/certs/_lego/certificates/__thefij_rocks_crt" }}{{ .contents }}{{ end -}}"
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/certs/_.thefij.rocks.crt"
+        change_mode = "noop"
+      }
+
+      template {
+        data = <<EOF
+{{- with nomadVar "secrets/certs/_lego/certificates/__thefij_rocks_key" }}{{ .contents }}{{ end -}}"
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/certs/_.thefij.rocks.key"
+        change_mode = "noop"
+      }
+
+      template {
+        data = <<EOH
+[[tls.certificates]]
+  certFile = "/etc/traefik/certs/_.thefij.rocks.crt"
+  keyFile = "/etc/traefik/certs/_.thefij.rocks.key"
+        EOH
+        destination = "${NOMAD_TASK_DIR}/config/conf/dynamic-tls.toml"
         change_mode = "noop"
       }
 
@@ -258,7 +265,7 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
 {{- end }}
 {{- end }}
         EOH
-        destination = "local/config/conf/middlewares.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/middlewares.toml"
         change_mode = "noop"
       }
 
@@ -268,7 +275,7 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
 {{ .usersfile }}
 {{- end }}
         EOH
-        destination = "secrets/usersfile"
+        destination = "${NOMAD_SECRETS_DIR}/usersfile"
         change_mode = "noop"
       }
 
diff --git a/core/traefik/traefik.tf b/core/traefik/traefik.tf
index dcd8ed6..9a98e66 100644
--- a/core/traefik/traefik.tf
+++ b/core/traefik/traefik.tf
@@ -1,3 +1,23 @@
 resource "nomad_job" "traefik" {
   jobspec = file("${path.module}/traefik.nomad")
 }
+
+resource "nomad_acl_policy" "treafik_secrets_certs_read" {
+  name        = "traefik-secrets-certs-read"
+  description = "Read certs to secrets store"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/certs/*" {
+      capabilities = ["read"]
+    }
+    path "secrets/certs" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+  job_acl {
+    job_id = resource.nomad_job.traefik.id
+  }
+}