Add playbook for recovering nomad cluster with peers.json

Remove query for nomad-clients on backup task
Not used anymore
2022-11-01 15:03:08 -07:00 · 2022-10-31 15:27:29 -07:00 · 2022-10-31 15:24:14 -07:00 · 2022-10-31 15:23:42 -07:00 · 2022-10-31 11:32:09 -07:00 · 2022-10-28 12:42:28 -07:00
70 changed files with 1123 additions and 386 deletions
--- a/.secrets-baseline
+++ b/.secrets-baseline
@ -115,10 +115,10 @@
    }
  ],
  "results": {
-    "nomad/metrics/grafana/grafana.ini": [
+    "nomad/core/metrics/grafana/grafana.ini": [
      {
        "type": "Basic Auth Credentials",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "e5e9fa1ba31ecd1ae84f75caaa474f3a663f05f4",
        "is_verified": false,
        "line_number": 78,
@ -126,7 +126,7 @@
      },
      {
        "type": "Secret Keyword",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "55ebda65c08313526e7ba08ad733e5ebea9900bd",
        "is_verified": false,
        "line_number": 109,
@ -134,7 +134,7 @@
      },
      {
        "type": "Secret Keyword",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997",
        "is_verified": false,
        "line_number": 151,
@ -142,7 +142,7 @@
      },
      {
        "type": "Secret Keyword",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "10bea62ff1e1a7540dc7a6bc10f5fa992349023f",
        "is_verified": false,
        "line_number": 154,
@ -150,7 +150,7 @@
      },
      {
        "type": "Secret Keyword",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "5718bce97710e6be87ea160b36eaefb5032857d3",
        "is_verified": false,
        "line_number": 239,
@ -158,28 +158,28 @@
      },
      {
        "type": "Secret Keyword",
-        "filename": "nomad/metrics/grafana/grafana.ini",
+        "filename": "nomad/core/metrics/grafana/grafana.ini",
        "hashed_secret": "10aed9d7ebef778a9b3033dba3f7813b639e0d50",
        "is_verified": false,
        "line_number": 252,
        "is_secret": false
      }
    ],
-    "nomad/syslogng.nomad": [
+    "nomad/core/syslogng.nomad": [
      {
        "type": "Base64 High Entropy String",
-        "filename": "nomad/syslogng.nomad",
+        "filename": "nomad/core/syslogng.nomad",
        "hashed_secret": "298b5925fe7c7458cb8a12a74621fdedafea5ad6",
        "is_verified": false,
-        "line_number": 171,
+        "line_number": 159,
        "is_secret": false
      },
      {
        "type": "Base64 High Entropy String",
-        "filename": "nomad/syslogng.nomad",
+        "filename": "nomad/core/syslogng.nomad",
        "hashed_secret": "3a1cec2d3c3de7e4da4d99c6731ca696c24b72b4",
        "is_verified": false,
-        "line_number": 171,
+        "line_number": 159,
        "is_secret": false
      }
    ],
@ -210,5 +210,5 @@
      }
    ]
  },
-  "generated_at": "2022-09-04T21:06:55Z"
+  "generated_at": "2022-10-27T21:28:03Z"
 }
--- a/nomad/Makefile
+++ b/nomad/Makefile
@ -75,6 +75,11 @@ bootstrap-values: venv/bin/ansible galaxy
 		$(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \
 		-i ansible_hosts.yml -M ./roles ./bootstrap-values.yml

+.PHONY: unseal-vault
+unseal-vault: venv/bin/ansible galaxy
+	env VIRTUAL_ENV=/Users/ifij/workspace/iamthefij/orchestration-tests/nomad/venv ./venv/bin/ansible-playbook -K -vv \
+		-e "@vault-keys.json" -i ansible_hosts.yml -M ./roles ./unseal-vault.yml
+
 .PHONY: init
 init:
 	@terraform init
--- a/nomad/acls/nomad_policies.tf
+++ b/nomad/acls/nomad_policies.tf
@ -10,7 +10,7 @@ resource "nomad_acl_policy" "admin" {
  rules_hcl   = file("${path.module}/nomad-admin-policy.hcl")
 }

-# TODO: Limit this scope
+# TODO: (security) Limit this scope
 resource "nomad_acl_policy" "deploy" {
  name        = "deploy"
  description = "Write for job deployments"
--- a/nomad/ansible_hosts.yml
+++ b/nomad/ansible_hosts.yml
@ -20,6 +20,7 @@ all:
              mode: "0755"
              read_only: false
        n2.thefij:
+          nomad_node_class: ingress
          nomad_node_role: both
          nomad_unique_host_volumes:
            - name: nextcloud-data
@ -34,17 +35,25 @@ all:
              group: "bin"
              mode: "0755"
              read_only: false
-            - name: authentik-data
-              path: /srv/volumes/gitea
+            - name: sonarr-data
+              path: /srv/volumes/sonarr
              owner: "root"
              group: "bin"
              mode: "0755"
              read_only: false
-        n3.thefij:
-          nomad_node_class: ingress
-          nomad_node_role: both
-        pi4:
-          nomad_node_role: client
+            - name: nzbget-data
+              path: /srv/volumes/nzbget
+              owner: "root"
+              group: "bin"
+              mode: "0755"
+              read_only: false
+        # n3.thefij:
+        #   nomad_node_class: ingress
+        #   nomad_node_role: both
+        # pi3:
+        #   nomad_node_role: client
+        # pi4:
+        #   nomad_node_role: client

    consul_instances:
      children:
--- a/nomad/bootstrap-values.yml
+++ b/nomad/bootstrap-values.yml
@ -40,9 +40,23 @@
            name: hvac
            extra_args: --index-url https://pypi.org/simple

-        # This fails on first run because `root_token` isn't found
-        # Fails after taht too because the kv/ space has not been created yet either! Oh noes!
-        # Maybe move data bootstrapping to after the cluster is bootstrapped
+        - name: Check mount
+          community.hashi_vault.vault_read:
+            url: "http://{{ inventory_hostname }}:8200"
+            token: "{{ root_token }}"
+            path: "/sys/mounts/kv"
+          ignore_errors: true
+          register: check_mount
+
+        - name: Create kv mount
+          community.hashi_vault.vault_write:
+            url: "http://{{ inventory_hostname }}:8200"
+            token: "{{ root_token }}"
+            path: "/sys/mounts/kv"
+            data:
+              type: kv-v2
+          when: check_mount is not succeeded
+
        - name: Write values
          no_log: true
          community.hashi_vault.vault_write:
@ -53,6 +67,8 @@
              data:
                "{{ item.value }}"
          loop: "{{ hashi_vault_values | default({}) | dict2items }}"
+          retries: 2
+          delay: 10

        - name: Write userpass
          no_log: true
--- a/nomad/clear-data.yml
+++ b/nomad/clear-data.yml
@ -0,0 +1,64 @@
+---
+- name: Delete Consul data
+  hosts: consul_instances
+
+  tasks:
+    - name: Stop consul
+      systemd:
+        name: consul
+        state: stopped
+      become: true
+
+    - name: Stop vault
+      systemd:
+        name: consul
+        state: stopped
+      become: true
+
+    - name: Remove data dir
+      file:
+        path: /opt/consul
+        state: absent
+      become: true
+
+- name: Delete Nomad data
+  hosts: nomad_instances
+
+  tasks:
+    - name: Stop nomad
+      systemd:
+        name: nomad
+        state: stopped
+      become: true
+
+    - name: Kill nomad
+      shell:
+        cmd: systemctl kill nomad
+      become: true
+
+    - name: Stop all containers
+      shell:
+        cmd: docker ps -a | awk '/^[0-9abcdef]/{print $1}' | xargs -r docker stop
+      become: true
+
+    - name: Remove all containers
+      shell:
+        cmd: docker ps -a | awk '/^[0-9abcdef]/{print $1}' | xargs -r docker rm
+      become: true
+
+    - name: Unmount secrets
+      shell:
+        cmd: mount | awk '/nomad/ {print $3}' | xargs -n1 -r umount
+      become: true
+
+    - name: Remove data dir
+      file:
+        path: /var/nomad
+        state: absent
+      become: true
+
+    - name: Remove data dir
+      file:
+        path: /opt/nomad/data
+        state: absent
+      become: true
--- a/nomad/core.tf
+++ b/nomad/core.tf
@ -2,119 +2,11 @@ module "databases" {
  source = "./databases"
 }

-module "blocky" {
-  source = "./blocky"
+module "core" {
+  source = "./core"

  base_hostname = var.base_hostname
+
+  # Metrics and Blocky depend on databases
  depends_on = [module.databases]
 }
-
-module "traefik" {
-  source = "./traefik"
-
-  base_hostname = var.base_hostname
-}
-
-module "metrics" {
-  source = "./metrics"
-
-  consul_address = var.consul_address
-}
-
-module "loki" {
-  source = "./levant"
-
-  template_path = "service.nomad"
-  variables = {
-    name         = "loki"
-    image        = "grafana/loki:2.2.1"
-    service_port = 3100
-    ingress      = true
-    sticky_disk  = true
-    healthcheck  = "/ready"
-    templates = jsonencode([
-      {
-        data = file("./loki-config.yml")
-        dest = "/etc/loki/local-config.yaml"
-      }
-    ])
-  }
-}
-
-resource "consul_config_entry" "loki_intent" {
-  name = "loki"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "grafana"
-        Precedence = 9
-        Type       = "consul"
-      },
-      {
-        Action     = "allow"
-        Name       = "promtail"
-        Precedence = 9
-        Type       = "consul"
-      },
-      {
-        Action     = "allow"
-        Name       = "syslogng-promtail"
-        Precedence = 9
-        Type       = "consul"
-      },
-    ]
-  })
-}
-
-resource "nomad_job" "syslog-ng" {
-  jobspec = file("${path.module}/syslogng.nomad")
-}
-
-resource "nomad_job" "ddclient" {
-  jobspec = file("${path.module}/ddclient.nomad")
-}
-
-resource "nomad_job" "lldap" {
-  jobspec = file("${path.module}/lldap.nomad")
-}
-
-resource "consul_config_entry" "syslogng_promtail_intent" {
-  name = "syslogng-promtail"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "syslogng"
-        Precedence = 9
-        Type       = "consul"
-      },
-    ]
-  })
-}
-
-resource "consul_config_entry" "global_access" {
-  name = "*"
-  kind = "service-intentions"
-
-  config_json = jsonencode({
-    Sources = [
-      {
-        Action     = "allow"
-        Name       = "traefik"
-        Precedence = 6
-        Type       = "consul"
-      },
-      {
-        Action     = "deny"
-        Name       = "*"
-        Precedence = 5
-        Type       = "consul"
-      },
-    ]
-  })
-}
--- a/nomad/core/.terraform.lock.hcl
+++ b/nomad/core/.terraform.lock.hcl
@ -0,0 +1,59 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/consul" {
+  version = "2.16.2"
+  hashes = [
+    "h1:epldE7sZPBTQHnWEA4WlNJIOVT1UEX+/02SMg5nniaE=",
+    "zh:0a2e11ca2ba650954951a087a1daec95eee2f3000456b295409a9880c4a10b1a",
+    "zh:34f6bda06a0d1c213fa8d87d4313687681e67bc8c40c4cbaa7dbe59ce24a4f7e",
+    "zh:5b85cf93db11ee890f720c317a38158927071feb634855786a0c0cd65825a43c",
+    "zh:75ef915f3d087e6045751a66fbb7066a852a0944ec8c97200d1134dd84df7ffc",
+    "zh:8a4a95697bd91ad51a581c12fe50ac61a114afba27895d027f77ac4154a7ea15",
+    "zh:973d538c8d72793861a1ac9718249a9493f417a2b5096846367560054fd843b9",
+    "zh:9feb2bdc06fdc2d8370cc9aad9a0c69e7e5ae38aac43f315c3f57507c57be030",
+    "zh:c5709672d0afecbbe298bf519741ebcb9d04f02a73b5ee0c186dfa241aa5a524",
+    "zh:c65c60570de6da7190e1e7762577655a463caeb59bc5d38e33034821ed0cbcb9",
+    "zh:c958d6282650fc472aade61d5df4300936033f43cfb898293ef86aceccdfdf1d",
+    "zh:cdd3632c81e1d11d3becd193aaa061688840f39147950c45c4301d042743ae6a",
+    "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/external" {
+  version = "2.2.2"
+  hashes = [
+    "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
+    "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
+    "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
+    "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
+    "zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327",
+    "zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955",
+    "zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb",
+    "zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0",
+    "zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a",
+    "zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372",
+    "zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/nomad" {
+  version = "1.4.19"
+  hashes = [
+    "h1:EdBny2gaLr/IE+l+6csyCKeIGFMYZ/4tHKpcbS7ArgE=",
+    "zh:2f3ceeb3318a6304026035b0ac9ee3e52df04913bb9ee78827e58c5398b41254",
+    "zh:3fbe76c7d957d20dfe3c8c0528b33084651f22a95be9e0452b658e0922916e2a",
+    "zh:595671a05828cfe6c42ef73aac894ac39f81a52cc662a76f37eb74ebe04ddf75",
+    "zh:5d76e8788d2af3e60daf8076babf763ec887480bbb9734baccccd8fcddf4f03e",
+    "zh:676985afeaca6e67b22d60d43fd0ed7055763029ffebc3026089fe2fd3b4a288",
+    "zh:69152ce6164ac999a640cff962ece45208270e1ac37c10dac484eeea5cf47275",
+    "zh:6da0b15c05b81f947ec8e139bd81eeeb05c0d36eb5a967b985d0625c60998b40",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:822c0a3bbada5e38099a379db8b2e339526843699627c3be3664cc3b3752bab7",
+    "zh:af23af2f98a84695b25c8eba7028a81ad4aad63c44aefb79e01bbe2dc82e7f78",
+    "zh:e36cac9960b7506d92925b667254322520966b9c3feb3ca6102e57a1fb9b1761",
+    "zh:ffd1e096c1cc35de879c740a91918e9f06b627818a3cb4b1d87b829b54a6985f",
+  ]
+}
--- a/nomad/core/blocky/.terraform.lock.hcl
+++ b/nomad/core/blocky/.terraform.lock.hcl
--- a/nomad/core/blocky/blocky.nomad
+++ b/nomad/core/blocky/blocky.nomad
@ -108,6 +108,7 @@ job "blocky" {
      template {
        data = var.config_data
        destination = "app/config.yml"
+        splay = "1m"
      }
    }
  }
--- a/nomad/core/blocky/blocky.tf
+++ b/nomad/core/blocky/blocky.tf
--- a/nomad/core/blocky/config.yml
+++ b/nomad/core/blocky/config.yml
--- a/nomad/core/ddclient.nomad
+++ b/nomad/core/ddclient.nomad
@ -11,7 +11,7 @@ job "ddclient" {

        mount {
          type = "bind"
-          source = "local/ddclient.conf"
+          source = "secrets/ddclient.conf"
          target = "/config/ddclient.conf"
        }
      }
@ -30,21 +30,25 @@ ssl=yes
 use=web

 protocol=cloudflare,
+zone={{ key "ddclient/zone" }},
 ttl=1,
 {{ with secret "kv/data/cloudflare" -}}
-password={{ .Data.data.api_token_dns_edit_all }}
+login={{ .Data.data.api_user }},
+password={{ .Data.data.api_key }}
+# login=token,
+# password={{ .Data.data.api_token_dns_edit_all }}
 {{ end -}}
-zone={{ key "ddclient/zone" }}

 {{ key "ddclient/domain" }}
        EOH
-        destination = "local/ddclient.conf"
+        destination = "secrets/ddclient.conf"
        change_mode = "restart"
      }

      resources {
        cpu = 50
-        memory = 20
+        memory = 50
+        memory_max = 100
      }
    }
  }
--- a/nomad/core/lldap.nomad
+++ b/nomad/core/lldap.nomad
--- a/nomad/core/loki-config.yml
+++ b/nomad/core/loki-config.yml
--- a/nomad/core/main.tf
+++ b/nomad/core/main.tf
@ -0,0 +1,134 @@
+
+module "blocky" {
+  source = "./blocky"
+
+  base_hostname = var.base_hostname
+  # Not in this module
+  # depends_on = [module.databases]
+}
+
+module "traefik" {
+  source = "./traefik"
+
+  base_hostname = var.base_hostname
+}
+
+module "nomad_login" {
+  source = "../levant"
+
+  template_path = "service.nomad"
+  variables = {
+    name         = "nomad-login"
+    image        = "iamthefij/nomad-vault-login"
+    service_port = 5000
+    ingress      = true
+    ingress_rule = "Host(`nomad.thefij.rocks`) && PathPrefix(`/login`)"
+    env = jsonencode({
+      VAULT_ADDR = "http://$${attr.unique.network.ip-address}:8200",
+    })
+  }
+}
+
+module "metrics" {
+  source = "./metrics"
+  # Not in this module
+  # depends_on = [module.databases]
+}
+
+module "loki" {
+  source = "../levant"
+
+  template_path = "service.nomad"
+  variables = {
+    name         = "loki"
+    image        = "grafana/loki:2.2.1"
+    service_port = 3100
+    ingress      = true
+    sticky_disk  = true
+    healthcheck  = "/ready"
+    templates = jsonencode([
+      {
+        data = file("${path.module}/loki-config.yml")
+        dest = "/etc/loki/local-config.yaml"
+      }
+    ])
+  }
+}
+
+resource "consul_config_entry" "loki_intent" {
+  name = "loki"
+  kind = "service-intentions"
+
+  config_json = jsonencode({
+    Sources = [
+      {
+        Action     = "allow"
+        Name       = "grafana"
+        Precedence = 9
+        Type       = "consul"
+      },
+      {
+        Action     = "allow"
+        Name       = "promtail"
+        Precedence = 9
+        Type       = "consul"
+      },
+      {
+        Action     = "allow"
+        Name       = "syslogng-promtail"
+        Precedence = 9
+        Type       = "consul"
+      },
+    ]
+  })
+}
+
+resource "nomad_job" "syslog-ng" {
+  jobspec = file("${path.module}/syslogng.nomad")
+}
+
+resource "nomad_job" "ddclient" {
+  jobspec = file("${path.module}/ddclient.nomad")
+}
+
+resource "nomad_job" "lldap" {
+  jobspec = file("${path.module}/lldap.nomad")
+}
+
+resource "consul_config_entry" "syslogng_promtail_intent" {
+  name = "syslogng-promtail"
+  kind = "service-intentions"
+
+  config_json = jsonencode({
+    Sources = [
+      {
+        Action     = "allow"
+        Name       = "syslogng"
+        Precedence = 9
+        Type       = "consul"
+      },
+    ]
+  })
+}
+
+resource "consul_config_entry" "global_access" {
+  name = "*"
+  kind = "service-intentions"
+
+  config_json = jsonencode({
+    Sources = [
+      {
+        Action     = "allow"
+        Name       = "traefik"
+        Precedence = 6
+        Type       = "consul"
+      },
+      {
+        Action     = "deny"
+        Name       = "*"
+        Precedence = 5
+        Type       = "consul"
+      },
+    ]
+  })
+}
--- a/nomad/core/metrics/.terraform.lock.hcl
+++ b/nomad/core/metrics/.terraform.lock.hcl
--- a/nomad/core/metrics/exporters.nomad
+++ b/nomad/core/metrics/exporters.nomad
--- a/nomad/core/metrics/grafana.nomad
+++ b/nomad/core/metrics/grafana.nomad
--- a/nomad/core/metrics/grafana/grafana.ini
+++ b/nomad/core/metrics/grafana/grafana.ini
--- a/nomad/core/metrics/grafana/provisioning/dashboards/default/minitor-monitor.json
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/default/minitor-monitor.json
--- a/nomad/core/metrics/grafana/provisioning/dashboards/default/nomad.json
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/default/nomad.json
--- a/nomad/core/metrics/grafana/provisioning/dashboards/default/prometheus-stats-2-r2.json
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/default/prometheus-stats-2-r2.json
--- a/nomad/core/metrics/grafana/provisioning/dashboards/default/traefik-2.json
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/default/traefik-2.json
--- a/nomad/core/metrics/grafana/provisioning/dashboards/default/traefik.json
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/default/traefik.json
--- a/nomad/core/metrics/grafana/provisioning/dashboards/main.yml
+++ b/nomad/core/metrics/grafana/provisioning/dashboards/main.yml
--- a/nomad/core/metrics/grafana/provisioning/datasources/loki.yml
+++ b/nomad/core/metrics/grafana/provisioning/datasources/loki.yml
--- a/nomad/core/metrics/grafana/provisioning/datasources/prometheus.yml
+++ b/nomad/core/metrics/grafana/provisioning/datasources/prometheus.yml
--- a/nomad/core/metrics/grafana/provisioning/notifiers/main.yml
+++ b/nomad/core/metrics/grafana/provisioning/notifiers/main.yml
--- a/nomad/core/metrics/grafana/provisioning/notifiers/slack.yml
+++ b/nomad/core/metrics/grafana/provisioning/notifiers/slack.yml
--- a/nomad/core/metrics/metrics.tf
+++ b/nomad/core/metrics/metrics.tf
@ -1,8 +1,3 @@
-variable "consul_address" {
-  type        = string
-  description = "address of consul server for dynamic scraping"
-}
-
 resource "nomad_job" "exporters" {
  hcl2 {
    enabled = true
@ -20,12 +15,6 @@ data "consul_nodes" "all-nodes" {
 resource "nomad_job" "prometheus" {
  hcl2 {
    enabled = true
-    vars = {
-      # TODO: May not need this because we have an env variable for that
-      # "consul_address" = "${var.consul_address}",
-      # TODO: Should this be a list?
-      "consul_address" = "http://${data.consul_nodes.all-nodes.nodes[0].address}:8500",
-    }
  }

  jobspec = file("${path.module}/prometheus.nomad")
--- a/nomad/core/metrics/prometheus.nomad
+++ b/nomad/core/metrics/prometheus.nomad
@ -1,9 +1,3 @@
-variable "consul_address" {
-  type = string
-  description = "Full address of Consul instance to get catalog from"
-  default = "http://127.0.0.1:5400"
-}
-
 job "prometheus" {
  datacenters = ["dc1"]

@ -65,7 +59,7 @@ job "prometheus" {
        ports = ["web"]
        args = [
          "--config.file=/etc/prometheus/config/prometheus.yml",
-          "--storage.tsdb.path=/prometheus",
+          "--storage.tsdb.path=${NOMAD_ALLOC_DIR}/data/tsdb",
          "--web.listen-address=0.0.0.0:9090",
          "--web.console.libraries=/usr/share/prometheus/console_libraries",
          "--web.console.templates=/usr/share/prometheus/consoles",
@ -97,8 +91,7 @@ scrape_configs:
      format:
        - "prometheus"
    consul_sd_configs:
-      - server: "${var.consul_address}"
-      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
+      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
        services:
          - "nomad-client"

@ -108,8 +101,7 @@ scrape_configs:
      format:
        - "prometheus"
    consul_sd_configs:
-      - server: "${var.consul_address}"
-      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
+      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
        services:
          - "consul"
    relabel_configs:
@ -120,8 +112,7 @@ scrape_configs:
  - job_name: "exporters"
    metrics_path: "/metrics"
    consul_sd_configs:
-      - server: "${var.consul_address}"
-      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
+      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: drop
@ -143,8 +134,7 @@ scrape_configs:
  - job_name: "envoy"
    metrics_path: "/metrics"
    consul_sd_configs:
-      - server: "${var.consul_address}"
-      # - server: "{{ env "CONSUL_HTTP_ADDR" }}"
+      - server: "http://{{env "attr.unique.network.ip-address"}}:8500"
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: keep
--- a/nomad/core/syslogng.nomad
+++ b/nomad/core/syslogng.nomad
@ -92,13 +92,6 @@ EOF
  group "syslogng" {
    count = 1

-    constraint {
-      attribute = "${node.unique.name}"
-      # Needs to be on a predictable node for routing
-      # Maybe a loadbalancer could be used for routing from any node
-      value = "n2"
-    }
-
    network {
      mode = "bridge"
      port "main" {
@ -113,6 +106,8 @@ EOF
      connect {
        sidecar_service {
          proxy {
+            local_service_port = 514
+
            upstreams {
              destination_name = "syslogng-promtail"
              local_bind_port = 1000
@ -128,13 +123,6 @@ EOF
          }
        }
      }
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.tcp.routers.syslogngtcp.entrypoints=syslogtcp",
-        "traefik.tcp.routers.syslogngtcp.rule=HostSNI(`*`)",
-        "traefik.udp.routers.syslogngudp.entrypoints=syslogudp",
-      ]
    }

    task "syslogng" {
--- a/nomad/core/traefik/.terraform.lock.hcl
+++ b/nomad/core/traefik/.terraform.lock.hcl
--- a/nomad/core/traefik/traefik.nomad
+++ b/nomad/core/traefik/traefik.nomad
@ -32,9 +32,14 @@ job "traefik" {
      port "web" {
        static = 80
      }
+
      port "websecure" {
        static = 443
      }
+
+      port "syslog" {
+        static = 514
+      }
    }

    ephemeral_disk {
@ -152,7 +157,8 @@ job "traefik" {
 << if keyExists "traefik/acme/email" ->>
 [certificatesResolvers.letsEncrypt.acme]
  email = "<< key "traefik/acme/email" >>"
-  storage = "/secrets/acme.json"
+  # Store in /local because /secrets doesn't persist with ephemeral disk
+  storage = "/local/acme.json"
  [certificatesResolvers.letsEncrypt.acme.dnsChallenge]
    provider = "cloudflare"
    resolvers = ["1.1.1.1:53", "8.8.8.8:53"]
@ -214,6 +220,7 @@ CF_ZONE_API_TOKEN={{ .Data.data.api_token_zone_read }}
    {{ with service "vault" -}}
    [http.services.vault]
      [http.services.vault.loadBalancer]
+        [http.services.vault.loadBalancer.sticky.cookie]
        {{ range . -}}
        [[http.services.vault.loadBalancer.servers]]
          url = "http://{{ .Address }}:{{ .Port }}"
@ -224,6 +231,43 @@ CF_ZONE_API_TOKEN={{ .Data.data.api_token_zone_read }}
        change_mode = "noop"
      }

+      template {
+        data = <<EOH
+{{ with service "syslogng" -}}
+[tcp.routers]
+  [tcp.routers.syslogtcp]
+    entryPoints = ["syslogtcp"]
+    service = "syslogngtcp"
+    rule = "HostSNI(`*`)"
+
+[tcp.services]
+  [tcp.services.syslogngtcp]
+    [tcp.services.syslogngtcp.loadBalancer]
+      {{ range . -}}
+      [[tcp.services.syslogngtcp.loadBalancer.servers]]
+        address = "{{ .Address }}:{{ .Port }}"
+      {{ end -}}
+{{ end }}
+
+{{ with service "syslogng" -}}
+[udp.routers]
+  [udp.routers.syslogudp]
+    entryPoints = ["syslogudp"]
+    service = "syslogngudp"
+
+[udp.services]
+  [udp.services.syslogngudp]
+    [udp.services.syslogngudp.loadBalancer]
+      {{ range . -}}
+      [[udp.services.syslogngudp.loadBalancer.servers]]
+        address = "{{ .Address }}:{{ .Port }}"
+      {{ end -}}
+{{ end }}
+        EOH
+        destination = "local/config/conf/route-syslog-ng.toml"
+        change_mode = "noop"
+      }
+
      template {
        data = <<EOH
 [http.middlewares]
--- a/nomad/core/traefik/traefik.tf
+++ b/nomad/core/traefik/traefik.tf
--- a/nomad/core/vars.tf
+++ b/nomad/core/vars.tf
@ -0,0 +1,5 @@
+variable "base_hostname" {
+  type        = string
+  description = "Base hostname to serve content from"
+  default     = "dev.homelab"
+}
--- a/nomad/databases/.terraform.lock.hcl
+++ b/nomad/databases/.terraform.lock.hcl
@ -0,0 +1,40 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/consul" {
+  version = "2.15.1"
+  hashes = [
+    "h1:PexyQBRLDA+SR+sWlzYBZswry5O5h/tTfj87CaECtLc=",
+    "zh:1806830a3cf103e65e772a7d28fd4df2788c29a029fb2def1326bc777ad107ed",
+    "zh:252be544fb4c9daf09cad7d3776daf5fa66b62740d3ea9d6d499a7b1697c3433",
+    "zh:50985fe02a8e5ae47c75d7c28c911b25d7dc4716cff2ed55ca05889ab77a1f73",
+    "zh:54cf0ec90538703c66937c77e8d72a38d5af47437eb0b8b55eb5836c5d288878",
+    "zh:704f536c621337e06fffef6d5f49ac81f52d249f937250527c12884cb83aefed",
+    "zh:896d8ef6d0b555299f124eb25bce8a17d735da14ef21f07582098d301f47da30",
+    "zh:976277a85b0a0baafe267cc494f766448d1da5b6936ddcb3ce393bd4d22f08d2",
+    "zh:c7faa9a2b11bc45833a3e8e340f22f1ecf01597eaeffa7669234b4549d7dfa85",
+    "zh:caf851ef9c8ce482864badf7058f9278d4537112fa236efd8f1a9315801d9061",
+    "zh:db203435d58b0ac842540861b3307a623423275d85754c171773f3b210ae5b24",
+    "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5",
+    "zh:f710a37190429045d109edd35de69db3b5f619919c2fa04c77a3a639fea9fd7d",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/nomad" {
+  version = "1.4.17"
+  hashes = [
+    "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
+    "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
+    "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
+    "zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
+    "zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
+    "zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
+    "zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
+    "zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
+    "zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
+    "zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
+    "zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
+  ]
+}
--- a/nomad/databases/adminer.nomad
+++ b/nomad/databases/adminer.nomad
@ -25,7 +25,6 @@ job "adminer" {

            upstreams {
              destination_name = "mysql-server"
-              # TODO: how do I get these to not bind to the host eth0 address
              local_bind_port = 4040
            }

--- a/nomad/databases/mysql.nomad
+++ b/nomad/databases/mysql.nomad
@ -1,6 +1,7 @@
 job "mysql-server" {
  datacenters = ["dc1"]
  type = "service"
+  priority = 80

  group "mysql-server" {
    count = 1
--- a/nomad/databases/redis.nomad
+++ b/nomad/databases/redis.nomad
@ -1,6 +1,7 @@
 job "redis" {
  datacenters = ["dc1"]
  type = "service"
+  priority = 60

  group "cache" {
    count = 1
@ -52,7 +53,7 @@ job "redis" {

      config {
        image = "redis:6"
-        args = ["redis-server", "--save", "60", "1", "--loglevel", "warning"]
+        args = ["redis-server", "--save", "60", "1", "--loglevel", "warning", "--dir", "${NOMAD_ALLOC_DIR}/data"]
        ports = ["main"]
      }

--- a/nomad/media/.terraform.lock.hcl
+++ b/nomad/media/.terraform.lock.hcl
@ -1,20 +0,0 @@
-# This file is maintained automatically by "terraform init".
-# Manual edits may be lost in future updates.
-
-provider "registry.terraform.io/hashicorp/nomad" {
-  version = "1.4.16"
-  hashes = [
-    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
-    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
-    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
-    "zh:0df88393271078533a217654b96f0672c60eb59570d72e6aefcb839eea87a7a0",
-    "zh:2883b335bb6044b0db6a00e602d6926c047c7f330294a73a90d089f98b24d084",
-    "zh:390158d928009a041b3a182bdd82376b50530805ae92be2b84ed7c3b0fa902a0",
-    "zh:7169b8f8df4b8e9659c49043848fd5f7f8473d0471f67815e8b04980f827f5ef",
-    "zh:9417ee1383b1edd137024882d7035be4dca51fb4f725ca00ed87729086ec1755",
-    "zh:a22910b5a29eeab5610350700b4899267c1b09b66cf21f7e4d06afc61d425800",
-    "zh:a6185c9cd7aa458cd81861058ba568b6411fbac344373a20155e20256f4a7557",
-    "zh:b6260ca9f034df1b47905b4e2a9c33b67dbf77224a694d5b10fb09ae92ffad4c",
-    "zh:d87c12a6a7768f2b6c2a59495c7dc00f9ecc52b1b868331d4c284f791e278a1e",
-  ]
-}
--- a/nomad/mysql/.terraform.lock.hcl
+++ b/nomad/mysql/.terraform.lock.hcl
@ -1,38 +0,0 @@
-# This file is maintained automatically by "terraform init".
-# Manual edits may be lost in future updates.
-
-provider "registry.terraform.io/hashicorp/consul" {
-  version = "2.15.0"
-  hashes = [
-    "h1:o+Su3YqeOkHgf86GEArIVDZfaZQphYFjAOwpi/b0bzs=",
-    "zh:0bd2a9873099d89bd52e9eee623dd20ccb275d1e2f750da229a53a4d5b23450c",
-    "zh:1c9f87d4d97b2c61d006c0bef159d61d2a661a103025f8276ebbeb000129f931",
-    "zh:25b73a34115255c464be10a53f2510c4a1db958a71be31974d30654d5472e624",
-    "zh:32fa31329731db2bf4b7d0f09096416ca146f05b58f4482bbd4ee0f28cefbbcc",
-    "zh:59136b73d3abe7cc5b06d9e12d123ad21298ca86ed49a4060a3cd7c2a28a74a1",
-    "zh:a191f3210773ca25c543a92f2d392b85e6a053d596293655b1f25b33eb843b4c",
-    "zh:b8b6033cf0687eadc1099f11d9fb2ca9429ff40c2d85bd6cb047c0f6bc5d5d8d",
-    "zh:bb7d67ed28aa9b28fc5154161af003383f940b2beda0d4577857cad700f39cd1",
-    "zh:be615288f59327b975532a1999deab60a022e6819fe80e5a32526155210ecbba",
-    "zh:de1e3d5c34eef87eb301e74717754babb6dc8e19e3a964919e1165c5a076a719",
-    "zh:eb8c61b20d8ce2bfff9f735ca8456a0d6368af13aa1f43866f61c70f88cc491c",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/nomad" {
-  version = "1.4.16"
-  hashes = [
-    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
-    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
-    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
-    "zh:0df88393271078533a217654b96f0672c60eb59570d72e6aefcb839eea87a7a0",
-    "zh:2883b335bb6044b0db6a00e602d6926c047c7f330294a73a90d089f98b24d084",
-    "zh:390158d928009a041b3a182bdd82376b50530805ae92be2b84ed7c3b0fa902a0",
-    "zh:7169b8f8df4b8e9659c49043848fd5f7f8473d0471f67815e8b04980f827f5ef",
-    "zh:9417ee1383b1edd137024882d7035be4dca51fb4f725ca00ed87729086ec1755",
-    "zh:a22910b5a29eeab5610350700b4899267c1b09b66cf21f7e4d06afc61d425800",
-    "zh:a6185c9cd7aa458cd81861058ba568b6411fbac344373a20155e20256f4a7557",
-    "zh:b6260ca9f034df1b47905b4e2a9c33b67dbf77224a694d5b10fb09ae92ffad4c",
-    "zh:d87c12a6a7768f2b6c2a59495c7dc00f9ecc52b1b868331d4c284f791e278a1e",
-  ]
-}
--- a/nomad/recover-consul.yaml
+++ b/nomad/recover-consul.yaml
@ -0,0 +1,45 @@
+---
+- name: Recovery Consul
+  hosts: consul_instances
+
+  tasks:
+    - name: Stop Consul
+      systemd:
+        name: consul
+        state: stopped
+      become: true
+
+    - name: Get node-id
+      slurp:
+        src: /opt/consul/node-id
+      register: consul_node_id
+      become: true
+
+    - name: Node Id
+      debug:
+        msg: "node_id: {{ consul_node_id.content }}"
+
+    - name: Address
+      debug:
+        msg: "address: {{ ansible_default_ipv4.address }}"
+
+    - name: Save
+      copy:
+        dest: "/opt/consul/raft/peers.json"
+        content: |
+          [
+          {% for host in ansible_play_hosts|reject('equalto', inventory_hostname) -%}
+            {
+            "id": "{{ hostvars[host].consul_node_id.content }}",
+            "address": "{{ hostvars[host].ansible_default_ipv4.address }}:8300",
+            "non_voter": false
+            }{% if not loop.last %},{% endif %}
+          {% endfor -%}
+          ]
+      become: true
+
+    - name: Restart Consul
+      systemd:
+        name: consul
+        state: restarted
+      become: true
--- a/nomad/recover-nomad.yaml
+++ b/nomad/recover-nomad.yaml
@ -0,0 +1,45 @@
+---
+- name: Recovery Nomad
+  hosts: nomad_instances
+
+  tasks:
+    - name: Stop Nomad
+      systemd:
+        name: nomad
+        state: stopped
+      become: true
+
+    - name: Get node-id
+      slurp:
+        src: /var/nomad/server/node-id
+      register: nomad_node_id
+      become: true
+
+    - name: Node Id
+      debug:
+        msg: "node_id: {{ nomad_node_id.content }}"
+
+    - name: Address
+      debug:
+        msg: "address: {{ ansible_default_ipv4.address }}"
+
+    - name: Save
+      copy:
+        dest: /var/nomad/server/raft/peers.json
+        content: |
+          [
+          {% for host in ansible_play_hosts|reject('equalto', inventory_hostname) -%}
+            {
+            "id": "{{ hostvars[host].nomad_node_id.content }}",
+            "address": "{{ hostvars[host].ansible_default_ipv4.address }}:4647",
+            "non_voter": false
+            }{% if not loop.last %},{% endif %}
+          {% endfor -%}
+          ]
+      become: true
+
+    - name: Restart Nomad
+      systemd:
+        name: nomad
+        state: restarted
+      become: true
--- a/nomad/service.nomad
+++ b/nomad/service.nomad
@ -21,6 +21,10 @@
 #   read_only = bool,
 # )))
 # healthcheck = "/"
+# upstreams = json(list(dict(
+#   destination_name = str,
+#   local_bind_port = int
+# )))
 # mysql = bool
 # redis = bool
 # vault = bool
@ -83,6 +87,14 @@ job "[[.name]]" {
              local_bind_port = 6379
            }
            [[ end -]]
+            [[ with .upstreams -]]
+            [[range $u := . | parseJSON -]]
+            upstreams {
+              destination_name = "[[ $u.destination_name ]]"
+              local_bind_port = [[ $u.local_bind_port ]]
+            }
+            [[ end ]]
+            [[ end -]]
          }
        }

@ -110,6 +122,9 @@ job "[[.name]]" {
        [[ if default false .ingress -]]
        "traefik.enable=true",
        "traefik.http.routers.[[.name]].entryPoints=websecure",
+        [[ if not (empty .ingress_rule) -]]
+        "traefik.http.routers.[[.name]].rule=[[.ingress_rule]]",
+        [[ end -]]
        [[ end -]]
      ]
    }
--- a/nomad/services.tf
+++ b/nomad/services.tf
@ -1,55 +1,5 @@
-# module "nextcloud" {
-#   source = "./nextcloud"
-#
-#   depends_on = [module.databases]
-# }
+module "services" {
+  source = "./services"

-module "backups" {
-  source = "./backups"
-
-  depends_on = [module.databases]
-}
-
-module "media" {
-  source = "./media"
-}
-
-resource "nomad_job" "whoami" {
-  hcl2 {
-    enabled = true
-    vars = {
-      "count" = 1,
-      # "count" = "${2 * length(data.consul_service.nomad.service)}",
-    }
-  }
-
-  jobspec = file("${path.module}/whoami.nomad")
-}
-
-module "sonarr" {
-  source = "./levant"
-
-  template_path = "service.nomad"
-  variables = {
-    name         = "sonarr"
-    image        = "linuxserver/sonarr"
-    service_port = 8989
-    ingress      = true
-    env = jsonencode({
-      PGID = 100
-      PUID = 1001
-    })
-    host_volumes = jsonencode([
-      {
-        name      = "tv-sonarr"
-        dest      = "/srv/volumes/media-write/TV Shows"
-        read_only = false
-      },
-      {
-        name      = "download"
-        dest      = "/srv/volumes/download"
-        read_only = false
-      },
-    ])
-  }
+  depends_on = [module.databases, module.core]
 }
--- a/nomad/services/.terraform.lock.hcl
+++ b/nomad/services/.terraform.lock.hcl
@ -0,0 +1,40 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/consul" {
+  version = "2.16.2"
+  hashes = [
+    "h1:epldE7sZPBTQHnWEA4WlNJIOVT1UEX+/02SMg5nniaE=",
+    "zh:0a2e11ca2ba650954951a087a1daec95eee2f3000456b295409a9880c4a10b1a",
+    "zh:34f6bda06a0d1c213fa8d87d4313687681e67bc8c40c4cbaa7dbe59ce24a4f7e",
+    "zh:5b85cf93db11ee890f720c317a38158927071feb634855786a0c0cd65825a43c",
+    "zh:75ef915f3d087e6045751a66fbb7066a852a0944ec8c97200d1134dd84df7ffc",
+    "zh:8a4a95697bd91ad51a581c12fe50ac61a114afba27895d027f77ac4154a7ea15",
+    "zh:973d538c8d72793861a1ac9718249a9493f417a2b5096846367560054fd843b9",
+    "zh:9feb2bdc06fdc2d8370cc9aad9a0c69e7e5ae38aac43f315c3f57507c57be030",
+    "zh:c5709672d0afecbbe298bf519741ebcb9d04f02a73b5ee0c186dfa241aa5a524",
+    "zh:c65c60570de6da7190e1e7762577655a463caeb59bc5d38e33034821ed0cbcb9",
+    "zh:c958d6282650fc472aade61d5df4300936033f43cfb898293ef86aceccdfdf1d",
+    "zh:cdd3632c81e1d11d3becd193aaa061688840f39147950c45c4301d042743ae6a",
+    "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/nomad" {
+  version = "1.4.19"
+  hashes = [
+    "h1:EdBny2gaLr/IE+l+6csyCKeIGFMYZ/4tHKpcbS7ArgE=",
+    "zh:2f3ceeb3318a6304026035b0ac9ee3e52df04913bb9ee78827e58c5398b41254",
+    "zh:3fbe76c7d957d20dfe3c8c0528b33084651f22a95be9e0452b658e0922916e2a",
+    "zh:595671a05828cfe6c42ef73aac894ac39f81a52cc662a76f37eb74ebe04ddf75",
+    "zh:5d76e8788d2af3e60daf8076babf763ec887480bbb9734baccccd8fcddf4f03e",
+    "zh:676985afeaca6e67b22d60d43fd0ed7055763029ffebc3026089fe2fd3b4a288",
+    "zh:69152ce6164ac999a640cff962ece45208270e1ac37c10dac484eeea5cf47275",
+    "zh:6da0b15c05b81f947ec8e139bd81eeeb05c0d36eb5a967b985d0625c60998b40",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:822c0a3bbada5e38099a379db8b2e339526843699627c3be3664cc3b3752bab7",
+    "zh:af23af2f98a84695b25c8eba7028a81ad4aad63c44aefb79e01bbe2dc82e7f78",
+    "zh:e36cac9960b7506d92925b667254322520966b9c3feb3ca6102e57a1fb9b1761",
+    "zh:ffd1e096c1cc35de879c740a91918e9f06b627818a3cb4b1d87b829b54a6985f",
+  ]
+}
--- a/nomad/services/backups/.terraform.lock.hcl
+++ b/nomad/services/backups/.terraform.lock.hcl
@ -2,26 +2,28 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/consul" {
-  version = "2.15.0"
+  version = "2.16.2"
  hashes = [
-    "h1:o+Su3YqeOkHgf86GEArIVDZfaZQphYFjAOwpi/b0bzs=",
-    "zh:0bd2a9873099d89bd52e9eee623dd20ccb275d1e2f750da229a53a4d5b23450c",
-    "zh:1c9f87d4d97b2c61d006c0bef159d61d2a661a103025f8276ebbeb000129f931",
-    "zh:25b73a34115255c464be10a53f2510c4a1db958a71be31974d30654d5472e624",
-    "zh:32fa31329731db2bf4b7d0f09096416ca146f05b58f4482bbd4ee0f28cefbbcc",
-    "zh:59136b73d3abe7cc5b06d9e12d123ad21298ca86ed49a4060a3cd7c2a28a74a1",
-    "zh:a191f3210773ca25c543a92f2d392b85e6a053d596293655b1f25b33eb843b4c",
-    "zh:b8b6033cf0687eadc1099f11d9fb2ca9429ff40c2d85bd6cb047c0f6bc5d5d8d",
-    "zh:bb7d67ed28aa9b28fc5154161af003383f940b2beda0d4577857cad700f39cd1",
-    "zh:be615288f59327b975532a1999deab60a022e6819fe80e5a32526155210ecbba",
-    "zh:de1e3d5c34eef87eb301e74717754babb6dc8e19e3a964919e1165c5a076a719",
-    "zh:eb8c61b20d8ce2bfff9f735ca8456a0d6368af13aa1f43866f61c70f88cc491c",
+    "h1:epldE7sZPBTQHnWEA4WlNJIOVT1UEX+/02SMg5nniaE=",
+    "zh:0a2e11ca2ba650954951a087a1daec95eee2f3000456b295409a9880c4a10b1a",
+    "zh:34f6bda06a0d1c213fa8d87d4313687681e67bc8c40c4cbaa7dbe59ce24a4f7e",
+    "zh:5b85cf93db11ee890f720c317a38158927071feb634855786a0c0cd65825a43c",
+    "zh:75ef915f3d087e6045751a66fbb7066a852a0944ec8c97200d1134dd84df7ffc",
+    "zh:8a4a95697bd91ad51a581c12fe50ac61a114afba27895d027f77ac4154a7ea15",
+    "zh:973d538c8d72793861a1ac9718249a9493f417a2b5096846367560054fd843b9",
+    "zh:9feb2bdc06fdc2d8370cc9aad9a0c69e7e5ae38aac43f315c3f57507c57be030",
+    "zh:c5709672d0afecbbe298bf519741ebcb9d04f02a73b5ee0c186dfa241aa5a524",
+    "zh:c65c60570de6da7190e1e7762577655a463caeb59bc5d38e33034821ed0cbcb9",
+    "zh:c958d6282650fc472aade61d5df4300936033f43cfb898293ef86aceccdfdf1d",
+    "zh:cdd3632c81e1d11d3becd193aaa061688840f39147950c45c4301d042743ae6a",
+    "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5",
  ]
 }

 provider "registry.terraform.io/hashicorp/nomad" {
  version = "1.4.16"
  hashes = [
+    "h1:PQxNPNmMVOErxryTWIJwr22k95DTSODmgRylqjc2TjI=",
    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
--- a/nomad/services/backups/backup.nomad
+++ b/nomad/services/backups/backup.nomad
@ -48,6 +48,7 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
    }

    service {
+      name = "backups"
      port = "metrics"

      # Add connect to mysql
@ -158,17 +159,33 @@ CONSUL_HTTP_ADDR={{ env "attr.unique.network.ip-address" }}:8500
 # Consul backup
 ${file("${module_path}/jobs/consul.hcl")}
 {{ end -}}
+
 {{ range service "nextcloud" -}}
 # Nextcloud .Node {{ .Node }}
 {{ if eq .Node (env "node.unique.name") -}}
 ${file("${module_path}/jobs/nextcloud.hcl")}
 {{ end -}}
 {{ end -}}
+
 {{ range service "lldap" -}}
 # Lldap .Node {{ .Node }}
 {{ if eq .Node (env "node.unique.name") -}}
 ${file("${module_path}/jobs/lldap.hcl")}
 {{ end -}}
+{{ end -}}
+
+{{ range service "sonarr" -}}
+# Lldap .Node {{ .Node }}
+{{ if eq .Node (env "node.unique.name") -}}
+${file("${module_path}/jobs/sonarr.hcl")}
+{{ end -}}
+{{ end -}}
+
+{{ range service "nzbget" -}}
+# Lldap .Node {{ .Node }}
+{{ if eq .Node (env "node.unique.name") -}}
+${file("${module_path}/jobs/nzbget.hcl")}
+{{ end -}}
 {{ end -}}
        EOF
        destination = "jobs/node-jobs.hcl"
--- a/nomad/services/backups/backups.tf
+++ b/nomad/services/backups/backups.tf
@ -5,8 +5,18 @@ resource "nomad_job" "backups" {
  })
 }

+# Get Nomad clients from Consul
+# data "consul_service" "nomad" {
+#   name = "nomad-client"
+# }
+
 resource "nomad_job" "backups-oneoff" {
-  for_each = toset(["n1", "n2", "n3"])
+  # TODO: Get list of nomad hosts dynamically
+  for_each = toset(["n1", "n2"])
+  # for_each = toset([
+  #   for node in data.consul_service.nomad.service :
+  #   node.node_name
+  # ])
  jobspec = templatefile("${path.module}/backup.nomad", {
    module_path = "${path.module}",
    batch_node  = each.key,
--- a/nomad/services/backups/jobs/consul.hcl
+++ b/nomad/services/backups/jobs/consul.hcl
--- a/nomad/services/backups/jobs/lldap.hcl
+++ b/nomad/services/backups/jobs/lldap.hcl
--- a/nomad/services/backups/jobs/nextcloud.hcl
+++ b/nomad/services/backups/jobs/nextcloud.hcl
--- a/nomad/services/backups/jobs/nzbget.hcl
+++ b/nomad/services/backups/jobs/nzbget.hcl
@ -0,0 +1,21 @@
+job "nzbget" {
+  schedule = "@daily"
+
+  config {
+    repo = "rclone::ftp,env_auth:/nomad/nzbget"
+    passphrase = env("BACKUP_PASSPHRASE")
+  }
+
+  backup {
+    paths = ["/data/nzbget"]
+    # Because path is absolute
+    restore_opts {
+      Target = "/"
+    }
+  }
+
+  forget {
+    KeepLast = 2
+    Prune = true
+  }
+}
--- a/nomad/services/backups/jobs/sonarr.hcl
+++ b/nomad/services/backups/jobs/sonarr.hcl
@ -0,0 +1,27 @@
+job "sonarr" {
+  schedule = "@daily"
+
+  config {
+    repo = "rclone::ftp,env_auth:/nomad/sonarr"
+    passphrase = env("BACKUP_PASSPHRASE")
+  }
+
+  # sqlite "Backup database" {
+  #   path    = "/data/lldap/users.db"
+  #   # sqlite3 /data/lldap/users.db .backup /data/lldap/users.db.bak
+  #   dump_to = "/data/lldap/users.db.bak"
+  # }
+
+  backup {
+    paths = ["/data/sonarr"]
+    # Because path is absolute
+    restore_opts {
+      Target = "/"
+    }
+  }
+
+  forget {
+    KeepLast = 2
+    Prune = true
+  }
+}
--- a/nomad/services/external.tf
+++ b/nomad/services/external.tf
@ -0,0 +1,29 @@
+resource "consul_service" "homeassistant" {
+  name = "hass"
+  node = consul_node.homeassistant.name
+  port = 8123
+  tags = [
+    "traefik.enable=true",
+    "traefik.consulcatalog.connect=false",
+    "traefik.http.routers.hass.entryPoints=websecure",
+  ]
+
+  check {
+    check_id = "homeassistant:hass"
+    status   = "passing"
+    name     = "Home Assistant Health Check"
+    http     = "192.168.3.65:8123"
+    interval = "30s"
+    timeout  = "10s"
+  }
+}
+
+resource "consul_node" "homeassistant" {
+  name    = "homeassistant"
+  address = "192.168.3.65"
+
+  meta = {
+    "external-node"  = "true"
+    "external-probe" = "true"
+  }
+}
--- a/nomad/services/ip-dvr.nomad
+++ b/nomad/services/ip-dvr.nomad
@ -0,0 +1,201 @@
+job "ipdvr" {
+  region = "global"
+  datacenters = ["dc1"]
+
+  type = "service"
+
+  group "nzbget" {
+    network {
+      mode = "bridge"
+      port "main" {
+        host_network = "loopback"
+        to = 6789
+      }
+    }
+
+    volume "nzbget-data" {
+      type = "host"
+      read_only = false
+      source = "nzbget-data"
+    }
+
+    volume "download" {
+      type = "host"
+      read_only = false
+      source = "download"
+    }
+
+    service {
+      name = "nzbget"
+      port = "main"
+
+      connect {
+        sidecar_service {
+          proxy {
+            local_service_port = 6789
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 20
+            memory_max = 50
+          }
+        }
+      }
+
+      # check {
+      #   type = "http"
+      #   path = "/"
+      #   port = "main"
+      #   interval = "10s"
+      #   timeout = "10s"
+      # }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.nzbget.entryPoints=websecure",
+      ]
+    }
+
+    task "nzbget" {
+      driver = "docker"
+
+      config {
+        image = "linuxserver/nzbget"
+        ports = ["main"]
+      }
+
+      env = {
+        "PGID" = 100
+        "PUID" = 1001
+        "TZ" = "America/Los_Angeles"
+      }
+
+      volume_mount {
+        volume = "nzbget-data"
+        destination = "/config"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "download"
+        destination = "/downloads"
+        read_only = false
+      }
+
+      resources {
+        cpu = 200
+        memory = 200
+        memory_max = 500
+      }
+    }
+  }
+
+
+  group "sonarr" {
+    network {
+      mode = "bridge"
+      port "main" {
+        host_network = "loopback"
+        to = 8989
+      }
+    }
+
+    volume "sonarr-data" {
+      type = "host"
+      read_only = false
+      source = "sonarr-data"
+    }
+
+    volume "tv-sonarr" {
+      type = "host"
+      read_only = false
+      source = "tv-sonarr"
+    }
+
+    volume "download" {
+      type = "host"
+      read_only = false
+      source = "download"
+    }
+
+    service {
+      name = "sonarr"
+      port = "main"
+
+      connect {
+        sidecar_service {
+          proxy {
+            local_service_port = 8989
+            upstreams {
+              destination_name = "nzbget"
+              local_bind_port = 6789
+            }
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 20
+            memory_max = 50
+          }
+        }
+      }
+
+      # check {
+      #   type = "http"
+      #   path = "/"
+      #   port = "main"
+      #   interval = "10s"
+      #   timeout = "10s"
+      # }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.sonarr.entryPoints=websecure",
+      ]
+    }
+
+    task "sonarr" {
+      driver = "docker"
+
+      config {
+        image = "linuxserver/sonarr"
+        ports = ["main"]
+      }
+
+      env = {
+        "PGID" = 100
+        "PUID" = 1001
+        "TZ" = "America/Los_Angeles"
+      }
+
+      volume_mount {
+        volume = "sonarr-data"
+        destination = "/config"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "tv-sonarr"
+        destination = "/tv"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "download"
+        destination = "/downloads"
+        read_only = false
+      }
+
+      resources {
+        cpu = 100
+        memory = 300
+        memory_max = 500
+      }
+    }
+  }
+}
--- a/nomad/services/main.tf
+++ b/nomad/services/main.tf
@ -0,0 +1,109 @@
+# module "nextcloud" {
+#   source = "./nextcloud"
+#
+#   depends_on = [module.databases]
+# }
+
+module "backups" {
+  source = "./backups"
+
+  # In parent module
+  # depends_on = [module.databases]
+}
+
+module "media" {
+  source = "./media"
+}
+
+resource "nomad_job" "whoami" {
+  hcl2 {
+    enabled = true
+    vars = {
+      "count" = 1,
+      # "count" = "${2 * length(data.consul_service.nomad.service)}",
+    }
+  }
+
+  jobspec = file("${path.module}/whoami.nomad")
+}
+
+resource "nomad_job" "ipdvr" {
+  jobspec = file("${path.module}/ip-dvr.nomad")
+}
+
+resource "consul_config_entry" "nzbget_intents" {
+  depends_on = [nomad_job.ipdvr]
+
+  name = "nzbget"
+  kind = "service-intentions"
+
+  config_json = jsonencode({
+    Sources = [
+      {
+        Action     = "allow"
+        Name       = "sonarr"
+        Precedence = 9
+        Type       = "consul"
+      },
+    ]
+  })
+}
+
+# module "nzbget" {
+#   source "./levant"
+#
+#   template_path = "service.nomad"
+#   variables = {
+#     name = "nzbget"
+#     image = "linuxserver/nzbget"
+#     service_port = 6789
+#     ingress = true
+#     env = jsonencode({
+#       PGID = 100
+#       PUID = 1001
+#       TZ = "America/Los_Angeles"
+#     })
+#     host_volumes = jsonencode([
+#       {
+#         name      = "download"
+#         dest      = "/srv/volumes/download"
+#         read_only = false
+#       },
+#     ])
+#   }
+# }
+#
+# module "sonarr" {
+#   source = "./levant"
+#
+#   template_path = "service.nomad"
+#   variables = {
+#     name         = "sonarr"
+#     image        = "linuxserver/sonarr"
+#     service_port = 8989
+#     ingress      = true
+#     env = jsonencode({
+#       PGID = 100
+#       PUID = 1001
+#       TZ = "America/Los_Angeles"
+#
+#     })
+#     host_volumes = jsonencode([
+#       {
+#         name      = "sonarr-data"
+#         dest      = "/config"
+#         read_only = false
+#       },
+#       {
+#         name      = "tv-sonarr"
+#         dest      = "/srv/volumes/media-write/TV Shows"
+#         read_only = false
+#       },
+#       {
+#         name      = "download"
+#         dest      = "/srv/volumes/download"
+#         read_only = false
+#       },
+#     ])
+#   }
+# }
--- a/nomad/services/media/.terraform.lock.hcl
+++ b/nomad/services/media/.terraform.lock.hcl
@ -4,6 +4,7 @@
 provider "registry.terraform.io/hashicorp/nomad" {
  version = "1.4.16"
  hashes = [
+    "h1:PQxNPNmMVOErxryTWIJwr22k95DTSODmgRylqjc2TjI=",
    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
--- a/nomad/services/media/caddy.nomad
+++ b/nomad/services/media/caddy.nomad
--- a/nomad/services/media/media.tf
+++ b/nomad/services/media/media.tf
--- a/nomad/services/nextcloud/.terraform.lock.hcl
+++ b/nomad/services/nextcloud/.terraform.lock.hcl
@ -4,6 +4,7 @@
 provider "registry.terraform.io/hashicorp/nomad" {
  version = "1.4.16"
  hashes = [
+    "h1:PQxNPNmMVOErxryTWIJwr22k95DTSODmgRylqjc2TjI=",
    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
--- a/nomad/services/nextcloud/nextcloud-backup.hcl
+++ b/nomad/services/nextcloud/nextcloud-backup.hcl
--- a/nomad/services/nextcloud/nextcloud.nomad
+++ b/nomad/services/nextcloud/nextcloud.nomad
--- a/nomad/services/nextcloud/nextcloud.tf
+++ b/nomad/services/nextcloud/nextcloud.tf
--- a/nomad/services/whoami.nomad
+++ b/nomad/services/whoami.nomad
--- a/nomad/setup-cluster.yml
+++ b/nomad/setup-cluster.yml
@ -9,13 +9,14 @@
  roles:
    - role: ansible-consul
      vars:
-        consul_version: "1.13.1-1"
+        consul_version: "1.13.3-1"
        consul_install_upgrade: true
        consul_install_from_repo: true
        consul_os_repo_prerequisites: []

        consul_node_role: server
        consul_bootstrap_expect: true
+        consul_bootstrap_expect_value: "{{ [(play_hosts | length), 3] | min }}"

        consul_user: consul
        consul_manage_user: true
@ -59,22 +60,12 @@

    # If DNS is broken after dnsmasq, then need to set /etc/resolv.conf to something
    # pointing to 127.0.0.1 and possibly restart Docker and Nomad
-
-    - name: Add values
-      delegate_to: localhost
-      run_once: true
-      block:
-        - name: Install python-consul
-          pip:
-            name: python-consul
-            extra_args: --index-url https://pypi.org/simple
-
-        - name: Write values
-          consul_kv:
-            host: "{{ inventory_hostname }}"
-            key: "{{ item.key }}"
-            value: "{{ item.value }}"
-          loop: "{{ consul_values | default({}) | dict2items }}"
+    - name: Update resolv.conf
+      lineinfile:
+        dest: /etc/resolv.conf
+        create: true
+        line: "nameserver 127.0.0.1"
+      become: true

 - name: Setup Vault cluster
  hosts: vault_instances
@ -85,7 +76,7 @@
  roles:
    - name: ansible-vault
      vars:
-        vault_version: 1.11.2-1
+        vault_version: 1.12.0-1
        vault_install_hashi_repo: true
        vault_harden_file_perms: true
        vault_bin_path: /usr/bin
@ -154,28 +145,24 @@
        - unseal_keys_hex is defined
        - vault_status.json["sealed"]

-    - name: Bootstrap Vault secrets
-      delegate_to: localhost
-      run_once: true
-      block:
-        - name: Install hvac
-          pip:
-            name: hvac
-            extra_args: --index-url https://pypi.org/simple
+- name: Install Docker
+  hosts: nomad_instances
+  become: true
+  vars:
+    docker_architecture_map:
+      x86_64: amd64
+      armv7l: armhf
+      aarch64: arm64
+    docker_apt_arch: "{{ docker_architecture_map[ansible_architecture] }}"
+    docker_compose_arch: "{{ (ansible_architecture == 'armv7l') | ternary('armv7', ansible_architecture) }}"
+  roles:
+    - geerlingguy.docker

-        # TODO: This fails on first run because `root_token` isn't found
-        # Fails after taht too because the kv/ space has not been created yet either! Oh noes!
-        # Maybe move data bootstrapping to after the cluster is bootstrapped
-        - name: Write values
-          no_log: true
-          community.hashi_vault.vault_write:
-            url: "http://{{ inventory_hostname }}:8200"
-            token: "{{ root_token }}"
-            path: "kv/data/{{ item.key }}"
-            data:
-              data:
-                "{{ item.value }}"
-          loop: "{{ hashi_vault_values | default({}) | dict2items }}"
+  tasks:
+    - name: Remove snapd
+      package:
+        name: snapd
+        state: absent

 # Not on Ubuntu 20.04
 # - name: Install Podman
@ -238,19 +225,6 @@
        state: mounted
        fstype: nfs4

- name: Install Docker
-  hosts: nomad_instances
-  become: true
-  vars:
-    docker_architecture_map:
-      x86_64: amd64
-      armv7l: armhf
-      aarch64: arm64
-    docker_apt_arch: "{{ docker_architecture_map[ansible_architecture] }}"
-    docker_compose_arch: "{{ (ansible_architecture == 'armv7l') | ternary('armv7', ansible_architecture) }}"
-  roles:
-    - geerlingguy.docker
-
 - name: Build Nomad cluster
  hosts: nomad_instances
  any_errors_fatal: true
@ -265,9 +239,7 @@
        mode: "0755"
        read_only: false
      - name: media-read
-        path: /srv/volumes/media-read
-        owner: "root"
-        group: "root"
+        path: /srv/volumes/media-write
        read_only: true
      - name: media-write
        path: /srv/volumes/media-write
@ -283,10 +255,16 @@
        read_only: false
      - name: download
        path: /srv/volumes/download
-        owner: "root"
-        group: "root"
+        owner: 1001
+        group: 100
        mode: "0755"
        read_only: false
+      - name: nzbget-data
+        path: /srv/volumes/container/nzbget/config
+        read_only: false
+      - name: gitea-data
+        path: /srv/volumes/container/gitea
+        read_only: false
      - name: all-volumes
        path: /srv/volumes
        owner: "root"
@ -297,14 +275,20 @@
  roles:
    - name: ansible-nomad
      vars:
-        nomad_version: "1.3.4-1"
+        nomad_version: "1.4.1-1"
        nomad_install_upgrade: true
        nomad_allow_purge_config: true

+        nomad_meta:
+          # There are issues with v1.23.0 on arm64
+          connect.sidecar_image: envoyproxy/envoy:v1.23.1
+
        # Where nomad gets installed to
        nomad_bin_dir: /usr/bin
        nomad_install_from_repo: true

+        nomad_bootstrap_expect: "{{ [(play_hosts | length), 3] | min }}"
+        nomad_raft_protocol: 3
        nomad_autopilot: true
        nomad_encrypt_enable: true
        # nomad_use_consul: true
@ -350,9 +334,6 @@

        # Create networks for binding task ports
        nomad_host_networks:
-          # - name: public
-          #   interface: eth0
-          #   reserved_ports: "22"
          - name: nomad-bridge
            interface: nomad
            reserved_ports: "22"
@ -364,15 +345,19 @@
        nomad_acl_enabled: true

        # Enable vault integration
+        # HACK: Only talk to local Vault for now because it doesn't have HTTPS
+        # TODO: Would be really great to have this over https and point to vault.consul.service
+        # nomad_vault_address: "https://vault.service.consul:8200"
+        # Right now, each node only talks to it's local Vault, so if that node is rebooted and
+        # that vault is sealed, it will not have access to vault. This is a problem if a node
+        # must reboot.
+        nomad_vault_address: "http://127.0.0.1:8200"
        # TODO: This fails on first run because the Nomad-Vault integration can't be set up
        # until Nomad has started. Could maybe figure out if ACLs have been set up and leave
        # these out until the later play, maybe just bootstrap the nomad-cluster role in Vault
        # befor Nomad is set up
-        # nomad_vault_address: "http://vault.service.consul:8200"
-        # Only talk to local Vault for now because it doesn't have HTTPS
-        nomad_vault_address: "http://127.0.0.1:8200"
        nomad_vault_create_from_role: "nomad-cluster"
-        # TODO: Probably want to restict this to a narrower scoped token
+        # TODO: (security) Probably want to restict this to a narrower scoped token
        nomad_vault_enabled: "{{ root_token is defined }}"
        nomad_vault_token: "{{ root_token | default('') }}"

@ -380,26 +365,36 @@
          ui:
            enabled: true
            consul:
-              ui_url: "http://{{ ansible_hostname }}:8500/ui"
+              ui_url: "https://{{ ansible_hostname }}:8500/ui"
            vault:
-              ui_url: "http://{{ ansible_hostname }}:8200/ui"
+              ui_url: "https://{{ ansible_hostname }}:8200/ui"
          consul:
            tags:
              - "traefik.enable=true"
              - "traefik.consulcatalog.connect=true"
              - "traefik.http.routers.nomadclient.entrypoints=websecure"

+- name: Bootstrap Nomad ACLs and scheduler
+  hosts: nomad_instances
+
  tasks:
    - name: Start Nomad
      systemd:
        state: started
        name: nomad

- name: Bootstrap Nomad ACLs
-  hosts: nomad_instances
+    - name: Nomad API reachable?
+      uri:
+        url: "http://127.0.0.1:4646/v1/status/leader"
+        method: GET
+        status_code: 200
+      register: nomad_check_result
+      retries: 6
+      until: nomad_check_result is succeeded
+      delay: 10
+      changed_when: false
+      run_once: true

-  tasks:
-    # Need to wait until nomad is running
    - name: Bootstrap ACLs
      command:
        argv:
@ -432,6 +427,20 @@
      changed_when: false
      register: read_secretid

+    - name: Enable service scheduler preemption
+      command:
+        argv:
+          - nomad
+          - operator
+          - scheduler
+          - set-config
+          - -preempt-system-scheduler=true
+          - -preempt-service-scheduler=true
+      environment:
+        NOMAD_TOKEN: "{{ read_secretid.stdout }}"
+      delegate_to: "{{ play_hosts[0] }}"
+      run_once: true
+
    - name: Look for policy
      command:
        argv:
@ -478,3 +487,13 @@
          nomad_secret_id: "{{ read_secretid.stdout }}"
      delegate_to: localhost
      run_once: true
+      notify:
+        - Restart Nomad
+
+  handlers:
+    - name: Restart Nomad
+      systemd:
+        state: restarted
+        name: nomad
+      retries: 6
+      delay: 5
--- a/nomad/unseal-vault.yml
+++ b/nomad/unseal-vault.yml
@ -0,0 +1,27 @@
+---
+- name: Unseal Vault
+  hosts: vault_instances
+
+  tasks:
+    - name: Get Vault status
+      uri:
+        url: http://127.0.0.1:8200/v1/sys/health
+        method: GET
+        status_code: 200, 429, 472, 473, 501, 503
+        body_format: json
+        return_content: true
+      register: vault_status
+
+    - name: Unseal Vault
+      no_log: true
+      command:
+        argv:
+          - "vault"
+          - "operator"
+          - "unseal"
+          - "-address=http://127.0.0.1:8200/"
+          - "{{ item }}"
+      loop: "{{ unseal_keys_hex }}"
+      when:
+        - unseal_keys_hex is defined
+        - vault_status.json["sealed"]
--- a/nomad/vault-kv.tf
+++ b/nomad/vault-kv.tf
@ -1,5 +0,0 @@
-resource "vault_mount" "kv" {
-  path        = "kv"
-  type        = "kv-v2"
-  description = "Catch all kv mount"
-}
Author	SHA1	Message	Date
Ian Fijolek	af05a96501	Add playbook for recovering nomad cluster with peers.json	2022-11-01 15:03:08 -07:00
Ian Fijolek	754c4393f3	Remove query for nomad-clients on backup task Not used anymore	2022-10-31 15:27:29 -07:00
Ian Fijolek	04adb5db04	Go back to hard coded node names for `for_each` For some reason this worked until it didn't	2022-10-31 15:24:14 -07:00
Ian Fijolek	fafe8f0103	Rename backup service fixing mysql access	2022-10-31 15:23:42 -07:00
Ian Fijolek	ca299db949	Fix sonarr dl folder	2022-10-31 11:32:09 -07:00
Ian Fijolek	e7edef528b	Refactor external service definitions	2022-10-28 12:42:28 -07:00
Ian Fijolek	2a733aff96	Increase sonarr memory Wow, this uses a lot of mem	2022-10-28 12:34:44 -07:00
Ian Fijolek	7493fb27ec	Big refactor to split core and services for better ordering	2022-10-27 14:28:34 -07:00
Ian Fijolek	c230e05aa8	Better first run bootstrap	2022-09-27 21:59:37 -07:00
Ian Fijolek	7647832434	Fix typo	2022-09-27 21:29:00 -07:00
Ian Fijolek	0a6a9be872	New playbook to reset server data	2022-09-27 21:28:37 -07:00
Ian Fijolek	3e89a101bf	Improve first run cluster setup	2022-09-27 21:28:02 -07:00
Ian Fijolek	bfa8633fcf	Remove some hosts	2022-09-27 21:27:34 -07:00
Ian Fijolek	7d25cd3098	Add local loopback as dns in resolv.conf	2022-09-26 16:52:59 -07:00
Ian Fijolek	165a8b3a40	Increase priority of stateful jobs	2022-09-26 16:40:25 -07:00
Ian Fijolek	e3ad6c4ff0	Add playbook to recover consul using peers	2022-09-16 16:46:37 -07:00
Ian Fijolek	1654fd7aef	Add another pi	2022-09-16 16:46:10 -07:00
Ian Fijolek	ff49e039da	Enable preemption on nomad scheduler	2022-09-16 16:45:26 -07:00
Ian Fijolek	845ea4d391	Make Nomad media-read volume point to rw nfs	2022-09-16 16:44:40 -07:00
Ian Fijolek	b9ea8eb0a2	Remove bootstrapping values from setup playbook This will be done in another playbook	2022-09-16 16:43:45 -07:00
Ian Fijolek	fbacdb2699	Bootstrap with 3 servers	2022-09-16 16:42:54 -07:00
Ian Fijolek	04c76bd5f8	Get ddclient working again It was failing due to oom and using cloudflare api tokens which aren't yet supported	2022-09-07 12:20:58 -07:00
Ian Fijolek	50bdb61575	Wait until Nomad is running before bootstrapping ACLs	2022-09-07 11:11:10 -07:00
Ian Fijolek	bd1280b970	Clean up comments in setup cluster playbook	2022-09-07 11:09:33 -07:00
Ian Fijolek	ecbd9626e8	Explicitly set envoy version The default v1.23.0 does not work on arm64	2022-09-07 11:06:26 -07:00
Ian Fijolek	a9bbc3705a	Update vault and nomad versions	2022-09-07 11:05:54 -07:00
Ian Fijolek	c8d77e553d	Add playbook to unseal Vault	2022-09-07 11:05:27 -07:00
Ian Fijolek	24c04f4217	Make vault load balancer sticky Assets like css and js were not proxying correctly. I think it may be because they were proxying to a different instance and that the paths are dynamic. This should route subsequent requests for the session to a single backend.	2022-09-06 17:17:14 -07:00
Ian Fijolek	8aed3a877e	Add homeassistant external	2022-09-06 17:15:43 -07:00
Ian Fijolek	47f8b18b46	add nomad login	2022-09-06 14:47:06 -07:00
Ian Fijolek	cfc0a45440	Update security todos and reference node IP for consul queries	2022-09-06 14:46:49 -07:00
Ian Fijolek	a57b1ddee5	Move redis data to ephemeral disk	2022-09-06 11:31:15 -07:00
Ian Fijolek	0e5181fcf0	Move prometheus tsdb data to emphemeral disk	2022-09-06 11:15:14 -07:00
Ian Fijolek	fa5f9e28e6	Move acme certs to /local so they will persit between allocs	2022-09-06 09:45:04 -07:00
Ian Fijolek	722b63260c	Add splay to blocky template render Avoid all instances going down at once when the template canges	2022-09-05 12:57:13 -07:00
Ian Fijolek	67df912755	Fix syslog proxy Apparently traefik only supports http proxy over connect. https://github.com/traefik/traefik/issues/7803	2022-09-04 20:21:02 -07:00