Add some more monitors to nomad minitor

Update diun config to read from task socket
Add ability to set task identities for service module
2024-05-06 14:29:17 -07:00 · 2024-05-01 10:18:54 -07:00 · 2024-05-01 10:18:24 -07:00 · 2024-05-01 09:43:28 -07:00 · 2024-05-01 09:40:21 -07:00 · 2024-04-27 19:33:10 -07:00
73 changed files with 1679 additions and 1668 deletions
--- a/.secrets-baseline
+++ b/.secrets-baseline
@ -132,7 +132,7 @@
        "filename": "core/authelia.yml",
        "hashed_secret": "a32b08d97b1615dc27f58b6b17f67624c04e2c4f",
        "is_verified": false,
-        "line_number": 185,
+        "line_number": 189,
        "is_secret": false
      }
    ],
@ -187,5 +187,5 @@
      }
    ]
  },
-  "generated_at": "2023-08-24T20:00:24Z"
+  "generated_at": "2024-02-20T18:04:29Z"
 }
--- a/.terraform.lock.hcl
+++ b/.terraform.lock.hcl
@ -2,39 +2,39 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/nomad" {
-  version = "2.0.0"
+  version = "2.2.0"
  hashes = [
-    "h1:lIHIxA6ZmfyTGL3J9YIddhxlfit4ipSS09BLxkwo6L0=",
-    "zh:09b897d64db293f9a904a4a0849b11ec1e3fff5c638f734d82ae36d8dc044b72",
-    "zh:435cc106799290f64078ec24b6c59cb32b33784d609088638ed32c6d12121199",
-    "zh:7073444bd064e8c4ec115ca7d9d7f030cc56795c0a83c27f6668bba519e6849a",
+    "h1:BAjqzVkuXxHtRKG+l9unaZJPk2kWZpSTCEcQPRcl2so=",
+    "zh:052f909d25121e93dc799290216292fca67943ccde12ba515068b838a6ff8c66",
+    "zh:20e29aeb9989f7a1e04bb4093817c7acc4e1e737bb21a3066f3ea46f2001feff",
+    "zh:2326d101ef427599b72cce30c0e0c1d18ae783f1a897c20f2319fbf54bab0a61",
+    "zh:3420cbe4fd19cdc96d715d0ae8e79c272608023a76033bbf582c30637f6d570f",
+    "zh:41ec570f87f578f1c57655e2e4fbdb9932d94cf92dc9cd11828cccedf36dd4a4",
+    "zh:5f90dcc58e3356ffead82ea211ecb4a2d7094d3c2fbd14ff85527c3652a595a2",
+    "zh:64aaa48609d2db868fcfd347490df0e12c6c3fcb8e4f12908c5d52b1a0adf73f",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:79d238c35d650d2d83a439716182da63f3b2767e72e4cbd0b69cb13d9b1aebfc",
-    "zh:7ef5f49344278fe0bbc5447424e6aa5425ff1821d010d944a444d7fa2c751acf",
-    "zh:92179091638c8ba03feef371c4361a790190f9955caea1fa59de2055c701a251",
-    "zh:a8a34398851761368eb8e7c171f24e55efa6e9fdbb5c455f6dec34dc17f631bc",
-    "zh:b38fd5338625ebace5a4a94cea1a28b11bd91995d834e318f47587cfaf6ec599",
-    "zh:b71b273a2aca7ad5f1e07c767b25b5a888881ba9ca93b30044ccc39c2937f03c",
-    "zh:cd14357e520e0f09fb25badfb4f2ee37d7741afdc3ed47c7bcf54c1683772543",
-    "zh:e05e025f4bb95138c3c8a75c636e97cd7cfd2fc1525b0c8bd097db8c5f02df6e",
+    "zh:86b4923e10e6ba407d1d2aab83740b702058e8b01460af4f5f0e4008f40e492c",
+    "zh:ae89dcba33097af33a306344d20e4e25181f15dcc1a860b42db5b7199a97c6a6",
+    "zh:ce56d68cdfba60891765e94f9c0bf69eddb985d44d97db9f91874bea027f08e2",
+    "zh:e993bcde5dbddaedf3331e3014ffab904f98ab0f5e8b5d6082b7ca5083e0a2f1",
  ]
 }

 provider "registry.terraform.io/hashicorp/random" {
-  version = "3.5.1"
+  version = "3.6.0"
  hashes = [
-    "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=",
-    "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
-    "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
-    "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
-    "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
-    "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
+    "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=",
+    "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d",
+    "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211",
+    "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829",
+    "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d",
+    "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
-    "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
-    "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
-    "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
-    "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
-    "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
+    "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17",
+    "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21",
+    "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839",
+    "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0",
+    "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c",
+    "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e",
  ]
 }
--- a/10
+++ b/10
@ -87,6 +87,16 @@ apply:
 		-auto-approve \
 		-var "nomad_secret_id=$(shell jq -r .SecretID nomad_bootstrap.json)" \

+.PHONY: refresh
+refresh:
+	@terraform refresh \
+		-var "nomad_secret_id=$(shell jq -r .SecretID nomad_bootstrap.json)" \
+
+.PHONY: destroy
+destroy:
+	@terraform destroy \
+		-var "nomad_secret_id=$(shell jq -r .SecretID nomad_bootstrap.json)" \
+
 .PHONY: clean
 clean:
 	env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \
--- a/ansible_playbooks/ansible_hosts.yml
+++ b/ansible_playbooks/ansible_hosts.yml
@ -1,62 +1,72 @@
 ---
 all:
-  children:
-    servers:
-      hosts:
-        n1.thefij:
-          nomad_node_role: both
-          # nomad_meta:
-          #   hw_transcode.device: /dev/dri
-          #   hw_transcode.type: intel
-          nfs_mounts:
-            - src: 10.50.250.2:/srv/volumes
-              path: /srv/volumes/moxy
-              opts: proto=tcp,rw
-          nomad_unique_host_volumes:
-            - name: mysql-data
-              path: /srv/volumes/mysql
-              owner: "999"
-              group: "100"
-              mode: "0755"
-              read_only: false
-            - name: postgres-data
-              path: /srv/volumes/postgres
-              owner: "999"
-              group: "999"
-              mode: "0755"
-              read_only: false
-        n2.thefij:
-          nfs_mounts:
-            - src: 10.50.250.2:/srv/volumes
-              path: /srv/volumes/moxy
-              opts: proto=tcp,rw
-          nomad_node_class: ingress
-          nomad_node_role: both
-          nomad_unique_host_volumes:
-            - name: nextcloud-data
-              path: /srv/volumes/nextcloud
-              owner: "root"
-              group: "bin"
-              mode: "0755"
-              read_only: false
-            - name: gitea-data
-              path: /srv/volumes/gitea
-              owner: "root"
-              group: "bin"
-              mode: "0755"
-              read_only: false
-            - name: sonarr-data
-              path: /srv/volumes/sonarr
-              owner: "root"
-              group: "bin"
-              mode: "0755"
-              read_only: false
-        pi4:
-          nomad_node_role: both
-          nomad_meta:
-            hw_transcode.device: /dev/video11
-            hw_transcode.type: raspberry
+  hosts:
+    n1.thefij:
+      nomad_node_class: ingress
+      nomad_reserved_memory: 1024
+      # nomad_meta:
+      #   hw_transcode.device: /dev/dri
+      #   hw_transcode.type: intel
+      nfs_mounts:
+        - src: 10.50.250.2:/srv/volumes
+          path: /srv/volumes/moxy
+          opts: proto=tcp,rw
+      nomad_unique_host_volumes:
+        - name: mysql-data
+          path: /srv/volumes/mysql
+          owner: "999"
+          group: "100"
+          mode: "0755"
+          read_only: false
+        - name: postgres-data
+          path: /srv/volumes/postgres
+          owner: "999"
+          group: "999"
+          mode: "0755"
+          read_only: false
+    n2.thefij:
+      nomad_node_class: ingress
+      nomad_reserved_memory: 1024
+      nfs_mounts:
+        - src: 10.50.250.2:/srv/volumes
+          path: /srv/volumes/moxy
+          opts: proto=tcp,rw
+      nomad_unique_host_volumes:
+        - name: nextcloud-data
+          path: /srv/volumes/nextcloud
+          owner: "root"
+          group: "bin"
+          mode: "0755"
+          read_only: false
+    pi4:
+      nomad_node_class: ingress
+      nomad_reserved_memory: 512
+      nomad_meta:
+        hw_transcode.device: /dev/video11
+        hw_transcode.type: raspberry
+    qnomad.thefij:
+      ansible_host: 192.168.2.234
+      nomad_reserved_memory: 1024
+      # This VM uses a non-standard interface
+      nomad_network_interface: ens3

-    nomad_instances:
-      children:
-        servers: {}
+nomad_instances:
+  vars:
+    nomad_network_interface: eth0
+  children:
+    nomad_servers: {}
+    nomad_clients: {}
+nomad_servers:
+  hosts:
+    nonopi.thefij:
+      ansible_host: 192.168.2.170
+    n1.thefij: {}
+    n2.thefij: {}
+    pi4: {}
+    qnomad.thefij: {}
+nomad_clients:
+  hosts:
+    n1.thefij: {}
+    n2.thefij: {}
+    pi4: {}
+    qnomad.thefij: {}
--- a/ansible_playbooks/nomad_wesher_restart.yml
+++ b/ansible_playbooks/nomad_wesher_restart.yml
@ -14,8 +14,14 @@
        state: restarted
      become: true

+    - name: Start Docker
+      systemd:
+        name: docker
+        state: started
+      become: true
+
    - name: Start Nomad
      systemd:
        name: nomad
-        state: stopped
+        state: started
      become: true
--- a/ansible_playbooks/recover-nomad.yaml
+++ b/ansible_playbooks/recover-nomad.yaml
@ -1,6 +1,6 @@
 ---
 - name: Recover Nomad
-  hosts: nomad_instances
+  hosts: nomad_servers
  any_errors_fatal: true

  tasks:
--- a/ansible_playbooks/setup-cluster.yml
+++ b/ansible_playbooks/setup-cluster.yml
@ -14,7 +14,7 @@
        line: "nameserver {{ non_nomad_dns }}"

 - name: Install Docker
-  hosts: nomad_instances
+  hosts: nomad_clients
  become: true
  vars:
    docker_architecture_map:
@ -44,7 +44,7 @@
 #         state: present

 - name: Create NFS mounts
-  hosts: nomad_instances
+  hosts: nomad_clients
  become: true
  vars:
    shared_nfs_mounts:
@ -112,12 +112,21 @@
      - name: nzbget-config
        path: /srv/volumes/nas-container/nzbget
        read_only: false
+      - name: sonarr-config
+        path: /srv/volumes/nas-container/sonarr
+        read_only: false
      - name: lidarr-config
        path: /srv/volumes/nas-container/lidarr
        read_only: false
+      - name: radarr-config
+        path: /srv/volumes/nas-container/radarr
+        read_only: false
      - name: bazarr-config
        path: /srv/volumes/nas-container/bazarr
        read_only: false
+      - name: gitea-data
+        path: /srv/volumes/nas-container/gitea
+        read_only: false
      - name: all-volumes
        path: /srv/volumes
        owner: "root"
@ -128,9 +137,10 @@
  roles:
    - name: ansible-nomad
      vars:
-        nomad_version: "1.6.1-1"
+        nomad_version: "1.7.6-1"
        nomad_install_upgrade: true
        nomad_allow_purge_config: true
+        nomad_node_role: "{% if 'nomad_clients' in group_names %}{% if 'nomad_servers' in group_names %}both{% else %}client{% endif %}{% else %}server{% endif %}"

        # Where nomad gets installed to
        nomad_bin_dir: /usr/bin
@ -184,7 +194,8 @@
        nomad_bind_address: 0.0.0.0

        # Default interface for binding tasks
-        nomad_network_interface: eth0
+        # This is now set at the inventory level
+        # nomad_network_interface: eth0

        # Create networks for binding task ports
        nomad_host_networks:
@ -203,7 +214,7 @@
            enabled: true

 - name: Bootstrap Nomad ACLs and scheduler
-  hosts: nomad_instances
+  hosts: nomad_servers

  tasks:
    - name: Start Nomad
@ -233,6 +244,7 @@
      run_once: true
      ignore_errors: true
      register: bootstrap_result
+      changed_when: bootstrap_result is succeeded

    - name: Save bootstrap result
      copy:
@ -264,13 +276,15 @@
          - list
      environment:
        NOMAD_TOKEN: "{{ read_secretid.stdout }}"
-      run_once: true
      register: policies
+      run_once: true
+      changed_when: false

    - name: Copy policy
      copy:
        src: ../acls/nomad-anon-policy.hcl
        dest: /tmp/anonymous.policy.hcl
+      delegate_to: "{{ play_hosts[0] }}"
      run_once: true
      register: anon_policy

@ -290,6 +304,18 @@
      delegate_to: "{{ play_hosts[0] }}"
      run_once: true

+    - name: Read scheduler config
+      command:
+        argv:
+          - nomad
+          - operator
+          - scheduler
+          - get-config
+          - -json
+      run_once: true
+      register: scheduler_config
+      changed_when: false
+
    - name: Enable service scheduler preemption
      command:
        argv:
@ -297,12 +323,24 @@
          - operator
          - scheduler
          - set-config
-          - -preempt-system-scheduler=true
          - -preempt-service-scheduler=true
      environment:
        NOMAD_TOKEN: "{{ read_secretid.stdout }}"
-      delegate_to: "{{ play_hosts[0] }}"
      run_once: true
+      when: (scheduler_config.stdout | from_json)["SchedulerConfig"]["PreemptionConfig"]["ServiceSchedulerEnabled"] is false
+
+    - name: Enable system scheduler preemption
+      command:
+        argv:
+          - nomad
+          - operator
+          - scheduler
+          - set-config
+          - -preempt-system-scheduler=true
+      environment:
+        NOMAD_TOKEN: "{{ read_secretid.stdout }}"
+      run_once: true
+      when: (scheduler_config.stdout | from_json)["SchedulerConfig"]["PreemptionConfig"]["SystemSchedulerEnabled"] is false

    # - name: Set up Nomad backend and roles in Vault
    #   community.general.terraform:
--- a/ansible_playbooks/vars/nomad_vars.sample.yml
+++ b/ansible_playbooks/vars/nomad_vars.sample.yml
@ -9,8 +9,6 @@ nomad/jobs/authelia:
  db_user: VALUE
  email_sender: VALUE
  jwt_secret: VALUE
-  lldap_admin_password: VALUE
-  lldap_admin_user: VALUE
  oidc_clients: VALUE
  oidc_hmac_secret: VALUE
  oidc_issuer_certificate_chain: VALUE
@ -24,21 +22,29 @@ nomad/jobs/backup:
  nas_ftp_host: VALUE
  nas_ftp_pass: VALUE
  nas_ftp_user: VALUE
+  nas_minio_access_key_id: VALUE
+  nas_minio_secret_access_key: VALUE
 nomad/jobs/backup-oneoff-n1:
  backup_passphrase: VALUE
  nas_ftp_host: VALUE
  nas_ftp_pass: VALUE
  nas_ftp_user: VALUE
+  nas_minio_access_key_id: VALUE
+  nas_minio_secret_access_key: VALUE
 nomad/jobs/backup-oneoff-n2:
  backup_passphrase: VALUE
  nas_ftp_host: VALUE
  nas_ftp_pass: VALUE
  nas_ftp_user: VALUE
+  nas_minio_access_key_id: VALUE
+  nas_minio_secret_access_key: VALUE
 nomad/jobs/backup-oneoff-pi4:
  backup_passphrase: VALUE
  nas_ftp_host: VALUE
  nas_ftp_pass: VALUE
  nas_ftp_user: VALUE
+  nas_minio_access_key_id: VALUE
+  nas_minio_secret_access_key: VALUE
 nomad/jobs/bazarr:
  db_name: VALUE
  db_pass: VALUE
@ -61,6 +67,7 @@ nomad/jobs/git:
  db_name: VALUE
  db_pass: VALUE
  db_user: VALUE
+  oidc_secret: VALUE
  secret_key: VALUE
  smtp_sender: VALUE
 nomad/jobs/grafana:
@ -83,17 +90,15 @@ nomad/jobs/immich:
  db_name: VALUE
  db_pass: VALUE
  db_user: VALUE
-nomad/jobs/ipdvr/radarr:
-  db_pass: VALUE
-  db_user: VALUE
+nomad/jobs/lego:
+  acme_email: VALUE
+  domain_lego_dns: VALUE
+  usersfile: VALUE
 nomad/jobs/lidarr:
  db_name: VALUE
  db_pass: VALUE
  db_user: VALUE
 nomad/jobs/lldap:
-  admin_email: VALUE
-  admin_password: VALUE
-  admin_user: VALUE
  db_name: VALUE
  db_pass: VALUE
  db_user: VALUE
@ -114,18 +119,32 @@ nomad/jobs/photoprism:
 nomad/jobs/postgres-server:
  superuser: VALUE
  superuser_pass: VALUE
+nomad/jobs/radarr:
+  db_name: VALUE
+  db_pass: VALUE
+  db_user: VALUE
 nomad/jobs/redis-authelia:
  allowed_psks: VALUE
 nomad/jobs/redis-blocky:
  allowed_psks: VALUE
 nomad/jobs/rediscommander:
  redis_stunnel_psk: VALUE
+nomad/jobs/sonarr:
+  db_name: VALUE
+  db_pass: VALUE
+  db_user: VALUE
 nomad/jobs/traefik:
-  acme_email: VALUE
-  domain_lego_dns: VALUE
+  external: VALUE
  usersfile: VALUE
+nomad/jobs/unifi-traffic-route-ips:
+  unifi_password: VALUE
+  unifi_username: VALUE
 nomad/oidc:
  secret: VALUE
+secrets/ldap:
+  admin_email: VALUE
+  admin_password: VALUE
+  admin_user: VALUE
 secrets/mysql:
  mysql_root_password: VALUE
 secrets/postgres:
--- a/backups/backup.nomad
+++ b/backups/backup.nomad
@ -44,6 +44,11 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
      source = "all-volumes"
    }

+    ephemeral_disk {
+      # Try to keep restic cache intact
+      sticky = true
+    }
+
    service {
      name = "backup"
      provider = "nomad"
@ -57,6 +62,8 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
    task "backup" {
      driver = "docker"

+      shutdown_delay = "5m"
+
      volume_mount {
        volume = "all-volumes"
        destination = "/data"
@ -64,7 +71,7 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
      }

      config {
-        image = "iamthefij/resticscheduler:0.2.0"
+        image = "iamthefij/resticscheduler:0.4.0"
        ports = ["metrics"]
        args = [
          %{ if batch_node != null ~}
@ -80,10 +87,27 @@ job "backup%{ if batch_node != null }-oneoff-${batch_node}%{ endif }" {
        ]
      }

+      action "unlockenv" {
+        command = "sh"
+        args = ["-c", "/bin/resticscheduler -once -unlock all $${NOMAD_TASK_DIR}/node-jobs.hcl"]
+      }
+
+      action "unlocktmpl" {
+        command = "/bin/resticscheduler"
+        args = ["-once", "-unlock", "all", "{{ env 'NOMAD_TASK_DIR' }}/node-jobs.hcl"]
+      }
+
+      action "unlockhc" {
+        command = "/bin/resticscheduler"
+        args = ["-once", "-unlock", "all", "/local/node-jobs.hcl"]
+      }
+
      env = {
-        "RCLONE_CHECKERS" = "2"
-        "RCLONE_TRANSFERS" = "2"
-        "RCLONE_FTP_CONCURRENCY" = "5"
+        RCLONE_CHECKERS = "2"
+        RCLONE_TRANSFERS = "2"
+        RCLONE_FTP_CONCURRENCY = "5"
+        RESTIC_CACHE_DIR = "$${NOMAD_ALLOC_DIR}/data"
+        TZ = "America/Los_Angeles"
      }

      template {
@ -107,13 +131,14 @@ RCLONE_FTP_USER={{ .nas_ftp_user }}
 RCLONE_FTP_PASS={{ .nas_ftp_pass.Value | toJSON }}
 RCLONE_FTP_EXPLICIT_TLS=true
 RCLONE_FTP_NO_CHECK_CERTIFICATE=true
+AWS_ACCESS_KEY_ID={{ .nas_minio_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ .nas_minio_secret_access_key }}
 {{ end -}}
        EOF
        destination = "secrets/db.env"
        env = true
      }

-
      template {
        # Build jobs based on node
        data = <<EOF
@ -128,13 +153,31 @@ ${file("${module_path}/${job_file}")}
 {{ end -}}
 {{ end -}}
 %{ endfor ~}
+
+# Dummy job to keep task healthy on node without any stateful services
+job "Dummy" {
+  schedule = "@daily"
+
+  config {
+    repo = "/local/dummy-repo"
+    passphrase = env("BACKUP_PASSPHRASE")
+  }
+
+  backup {
+    paths = ["/local/node-jobs.hcl"]
+  }
+
+  forget {
+    KeepLast = 1
+  }
+}
        EOF
        destination = "local/node-jobs.hcl"
      }

      resources {
        cpu = 50
-        memory = 256
+        memory = 500
      }
    }

@ -147,8 +190,8 @@ ${file("${module_path}/${job_file}")}
      }

      config {
-        image = "alpine:3.17"
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
      }

      resources {
@ -156,15 +199,6 @@ ${file("${module_path}/${job_file}")}
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel {{ env "NOMAD_TASK_DIR" }}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
--- a/backups/backups.tf
+++ b/backups/backups.tf
@ -22,7 +22,9 @@ resource "nomad_job" "backup-oneoff" {
 }

 locals {
-  all_job_ids = toset(flatten([[for job in resource.nomad_job.backup-oneoff : job.id], [resource.nomad_job.backup.id]]))
+  # NOTE: This can't be dynamic in first deploy since these values are not known
+  # all_job_ids = toset(flatten([[for job in resource.nomad_job.backup-oneoff : job.id], [resource.nomad_job.backup.id]]))
+  all_job_ids = toset(["backup", "backup-oneoff-n1", "backup-oneoff-n2", "backup-oneoff-pi4"])
 }

 resource "nomad_acl_policy" "secrets_mysql" {
--- a/backups/jobs/authelia.hcl
+++ b/backups/jobs/authelia.hcl
@ -2,8 +2,12 @@ job "authelia" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/authelia"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/authelia"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Create local authelia dir" {
--- a/backups/jobs/git.hcl
+++ b/backups/jobs/git.hcl
@ -0,0 +1,57 @@
+job "git" {
+  schedule = "@daily"
+
+  config {
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/gitea"
+    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
+  }
+
+  task "Create local gitea dir" {
+    pre_script {
+      on_backup = "mkdir -p /local/gitea"
+    }
+  }
+
+  task "Backup database" {
+    mysql "Backup database" {
+      hostname       = env("MYSQL_HOST")
+      port           = env("MYSQL_PORT")
+      database       = "gitea"
+      username       = env("MYSQL_USER")
+      password       = env("MYSQL_PASSWORD")
+      no_tablespaces = true
+      dump_to        = "/local/gitea/dump.sql"
+    }
+  }
+
+  backup {
+    paths = [
+      "/local/gitea",
+      "/data/nas-container/gitea",
+    ]
+
+    backup_opts {
+      Host = "nomad"
+    }
+
+    restore_opts {
+      Host = ["nomad"]
+      # Because path is absolute
+      Target = "/"
+    }
+  }
+
+  forget {
+    KeepLast = 2
+    KeepHourly = 24
+    KeepDaily = 30
+    KeepWeekly = 8
+    KeepMonthly = 6
+    KeepYearly = 2
+    Prune = true
+  }
+}
--- a/backups/jobs/grafana.hcl
+++ b/backups/jobs/grafana.hcl
@ -2,8 +2,12 @@ job "grafana" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/grafana"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/grafana"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Create local grafana dir" {
--- a/backups/jobs/lidarr.hcl
+++ b/backups/jobs/lidarr.hcl
@ -2,8 +2,12 @@ job "lidarr" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/lidarr"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/lidarr"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Backup main database" {
@ -34,7 +38,11 @@ job "lidarr" {
    paths = ["/data/nas-container/lidarr"]

    backup_opts {
-      Exclude = ["lidarr_backup_*.zip"]
+      Exclude = [
+        "lidarr_backup_*.zip",
+        "/data/nas-container/lidarr/MediaCover",
+        "/data/nas-container/lidarr/logs",
+      ]
      Host = "nomad"
    }

--- a/backups/jobs/lldap.hcl
+++ b/backups/jobs/lldap.hcl
@ -2,8 +2,12 @@ job "lldap" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/lldap"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/lldap"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Create local backup dir" {
--- a/backups/jobs/nzbget.hcl
+++ b/backups/jobs/nzbget.hcl
@ -2,8 +2,12 @@ job "nzbget" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/nzbget"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/nzbget"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  backup {
--- a/backups/jobs/photoprism.hcl
+++ b/backups/jobs/photoprism.hcl
@ -2,8 +2,12 @@ job "photoprism" {
  schedule = "10 * * * *"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/photoprism"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/photoprism"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Create local photoprism dir" {
@ -32,6 +36,9 @@ job "photoprism" {

    backup_opts {
      Host = "nomad"
+      Exclude = [
+        "/data/nas-container/photoprism/cache",
+       ]
    }

    restore_opts {
--- a/backups/jobs/radarr.hcl
+++ b/backups/jobs/radarr.hcl
@ -0,0 +1,64 @@
+job "radarr" {
+  schedule = "@daily"
+
+  config {
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/radarr"
+    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
+  }
+
+  task "Backup main database" {
+    postgres "Backup database" {
+      hostname       = env("POSTGRES_HOST")
+      port           = env("POSTGRES_PORT")
+      username       = env("POSTGRES_USER")
+      password       = env("POSTGRES_PASSWORD")
+      database       = "radarr"
+      no_tablespaces = true
+      dump_to        = "/data/nas-container/radarr/Backups/dump-radarr.sql"
+    }
+  }
+
+  task "Backup logs database" {
+    postgres "Backup database" {
+      hostname       = env("POSTGRES_HOST")
+      port           = env("POSTGRES_PORT")
+      username       = env("POSTGRES_USER")
+      password       = env("POSTGRES_PASSWORD")
+      database       = "radarr-logs"
+      no_tablespaces = true
+      dump_to        = "/data/nas-container/radarr/Backups/dump-radarr-logs.sql"
+    }
+  }
+
+  backup {
+    paths = ["/data/nas-container/radarr"]
+
+    backup_opts {
+      Exclude = [
+        "radarr_backup_*.zip",
+        "/data/nas-container/radarr/MediaCover",
+        "/data/nas-container/radarr/logs",
+      ]
+      Host = "nomad"
+    }
+
+    restore_opts {
+      Host = ["nomad"]
+      # Because path is absolute
+      Target = "/"
+    }
+  }
+
+  forget {
+    KeepLast = 2
+    KeepDaily = 30
+    KeepWeekly = 8
+    KeepMonthly = 6
+    KeepYearly = 2
+    Prune = true
+  }
+}
--- a/backups/jobs/sabnzbd.hcl
+++ b/backups/jobs/sabnzbd.hcl
@ -2,8 +2,12 @@ job "sabnzbd" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/sabnzbd"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/sabnzbd"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  backup {
--- a/backups/jobs/sonarr.hcl
+++ b/backups/jobs/sonarr.hcl
@ -2,30 +2,46 @@ job "sonarr" {
  schedule = "@daily"

  config {
-    repo = "rclone::ftp,env_auth:/nomad/sonarr"
+    repo = "s3://backups-minio.agnosticfront.thefij:8443/nomad/sonarr"
    passphrase = env("BACKUP_PASSPHRASE")
+
+    options {
+      InsecureTls = true
+    }
  }

  task "Backup main database" {
-    sqlite "Backup database" {
-      path = "/data/sonarr/sonarr.db"
-      dump_to = "/data/sonarr/Backups/sonarr.db.bak"
+    postgres "Backup database" {
+      hostname       = env("POSTGRES_HOST")
+      port           = env("POSTGRES_PORT")
+      username       = env("POSTGRES_USER")
+      password       = env("POSTGRES_PASSWORD")
+      database       = "sonarr"
+      no_tablespaces = true
+      dump_to        = "/data/nas-container/sonarr/Backups/dump-sonarr.sql"
    }
  }

  task "Backup logs database" {
-    sqlite "Backup database" {
-      path = "/data/sonarr/logs.db"
-      dump_to = "/data/sonarr/Backups/logs.db.bak"
+    postgres "Backup database" {
+      hostname       = env("POSTGRES_HOST")
+      port           = env("POSTGRES_PORT")
+      username       = env("POSTGRES_USER")
+      password       = env("POSTGRES_PASSWORD")
+      database       = "sonarr-logs"
+      no_tablespaces = true
+      dump_to        = "/data/nas-container/sonarr/Backups/dump-sonarr-logs.sql"
    }
  }

  backup {
-    paths = ["/data/sonarr"]
+    paths = ["/data/nas-container/sonarr"]

    backup_opts {
      Exclude = [
        "sonarr_backup_*.zip",
+        "/data/nas-container/sonarr/MediaCover",
+        "/data/nas-container/sonarr/logs",
        "*.db",
        "*.db-shm",
        "*.db-wal",
--- a/core/.terraform.lock.hcl
+++ b/core/.terraform.lock.hcl
@ -2,39 +2,39 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/nomad" {
-  version = "2.0.0"
+  version = "2.1.1"
  hashes = [
-    "h1:lIHIxA6ZmfyTGL3J9YIddhxlfit4ipSS09BLxkwo6L0=",
-    "zh:09b897d64db293f9a904a4a0849b11ec1e3fff5c638f734d82ae36d8dc044b72",
-    "zh:435cc106799290f64078ec24b6c59cb32b33784d609088638ed32c6d12121199",
-    "zh:7073444bd064e8c4ec115ca7d9d7f030cc56795c0a83c27f6668bba519e6849a",
+    "h1:liQBgBXfQEYmwpoGZUfSsu0U0t/nhvuRZbMhaMz7wcQ=",
+    "zh:28bc6922e8a21334568410760150d9d413d7b190d60b5f0b4aab2f4ef52efeeb",
+    "zh:2d4283740e92ce1403875486cd5ff2c8acf9df28c190873ab4d769ce37db10c1",
+    "zh:457e16d70075eae714a7df249d3ba42c2176f19b6750650152c56f33959028d9",
+    "zh:49ee88371e355c00971eefee6b5001392431b47b3e760a5c649dda76f59fb8fa",
+    "zh:614ad3bf07155ed8a5ced41dafb09042afbd1868490a379654b3e970def8e33d",
+    "zh:75be7199d76987e7549e1f27439922973d1bf27353b44a593bfbbc2e3b9f698f",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:79d238c35d650d2d83a439716182da63f3b2767e72e4cbd0b69cb13d9b1aebfc",
-    "zh:7ef5f49344278fe0bbc5447424e6aa5425ff1821d010d944a444d7fa2c751acf",
-    "zh:92179091638c8ba03feef371c4361a790190f9955caea1fa59de2055c701a251",
-    "zh:a8a34398851761368eb8e7c171f24e55efa6e9fdbb5c455f6dec34dc17f631bc",
-    "zh:b38fd5338625ebace5a4a94cea1a28b11bd91995d834e318f47587cfaf6ec599",
-    "zh:b71b273a2aca7ad5f1e07c767b25b5a888881ba9ca93b30044ccc39c2937f03c",
-    "zh:cd14357e520e0f09fb25badfb4f2ee37d7741afdc3ed47c7bcf54c1683772543",
-    "zh:e05e025f4bb95138c3c8a75c636e97cd7cfd2fc1525b0c8bd097db8c5f02df6e",
+    "zh:888e14a24410d56b37212fbea373a3e0401d0ff8f8e4f4dd00ba8b29de9fed39",
+    "zh:aa261925e8b152636a0886b3a2149864707632d836e98a88dacea6cfb6302082",
+    "zh:ac10cefb4064b3bb63d4b0379624a416a45acf778eac0004466f726ead686196",
+    "zh:b1a3c8b4d5b2dc9b510eac5e9e02665582862c24eb819ab74f44d3d880246d4f",
+    "zh:c552e2fe5670b6d3ad9a5faf78e3a27197eeedbe2b13928d2c491fa509bc47c7",
  ]
 }

 provider "registry.terraform.io/hashicorp/random" {
-  version = "3.5.1"
+  version = "3.6.0"
  hashes = [
-    "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=",
-    "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
-    "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
-    "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
-    "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
-    "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
+    "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=",
+    "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d",
+    "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211",
+    "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829",
+    "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d",
+    "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
-    "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
-    "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
-    "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
-    "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
-    "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
+    "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17",
+    "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21",
+    "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839",
+    "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0",
+    "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c",
+    "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e",
  ]
 }
--- a/core/authelia.tf
+++ b/core/authelia.tf
@ -49,7 +49,7 @@ module "authelia" {
      mount = false
    },
    {
-      data        = "{{ with nomadVar \"nomad/jobs/authelia\" }}{{ .lldap_admin_password }}{{ end }}"
+      data        = "{{ with nomadVar \"secrets/ldap\" }}{{ .admin_password }}{{ end }}"
      dest_prefix = "$${NOMAD_SECRETS_DIR}"
      dest        = "ldap_password.txt"
      mount       = false
@ -105,6 +105,43 @@ module "authelia" {
  ]
 }

+resource "nomad_acl_policy" "authelia" {
+  name        = "authelia"
+  description = "Give access to shared authelia variables"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "authelia/*" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+
+  job_acl {
+    job_id = module.authelia.job_id
+  }
+}
+
+# Give access to ldap secrets
+resource "nomad_acl_policy" "authelia_ldap_secrets" {
+  name        = "authelia-secrets-ldap"
+  description = "Give access to LDAP secrets"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/ldap" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+
+  job_acl {
+    job_id = module.authelia.job_id
+  }
+}
+
 resource "nomad_acl_auth_method" "nomad_authelia" {
  name           = "authelia"
  type           = "OIDC"
--- a/core/authelia.yml
+++ b/core/authelia.yml
@ -89,8 +89,8 @@ authentication_backend:
    groups_filter: (member={dn})

    ## The username and password of the admin user.
-    {{ with nomadVar "nomad/jobs/authelia" }}
-    user: uid={{ .lldap_admin_user }},ou=people,{{ with nomadVar "nomad/jobs" }}{{ .ldap_base_dn }}{{ end }}
+    {{ with nomadVar "secrets/ldap" }}
+    user: uid={{ .admin_user }},ou=people,{{ with nomadVar "nomad/jobs" }}{{ .ldap_base_dn }}{{ end }}
    {{ end }}
    # password set using secrets file
    # password: <secret>
@ -151,6 +151,10 @@ access_control:
      networks: 192.168.5.0/24

  rules:
+    {{ range nomadVarList "authelia/access_control/service_rules" }}{{ with nomadVar .Path }}
+    - domain: '{{ .name }}.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{{ end }}'
+{{ .rule.Value | indent 6 }}
+    {{ end }}{{ end }}
    ## Rules applied to everyone
    - domain: '*.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{{ end }}'
      networks:
--- a/core/blocky/blocky.nomad
+++ b/core/blocky/blocky.nomad
@ -32,7 +32,9 @@ job "blocky" {

      dns {
        # Set expclicit DNS servers because tasks, by default, use this task
-        servers = ["1.1.1.1", "1.0.0.1"]
+        servers = [
+          "192.168.2.1",
+        ]
      }
    }

@ -74,8 +76,8 @@ job "blocky" {

      resources {
        cpu = 50
-        memory = 50
-        memory_max = 100
+        memory = 75
+        memory_max = 150
      }

      template {
@ -116,9 +118,9 @@ job "blocky" {
      }

      config {
-        image = "alpine:3.17"
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
        ports = ["tls"]
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
      }

      resources {
@ -126,36 +128,27 @@ job "blocky" {
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel {{ env "NOMAD_TASK_DIR" }}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
 foreground = yes
 delay = yes

+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-tls" -}}
 [mysql_client]
 client = yes
 accept = 127.0.0.1:3306
-{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "mysql-tls" -}}
 connect = {{ .Address }}:{{ .Port }}
-{{- end }}
 PSKsecrets = {{ env "NOMAD_SECRETS_DIR" }}/mysql_stunnel_psk.txt
+{{- end }}

+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "redis-blocky" -}}
 [redis_client]
 client = yes
 accept = 127.0.0.1:6379
-{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "redis-blocky" -}}
 connect = {{ .Address }}:{{ .Port }}
-{{- end }}
 PSKsecrets = {{ env "NOMAD_SECRETS_DIR" }}/stunnel_psk.txt
+{{- end }}
        EOF
        destination = "$${NOMAD_TASK_DIR}/stunnel.conf"
      }
--- a/core/blocky/config.yml
+++ b/core/blocky/config.yml
@ -5,25 +5,45 @@ ports:
 bootstrapDns:
  - upstream: 1.1.1.1
  - upstream: 1.0.0.1
+  - upstream: 9.9.9.9
+  - upstream: 149.112.112.112

-upstream:
-  default:
-    - 1.1.1.1
-    - 1.0.0.1
-  quad9:
-    - 9.9.9.9
-    - 149.112.112.112
-    - 2620:fe::fe
-    - 2620:fe::9
-    - https://dns.quad9.net/dns-query
-    - tcp-tls:dns.quad9.net
-  quad9-unsecured:
-    - 9.9.9.10
-    - 149.112.112.10
-    - 2620:fe::10
-    - 2620:fe::fe:10
-    - https://dns10.quad9.net/dns-query
-    - tcp-tls:dns10.quad9.net
+
+upstreams:
+  groups:
+    default:
+      - https://dns.quad9.net/dns-query
+      - tcp-tls:dns.quad9.net
+      - https://one.one.one.one/dns-query
+      - tcp-tls:one.one.one.one
+    cloudflare:
+      - 1.1.1.1
+      - 1.0.0.1
+      - 2606:4700:4700::1111
+      - 2606:4700:4700::1001
+      - https://one.one.one.one/dns-query
+      - tcp-tls:one.one.one.one
+    quad9:
+      - 9.9.9.9
+      - 149.112.112.112
+      - 2620:fe::fe
+      - 2620:fe::9
+      - https://dns.quad9.net/dns-query
+      - tcp-tls:dns.quad9.net
+    quad9-secured:
+      - 9.9.9.11
+      - 149.112.112.11
+      - 2620:fe::11
+      - 2620:fe::fe:11
+      - https://dns11.quad9.net/dns-query
+      - tcp-tls:dns11.quad9.net
+    quad9-unsecured:
+      - 9.9.9.10
+      - 149.112.112.10
+      - 2620:fe::10
+      - 2620:fe::fe:10
+      - https://dns10.quad9.net/dns-query
+      - tcp-tls:dns10.quad9.net

 conditional:
  fallbackUpstream: false
@ -36,9 +56,11 @@ conditional:
    .: 192.168.2.1

 hostsFile:
-  filePath: {{ env "NOMAD_TASK_DIR" }}/nomad.hosts
+  sources:
+    - {{ env "NOMAD_TASK_DIR" }}/nomad.hosts
  hostsTTL: 30s
-  refreshPeriod: 30s
+  loading:
+    refreshPeriod: 30s

 clientLookup:
  upstream: 192.168.2.1
@ -50,10 +72,10 @@ blocking:
      - http://sysctl.org/cameleon/hosts
      - https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
      - https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
-      - https://hosts-file.net/ad_servers.txt
+      # - https://hosts-file.net/ad_servers.txt
    smarttv:
      - https://perflyst.github.io/PiHoleBlocklist/SmartTV.txt
-      - https://perflyst.github.io/PiHoleBlocklist/regex.list
+      # - https://perflyst.github.io/PiHoleBlocklist/regex.list
    wemo:
      - |
        # Remote commands
@ -64,8 +86,6 @@ blocking:
        nat.wemo2.com
        # Connectivity checks
        heartbeat.xwemo.com
-    malware:
-      - https://mirror1.malwaredomains.com/files/justdomains
    antisocial:
      - |
        facebook.com
@ -73,18 +93,21 @@ blocking:
        reddit.com
        twitter.com
        youtube.com
+    custom:
+      - https://git.thefij.rocks/iamthefij/blocklists/raw/branch/main/block

  whiteLists:
-    # Move to Gitea when deployed internally
    ads:
 {{ with nomadVar "nomad/jobs/blocky" -}}
 {{ .whitelists_ads.Value | indent 6 }}
 {{- end }}
+    custom:
+      - https://git.thefij.rocks/iamthefij/blocklists/raw/branch/main/allow

  clientGroupsBlock:
    default:
      - ads
-      - malware
+      - custom
      - smarttv
      - wemo

@ -105,7 +128,7 @@ customDNS:
 prometheus:
  enable: true

-{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "redis-tls" -}}
+{{ range nomadService 1 (env "NOMAD_ALLOC_ID") "redis-blocky" -}}
 redis:
    address: 127.0.0.1:6379
    # password: ""
--- a/core/grafana.nomad
+++ b/core/grafana.nomad
@ -28,7 +28,6 @@ job "grafana" {
      tags = [
        "traefik.enable=true",
        "traefik.http.routers.grafana.entryPoints=websecure",
-        # "traefik.http.routers.grafana.middlewares=authelia@nomad",
      ]
    }

@ -41,8 +40,8 @@ job "grafana" {
      }

      config {
-        image = "alpine:3.17"
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
      }

      resources {
@ -50,15 +49,6 @@ job "grafana" {
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel {{ env "NOMAD_TASK_DIR" }}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
@ -143,14 +133,15 @@ SELECT 'NOOP';
      driver = "docker"

      config {
-        image = "grafana/grafana:9.4.2"
+        image = "grafana/grafana:10.0.10"
        args = ["--config", "$${NOMAD_ALLOC_DIR}/config/grafana.ini"]
        ports = ["web"]
      }

      env = {
        "GF_INSTALL_PLUGINS" = "grafana-clock-panel,grafana-piechart-panel,grafana-polystat-panel,natel-discrete-panel",
-        "GF_PATHS_PROVISIONING" = "$${NOMAD_ALLOC_DIR}/config/provisioning"
+        "GF_PATHS_CONFIG" = "$${NOMAD_ALLOC_DIR}/config/grafana.ini",
+        "GF_PATHS_PROVISIONING" = "$${NOMAD_ALLOC_DIR}/config/provisioning",
      }

      template {
@ -265,7 +256,7 @@ ${file(join("/", [module_path, "grafana", config_file]))}
        # Set owner to grafana uid
        # uid = 472
        # Change template delimeter for dashboard files that use json and have double curly braces and square braces
-        %{ if length(regexall("dashboard", config_file)) > 0 ~}
+        %{ if endswith(config_file, ".json") ~}
        left_delimiter = "<<<<"
        right_delimiter = ">>>>"
        %{ endif }
@ -281,6 +272,11 @@ ${file(join("/", [module_path, "grafana", config_file]))}
    task "grafana-image-renderer" {
      driver = "docker"

+      constraint {
+        attribute = "$${attr.cpu.arch}"
+        value = "amd64"
+      }
+
      config {
        image = "grafana/grafana-image-renderer:3.6.1"
        ports = ["renderer"]
--- a/core/grafana/grafana.ini
+++ b/core/grafana/grafana.ini
@ -20,8 +20,8 @@ data = /var/lib/grafana
 # Directory where grafana will automatically scan and look for plugins
 ;plugins = /var/lib/grafana/plugins

-# folder that contains provisioning config files that grafana will apply on startup and while running.
-; provisioning = /etc/grafana/provisioning
+# folder that contains PROVISIONING config files that grafana will apply on startup and while running.
+provisioning = from_env

 #################################### Server ####################################
 [server]
@ -270,6 +270,10 @@ api_url = https://authelia.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{
 login_attribute_path = preferred_username
 groups_attribute_path = groups
 name_attribute_path = name
+# Role attribute path is not working
+role_attribute_path = contains(groups[*], 'admin') && 'Admin' || contains(groups[*], 'grafana-admin') && 'Admin' || contains(groups[*], 'grafana-editor') && 'Editor' || contains(groups[*], 'developer') && 'Editor'
+allow_assign_grafana_admin = true
+skip_org_role_sync = true
 use_pkce = true

 ;team_ids =
--- a/core/grafana/provisioning/dashboards/default/blocky.json
+++ b/core/grafana/provisioning/dashboards/default/blocky.json
@ -104,7 +104,7 @@
            "uid": "Prometheus"
          },
          "exemplar": false,
-          "expr": "sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "instant": true,
          "interval": "",
@ -458,7 +458,7 @@
            "uid": "Prometheus"
          },
          "exemplar": true,
-          "expr": "sum(blocky_blacklist_cache) / sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(blocky_blacklist_cache) / sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "instant": false,
          "interval": "",
@ -533,7 +533,7 @@
            "uid": "Prometheus"
          },
          "exemplar": true,
-          "expr": "sum(go_memstats_sys_bytes{job=\"exporters\", consul_service=\"blocky-api\"})/sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(go_memstats_sys_bytes{job=\"exporters\", consul_service=\"blocky-api\"})/sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "instant": false,
          "interval": "",
@ -753,7 +753,7 @@
            "uid": "Prometheus"
          },
          "exemplar": true,
-          "expr": "sum(blocky_cache_entry_count)/ sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(blocky_cache_entry_count)/ sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "instant": false,
          "interval": "",
@ -1162,7 +1162,7 @@
            "uid": "Prometheus"
          },
          "exemplar": false,
-          "expr": "sum(time() -blocky_last_list_group_refresh)/ sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(time() -blocky_last_list_group_refresh)/ sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "instant": true,
          "interval": "",
@ -1224,7 +1224,7 @@
            "uid": "Prometheus"
          },
          "exemplar": true,
-          "expr": "sum(blocky_prefetch_domain_name_cache_count)/ sum(up{job=\"exporters\", consul_service=\"blocky-api\"})",
+          "expr": "sum(blocky_prefetch_domain_name_cache_count)/ sum(nomad_client_allocs_running{exported_job=\"blocky\", task=\"blocky\"})",
          "format": "table",
          "interval": "",
          "legendFormat": "",
--- a/core/grafana/provisioning/dashboards/default/traefik.json
+++ b/core/grafana/provisioning/dashboards/default/traefik.json
@ -1,783 +0,0 @@
-{
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS",
-      "label": "Prometheus",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "7.5.5"
-    },
-    {
-      "type": "panel",
-      "id": "graph",
-      "name": "Graph",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "piechart",
-      "name": "Pie chart v2",
-      "version": ""
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "singlestat",
-      "name": "Singlestat",
-      "version": ""
-    }
-  ],
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "description": "Traefik dashboard prometheus",
-  "editable": true,
-  "gnetId": 4475,
-  "graphTooltip": 0,
-  "id": null,
-  "iteration": 1620932097756,
-  "links": [],
-  "panels": [
-    {
-      "datasource": null,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 10,
-      "title": "$backend stats",
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "decimals": 0,
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "id": 2,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 3,
-      "options": {
-        "displayLabels": [],
-        "legend": {
-          "calcs": [],
-          "displayMode": "table",
-          "placement": "right",
-          "values": [
-            "value",
-            "percent"
-          ]
-        },
-        "pieType": "pie",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "text": {}
-      },
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "traefik_service_requests_total{service=\"$service\"}",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "{{method}} : {{code}}",
-          "refId": "A"
-        }
-      ],
-      "title": "$service return code",
-      "type": "piechart"
-    },
-    {
-      "cacheTimeout": null,
-      "colorBackground": false,
-      "colorValue": false,
-      "colors": [
-        "#299c46",
-        "rgba(237, 129, 40, 0.89)",
-        "#d44a3a"
-      ],
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "format": "ms",
-      "gauge": {
-        "maxValue": 100,
-        "minValue": 0,
-        "show": false,
-        "thresholdLabels": false,
-        "thresholdMarkers": true
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "id": 4,
-      "interval": null,
-      "links": [],
-      "mappingType": 1,
-      "mappingTypes": [
-        {
-          "name": "value to text",
-          "value": 1
-        },
-        {
-          "name": "range to text",
-          "value": 2
-        }
-      ],
-      "maxDataPoints": 100,
-      "nullPointMode": "connected",
-      "nullText": null,
-      "postfix": "",
-      "postfixFontSize": "50%",
-      "prefix": "",
-      "prefixFontSize": "50%",
-      "rangeMaps": [
-        {
-          "from": "null",
-          "text": "N/A",
-          "to": "null"
-        }
-      ],
-      "sparkline": {
-        "fillColor": "rgba(31, 118, 189, 0.18)",
-        "full": false,
-        "lineColor": "rgb(31, 120, 193)",
-        "show": true
-      },
-      "tableColumn": "",
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(traefik_service_request_duration_seconds_sum{service=\"$service\"}) / sum(traefik_service_requests_total{service=\"$service\"}) * 1000",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "",
-          "refId": "A"
-        }
-      ],
-      "thresholds": "",
-      "title": "$service response time",
-      "type": "singlestat",
-      "valueFontSize": "80%",
-      "valueMaps": [
-        {
-          "op": "=",
-          "text": "N/A",
-          "value": "null"
-        }
-      ],
-      "valueName": "avg"
-    },
-    {
-      "aliasColors": {},
-      "bars": true,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 8
-      },
-      "hiddenSeries": false,
-      "id": 3,
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": false,
-        "max": true,
-        "min": true,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": false,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.5.5",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(rate(traefik_service_requests_total{service=\"$service\"}[5m]))",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "Total requests $service",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Total requests over 5min $service",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "collapsed": false,
-      "datasource": null,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 15
-      },
-      "id": 12,
-      "panels": [],
-      "title": "Global stats",
-      "type": "row"
-    },
-    {
-      "aliasColors": {},
-      "bars": true,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 16
-      },
-      "hiddenSeries": false,
-      "id": 5,
-      "legend": {
-        "alignAsTable": true,
-        "avg": false,
-        "current": true,
-        "max": true,
-        "min": true,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": false,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.5.5",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=\"200\"}[5m])",
-          "format": "time_series",
-          "intervalFactor": 2,
-          "legendFormat": "{{method}} : {{code}}",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Status code 200 over 5min",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": true,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 16
-      },
-      "hiddenSeries": false,
-      "id": 6,
-      "legend": {
-        "alignAsTable": true,
-        "avg": false,
-        "current": true,
-        "max": true,
-        "min": true,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": false,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.5.5",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code!=\"200\"}[5m])",
-          "format": "time_series",
-          "intervalFactor": 2,
-          "legendFormat": "{{ method }} : {{code}}",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Others status code over 5min",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "decimals": 0,
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 0,
-        "y": 23
-      },
-      "id": 7,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 3,
-      "options": {
-        "displayLabels": [],
-        "legend": {
-          "calcs": [],
-          "displayMode": "table",
-          "placement": "right",
-          "values": [
-            "value"
-          ]
-        },
-        "pieType": "pie",
-        "reduceOptions": {
-          "calcs": [
-            "sum"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "text": {}
-      },
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(rate(traefik_service_requests_total[5m])) by (service) ",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "{{ service }}",
-          "refId": "A"
-        }
-      ],
-      "title": "Requests by service",
-      "type": "piechart"
-    },
-    {
-      "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "decimals": 0,
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 12,
-        "x": 12,
-        "y": 23
-      },
-      "id": 8,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 3,
-      "options": {
-        "displayLabels": [],
-        "legend": {
-          "calcs": [],
-          "displayMode": "table",
-          "placement": "right",
-          "values": [
-            "value"
-          ]
-        },
-        "pieType": "pie",
-        "reduceOptions": {
-          "calcs": [
-            "sum"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "text": {}
-      },
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint =~ \"$entrypoint\"}[5m])) by (entrypoint) ",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "{{ entrypoint }}",
-          "refId": "A"
-        }
-      ],
-      "title": "Requests by protocol",
-      "type": "piechart"
-    }
-  ],
-  "schemaVersion": 27,
-  "style": "dark",
-  "tags": [
-    "traefik",
-    "prometheus"
-  ],
-  "templating": {
-    "list": [
-      {
-        "allValue": null,
-        "current": {},
-        "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(service)",
-        "description": null,
-        "error": null,
-        "hide": 0,
-        "includeAll": false,
-        "label": null,
-        "multi": false,
-        "name": "service",
-        "options": [],
-        "query": {
-          "query": "label_values(service)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      },
-      {
-        "allValue": null,
-        "current": {},
-        "datasource": "${DS_PROMETHEUS}",
-        "definition": "",
-        "description": null,
-        "error": null,
-        "hide": 0,
-        "includeAll": true,
-        "label": null,
-        "multi": true,
-        "name": "entrypoint",
-        "options": [],
-        "query": {
-          "query": "label_values(entrypoint)",
-          "refId": "Prometheus-entrypoint-Variable-Query"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ],
-    "time_options": [
-      "5m",
-      "15m",
-      "1h",
-      "6h",
-      "12h",
-      "24h",
-      "2d",
-      "7d",
-      "30d"
-    ]
-  },
-  "timezone": "",
-  "title": "Traefik",
-  "uid": "qPdAviJmz",
-  "version": 10
-}
--- a/core/grafana/provisioning/dashboards/main.yml
+++ b/core/grafana/provisioning/dashboards/main.yml
@ -5,4 +5,4 @@ providers:
    type: file
    disableDeletion: false
    options:
-      path: /etc/grafana/provisioning/dashboards/default
+      path: {{ env "NOMAD_ALLOC_DIR" }}/config/provisioning/dashboards/default
--- a/core/grafana/provisioning/datasources/influxdb.yml
+++ b/core/grafana/provisioning/datasources/influxdb.yml
@ -0,0 +1,19 @@
+---
+apiVersion: 1
+
+datasources:
+  - name: HASS Metrics
+    url: "http://192.168.2.75:8086"
+    type: influxdb
+    access: proxy
+    database: hass
+    jsonData:
+      dbName: hass
+
+  - name: Proxmox Metrics
+    url: "http://192.168.2.75:8086"
+    type: influxdb
+    access: proxy
+    database: proxmox
+    jsonData:
+      dbName: proxmox
--- a/core/lego.nomad
+++ b/core/lego.nomad
@ -0,0 +1,97 @@
+variable "lego_version" {
+  default = "4.14.2"
+  type = string
+}
+
+variable "nomad_var_dirsync_version" {
+  default = "0.0.2"
+  type = string
+}
+
+job "lego" {
+
+  type = "batch"
+
+  periodic {
+    cron = "@weekly"
+    prohibit_overlap = true
+  }
+
+  group "main" {
+
+    network {
+      dns {
+        servers = ["1.1.1.1", "1.0.0.1"]
+      }
+    }
+
+    task "main" {
+      driver = "exec"
+
+      config {
+        # image = "alpine:3"
+        command = "/bin/bash"
+        args = ["${NOMAD_TASK_DIR}/start.sh"]
+      }
+
+      artifact {
+        source = "https://github.com/go-acme/lego/releases/download/v${var.lego_version}/lego_v${var.lego_version}_linux_${attr.cpu.arch}.tar.gz"
+      }
+
+      artifact {
+        source = "https://git.iamthefij.com/iamthefij/nomad-var-dirsync/releases/download/v${var.nomad_var_dirsync_version}/nomad-var-dirsync-linux-${attr.cpu.arch}.tar.gz"
+      }
+
+      template {
+        data = <<EOH
+#! /bin/sh
+set -ex
+
+cd ${NOMAD_TASK_DIR}
+
+echo "Read certs from nomad vars"
+${NOMAD_TASK_DIR}/nomad-var-dirsync-linux-{{ env "attr.cpu.arch" }} -root-var=secrets/certs read .
+
+action=run
+if [ -f /.lego/certificates/_.thefij.rocks.crt ]; then
+    action=renew
+fi
+
+echo "Attempt to $action certificates"
+${NOMAD_TASK_DIR}/lego \
+    --accept-tos --pem \
+    --email=iamthefij@gmail.com \
+    --domains="*.thefij.rocks" \
+    --dns="cloudflare" \
+    $action \
+    --$action-hook="${NOMAD_TASK_DIR}/nomad-var-dirsync-linux-{{ env "attr.cpu.arch" }} -root-var=secrets/certs write .lego" \
+EOH
+        destination = "${NOMAD_TASK_DIR}/start.sh"
+      }
+
+      template {
+        data = <<EOH
+{{ with nomadVar "nomad/jobs/lego" -}}
+CF_DNS_API_TOKEN={{ .domain_lego_dns }}
+CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
+{{- end }}
+        EOH
+        destination = "secrets/cloudflare.env"
+        env = true
+      }
+
+      env = {
+        NOMAD_ADDR = "unix:///secrets/api.sock"
+      }
+
+      identity {
+        env = true
+      }
+
+      resources {
+        cpu = 50
+        memory = 100
+      }
+    }
+  }
+}
--- a/core/lego.tf
+++ b/core/lego.tf
@ -0,0 +1,23 @@
+resource "nomad_job" "lego" {
+  jobspec = file("${path.module}/lego.nomad")
+}
+
+resource "nomad_acl_policy" "secrets_certs_write" {
+  name        = "secrets-certs-write"
+  description = "Write certs to secrets store"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/certs/*" {
+      capabilities = ["write", "read"]
+    }
+    path "secrets/certs" {
+      capabilities = ["write", "read"]
+    }
+  }
+}
+EOH
+  job_acl {
+    job_id = "lego/*"
+  }
+}
--- a/core/loki-config.yml
+++ b/core/loki-config.yml
@ -3,31 +3,27 @@ auth_enabled: false
 server:
  http_listen_port: 3100

-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  chunk_idle_period: 5m
-  chunk_retain_period: 30s
-  max_transfer_retries: 0
+common:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki

 schema_config:
  configs:
-    - from: 2018-04-15
-      store: boltdb
+    - from: 2020-05-15
+      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
-        period: 168h
+        period: 24h

 storage_config:
-  boltdb:
-    directory: {{ env "NOMAD_TASK_DIR" }}/index
+  boltdb_shipper:
+    active_index_directory: {{ env "NOMAD_TASK_DIR" }}/index

  filesystem:
    directory: {{ env "NOMAD_TASK_DIR" }}/chunks
@ -38,8 +34,8 @@ limits_config:
  reject_old_samples_max_age: 168h

 chunk_store_config:
-  max_look_back_period: 0s
+  max_look_back_period: 168h

 table_manager:
-  retention_deletes_enabled: false
-  retention_period: 0s
+  retention_deletes_enabled: true
+  retention_period: 168h
--- a/core/loki.tf
+++ b/core/loki.tf
@ -3,15 +3,17 @@ module "loki" {
  detach = false

  name  = "loki"
-  image = "grafana/loki:2.2.1"
+  image = "grafana/loki:2.8.7"
  args  = ["--config.file=$${NOMAD_TASK_DIR}/loki-config.yml"]

  service_port = 3100
  ingress      = true
  use_wesher   = var.use_wesher
+  service_check = {
+    path = "/ready"
+  }

  sticky_disk = true
-  # healthcheck  = "/ready"
  templates = [
    {
      data  = file("${path.module}/loki-config.yml")
--- a/core/nomad-client-stalker.nomad
+++ b/core/nomad-client-stalker.nomad
@ -24,7 +24,8 @@ job "nomad-client-stalker" {

      resources {
        cpu = 10
-        memory = 10
+        memory = 15
+        memory_max = 30
      }
    }
  }
--- a/core/syslogng.nomad
+++ b/core/syslogng.nomad
@ -27,9 +27,7 @@ job "syslogng" {
      driver = "docker"

      meta = {
-        "diun.sort_tags" = "semver"
        "diun.watch_repo" = true
-        "diun.include_tags" = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
      }

      config {
--- a/core/traefik/.terraform.lock.hcl
+++ b/core/traefik/.terraform.lock.hcl
@ -2,20 +2,20 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/nomad" {
-  version = "1.4.17"
+  version = "2.1.0"
  hashes = [
-    "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
-    "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
-    "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
-    "zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
-    "zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
-    "zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
-    "zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
+    "h1:ek0L7fA+4R1/BXhbutSRqlQPzSZ5aY/I2YfVehuYeEU=",
+    "zh:39ba4d4fc9557d4d2c1e4bf866cf63973359b73e908cce237c54384512bdb454",
+    "zh:40d2b66e3f3675e6b88000c145977c1d5288510c76b702c6c131d9168546c605",
+    "zh:40fbe575d85a083f96d4703c6b7334e9fc3e08e4f1d441de2b9513215184ebcc",
+    "zh:42ce6db79e2f94557fae516ee3f22e5271f0b556638eb45d5fbad02c99fc7af3",
+    "zh:4acf63dfb92f879b3767529e75764fef68886521b7effa13dd0323c38133ce88",
+    "zh:72cf35a13c2fb542cd3c8528826e2390db9b8f6f79ccb41532e009ad140a3269",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
-    "zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
-    "zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
-    "zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
-    "zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
+    "zh:8b8bcc136c05916234cb0c3bcc3d48fda7ca551a091ad8461ea4ab16fb6960a3",
+    "zh:8e1c2f924eae88afe7ac83775f000ae8fd71a04e06228edf7eddce4df2421169",
+    "zh:abc6e725531fc06a8e02e84946aaabc3453ecafbc1b7a442ea175db14fd9c86a",
+    "zh:b735fcd1fb20971df3e92f81bb6d73eef845dcc9d3d98e908faa3f40013f0f69",
+    "zh:ce59797282505d872903789db8f092861036da6ec3e73f6507dac725458a5ec9",
  ]
 }
--- a/core/traefik/traefik.nomad
+++ b/core/traefik/traefik.nomad
@ -20,7 +20,7 @@ job "traefik" {
  }

  group "traefik" {
-    count = 1
+    count = 2

    network  {
      port "web" {
@ -35,12 +35,17 @@ job "traefik" {
        static = 514
      }

+      port "gitssh" {
+        static = 2222
+      }
+
+      port "metrics" {}
+
      dns {
        servers = [
          "192.168.2.101",
          "192.168.2.102",
          "192.168.2.30",
-          "192.168.2.170",
        ]
      }
    }
@ -50,39 +55,42 @@ job "traefik" {
      sticky = true
    }

-    service {
-      name = "traefik"
-      provider = "nomad"
-      port = "web"
-
-      check {
-        type = "http"
-        path = "/ping"
-        port = "web"
-        interval = "10s"
-        timeout = "2s"
-      }
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.http.routers.traefik.entryPoints=websecure",
-        "traefik.http.routers.traefik.service=api@internal",
-      ]
-    }
-
    task "traefik" {
      driver = "docker"

-      meta = {
-        "diun.sort_tags" = "semver"
-        "diun.watch_repo" = true
-        "diun.include_tags" = "^[0-9]+\\.[0-9]+$"
+      service {
+        name = "traefik"
+        provider = "nomad"
+        port = "web"
+
+        check {
+          type = "http"
+          path = "/ping"
+          interval = "10s"
+          timeout = "2s"
+        }
+
+        tags = [
+          "traefik.enable=true",
+          "traefik.http.routers.traefik.entryPoints=websecure",
+          "traefik.http.routers.traefik.service=api@internal",
+        ]
+      }
+
+      service {
+        name = "traefik-metrics"
+        provider = "nomad"
+        port = "metrics"
+
+        tags = [
+          "prometheus.scrape",
+        ]
      }

      config {
-        image = "traefik:2.9"
+        image = "traefik:2.10"

-        ports = ["web", "websecure"]
+        ports = ["web", "websecure", "syslog", "gitssh", "metrics"]
        network_mode = "host"

        mount {
@ -96,6 +104,12 @@ job "traefik" {
          target = "/etc/traefik/usersfile"
          source = "secrets/usersfile"
        }
+
+        mount {
+          type = "bind"
+          target = "/etc/traefik/certs"
+          source = "secrets/certs"
+        }
      }

      template {
@ -118,12 +132,9 @@ job "traefik" {
  [entryPoints.websecure]
    address = ":443"
    [entryPoints.websecure.http.tls]
-      certResolver = "letsEncrypt"
-      [[entryPoints.websecure.http.tls.domains]]
-        main = "*.<< with nomadVar "nomad/jobs" >><< .base_hostname >><< end >>"

  [entryPoints.metrics]
-    address = ":8989"
+    address = ":<< env "NOMAD_PORT_metrics" >>"

  [entryPoints.syslogtcp]
    address = ":514"
@ -131,6 +142,9 @@ job "traefik" {
  [entryPoints.syslogudp]
    address = ":514/udp"

+  [entryPoints.gitssh]
+    address = ":2222"
+
 [api]
  dashboard = true

@ -150,31 +164,9 @@ job "traefik" {
  exposedByDefault = false
  defaultRule = "Host(`{{normalize .Name}}.<< with nomadVar "nomad/jobs" >><< .base_hostname >><< end >>`)"
  [providers.nomad.endpoint]
-  address = "http://<< env "attr.unique.network.ip-address" >>:4646"
-
-<< if nomadVarExists "nomad/jobs/traefik" ->>
-[certificatesResolvers.letsEncrypt.acme]
-  email = "<< with nomadVar "nomad/jobs/traefik" >><< .acme_email >><< end >>"
-  # Store in /local because /secrets doesn't persist with ephemeral disk
-  storage = "/local/acme.json"
-  [certificatesResolvers.letsEncrypt.acme.dnsChallenge]
-    provider = "cloudflare"
-    resolvers = ["1.1.1.1:53", "8.8.8.8:53"]
-    delayBeforeCheck = 0
-<<- end >>
+  address = "http://127.0.0.1:4646"
        EOH
-        destination = "local/config/traefik.toml"
-      }
-
-      template {
-        data = <<EOH
-{{ with nomadVar "nomad/jobs/traefik" -}}
-CF_DNS_API_TOKEN={{ .domain_lego_dns }}
-CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
-{{- end }}
-        EOH
-        destination = "secrets/cloudflare.env"
-        env = true
+        destination = "${NOMAD_TASK_DIR}/config/traefik.toml"
      }

      template {
@ -185,23 +177,39 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
      entryPoints = ["websecure"]
      service = "nomad"
      rule = "Host(`nomad.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{{ end }}`)"
-    [http.routers.hass]
+
+    {{ with nomadVar "nomad/jobs/traefik" }}{{ with .external }}{{ with .Value | parseYAML -}}
+    {{ range $service, $url := . }}
+    [http.routers.{{ $service }}]
      entryPoints = ["websecure"]
-      service = "hass"
-      rule = "Host(`hass.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{{ end }}`)"
+      service = "{{ $service }}"
+      rule = "Host(`{{ $service }}.{{ with nomadVar "nomad/jobs" }}{{ .base_hostname }}{{ end }}`)"
+    {{ end }}
+    {{- end }}{{ end }}{{ end }}

  [http.services]
    [http.services.nomad]
      [http.services.nomad.loadBalancer]
        [[http.services.nomad.loadBalancer.servers]]
          url = "http://127.0.0.1:4646"
-    [http.services.hass]
-      [http.services.hass.loadBalancer]
-        [[http.services.hass.loadBalancer.servers]]
-          url = "http://192.168.3.65:8123"
+
+    {{ with nomadVar "nomad/jobs/traefik" }}{{ with .external }}{{ with .Value | parseYAML -}}
+    {{ range $service, $url := . }}
+    [http.services.{{ $service }}]
+      [http.services.{{ $service }}.loadBalancer]
+        [[http.services.{{ $service }}.loadBalancer.servers]]
+          url = "{{ $url }}"
+    {{ end }}
+    {{- end }}{{ end }}{{ end }}
        EOH
-        destination = "local/config/conf/route-hashi.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/route-hashi.toml"
        change_mode = "noop"
+        splay = "1m"
+
+        wait {
+          min = "10s"
+          max = "20s"
+        }
      }

      template {
@ -237,7 +245,39 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
      {{ end -}}
 {{- end }}
        EOH
-        destination = "local/config/conf/route-syslog-ng.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/route-syslog-ng.toml"
+        change_mode = "noop"
+        splay = "1m"
+
+        wait {
+          min = "10s"
+          max = "20s"
+        }
+      }
+
+      template {
+        data = <<EOF
+{{- with nomadVar "secrets/certs/_lego/certificates/__thefij_rocks_crt" }}{{ .contents }}{{ end -}}"
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/certs/_.thefij.rocks.crt"
+        change_mode = "noop"
+      }
+
+      template {
+        data = <<EOF
+{{- with nomadVar "secrets/certs/_lego/certificates/__thefij_rocks_key" }}{{ .contents }}{{ end -}}"
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/certs/_.thefij.rocks.key"
+        change_mode = "noop"
+      }
+
+      template {
+        data = <<EOH
+[[tls.certificates]]
+  certFile = "/etc/traefik/certs/_.thefij.rocks.crt"
+  keyFile = "/etc/traefik/certs/_.thefij.rocks.key"
+        EOH
+        destination = "${NOMAD_TASK_DIR}/config/conf/dynamic-tls.toml"
        change_mode = "noop"
      }

@ -247,12 +287,11 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
 {{ with nomadVar "nomad/jobs/traefik" }}
 {{ if .usersfile }}
  [http.middlewares.basic-auth.basicAuth]
-  # TODO: Reference secrets mount
    usersFile = "/etc/traefik/usersfile"
 {{- end }}
 {{- end }}
        EOH
-        destination = "local/config/conf/middlewares.toml"
+        destination = "${NOMAD_TASK_DIR}/config/conf/middlewares.toml"
        change_mode = "noop"
      }

@ -262,7 +301,7 @@ CF_ZONE_API_TOKEN={{ .domain_lego_dns }}
 {{ .usersfile }}
 {{- end }}
        EOH
-        destination = "secrets/usersfile"
+        destination = "${NOMAD_SECRETS_DIR}/usersfile"
        change_mode = "noop"
      }

--- a/core/traefik/traefik.tf
+++ b/core/traefik/traefik.tf
@ -1,3 +1,23 @@
 resource "nomad_job" "traefik" {
  jobspec = file("${path.module}/traefik.nomad")
 }
+
+resource "nomad_acl_policy" "treafik_secrets_certs_read" {
+  name        = "traefik-secrets-certs-read"
+  description = "Read certs to secrets store"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/certs/*" {
+      capabilities = ["read"]
+    }
+    path "secrets/certs" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+  job_acl {
+    job_id = resource.nomad_job.traefik.id
+  }
+}
--- a/databases/lldap.nomad
+++ b/databases/lldap.nomad
@ -70,10 +70,12 @@ job "lldap" {
        data = <<EOH
 ldap_base_dn = "{{ with nomadVar "nomad/jobs" }}{{ .ldap_base_dn }}{{ end }}"

-{{ with nomadVar "nomad/jobs/lldap" -}}
+{{ with nomadVar "secrets/ldap" -}}
 ldap_user_dn = "{{ .admin_user }}"
 ldap_user_email = "{{ .admin_email }}"
+{{ end -}}

+{{ with nomadVar "nomad/jobs/lldap" -}}
 [smtp_options]
 from = "{{ .smtp_from }}"
 reply_to = "{{ .smtp_reply_to }}"
@ -109,7 +111,7 @@ user = "{{ .user }}"
      }

      template {
-        data = "{{ with nomadVar \"nomad/jobs/lldap\" }}{{ .admin_password }}{{ end }}"
+        data = "{{ with nomadVar \"secrets/ldap\" }}{{ .admin_password }}{{ end }}"
        destination = "$${NOMAD_SECRETS_DIR}/user_pass.txt"
        change_mode = "restart"
      }
@ -193,9 +195,9 @@ SELECT 'NOOP';
      }

      config {
-        image = "alpine:3.17"
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
        ports = ["tls"]
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
      }

      resources {
@ -203,15 +205,6 @@ SELECT 'NOOP';
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel {{ env "NOMAD_TASK_DIR" }}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
@ -222,7 +215,7 @@ delay = yes
 accept = {{ env "NOMAD_PORT_tls" }}
 connect = 127.0.0.1:{{ env "NOMAD_PORT_ldap" }}
 ciphers = PSK
-PSKsecrets = {{ env "NOMAD_TASK_DIR" }}/stunnel_psk.txt
+PSKsecrets = {{ env "NOMAD_SECRETS_DIR" }}/stunnel_psk.txt

 [mysql_client]
 client = yes
@ -241,7 +234,7 @@ PSKsecrets = {{ env "NOMAD_SECRETS_DIR" }}/mysql_stunnel_psk.txt
 {{ with nomadVar .Path }}{{ .psk }}{{ end }}
 {{ end -}}
 EOF
-        destination = "$${NOMAD_TASK_DIR}/stunnel_psk.txt"
+        destination = "$${NOMAD_SECRETS_DIR}/stunnel_psk.txt"
      }

      template {
--- a/databases/lldap.tf
+++ b/databases/lldap.tf
@ -9,6 +9,42 @@ resource "nomad_job" "lldap" {
  detach = false
 }

+# Give access to ldap secrets
+resource "nomad_acl_policy" "lldap_ldap_secrets" {
+  name        = "lldap-secrets-ldap"
+  description = "Give access to LDAP secrets"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/ldap/*" {
+      capabilities = ["read"]
+    }
+    path "secrets/ldap" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+
+  job_acl {
+    # job_id = resource.nomad_job.lldap.id
+    job_id = "lldap"
+  }
+}
+
+# Create self-scoped psk so that config is valid at first start
+resource "random_password" "lldap_ldap_psk" {
+  length           = 32
+  override_special = "!@#%&*-_="
+}
+
+resource "nomad_variable" "lldap_ldap_psk" {
+  path = "secrets/ldap/allowed_psks/ldap"
+  items = {
+    psk = "lldap:${resource.random_password.lldap_ldap_psk.result}"
+  }
+}
+
 # Give access to smtp secrets
 resource "nomad_acl_policy" "lldap_smtp_secrets" {
  name        = "lldap-secrets-smtp"
@ -24,6 +60,7 @@ namespace "default" {
 EOH

  job_acl {
+    # job_id = resource.nomad_job.lldap.id
    job_id = "lldap"
    group  = "lldap"
    task   = "lldap"
@ -45,6 +82,7 @@ namespace "default" {
 EOH

  job_acl {
+    # job_id = resource.nomad_job.lldap.id
    job_id = "lldap"
    group  = "lldap"
    task   = "bootstrap"
@ -77,27 +115,9 @@ namespace "default" {
 EOH

  job_acl {
+    # job_id = resource.nomad_job.lldap.id
    job_id = "lldap"
    group  = "lldap"
    task   = "stunnel"
  }
 }
-
-# Give access to all ldap secrets
-resource "nomad_acl_policy" "secrets_ldap" {
-  name        = "secrets-ldap"
-  description = "Give access to Postgres secrets"
-  rules_hcl   = <<EOH
-namespace "default" {
-  variables {
-    path "secrets/ldap/*" {
-      capabilities = ["read"]
-    }
-  }
-}
-EOH
-
-  job_acl {
-    job_id = resource.nomad_job.lldap.id
-  }
-}
--- a/databases/main.tf
+++ b/databases/main.tf
@ -1,62 +0,0 @@
-resource "nomad_job" "mysql-server" {
-  jobspec = file("${path.module}/mysql.nomad")
-
-  # Block until deployed as there are servics dependent on this one
-  detach = false
-}
-
-resource "nomad_acl_policy" "secrets_mysql" {
-  name        = "secrets-mysql"
-  description = "Give access to MySQL secrets"
-  rules_hcl   = <<EOH
-namespace "default" {
-  variables {
-    path "secrets/mysql/*" {
-      capabilities = ["read"]
-    }
-  }
-}
-EOH
-
-  job_acl {
-    job_id = resource.nomad_job.mysql-server.id
-  }
-}
-
-resource "nomad_job" "postgres-server" {
-  jobspec = file("${path.module}/postgres.nomad")
-
-  # Block until deployed as there are servics dependent on this one
-  detach = false
-}
-
-resource "nomad_acl_policy" "secrets_postgres" {
-  name        = "secrets-postgres"
-  description = "Give access to Postgres secrets"
-  rules_hcl   = <<EOH
-namespace "default" {
-  variables {
-    path "secrets/postgres/*" {
-      capabilities = ["read"]
-    }
-  }
-}
-EOH
-
-  job_acl {
-    job_id = resource.nomad_job.postgres-server.id
-  }
-}
-
-resource "nomad_job" "redis" {
-  for_each = toset(["blocky", "authelia"])
-
-  jobspec = templatefile("${path.module}/redis.nomad",
-    {
-      name = each.key,
-    }
-  )
-
-  # Block until deployed as there are servics dependent on this one
-  detach = false
-}
--- a/databases/mysql.nomad
+++ b/databases/mysql.nomad
@ -81,9 +81,9 @@ MYSQL_ROOT_PASSWORD={{ .mysql_root_password }}
      driver = "docker"

      config {
-        image = "alpine:3.17"
+        image = "iamthefij/stunnel:latest"
+        args = ["${NOMAD_TASK_DIR}/stunnel.conf"]
        ports = ["tls"]
-        args = ["/bin/sh", "${NOMAD_TASK_DIR}/start.sh"]
      }

      resources {
@ -91,15 +91,6 @@ MYSQL_ROOT_PASSWORD={{ .mysql_root_password }}
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel ${NOMAD_TASK_DIR}/stunnel.conf
-        EOF
-        destination = "${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
--- a/databases/mysql.tf
+++ b/databases/mysql.tf
@ -0,0 +1,41 @@
+resource "nomad_job" "mysql-server" {
+  jobspec = file("${path.module}/mysql.nomad")
+
+  # Block until deployed as there are servics dependent on this one
+  detach = false
+}
+
+resource "nomad_acl_policy" "secrets_mysql" {
+  name        = "secrets-mysql"
+  description = "Give access to MySQL secrets"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/mysql" {
+      capabilities = ["read"]
+    }
+    path "secrets/mysql/*" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+
+  job_acl {
+    # job_id = resource.nomad_job.mysql-server.id
+    job_id = "mysql-server"
+  }
+}
+
+# Create self-scoped psk so that config is valid at first start
+resource "random_password" "mysql_mysql_psk" {
+  length           = 32
+  override_special = "!@#%&*-_="
+}
+
+resource "nomad_variable" "mysql_mysql_psk" {
+  path = "secrets/mysql/allowed_psks/mysql"
+  items = {
+    psk = "mysql:${resource.random_password.mysql_mysql_psk.result}"
+  }
+}
--- a/databases/postgres.nomad
+++ b/databases/postgres.nomad
@ -73,7 +73,8 @@ POSTGRES_PASSWORD={{ .superuser_pass }}

      resources {
        cpu = 500
-        memory = 400
+        memory = 700
+        memory_max = 1200
      }
    }

@ -81,9 +82,9 @@ POSTGRES_PASSWORD={{ .superuser_pass }}
      driver = "docker"

      config {
-        image = "alpine:3.17"
+        image = "iamthefij/stunnel:latest"
+        args = ["${NOMAD_TASK_DIR}/stunnel.conf"]
        ports = ["tls"]
-        args = ["/bin/sh", "${NOMAD_TASK_DIR}/start.sh"]
      }

      resources {
@ -91,15 +92,6 @@ POSTGRES_PASSWORD={{ .superuser_pass }}
        memory = 100
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel ${NOMAD_TASK_DIR}/stunnel.conf
-        EOF
-        destination = "${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
--- a/databases/postgres.tf
+++ b/databases/postgres.tf
@ -0,0 +1,41 @@
+resource "nomad_job" "postgres-server" {
+  jobspec = file("${path.module}/postgres.nomad")
+
+  # Block until deployed as there are servics dependent on this one
+  detach = false
+}
+
+resource "nomad_acl_policy" "secrets_postgres" {
+  name        = "secrets-postgres"
+  description = "Give access to Postgres secrets"
+  rules_hcl   = <<EOH
+namespace "default" {
+  variables {
+    path "secrets/postgres" {
+      capabilities = ["read"]
+    }
+    path "secrets/postgres/*" {
+      capabilities = ["read"]
+    }
+  }
+}
+EOH
+
+  job_acl {
+    # job_id = resource.nomad_job.postgres-server.id
+    job_id = "postgres-server"
+  }
+}
+
+# Create self-scoped psk so that config is valid at first start
+resource "random_password" "postgres_postgres_psk" {
+  length           = 32
+  override_special = "!@#%&*-_="
+}
+
+resource "nomad_variable" "postgres_postgres_psk" {
+  path = "secrets/postgres/allowed_psks/postgres"
+  items = {
+    psk = "postgres:${resource.random_password.postgres_postgres_psk.result}"
+  }
+}
--- a/databases/redis.nomad
+++ b/databases/redis.nomad
@ -35,7 +35,7 @@ job "redis-${name}" {

      resources {
        cpu = 100
-        memory = 128
+        memory = 64
        memory_max = 512
      }
    }
@ -44,23 +44,14 @@ job "redis-${name}" {
      driver = "docker"

      config {
-        image = "alpine:3.17"
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
        ports = ["tls"]
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
      }

      resources {
-        cpu = 100
-        memory = 100
-      }
-
-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel $${NOMAD_TASK_DIR}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
+        cpu = 50
+        memory = 15
      }

      template {
--- a/databases/redis.tf
+++ b/databases/redis.tf
@ -0,0 +1,12 @@
+resource "nomad_job" "redis" {
+  for_each = toset(["blocky", "authelia"])
+
+  jobspec = templatefile("${path.module}/redis.nomad",
+    {
+      name = each.key,
+    }
+  )
+
+  # Block until deployed as there are servics dependent on this one
+  detach = false
+}
--- a/scripts/nomad_missing_services.py
+++ b/scripts/nomad_missing_services.py
@ -56,6 +56,10 @@ for job in nomad_req("jobs"):
    if job["Type"] in ("batch", "sysbatch"):
        continue

+    if job["Status"] != "running":
+        print(f"WARNING: job {job['Name']} is {job['Status']}")
+        continue
+
    job_detail = nomad_req("job", job["ID"])
    job_detail = cast(dict[str, Any], job_detail)

@ -83,7 +87,7 @@ for job in nomad_req("jobs"):
    restart_allocs: set[str] = set()
    for allocation in nomad_req("job", job_detail["ID"], "allocations"):
        allocation = cast(dict[str, Any], allocation)
-        if allocation["TaskGroup"] in restart_groups:
+        if allocation["ClientStatus"] == "running" and allocation["TaskGroup"] in restart_groups:
            restart_allocs.add(allocation["ID"])

    # Restart allocs associated with missing services
--- a/scripts/nomad_orphan_services.py
+++ b/scripts/nomad_orphan_services.py
@ -57,9 +57,11 @@ for namespace in nomad_req("services"):
            except requests.exceptions.HTTPError as e:
                if e.response.status_code == 404:
                    alloc_found = False
-                    print(
-                        f"alloc {alloc_id} not found for {service_name}. Deleting {service_id}"
-                    )
+                    message = f"alloc {alloc_id} not found for {service_name}."
+                    if args.delete:
+                        message += f" Deleting {service_id}"
+
+                    print(message)
                else:
                    raise e

--- a/services/.terraform.lock.hcl
+++ b/services/.terraform.lock.hcl
@ -2,39 +2,39 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/nomad" {
-  version = "2.0.0"
+  version = "2.1.1"
  hashes = [
-    "h1:lIHIxA6ZmfyTGL3J9YIddhxlfit4ipSS09BLxkwo6L0=",
-    "zh:09b897d64db293f9a904a4a0849b11ec1e3fff5c638f734d82ae36d8dc044b72",
-    "zh:435cc106799290f64078ec24b6c59cb32b33784d609088638ed32c6d12121199",
-    "zh:7073444bd064e8c4ec115ca7d9d7f030cc56795c0a83c27f6668bba519e6849a",
+    "h1:liQBgBXfQEYmwpoGZUfSsu0U0t/nhvuRZbMhaMz7wcQ=",
+    "zh:28bc6922e8a21334568410760150d9d413d7b190d60b5f0b4aab2f4ef52efeeb",
+    "zh:2d4283740e92ce1403875486cd5ff2c8acf9df28c190873ab4d769ce37db10c1",
+    "zh:457e16d70075eae714a7df249d3ba42c2176f19b6750650152c56f33959028d9",
+    "zh:49ee88371e355c00971eefee6b5001392431b47b3e760a5c649dda76f59fb8fa",
+    "zh:614ad3bf07155ed8a5ced41dafb09042afbd1868490a379654b3e970def8e33d",
+    "zh:75be7199d76987e7549e1f27439922973d1bf27353b44a593bfbbc2e3b9f698f",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:79d238c35d650d2d83a439716182da63f3b2767e72e4cbd0b69cb13d9b1aebfc",
-    "zh:7ef5f49344278fe0bbc5447424e6aa5425ff1821d010d944a444d7fa2c751acf",
-    "zh:92179091638c8ba03feef371c4361a790190f9955caea1fa59de2055c701a251",
-    "zh:a8a34398851761368eb8e7c171f24e55efa6e9fdbb5c455f6dec34dc17f631bc",
-    "zh:b38fd5338625ebace5a4a94cea1a28b11bd91995d834e318f47587cfaf6ec599",
-    "zh:b71b273a2aca7ad5f1e07c767b25b5a888881ba9ca93b30044ccc39c2937f03c",
-    "zh:cd14357e520e0f09fb25badfb4f2ee37d7741afdc3ed47c7bcf54c1683772543",
-    "zh:e05e025f4bb95138c3c8a75c636e97cd7cfd2fc1525b0c8bd097db8c5f02df6e",
+    "zh:888e14a24410d56b37212fbea373a3e0401d0ff8f8e4f4dd00ba8b29de9fed39",
+    "zh:aa261925e8b152636a0886b3a2149864707632d836e98a88dacea6cfb6302082",
+    "zh:ac10cefb4064b3bb63d4b0379624a416a45acf778eac0004466f726ead686196",
+    "zh:b1a3c8b4d5b2dc9b510eac5e9e02665582862c24eb819ab74f44d3d880246d4f",
+    "zh:c552e2fe5670b6d3ad9a5faf78e3a27197eeedbe2b13928d2c491fa509bc47c7",
  ]
 }

 provider "registry.terraform.io/hashicorp/random" {
-  version = "3.5.1"
+  version = "3.6.0"
  hashes = [
-    "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=",
-    "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
-    "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
-    "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
-    "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
-    "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
+    "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=",
+    "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d",
+    "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211",
+    "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829",
+    "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d",
+    "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
-    "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
-    "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
-    "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
-    "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
-    "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
+    "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17",
+    "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21",
+    "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839",
+    "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0",
+    "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c",
+    "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e",
  ]
 }
--- a/services/bazarr.tf
+++ b/services/bazarr.tf
@ -1,58 +0,0 @@
-module "bazarr" {
-  source = "./service"
-
-  name  = "bazarr"
-  image = "lscr.io/linuxserver/bazarr:1.2.4"
-
-  resources = {
-    cpu    = 150
-    memory = 400
-  }
-
-  ingress      = true
-  service_port = 6767
-  use_wesher   = var.use_wesher
-
-  use_postgres = true
-  postgres_bootstrap = {
-    enabled = true
-  }
-
-  env = {
-    PGID = 100
-    PUID = 1001
-    TZ   = "America/Los_Angeles"
-  }
-
-  host_volumes = [
-    {
-      name      = "bazarr-config"
-      dest      = "/config"
-      read_only = false
-    },
-    {
-      name      = "media-write"
-      dest      = "/media"
-      read_only = false
-    },
-  ]
-
-  templates = [
-    {
-      data        = <<EOF
-      {{ with nomadVar "nomad/jobs/bazarr" -}}
-      POSTGRES_ENABLED=True
-      POSTGRES_HOST=127.0.0.1
-      POSTGRES_PORT=5432
-      POSTGRES_DATABASE={{ .db_name }}
-      POSTGRES_USERNAME={{ .db_user }}
-      POSTGRES_PASSWORD={{ .db_pass }}
-      {{- end }}
-      EOF
-      dest_prefix = "$${NOMAD_SECRETS_DIR}/"
-      dest        = "env"
-      env         = true
-      mount       = false
-    },
-  ]
-}
--- a/services/diun.tf
+++ b/services/diun.tf
@ -2,7 +2,7 @@ module "diun" {
  source = "./service"

  name  = "diun"
-  image = "crazymax/diun:4.26"
+  image = "crazymax/diun:4.27"
  args  = ["serve", "--log-level=debug"]

  sticky_disk = true
@ -13,13 +13,16 @@ module "diun" {
    DIUN_PROVIDERS_NOMAD_WATCHBYDEFAULT = true
    DIUN_DEFAULTS_WATCHREPO             = true
    DIUN_DEFAULTS_SORTTAGS              = "semver"
-    DIUN_DEFAUTLS_INCLUDETAGS           = "^\\d+(\\.\\d+){0,2}$"
+    DIUN_DEFAULTS_INCLUDETAGS           = "^\\d+(\\.\\d+){0,2}$"

    # Nomad API
-    # TODO: Use socket in $NOMAD_SECRETS_DIR/api.sock when we can assign workload ACLs with Terraform to
-    # allow read access. Will need to update template to allow passing token by env
-    NOMAD_ADDR           = "http://$${attr.unique.network.ip-address}:4646/"
-    DIUN_PROVIDERS_NOMAD = true
+    NOMAD_ADDR                    = "unix:///secrets/api.sock"
+    DIUN_PROVIDERS_NOMAD          = true
+    DIUN_PROVIDERS_NOMAD_SECRETID = "$${NOMAD_TOKEN}"
+  }
+
+  task_identity = {
+    env = true
  }

  templates = [
@ -36,3 +39,16 @@ module "diun" {
    },
  ]
 }
+
+resource "nomad_acl_policy" "diun_query_jobs" {
+  name        = "diun-query-jobs"
+  description = "Allow diun to query jobs"
+  rules_hcl   = <<EOH
+namespace "default" {
+  capabilities = ["list-jobs", "read-job"]
+}
+EOH
+  job_acl {
+    job_id = module.diun.job_id
+  }
+}
--- a/services/gitea.tf
+++ b/services/gitea.tf
@ -0,0 +1,116 @@
+module "gitea" {
+  source = "./service"
+
+  name  = "git"
+  image = "gitea/gitea:1.21"
+
+  resources = {
+    cpu    = 200
+    memory = 512
+  }
+
+  env = {
+    # Custom files should be part of the task
+    GITEA_WORK_DIR = "$${NOMAD_TASK_DIR}"
+    GITEA_CUSTOM   = "$${NOMAD_TASK_DIR}/custom"
+  }
+
+  ingress      = true
+  service_port = 3000
+  use_wesher   = var.use_wesher
+  ports = [
+    {
+      name = "ssh"
+      to   = 22
+    }
+  ]
+  service_check = {
+    path = "/api/healthz"
+  }
+
+  custom_services = [
+    {
+      name = "git-ssh"
+      port = "ssh"
+      tags = [
+        "traefik.enable=true",
+        "traefik.tcp.routers.git-ssh.entryPoints=gitssh",
+        "traefik.tcp.routers.git-ssh.rule=HostSNI(`*`)",
+        "traefik.tcp.routers.git-ssh.tls=false",
+      ]
+    },
+  ]
+
+  use_smtp = true
+  mysql_bootstrap = {
+    enabled = true
+  }
+
+  host_volumes = [
+    {
+      name      = "gitea-data"
+      dest      = "/data"
+      read_only = false
+    },
+  ]
+
+  # TODO: Bootstrap OIDC with
+  # su -- git gitea admin auth add-oauth --name authelia --provider openidConnect --key gitea --secret "{{ .oidc_secret }}" --auto-discover-url https://authelia.thefij.rocks/.well-known/openid-configuration --skip-local-2fa
+
+  templates = [
+    {
+      data  = <<EOF
+{{ with nomadVar "nomad/jobs/git" }}
+GITEA__server__DOMAIN=git.thefij.rocks
+GITEA__server__SSH_PORT=2222
+GITEA__server__ROOT_URL=https://git.thefij.rocks
+
+GITEA__security__INSTALL_LOCK=true
+
+GITEA__database__DB_TYPE=mysql
+GITEA__database__HOST=127.0.0.1:3306
+GITEA__database__NAME={{ .db_name }}
+GITEA__database__USER={{ .db_user }}
+
+GITEA__service__DISABLE_REGISTRATION=false
+GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true
+GITEA__service__SHOW_REGISTRATION_BUTTON=false
+
+GITEA__openid__ENABLE_OPENID_SIGNIN=true
+GITEA__openid__ENABLE_OPENID_SIGNUP=true
+GITEA__openid__WHITELISTED_URIS=authelia.thefij.rocks
+
+GITEA__log__ROOT_PATH={{ env "NOMAD_TASK_DIR" }}/log
+
+GITEA__mailer__ENABLED=true
+GITEA__mailer__FROM={{ .smtp_sender }}
+
+GITEA__session__provider=db
+{{ end }}
+EOF
+      env   = true
+      mount = false
+      dest  = "env"
+    },
+    # TODO: Gitea writes these out to the ini file in /local anyway
+    # Find some way to get it to write to /secrets
+    {
+      data        = <<EOF
+{{ with nomadVar "nomad/jobs/git" }}
+GITEA__security__SECRET_KEY="{{ .secret_key }}"
+GITEA__database__PASSWD={{ .db_pass }}
+{{ end }}
+{{ with nomadVar "secrets/smtp" }}
+GITEA__mailer__SMTP_ADDR={{ .server }}
+GITEA__mailer__SMTP_PORT={{ .port }}
+GITEA__mailer__USER={{ .user }}
+GITEA__mailer__PASSWD={{ .password }}
+{{ end }}
+      EOF
+      env         = true
+      mount       = false
+      dest        = "env"
+      dest_prefix = "$${NOMAD_SECRETS_DIR}"
+    }
+  ]
+}
--- a/services/ip-dvr.nomad
+++ b/services/ip-dvr.nomad
@ -1,212 +0,0 @@
-job "ipdvr" {
-  region = "global"
-  datacenters = ["dc1"]
-
-  type = "service"
-
-  group "sabnzbd" {
-    network {
-      mode = "bridge"
-
-      port "main" {
-        %{~ if use_wesher ~}
-        host_network = "wesher"
-        %{~ endif ~}
-        to = 8080
-      }
-    }
-
-    volume "sabnzbd-config" {
-      type = "host"
-      read_only = false
-      source = "sabnzbd-config"
-    }
-
-    volume "media-downloads" {
-      type = "host"
-      read_only = false
-      source = "media-downloads"
-    }
-
-    service {
-      name = "sabnzbd"
-      provider = "nomad"
-      port = "main"
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.http.routers.sabnzbd.entryPoints=websecure",
-      ]
-    }
-
-    task "sabnzbd" {
-      driver = "docker"
-
-      config {
-        image = "linuxserver/sabnzbd"
-        ports = ["main"]
-      }
-
-      env = {
-        "PGID" = 100
-        "PUID" = 1001
-        "TZ" = "America/Los_Angeles"
-      }
-
-      volume_mount {
-        volume = "sabnzbd-config"
-        destination = "/config"
-        read_only = false
-      }
-
-      volume_mount {
-        volume = "media-downloads"
-        destination = "/downloads"
-        read_only = false
-      }
-
-      resources {
-        cpu = 400
-        memory = 500
-        memory_max = 800
-      }
-    }
-  }
-
-  group "nzbget" {
-    network {
-      mode = "bridge"
-      port "main" {
-        %{~ if use_wesher ~}
-        host_network = "wesher"
-        %{~ endif ~}
-        static = 6789
-      }
-    }
-
-    volume "nzbget-config" {
-      type = "host"
-      read_only = false
-      source = "nzbget-config"
-    }
-
-    volume "media-downloads" {
-      type = "host"
-      read_only = false
-      source = "media-downloads"
-    }
-
-    service {
-      name = "nzbget"
-      provider = "nomad"
-      port = "main"
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.http.routers.nzbget.entryPoints=websecure",
-      ]
-    }
-
-    task "nzbget" {
-      driver = "docker"
-
-      config {
-        image = "linuxserver/nzbget"
-        ports = ["main"]
-      }
-
-      env = {
-        "PGID" = 100
-        "PUID" = 1001
-        "TZ" = "America/Los_Angeles"
-      }
-
-      volume_mount {
-        volume = "nzbget-config"
-        destination = "/config"
-        read_only = false
-      }
-
-      volume_mount {
-        volume = "media-downloads"
-        destination = "/downloads"
-        read_only = false
-      }
-
-      resources {
-        cpu = 200
-        memory = 300
-        memory_max = 500
-      }
-    }
-  }
-
-
-  group "sonarr" {
-    network {
-      mode = "bridge"
-      port "main" {
-        %{~ if use_wesher ~}
-        host_network = "wesher"
-        %{~ endif ~}
-        to = 8989
-      }
-    }
-
-    volume "sonarr-data" {
-      type = "host"
-      read_only = false
-      source = "sonarr-data"
-    }
-
-    volume "media-write" {
-      type = "host"
-      read_only = false
-      source = "media-write"
-    }
-
-    service {
-      name = "sonarr"
-      provider = "nomad"
-      port = "main"
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.http.routers.sonarr.entryPoints=websecure",
-      ]
-    }
-
-    task "sonarr" {
-      driver = "docker"
-
-      config {
-        image = "lscr.io/linuxserver/sonarr:3.0.10"
-        ports = ["main"]
-      }
-
-      env = {
-        "PGID" = 100
-        "PUID" = 1001
-        "TZ" = "America/Los_Angeles"
-      }
-
-      volume_mount {
-        volume = "sonarr-data"
-        destination = "/config"
-        read_only = false
-      }
-
-      volume_mount {
-        volume = "media-write"
-        destination = "/media"
-        read_only = false
-      }
-
-      resources {
-        cpu = 100
-        memory = 500
-        memory_max = 700
-      }
-    }
-  }
-}
--- a/services/languagetool.tf
+++ b/services/languagetool.tf
@ -0,0 +1,25 @@
+module "languagetool" {
+  source = "./service"
+
+  name         = "languagetool"
+  image        = "ghcr.io/erikvl87/docker-languagetool/languagetool:4.8"
+  ingress      = true
+  service_port = 8010
+  use_wesher   = var.use_wesher
+
+  env = {
+    Java_Xmx = "512m"
+  }
+
+  service_check = {
+    path = "/v2/healthcheck"
+  }
+
+  # Possibility to use a volume over nfs to host n-gram datasets
+  # https://github.com/Erikvl87/docker-languagetool/pkgs/container/docker-languagetool%2Flanguagetool#using-n-gram-datasets
+
+  resources = {
+    cpu    = 100
+    memory = 512
+  }
+}
--- a/services/lidarr.tf
+++ b/services/lidarr.tf
@ -40,9 +40,4 @@ module "lidarr" {
    cpu    = 500
    memory = 1500
  }
-
-  stunnel_resources = {
-    cpu    = 100
-    memory = 100
-  }
 }
--- a/services/main.tf
+++ b/services/main.tf
@ -1,5 +0,0 @@
-resource "nomad_job" "ipdvr" {
-  jobspec = templatefile("${path.module}/ip-dvr.nomad", {
-    use_wesher = var.use_wesher,
-  })
-}
--- a/services/minitor-config.yml
+++ b/services/minitor-config.yml
@ -21,15 +21,6 @@ monitors:
      - '/app/scripts/curl_ok.sh'
      - 'https://grafana.thefij.rocks'

-  - name: Plex
-    command:
-      - 'curl'
-      - '--silent'
-      - '--show-error'
-      - '-o'
-      - '/dev/null'
-      - 'http://192.168.2.10:32400'
-
  - name: NZBget
    command:
      - '/app/scripts/curl_ok.sh'
@ -45,6 +36,11 @@ monitors:
      - '/app/scripts/curl_ok.sh'
      - 'https://lidarr.thefij.rocks'

+  - name: Radarr
+    command:
+      - '/app/scripts/curl_ok.sh'
+      - 'https://radarr.thefij.rocks'
+
  - name: Authelia
    command:
      - '/app/scripts/curl_ok.sh'
@ -55,6 +51,20 @@ monitors:
      - '/app/scripts/curl_ok.sh'
      - 'https://photoprism.thefij.rocks'

+  - name: Prometheus
+    command:
+      - '/app/scripts/curl_ok.sh'
+      - 'https://prometheus.thefij.rocks'
+
+  - name: Plex
+    command:
+      - 'curl'
+      - '--silent'
+      - '--show-error'
+      - '-o'
+      - '/dev/null'
+      - 'http://192.168.2.10:32400'
+
 alerts:
  log:
    command:
--- a/services/minitor.tf
+++ b/services/minitor.tf
@ -1,12 +1,13 @@
 module "minitor" {
  source = "./service"

-  name         = "minitor"
-  image        = "iamthefij/minitor-go:1.4.1"
-  args         = ["-metrics", "-config=$${NOMAD_TASK_DIR}/config.yml"]
-  service_port = 8080
-  use_wesher   = var.use_wesher
-  prometheus   = true
+  name          = "minitor"
+  image         = "iamthefij/minitor-go:1.4.1"
+  args          = ["-metrics", "-config=$${NOMAD_TASK_DIR}/config.yml"]
+  service_port  = 8080
+  service_check = null
+  use_wesher    = var.use_wesher
+  prometheus    = true

  env = {
    TZ = "America/Los_Angeles",
--- a/services/nomad-fixers.nomad
+++ b/services/nomad-fixers.nomad
@ -0,0 +1,59 @@
+job "fixers" {
+
+  type = "batch"
+
+  periodic {
+    cron = "*/15 * * * * *"
+    prohibit_overlap = true
+  }
+
+  group "orphaned_services" {
+    task "orphaned_services" {
+      driver = "docker"
+
+      config {
+        image = "iamthefij/nomad-service-fixers:0.1.1"
+        command = "/scripts/nomad_orphan_services.py"
+        args = ["--delete"]
+      }
+
+      env = {
+        NOMAD_ADDR = "http+unix://%2Fsecrets%2Fapi.sock"
+      }
+
+      identity {
+        env = true
+      }
+
+      resources {
+        cpu = 50
+        memory = 100
+      }
+    }
+  }
+
+  group "missing_services" {
+    task "missing_services" {
+      driver = "docker"
+
+      config {
+        image = "iamthefij/nomad-service-fixers:0.1.0"
+        command = "/scripts/nomad_missing_services.py"
+        args = ["--restart"]
+      }
+
+      env = {
+        NOMAD_ADDR = "http+unix://%2Fsecrets%2Fapi.sock"
+      }
+
+      identity {
+        env = true
+      }
+
+      resources {
+        cpu = 50
+        memory = 100
+      }
+    }
+  }
+}
--- a/services/nomad-fixers.tf
+++ b/services/nomad-fixers.tf
@ -0,0 +1,22 @@
+resource "nomad_job" "nomad-fixers" {
+  jobspec = file("${path.module}/nomad-fixers.nomad")
+}
+
+resource "nomad_acl_policy" "nomad_fixers_workload" {
+  name        = "nomad-fixers-workload"
+  description = "Give nomad fixers access to the Nomad api for fixing things"
+  rules_hcl   = <<EOH
+namespace "default" {
+  capabilities = [
+    "list-jobs",
+    "read-job",
+    "submit-job",  # This allows deleting a service registeration
+    "alloc-lifecycle",
+  ]
+}
+  EOH
+
+  job_acl {
+    job_id = "fixers/*"
+  }
+}
--- a/services/nzbget.tf
+++ b/services/nzbget.tf
@ -0,0 +1,35 @@
+module "nzbget" {
+  source = "./service"
+
+  name  = "nzbget"
+  image = "linuxserver/nzbget:v21.1-ls138"
+
+  ingress      = true
+  service_port = 6789
+  use_wesher   = var.use_wesher
+
+  env = {
+    PGID = 100
+    PUID = 1001
+    TZ   = "America/Los_Angeles"
+  }
+
+  host_volumes = [
+    {
+      name      = "nzbget-config"
+      dest      = "/config"
+      read_only = false
+    },
+    {
+      name      = "media-downloads"
+      dest      = "/downloads"
+      read_only = false
+    },
+  ]
+
+  resources = {
+    cpu        = 200
+    memory     = 300
+    memory_max = 500
+  }
+}
--- a/services/photoprism.tf
+++ b/services/photoprism.tf
@ -2,27 +2,24 @@ module "photoprism_module" {
  source = "./service"

  name               = "photoprism"
-  image              = "photoprism/photoprism:221118-jammy"
+  image              = "photoprism/photoprism:231128"
  image_pull_timeout = "10m"
-  constraints = [{
-    attribute = "$${meta.hw_transcode.type}"
-    # operator = "is_set"
-    value = "raspberry"
-  }]
+  # constraints = [{
+  #   attribute = "$${meta.hw_transcode.type}"
+  #   # operator = "is_set"
+  #   value = "raspberry"
+  # }]
+  priority = 60

-  docker_devices = [{
-    host_path      = "$${meta.hw_transcode.device}"
-    container_path = "$${meta.hw_transcode.device}"
-  }]
+  # docker_devices = [{
+  #   host_path      = "$${meta.hw_transcode.device}"
+  #   container_path = "$${meta.hw_transcode.device}"
+  # }]
  resources = {
-    cpu        = 2000
-    memory     = 2500
+    cpu        = 1500
+    memory     = 2200
    memory_max = 4000
  }
-  stunnel_resources = {
-    cpu    = 100
-    memory = 100
-  }
  sticky_disk = true
  host_volumes = [
    {
--- a/services/radarr.tf
+++ b/services/radarr.tf
@ -0,0 +1,59 @@
+module "radarr" {
+  source = "./service"
+
+  name  = "radarr"
+  image = "lscr.io/linuxserver/radarr:5.2.6"
+
+  ingress      = true
+  service_port = 7878
+  use_wesher   = var.use_wesher
+  ingress_middlewares = [
+    "authelia@nomad"
+  ]
+
+  use_postgres = true
+  postgres_bootstrap = {
+    enabled = true
+    databases = [
+      "radarr",
+      "radarr-logs",
+    ]
+  }
+
+  env = {
+    PGID = 100
+    PUID = 1001
+    TZ   = "America/Los_Angeles"
+  }
+
+  host_volumes = [
+    {
+      name      = "radarr-config"
+      dest      = "/config"
+      read_only = false
+    },
+    {
+      name      = "media-write"
+      dest      = "/media"
+      read_only = false
+    },
+  ]
+
+  resources = {
+    cpu        = 500
+    memory     = 500
+    memory_max = 700
+  }
+}
+
+resource "nomad_variable" "authelia_service_rules_radarr" {
+  path = "authelia/access_control/service_rules/radarr"
+  items = {
+    name = "radarr"
+    rule = <<EOH
+policy: bypass
+resources:
+  - '^/api([/?].*)?$'
+    EOH
+  }
+}
--- a/services/service/main.tf
+++ b/services/service/main.tf
@ -8,6 +8,7 @@ resource "nomad_job" "service" {
    args               = var.args
    env                = var.env
    task_meta          = var.task_meta
+    task_identity      = var.task_identity
    group_meta         = var.group_meta
    job_meta           = var.job_meta
    constraints        = var.constraints
@ -15,6 +16,7 @@ resource "nomad_job" "service" {

    service_port        = var.service_port
    service_port_static = var.service_port_static
+    service_check       = var.service_check
    ports               = var.ports
    sticky_disk         = var.sticky_disk
    resources           = var.resources
@ -59,7 +61,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = "mysql-bootstrap"
  }
@ -97,7 +99,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = "stunnel"
  }
@ -119,7 +121,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = "postgres-bootstrap"
  }
@ -157,7 +159,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = "stunnel"
  }
@ -195,7 +197,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = "stunnel"
  }
@ -217,7 +219,7 @@ namespace "default" {
 EOH

  job_acl {
-    job_id = var.name
+    job_id = resource.nomad_job.service.id
    group  = var.name
    task   = var.name
  }
--- a/services/service/output.tf
+++ b/services/service/output.tf
@ -0,0 +1,3 @@
+output "job_id" {
+  value = resource.nomad_job.service.id
+}
--- a/services/service/service_template.nomad
+++ b/services/service/service_template.nomad
@ -31,10 +31,18 @@ job "${name}" {
      %{~ endif ~}
      %{~ for port in ports }
      port "${port.name}" {
-        %{ if port.host_network != null }host_network = "${port.host_network}"%{ endif ~}
-        %{ if port.from != null }to = ${port.from}%{ endif ~}
-        %{ if port.to != null }to = ${port.to}%{ endif ~}
-        %{ if port.static != null }static = ${port.static}%{ endif ~}
+        %{~ if port.host_network != null ~}
+        host_network = "${port.host_network}"
+        %{~ endif ~}
+        %{~ if port.from != null ~}
+        from = ${port.from}
+        %{~ endif ~}
+        %{~ if port.to != null ~}
+        to = ${port.to}
+        %{~ endif ~}
+        %{~ if port.static != null ~}
+        static = ${port.static}
+        %{~ endif ~}
      }
      %{~ endfor ~}
    }
@ -65,43 +73,7 @@ job "${name}" {
      source = "${host_volume.name}"
    }
    %{~ endfor ~}
-    %{~ if service_port != null }
-    service {
-      name = "${replace(name, "_", "-")}"
-      provider = "nomad"
-      port = "main"

-      tags = [
-        %{~ if prometheus == true ~}
-        "prometheus.scrape",
-        %{~ endif ~}
-        %{~ if ingress ~}
-        "traefik.enable=true",
-        "traefik.http.routers.${name}.entryPoints=websecure",
-        %{~ if try(ingress_rule, null) != null ~}
-        "traefik.http.routers.${name}.rule=${ingress_rule}",
-        %{~ endif ~}
-        %{~ for middleware in ingress_middlewares ~}
-        "traefik.http.routers.${name}.middlewares=${middleware}",
-        %{~ endfor ~}
-        %{~ endif ~}
-        %{~ for tag in service_tags ~}
-        "${tag}",
-        %{~ endfor ~}
-      ]
-    }
-    %{~ endif ~}
-
-    %{~ for custom_service in custom_services ~}
-    service {
-      name     = "${custom_service.name}"
-      provider = "nomad"
-      port     = "${custom_service.port}"
-
-      tags = ${jsonencode(custom_service.tags)}
-    }
-
-    %{~ endfor ~}
    task "${name}" {
      driver = "docker"
      %{~ if length(task_meta) > 0 }
@ -111,7 +83,63 @@ job "${name}" {
        %{ endfor ~}
      }
      %{~ endif ~}
+      %{~ if service_port != null }
+      service {
+        name = "${replace(name, "_", "-")}"
+        provider = "nomad"
+        port = "main"

+        tags = [
+          %{~ if prometheus == true ~}
+          "prometheus.scrape",
+          %{~ endif ~}
+          %{~ if ingress ~}
+          "traefik.enable=true",
+          "traefik.http.routers.${name}.entryPoints=websecure",
+          %{~ if try(ingress_rule, null) != null ~}
+          "traefik.http.routers.${name}.rule=${ingress_rule}",
+          %{~ endif ~}
+          %{~ for middleware in ingress_middlewares ~}
+          "traefik.http.routers.${name}.middlewares=${middleware}",
+          %{~ endfor ~}
+          %{~ endif ~}
+          %{~ for tag in service_tags ~}
+          "${tag}",
+          %{~ endfor ~}
+        ]
+
+        %{~ if service_check != null ~}
+        check {
+          %{~ if service_check.name != "" ~}
+          name = "${service_check.name}"
+          %{~ endif ~}
+          %{~ if service_check.name != "" ~}
+          port = "${service_check.port}"
+          %{~ endif ~}
+          type = "${service_check.type}"
+          path = "${service_check.path}"
+          interval = "${service_check.interval}"
+          timeout = "${service_check.timeout}"
+
+          check_restart {
+            limit = 5
+            grace = "90s"
+          }
+        }
+        %{~ endif ~}
+      }
+
+      %{~ endif ~}
+      %{~ for custom_service in custom_services ~}
+      service {
+        name     = "${custom_service.name}"
+        provider = "nomad"
+        port     = "${custom_service.port}"
+
+        tags = ${jsonencode(custom_service.tags)}
+      }
+
+      %{~ endfor ~}
      config {
        image = "${image}"
        %{~if image_pull_timeout != null ~}
@ -175,6 +203,14 @@ EOF
        %{~ if template.change_signal != null }
        change_signal = "${template.change_signal}"
        %{~ endif ~}
+        %{~ if template.change_script != null }
+        change_script {
+          command = "${template.change_script.command}"
+          args = ${jsonencode(template.change_script.args)}
+          timeout = "${template.change_script.timeout}"
+          fail_on_error = ${template.change_script.fail_on_error}
+        }
+        %{~ endif ~}
        %{~ if template.env != null }
        env = ${template.env}
        %{~ endif ~}
@ -189,6 +225,12 @@ EOF
        %{~ endif ~}
      }
      %{~ endif ~}
+      %{~ if task_identity != null }
+      identity {
+        env = ${task_identity.env}
+        file = ${task_identity.file}
+      }
+      %{~ endif ~}
    }
    %{~ if mysql_bootstrap != null }
    task "mysql-bootstrap" {
@ -338,8 +380,8 @@ $$;
      }

      config {
-        image = "alpine:3.17"
-        args = ["/bin/sh", "$${NOMAD_TASK_DIR}/start.sh"]
+        image = "iamthefij/stunnel:latest"
+        args = ["$${NOMAD_TASK_DIR}/stunnel.conf"]
      }

      resources {
@ -350,15 +392,6 @@ $$;
        %{~ endif ~}
      }

-      template {
-        data = <<EOF
-set -e
-apk add stunnel
-exec stunnel {{ env "NOMAD_TASK_DIR" }}/stunnel.conf
-        EOF
-        destination = "$${NOMAD_TASK_DIR}/start.sh"
-      }
-
      template {
        data = <<EOF
 syslog = no
--- a/services/service/vars.tf
+++ b/services/service/vars.tf
@ -21,7 +21,6 @@ variable "priority" {
  description = "Scheduler priority of the service"
 }

-
 variable "image" {
  type        = string
  description = "Image that should be run"
@ -39,6 +38,15 @@ variable "task_meta" {
  description = "Meta attributes to attach to the task"
 }

+variable "task_identity" {
+  description = "Task workload identity"
+  type = object({
+    env  = optional(bool, false)
+    file = optional(bool, false)
+  })
+  default = null
+}
+
 variable "group_meta" {
  type        = map(string)
  default     = {}
@ -110,7 +118,7 @@ variable "stunnel_resources" {

  default = {
    cpu        = 50
-    memory     = 50
+    memory     = 15
    memory_max = null
  }

@ -158,12 +166,18 @@ variable "templates" {
    data            = string
    dest            = string
    dest_prefix     = optional(string, "$${NOMAD_TASK_DIR}")
-    change_mode     = optional(string)
-    change_signal   = optional(string)
    left_delimiter  = optional(string)
    right_delimiter = optional(string)
    mount           = optional(bool, true)
    env             = optional(bool, false)
+    change_mode     = optional(string)
+    change_signal   = optional(string)
+    change_script = optional(object({
+      command       = optional(string, "")
+      args          = optional(list(string), [])
+      timeout       = optional(string, "5s")
+      fail_on_error = optional(bool, false)
+    }))
  }))
  default     = []
  description = "Templates to be used"
@ -262,3 +276,17 @@ variable "use_wesher" {
  description = "Indicates whether or not services should expose themselves on the wesher network"
  default     = true
 }
+
+variable "service_check" {
+  description = "Health check for main ingress service"
+  type = object({
+    name     = optional(string, "")
+    port     = optional(string, "")
+    path     = optional(string, "/")
+    interval = optional(string, "30s")
+    timeout  = optional(string, "2s")
+    type     = optional(string, "http")
+  })
+
+  default = {}
+}
--- a/services/sonarr.tf
+++ b/services/sonarr.tf
@ -0,0 +1,61 @@
+module "sonarr" {
+  source = "./service"
+
+  name  = "sonarr"
+  image = "lscr.io/linuxserver/sonarr:4.0.2"
+
+  priority = 55
+
+  ingress      = true
+  service_port = 8989
+  use_wesher   = var.use_wesher
+  ingress_middlewares = [
+    "authelia@nomad"
+  ]
+
+  use_postgres = true
+  postgres_bootstrap = {
+    enabled = true
+    databases = [
+      "sonarr",
+      "sonarr-logs",
+    ]
+  }
+
+  env = {
+    PGID = 100
+    PUID = 1001
+    TZ   = "America/Los_Angeles"
+  }
+
+  host_volumes = [
+    {
+      name      = "sonarr-config"
+      dest      = "/config"
+      read_only = false
+    },
+    {
+      name      = "media-write"
+      dest      = "/media"
+      read_only = false
+    },
+  ]
+
+  resources = {
+    cpu        = 100
+    memory     = 500
+    memory_max = 700
+  }
+}
+
+resource "nomad_variable" "authelia_service_rules_sonarr" {
+  path = "authelia/access_control/service_rules/sonarr"
+  items = {
+    name = "sonarr"
+    rule = <<EOH
+policy: bypass
+resources:
+  - '^/api([/?].*)?$'
+    EOH
+  }
+}
--- a/services/unifi-traffic-route-ips.nomad
+++ b/services/unifi-traffic-route-ips.nomad
@ -0,0 +1,36 @@
+job "unifi-traffic-route-ips" {
+
+  type = "batch"
+
+  periodic {
+    cron = "*/15 * * * * *"
+    prohibit_overlap = true
+  }
+
+  group "main" {
+
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "iamthefij/unifi-traffic-routes:0.0.2"
+      }
+
+      env = {
+        UNIFI_HOST = "192.168.2.1",
+        UNIFI_PORT = "443",
+      }
+
+      template {
+      data        = <<EOF
+{{ with nomadVar "nomad/jobs/unifi-traffic-route-ips" -}}
+UNIFI_USER={{ .unifi_username }}
+UNIFI_PASS={{ .unifi_password }}
+{{ end -}}
+      EOF
+      destination = "$${NOMAD_SECRETS_DIR}/env"
+      env         = true
+      }
+    }
+  }
+}
--- a/services/unifi-traffic-route-ips.tf
+++ b/services/unifi-traffic-route-ips.tf
@ -0,0 +1,3 @@
+resource "nomad_job" "unifi-traffic-route-ips" {
+  jobspec = file("${path.module}/unifi-traffic-route-ips.nomad")
+}
Author	SHA1	Message	Date
IamTheFij	bdfde48bec	Add some more monitors to nomad minitor	2024-05-06 14:29:17 -07:00
IamTheFij	9af55580e7	Update diun config to read from task socket	2024-05-01 10:18:54 -07:00
IamTheFij	b9c35bf18f	Add ability to set task identities for service module	2024-05-01 10:18:24 -07:00
IamTheFij	e7f740a2d9	Add languagetool server	2024-05-01 09:43:28 -07:00
IamTheFij	57efee14e9	Update Ansible inventory to split node roles Splits servers and clients to their own groups so that plays can target specific roles. Prior, everything was "both", but i want to and another server for recovery purposes but not host containers on it.	2024-05-01 09:40:21 -07:00
IamTheFij	c711c25737	Always use CF for dns when renewing lego certs Makes it more resilient if my servers are down, but also cuts out a hop because CF is the nameserver as well.	2024-04-27 19:33:10 -07:00
IamTheFij	24122c2a3e	Split fixers to their own groups Allow them to deploy as different allocs on different hosts	2024-04-22 09:07:03 -07:00
IamTheFij	13121862ec	Add new host on qnap nas	2024-04-22 09:06:33 -07:00
IamTheFij	28da3f425b	Move nomad default interface to host vars	2024-04-22 09:06:11 -07:00
IamTheFij	2d59886378	Update diun to include ability to read nomad socket	2024-04-17 10:46:28 -07:00
IamTheFij	da0f52dab3	Improve change detection for cluster bootstrap	2024-04-17 10:46:10 -07:00
IamTheFij	beac302a53	Upgrade nomad to 1.7.6	2024-04-17 10:45:27 -07:00
IamTheFij	5edcb86e7e	Remove traefik grafana dashboard Now in data backups rather than git.	2024-03-26 14:56:14 -07:00
IamTheFij	3dcd4c44b3	Tune memory after reviewing grafana	2024-03-26 09:48:31 -07:00
IamTheFij	e6653f6495	Migrate sonarr to postgresql And increase postgresql memory to accomodate	2024-03-25 16:05:58 -07:00
IamTheFij	a9a919b8f2	Increase priority for sevices with highee resources Photoprism requires lots if mem and sonar a specific volume	2024-03-22 21:09:19 -07:00
IamTheFij	cc66bfdbcb	Update photoprism	2024-03-22 21:07:55 -07:00
IamTheFij	b02050112e	Tune some service memeory	2024-03-22 21:07:07 -07:00
IamTheFij	d5c2a0d185	Use default diun for syslogng	2024-03-22 21:05:53 -07:00
IamTheFij	6a3ae49d8e	Update terraform modules	2024-03-11 22:02:07 -07:00
IamTheFij	75ee09d7e6	Remove bazarr Plex does this automatically now	2024-02-20 10:13:40 -08:00
IamTheFij	8b90aa0d74	Add 1.1.1.1 dns back to blocky for better resiliance	2024-02-20 10:10:41 -08:00
IamTheFij	62e120ce51	Add radarr	2024-02-20 10:09:48 -08:00
IamTheFij	5fb510202d	Fix indent for Authelia rules	2024-02-20 10:05:25 -08:00
IamTheFij	64a085ef80	Reatart failing services Restart services that fail checks	2024-02-18 07:49:16 -08:00
IamTheFij	f2f415aeac	Fix traefik metrics	2024-02-18 07:47:31 -08:00
IamTheFij	bb291b1f01	Move databases to their own tf files and improve first start	2024-02-13 12:05:55 -08:00
IamTheFij	056eac976c	lldap: Make it work on first bootstrap Can't use the job id for creating the variables and permissions because we end up with circular dependencies. The job won't return until it's successful in Nomad and it won't start in nomad without access to varibles	2024-02-13 12:05:21 -08:00
IamTheFij	198f96f3f7	Add back other traefik ports and metrics	2024-02-13 12:03:03 -08:00
IamTheFij	6b5adbdf39	Remove 404 block list	2024-02-13 12:02:35 -08:00
IamTheFij	77ef4b4167	Use quad9 encrypted dns	2024-02-13 12:02:14 -08:00
IamTheFij	b35b8cecd5	Blocky: Remove mysql and redis configs from stunnel if server isn't found	2024-02-13 12:01:45 -08:00
IamTheFij	b9dfeff6d8	Have blocky use router for upstream in nomad	2024-02-13 12:01:08 -08:00
IamTheFij	2ff954b4b5	Bump nomad	2024-02-13 12:00:43 -08:00
IamTheFij	2528dafcc6	Make nomad restart playbook more resilient	2024-02-13 12:00:24 -08:00
IamTheFij	0e168376b8	Add terraform destroy to makefile	2024-02-13 11:59:47 -08:00
IamTheFij	a16dc204fe	Run dummy backup more frequently to make graphs easier to read	2024-01-24 20:10:14 -08:00
IamTheFij	93d340c182	Make sure gitea ingress uses system wesher config It was always using wesher	2024-01-23 12:09:59 -08:00
IamTheFij	37ee67b2e6	fix: Add job_id output to services This should be earlier in history	2024-01-23 12:09:29 -08:00
IamTheFij	35dfeb3093	Add service healthchecks	2024-01-23 12:08:47 -08:00
IamTheFij	0a2eace3dd	Fix lldap secrets	2024-01-23 12:07:42 -08:00
IamTheFij	6fe1b200f2	Update loki	2024-01-23 12:06:25 -08:00
IamTheFij	c5d5ab42b8	Add some nomad actions for backups to test different formatting	2024-01-23 12:05:56 -08:00
IamTheFij	efe7864cc9	Delay shutdowns of backup jobs to reduce killing those in progress	2024-01-23 12:05:20 -08:00
IamTheFij	9ba74ce698	Use return vars for service acl	2024-01-16 14:16:21 -08:00
IamTheFij	4fe3d46d5f	Add external service acls for authelia	2024-01-16 14:15:56 -08:00
IamTheFij	cf8bde7920	Add external traefik routes to nomad vars	2024-01-16 14:15:18 -08:00
IamTheFij	bc87688f1a	Move ldap secrets	2024-01-16 14:14:39 -08:00
IamTheFij	3491c1f679	Add refresh make target	2024-01-16 14:04:44 -08:00
IamTheFij	7b019e0787	Add auth to sonarr	2024-01-08 14:57:06 -08:00
IamTheFij	0f19e2433f	Upgrade sonarr to version 4	2024-01-08 10:14:53 -08:00
IamTheFij	c01d45c7a2	Upgrade grafana to version 10	2024-01-08 10:11:42 -08:00
IamTheFij	d07afe2319	Update traffic routes to handle null IPs Eg. 0.0.0.0 for blocked domains	2024-01-06 16:23:45 -08:00
IamTheFij	b025e4a87e	Add repo unlock via Nomad action to backups	2024-01-06 16:22:20 -08:00
IamTheFij	9be16fef1f	Upgrade traefik to 2.10	2024-01-04 13:25:10 -08:00
IamTheFij	c26da678b3	Small traefik cleanup Remove fallback DNS since we only care about internal DNS Use loopback address for accessing Nomad UI	2024-01-04 13:24:49 -08:00
IamTheFij	6b9533ef71	Run traefik on multiple hosts	2024-01-04 13:24:15 -08:00
IamTheFij	0bd995ec2b	Traefik: Use nomad vars for dynamic certs Rather than having Traefik handle cert fetching, instead it is delegated to a separate job so that multiple Traefik instances can share certs	2024-01-04 10:55:49 -08:00
IamTheFij	0d340f3349	Periodic job to renew lego certs and store them in Nomad Variables This will allow multiple instance of Traefik to serve certs.	2024-01-04 10:53:25 -08:00
IamTheFij	bcad131aa7	Use job id for lldap acls	2024-01-04 10:53:23 -08:00
IamTheFij	cda2842f8f	Switch to image containing stunnel Rather than installing on container startup, using an image with stunnel pre-installed. This avoids issues with DNS breaking the container on startup.	2024-01-03 13:50:49 -08:00
IamTheFij	9544222961	Bump to 1.7.2	2023-12-29 20:47:58 -08:00
IamTheFij	7bc4ae1f8b	Reserve node memory to reduce OOM kills	2023-12-29 07:36:23 -08:00
IamTheFij	1a3c096b65	Fix nomad fixers	2023-12-29 07:35:07 -08:00
IamTheFij	25e533287d	Fix gitea backups syntax	2023-12-18 12:23:21 -08:00
IamTheFij	7e87002be2	Nomad 1.7	2023-12-18 12:22:19 -08:00
IamTheFij	ab6906e989	Gitea backups	2023-12-10 20:39:33 -08:00
IamTheFij	ca55209316	Fix blocky redis	2023-12-10 20:37:43 -08:00
IamTheFij	1b49f015c5	Update blocky config to v0.22 schema	2023-11-30 14:00:27 -08:00
IamTheFij	eb25138675	Remove defunct lists	2023-11-30 13:39:22 -08:00
IamTheFij	69a0f760b4	Remove defunct lists	2023-11-30 13:39:01 -08:00
IamTheFij	3fcedaddb7	Remove todo from traefik	2023-11-30 13:26:15 -08:00
IamTheFij	bb34b434b8	Add custom blocklists hosted on my gitea server	2023-11-30 13:23:54 -08:00
IamTheFij	36cdb8f41b	Add Gitea Currently it won't auto bootstrap auth. A command has to be executed one time to get it to be added to the database.	2023-11-30 13:22:54 -08:00
IamTheFij	cdd4e9b5d5	Fix custom ports for services	2023-11-30 13:22:53 -08:00
IamTheFij	f06e90ab0d	Remove hw transcode constraints from photoprism	2023-11-30 10:05:39 -08:00
IamTheFij	2d733b278c	Make backup jobids static so they work on clean deploy	2023-11-30 09:55:08 -08:00
IamTheFij	b218633c2c	Add scheduled job to update UniFi Traffic Routes Because I use a custom DNS server, Domain based routing rules don't work. This instead resolves the domains and then adds the IP addresses to the rules.	2023-11-20 10:37:03 -08:00
IamTheFij	e21ec11eb5	Fix grafana Broken template	2023-11-20 10:35:49 -08:00
IamTheFij	d6f9c2a7e4	Fix diun include tags variable This fixes a configuration bug causing diun to include all tags by default.	2023-11-16 12:22:44 -08:00
IamTheFij	891cfa7b2d	Update blocky dashboard to not use consul tags	2023-11-16 12:21:59 -08:00
IamTheFij	c11b8e157b	Fix grafana dashboard provisioning A path mismatch existed after migrating to alloc storage	2023-11-16 12:21:40 -08:00
IamTheFij	0d208b7394	Add dummy backup job to keep backup task running on all hosts Otherwise, if a client is not running any stateful services, the task will fail and Nomad will eventually stop retrying. If a service gets relocated to the host, the task is not restarted. This makes sure the task will cover moved services and make it more easy to determine that backups are healthy.	2023-11-16 12:19:19 -08:00
IamTheFij	9b347880cc	Ignore some regeneratable files in backups Reduces the size of Lidarr and Photoprism backups	2023-11-07 16:50:02 -08:00
IamTheFij	a0185d9642	Bump resources for backups to allow more memory Was getting OOM killed	2023-11-07 16:49:27 -08:00
IamTheFij	f2f5f4407c	Add TZ to restic-scheduler	2023-11-07 16:48:57 -08:00
IamTheFij	0b3d3caff6	Update restic-scheduler to fix index out of range error	2023-11-07 16:48:41 -08:00
IamTheFij	52abd94a38	Use minio as restic repo rather than sftp I've been getting a lot of restic lock errors using sftp	2023-11-06 16:35:13 -08:00
IamTheFij	0391fd95ad	Allow fixers to actually fix things	2023-11-06 14:41:54 -08:00
IamTheFij	df1ae60936	Add change_script to service module	2023-11-06 14:41:13 -08:00
IamTheFij	a2d33ac309	Add proxmox influxdb to Grafana	2023-10-23 13:10:01 -07:00
IamTheFij	1b48892172	Add read-only implementation of fixers as scheduled batches	2023-10-23 12:59:45 -07:00
IamTheFij	48a48bb080	Move sonarr and nzbget to their own jobs	2023-10-23 12:59:11 -07:00
IamTheFij	bd2c5ca3db	Put restic cache on ephemeral disk	2023-10-23 08:54:05 -07:00
IamTheFij	48074bdc39	Fix log from orphaned services to not say deleting when it's in dry run.	2023-10-19 12:08:28 -07:00
IamTheFij	2f3fc87f12	Consider job status when detecting missing services Prevents false alarms and attempts to restart failed or stopped allocs.	2023-10-19 12:07:57 -07:00
IamTheFij	369802cacc	Bump Postgres memory to 500mb	2023-10-19 12:07:14 -07:00
IamTheFij	0c3f98d5c3	Pin Grafana to amd64 since renderer requires it. This could be mitigated by moving the renderer to another task group.	2023-10-19 12:06:47 -07:00
IamTheFij	b97cfb68ad	Minor Nomad bmp 1.6.2	2023-10-19 12:05:52 -07:00