From 90b77403435f42a8be21a393602ae9ff8db7f5a5 Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Thu, 30 May 2024 11:41:40 -0700
Subject: [PATCH] Move Blocky and Exporters away from system to service jobs

This is because service jobs do not get rescheduled when allocs fail
---
 core/blocky/blocky.nomad | 18 ++++++++++++++++--
 core/exporters.nomad     |  6 +++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/core/blocky/blocky.nomad b/core/blocky/blocky.nomad
index 5bf7f19..5e0fc91 100644
--- a/core/blocky/blocky.nomad
+++ b/core/blocky/blocky.nomad
@@ -5,16 +5,25 @@ variable "config_data" {
 
 job "blocky" {
   datacenters = ["dc1"]
-  type = "system"
+  type = "service"
   priority = 100
 
+  constraint {
+    distinct_hosts = true
+  }
+
   update {
     max_parallel = 1
     # TODO: maybe switch to service job from system so we can use canary and autorollback
-    # auto_revert = true
+    auto_revert = true
+    min_healthy_time = "60s"
+    healthy_deadline = "5m"
   }
 
   group "blocky" {
+    # TODO: This must be updated to match the nubmer of servers (possibly grabbed from TF)
+    # I am moving away from `system` jobs because of https://github.com/hashicorp/nomad/issues/12023
+    count = 3
 
     network {
       mode = "bridge"
@@ -62,6 +71,11 @@ job "blocky" {
         path     = "/"
         interval = "10s"
         timeout  = "3s"
+
+        check_restart {
+          limit = 3
+          grace = "5m"
+        }
       }
     }
 
diff --git a/core/exporters.nomad b/core/exporters.nomad
index 38a9262..99e1a70 100644
--- a/core/exporters.nomad
+++ b/core/exporters.nomad
@@ -1,8 +1,12 @@
 job "exporters" {
   datacenters = ["dc1"]
-  type = "system"
+  type = "service"
+  priority = 55
 
   group "promtail" {
+    # TODO: This must be updated to match the nubmer of servers (possibly grabbed from TF)
+    # I am moving away from `system` jobs because of https://github.com/hashicorp/nomad/issues/12023
+    count = 3
 
     network {
       mode = "bridge"