Merge loki and promtail to one job

core/exporters.tf

@@ -1,5 +0,0 @@
-resource "nomad_job" "exporters" {
-  jobspec = templatefile("${path.module}/exporters.nomad", {
-    use_wesher = var.use_wesher,
-  })
-}

core/loki-config.yml

@@ -1,41 +0,0 @@
-auth_enabled: false
-
-server:
-  http_listen_port: 3100
-
-common:
-  ring:
-    instance_addr: 127.0.0.1
-    kvstore:
-      store: inmemory
-  replication_factor: 1
-  path_prefix: /tmp/loki
-
-schema_config:
-  configs:
-    - from: 2020-05-15
-      store: boltdb-shipper
-      object_store: filesystem
-      schema: v11
-      index:
-        prefix: index_
-        period: 24h
-
-storage_config:
-  boltdb_shipper:
-    active_index_directory: {{ env "NOMAD_TASK_DIR" }}/index
-
-  filesystem:
-    directory: {{ env "NOMAD_TASK_DIR" }}/chunks
-
-limits_config:
-  enforce_metric_name: false
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-
-chunk_store_config:
-  max_look_back_period: 168h
-
-table_manager:
-  retention_deletes_enabled: true
-  retention_period: 168h

core/loki.nomad

@@ -1,10 +1,116 @@
-job "exporters" {
+job "loki" {
   datacenters = ["dc1"]
   type = "service"
   priority = 55
 
-  constraint {
-    distinct_hosts = true
+  group "loki" {
+    count = 1
+
+    network {
+      mode = "bridge"
+
+      port "main" {
+        %{~ if use_wesher ~}
+        host_network = "wesher"
+        %{~ endif ~}
+        to = 3100
+      }
+    }
+
+    # Loki uses disk storage. We want to preserve, if possible
+    # but some loss is ok.
+    ephemeral_disk {
+      migrate = true
+      sticky = true
+    }
+
+    task "loki" {
+      driver = "docker"
+
+      service {
+        name = "loki"
+        provider = "nomad"
+        port = "main"
+
+        tags = [
+          "traefik.enable=true",
+          "traefik.http.routers.loki.entryPoints=websecure",
+        ]
+
+        check {
+          type = "http"
+          path = "/ready"
+          interval = "30s"
+          timeout = "2s"
+
+          check_restart {
+            limit = 5
+            grace = "90s"
+          }
+        }
+      }
+
+      config {
+        image = "grafana/loki:2.8.7"
+        ports = ["main"]
+        args = ["--config.file=$${NOMAD_TASK_DIR}/loki-config.yml"]
+      }
+
+      template {
+        data = <<EOF
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki
+
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: {{ env "NOMAD_TASK_DIR" }}/index
+
+  filesystem:
+    directory: {{ env "NOMAD_TASK_DIR" }}/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 168h
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 168h
+
+EOF
+        destination = "$${NOMAD_TASK_DIR}/loki-config.yml"
+
+        env = false
+      }
+
+      resources {
+        cpu = 50
+        memory = 100
+      }
+    }
   }
 
   group "promtail" {
@@ -12,6 +118,10 @@ job "exporters" {
     # I am moving away from `system` jobs because of https://github.com/hashicorp/nomad/issues/1202
     count = 2
 
+    constraint {
+      distinct_hosts = true
+    }
+
     network {
       mode = "bridge"
 

core/loki.tf

@@ -1,24 +1,5 @@
-module "loki" {
-  source = "../services/service"
-  detach = false
-
-  name  = "loki"
-  image = "grafana/loki:2.8.7"
-  args  = ["--config.file=$${NOMAD_TASK_DIR}/loki-config.yml"]
-
-  service_port = 3100
-  ingress      = true
-  use_wesher   = var.use_wesher
-  service_check = {
-    path = "/ready"
-  }
-
-  sticky_disk = true
-  templates = [
-    {
-      data  = file("${path.module}/loki-config.yml")
-      dest  = "loki-config.yml"
-      mount = false
-    }
-  ]
+resource "nomad_job" "loki" {
+  jobspec = templatefile("${path.module}/loki.nomad", {
+    use_wesher = var.use_wesher,
+  })
 }

core/main.tf

@@ -19,7 +19,7 @@ resource "nomad_job" "nomad-client-stalker" {
 resource "nomad_job" "syslog-ng" {
   jobspec = file("${path.module}/syslogng.nomad")
 
-  depends_on = [module.loki]
+  depends_on = [nomad_job.loki]
 }
 
 resource "nomad_job" "ddclient" {