Add loki, promtail, and syslog-ng

nomad/core.tf

@@ -21,3 +21,30 @@ module "metrics" {
 
   consul_address = var.consul_address
 }
+
+module "loki" {
+  source = "./levant"
+
+  template_path = "service.nomad"
+  variables = {
+    name         = "loki"
+    image        = "grafana/loki:2.2.1"
+    service_port = 3100
+    ingress      = true
+    sticky_disk  = true
+    templates = jsonencode([
+      {
+        data = file("./loki-config.yml")
+        dest = "/etc/loki/local-config.yaml"
+      }
+    ])
+  }
+}
+
+resource "nomad_job" "syslog-ng" {
+  hcl2 {
+    enabled = true
+  }
+
+  jobspec = file("${path.module}/syslogng.nomad")
+}

nomad/loki-config.yml

@@ -0,0 +1,45 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+    - from: 2018-04-15
+      store: boltdb
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 168h
+
+storage_config:
+  boltdb:
+    directory: /loki/index
+
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s

nomad/metrics/exporters.nomad

@@ -204,5 +204,140 @@ job "metrics" {
         memory = 50
       }
     }
+
+    service {
+      name = "promtail"
+      port = "promtail"
+
+      meta {
+        metrics_addr = "${NOMAD_ADDR_promtail}"
+        nomad_dc = "${NOMAD_DC}"
+        nomad_node_name = "${node.unique.name}"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            local_service_port = 9080
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 50
+          }
+        }
+      }
+
+      check {
+        type = "http"
+        path = "/metrics"
+        port = "promtail"
+        interval = "10s"
+        timeout = "10s"
+      }
+    }
+
+    task "promtail" {
+      driver = "docker"
+
+      config {
+        image = "grafana/promtail:2.2.1"
+        args = ["-config.file=/etc/promtail/promtail.yml"]
+        ports = ["promtail"]
+
+        # Mount config
+        mount {
+          type = "bind"
+          target = "/etc/promtail/promtail.yml"
+          source = "local/promtail.yml"
+        }
+
+        # Bind mount host machine-id and log directories
+        mount {
+          type = "bind"
+          source = "/etc/machine-id"
+          target = "/etc/machine-id"
+          readonly = true
+        }
+
+        mount {
+          type = "bind"
+          source = "/etc/machine-id"
+          target = "/etc/machine-id"
+          readonly = true
+        }
+
+        mount {
+          type = "bind"
+          source = "/var/log/journal/"
+          target = "/var/log/journal/"
+          readonly = true
+        }
+
+        mount {
+          type = "bind"
+          source = "/run/log/journal/"
+          target = "/run/log/journal/"
+          readonly = true
+        }
+
+        mount {
+          type = "bind"
+          source = "/var/log/audit"
+          target = "/var/log/audit"
+          readonly = true
+        }
+      }
+
+      template {
+        data = <<EOF
+---
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 9080
+
+clients:
+  - url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push
+
+scrape_configs:
+
+  - job_name: journal
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        target_label: unit
+      - source_labels: ['__journal__hostname']
+        target_label: hostname
+      - source_labels: ['__journal__transport']
+        target_label: journal_transport
+      # Docker log labels
+      - source_labels: ['__journal_syslog_identifier']
+        target_label: syslog_identifier
+      - source_labels: ['__journal_image_name']
+        target_label: docker_image_name
+      - source_labels: ['__journal_container_name']
+        target_label: docker_container_name
+      - source_labels: ['__journal_container_id']
+        target_label: docker_container_id
+      - source_labels: ['__journal_com_docker_compose_project']
+        target_label: docker_compose_project
+      - source_labels: ['__journal_com_docker_compose_service']
+        target_label: docker_compose_service
+EOF
+        destination = "local/promtail.yml"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
   }
 }

nomad/syslogng.nomad

@@ -0,0 +1,183 @@
+job "syslogng" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "promtail" {
+    count = 1
+
+    network {
+      mode = "bridge"
+
+      port "main" {
+        to = 1514
+      }
+
+      port "metrics" {
+        to = 9080
+      }
+    }
+
+    service {
+      name = "syslogng-promtail"
+      port = "main"
+
+      connect {
+        sidecar_service {
+          local_service_port = 1514
+          proxy {
+            upstreams {
+              destination_name = "loki"
+              local_bind_port = 1000
+            }
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 50
+          }
+        }
+      }
+    }
+
+    task "promtail" {
+      driver = "docker"
+
+      config {
+        image = "grafana/promtail:2.2.1"
+        ports = ["main", "metrics"]
+        args = ["--config.file=/etc/promtail/promtail.yml"]
+
+        mount {
+          type = "bind"
+          target = "/etc/promtail/promtail.yml"
+          source = "local/promtail.yml"
+        }
+      }
+
+      template {
+        data = <<EOF
+---
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 9080
+
+clients:
+  - url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push
+
+scrape_configs:
+  # TCP syslog receiver
+  - job_name: syslog
+    syslog:
+      listen_address: 0.0.0.0:${NOMAD_PORT_main}
+      labels:
+        job: syslog
+    relabel_configs:
+      - source_labels: ['__syslog_message_hostname']
+        target_label: hostname
+EOF
+        destination = "local/promtail.yml"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
+  }
+
+  group "syslogng" {
+    count = 1
+
+    constraint {
+      attribute = "${node.unique.name}"
+      # Needs to be on a predictable node for routing
+      # Maybe a loadbalancer could be used for routing from any node
+      value = "n2"
+    }
+
+    network {
+      mode = "bridge"
+      port "main" {
+        static = 1514
+      }
+    }
+
+    service {
+      name = "syslogng"
+      port = "main"
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "syslogng-promtail"
+              local_bind_port = 1000
+            }
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 50
+          }
+        }
+      }
+
+      check {
+        type = "tcp"
+        port = "main"
+        interval = "10s"
+        timeout = "10s"
+      }
+
+    }
+
+    task "syslogng" {
+      driver = "docker"
+
+      config {
+        image = "balbit/syslog-ng:latest"
+        ports = ["main"]
+        args = ["--no-caps"]
+
+        mount {
+          type = "bind"
+          target = "/etc/syslog-ng/syslog-ng.conf"
+          source = "local/syslog-ng.conf"
+        }
+      }
+
+      template {
+        data = <<EOF
+@version: 3.22
+
+source s_external {
+    syslog(ip(0.0.0.0) port(1514) transport("tcp"));
+    syslog(ip(0.0.0.0) port(1514) transport("udp"));
+};
+
+source s_internal {
+    internal();
+};
+
+destination d_loki {
+    # Forward to Connect proxy to Promtail
+    syslog("${NOMAD_UPSTREAM_IP_syslogngpromtail}" transport("tcp") port(${NOMAD_UPSTREAM_PORT_syslogngpromtail}));
+};
+
+log { source(s_internal); destination(d_loki); };
+log { source(s_external); destination(d_loki); };
+EOF
+        destination = "local/syslog-ng.conf"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
+  }
+}