homelab-nomad/syslogng.nomad

job "syslogng" {
  datacenters = ["dc1"]
  type = "service"

  group "promtail" {
    count = 1

    network {
      mode = "bridge"

      port "main" {
        to = 1514
      }

      port "metrics" {
        to = 9080
      }
    }

    service {
      name = "syslogng-promtail"
      port = "main"

      connect {
        sidecar_service {
          proxy {
            local_service_port = 1514

            upstreams {
              destination_name = "loki"
              local_bind_port = 1000
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }
    }

    task "promtail" {
      driver = "docker"

      config {
        image = "grafana/promtail:2.2.1"
        ports = ["main", "metrics"]
        args = ["--config.file=/etc/promtail/promtail.yml"]

        mount {
          type = "bind"
          target = "/etc/promtail/promtail.yml"
          source = "local/promtail.yml"
        }
      }

      template {
        data = <<EOF
---
server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

clients:
  - url: http://{{ env "NOMAD_UPSTREAM_ADDR_loki" }}/loki/api/v1/push

scrape_configs:
  # TCP syslog receiver
  - job_name: syslog
    syslog:
      listen_address: 0.0.0.0:{{ env "NOMAD_PORT_main" }}
      labels:
        job: syslog
    relabel_configs:
      - source_labels: ['__syslog_message_hostname']
        target_label: hostname
EOF
        destination = "local/promtail.yml"
      }

      resources {
        cpu = 50
        memory = 50
      }
    }
  }

  group "syslogng" {
    count = 1

    constraint {
      attribute = "${node.unique.name}"
      # Needs to be on a predictable node for routing
      # Maybe a loadbalancer could be used for routing from any node
      value = "n2"
    }

    network {
      mode = "bridge"
      port "main" {
        static = 1514
      }
    }

    service {
      name = "syslogng"
      port = "main"

      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "syslogng-promtail"
              local_bind_port = 1000
            }
          }
        }

        sidecar_task {
          resources {
            cpu    = 50
            memory = 50
          }
        }
      }
    }

    task "syslogng" {
      driver = "docker"

      config {
        image = "balabit/syslog-ng:3.37.1"
        ports = ["main"]
        args = ["--no-caps"]

        mount {
          type = "bind"
          target = "/etc/syslog-ng/syslog-ng.conf"
          source = "local/syslog-ng.conf"
        }
      }

      template {
        data = <<EOF
@version: 3.22

source s_external {
    syslog(ip(0.0.0.0) port(1514) transport("tcp"));
    syslog(ip(0.0.0.0) port(1514) transport("udp"));
};

source s_internal {
    internal();
};

destination d_loki {
    # Forward to Connect proxy to Promtail
    syslog("{{ env "NOMAD_UPSTREAM_IP_syslogng-promtail" }}" transport("tcp") port({{ env "NOMAD_UPSTREAM_PORT_syslogng-promtail" }}));
};

log { source(s_internal); destination(d_loki); };
log { source(s_external); destination(d_loki); };
EOF
        destination = "local/syslog-ng.conf"
      }

      resources {
        cpu = 50
        memory = 50
      }
    }
  }
}