Add loki, promtail, and syslog-ng

This commit is contained in:
IamTheFij 2022-07-25 10:46:16 -07:00
parent 816d6b7097
commit 6988e19014
4 changed files with 390 additions and 0 deletions

27
core.tf
View File

@ -21,3 +21,30 @@ module "metrics" {
consul_address = var.consul_address consul_address = var.consul_address
} }
module "loki" {
source = "./levant"
template_path = "service.nomad"
variables = {
name = "loki"
image = "grafana/loki:2.2.1"
service_port = 3100
ingress = true
sticky_disk = true
templates = jsonencode([
{
data = file("./loki-config.yml")
dest = "/etc/loki/local-config.yaml"
}
])
}
}
resource "nomad_job" "syslog-ng" {
hcl2 {
enabled = true
}
jobspec = file("${path.module}/syslogng.nomad")
}

45
loki-config.yml Normal file
View File

@ -0,0 +1,45 @@
auth_enabled: false
server:
http_listen_port: 3100
ingester:
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 5m
chunk_retain_period: 30s
max_transfer_retries: 0
schema_config:
configs:
- from: 2018-04-15
store: boltdb
object_store: filesystem
schema: v11
index:
prefix: index_
period: 168h
storage_config:
boltdb:
directory: /loki/index
filesystem:
directory: /loki/chunks
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s

View File

@ -204,5 +204,140 @@ job "metrics" {
memory = 50 memory = 50
} }
} }
service {
name = "promtail"
port = "promtail"
meta {
metrics_addr = "${NOMAD_ADDR_promtail}"
nomad_dc = "${NOMAD_DC}"
nomad_node_name = "${node.unique.name}"
}
connect {
sidecar_service {
proxy {
local_service_port = 9080
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "http"
path = "/metrics"
port = "promtail"
interval = "10s"
timeout = "10s"
}
}
task "promtail" {
driver = "docker"
config {
image = "grafana/promtail:2.2.1"
args = ["-config.file=/etc/promtail/promtail.yml"]
ports = ["promtail"]
# Mount config
mount {
type = "bind"
target = "/etc/promtail/promtail.yml"
source = "local/promtail.yml"
}
# Bind mount host machine-id and log directories
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
mount {
type = "bind"
source = "/etc/machine-id"
target = "/etc/machine-id"
readonly = true
}
mount {
type = "bind"
source = "/var/log/journal/"
target = "/var/log/journal/"
readonly = true
}
mount {
type = "bind"
source = "/run/log/journal/"
target = "/run/log/journal/"
readonly = true
}
mount {
type = "bind"
source = "/var/log/audit"
target = "/var/log/audit"
readonly = true
}
}
template {
data = <<EOF
---
server:
http_listen_address: 0.0.0.0
http_listen_port: 9080
clients:
- url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push
scrape_configs:
- job_name: journal
journal:
json: false
max_age: 12h
path: /var/log/journal
labels:
job: systemd-journal
relabel_configs:
- source_labels: ['__journal__systemd_unit']
target_label: unit
- source_labels: ['__journal__hostname']
target_label: hostname
- source_labels: ['__journal__transport']
target_label: journal_transport
# Docker log labels
- source_labels: ['__journal_syslog_identifier']
target_label: syslog_identifier
- source_labels: ['__journal_image_name']
target_label: docker_image_name
- source_labels: ['__journal_container_name']
target_label: docker_container_name
- source_labels: ['__journal_container_id']
target_label: docker_container_id
- source_labels: ['__journal_com_docker_compose_project']
target_label: docker_compose_project
- source_labels: ['__journal_com_docker_compose_service']
target_label: docker_compose_service
EOF
destination = "local/promtail.yml"
}
resources {
cpu = 50
memory = 50
}
}
} }
} }

183
syslogng.nomad Normal file
View File

@ -0,0 +1,183 @@
job "syslogng" {
datacenters = ["dc1"]
type = "service"
group "promtail" {
count = 1
network {
mode = "bridge"
port "main" {
to = 1514
}
port "metrics" {
to = 9080
}
}
service {
name = "syslogng-promtail"
port = "main"
connect {
sidecar_service {
local_service_port = 1514
proxy {
upstreams {
destination_name = "loki"
local_bind_port = 1000
}
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
}
task "promtail" {
driver = "docker"
config {
image = "grafana/promtail:2.2.1"
ports = ["main", "metrics"]
args = ["--config.file=/etc/promtail/promtail.yml"]
mount {
type = "bind"
target = "/etc/promtail/promtail.yml"
source = "local/promtail.yml"
}
}
template {
data = <<EOF
---
server:
http_listen_address: 0.0.0.0
http_listen_port: 9080
clients:
- url: http://${NOMAD_UPSTREAM_ADDR_loki}/loki/api/v1/push
scrape_configs:
# TCP syslog receiver
- job_name: syslog
syslog:
listen_address: 0.0.0.0:${NOMAD_PORT_main}
labels:
job: syslog
relabel_configs:
- source_labels: ['__syslog_message_hostname']
target_label: hostname
EOF
destination = "local/promtail.yml"
}
resources {
cpu = 50
memory = 50
}
}
}
group "syslogng" {
count = 1
constraint {
attribute = "${node.unique.name}"
# Needs to be on a predictable node for routing
# Maybe a loadbalancer could be used for routing from any node
value = "n2"
}
network {
mode = "bridge"
port "main" {
static = 1514
}
}
service {
name = "syslogng"
port = "main"
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "syslogng-promtail"
local_bind_port = 1000
}
}
}
sidecar_task {
resources {
cpu = 50
memory = 50
}
}
}
check {
type = "tcp"
port = "main"
interval = "10s"
timeout = "10s"
}
}
task "syslogng" {
driver = "docker"
config {
image = "balbit/syslog-ng:latest"
ports = ["main"]
args = ["--no-caps"]
mount {
type = "bind"
target = "/etc/syslog-ng/syslog-ng.conf"
source = "local/syslog-ng.conf"
}
}
template {
data = <<EOF
@version: 3.22
source s_external {
syslog(ip(0.0.0.0) port(1514) transport("tcp"));
syslog(ip(0.0.0.0) port(1514) transport("udp"));
};
source s_internal {
internal();
};
destination d_loki {
# Forward to Connect proxy to Promtail
syslog("${NOMAD_UPSTREAM_IP_syslogngpromtail}" transport("tcp") port(${NOMAD_UPSTREAM_PORT_syslogngpromtail}));
};
log { source(s_internal); destination(d_loki); };
log { source(s_external); destination(d_loki); };
EOF
destination = "local/syslog-ng.conf"
}
resources {
cpu = 50
memory = 50
}
}
}
}