Switch services to module based Terraform template from levant

This ends up with a better experience in dealing with tfstate for some
services. Not sure why.
This commit is contained in:
IamTheFij 2022-11-11 13:34:08 -08:00
parent 49c8a73ac9
commit a30749f357
7 changed files with 606 additions and 107 deletions

View File

@ -211,17 +211,7 @@
"line_number": 159,
"is_secret": false
}
],
"services/main.tf": [
{
"type": "Secret Keyword",
"filename": "services/main.tf",
"hashed_secret": "f84864c6bffa2e0843a4ab2abdca91df7995c462",
"is_verified": false,
"line_number": 168,
"is_secret": false
}
]
},
"generated_at": "2022-11-10T21:36:17Z"
"generated_at": "2022-11-11T21:26:53Z"
}

View File

@ -20,25 +20,6 @@ provider "registry.terraform.io/hashicorp/consul" {
]
}
provider "registry.terraform.io/hashicorp/external" {
version = "2.2.3"
hashes = [
"h1:uvOYRWcVIqOZSl8YjjaB18yZFz1AWIt2CnK7O45rckg=",
"zh:184ecd339d764de845db0e5b8a9c87893dcd0c9d822167f73658f89d80ec31c9",
"zh:2661eaca31d17d6bbb18a8f673bbfe3fe1b9b7326e60d0ceb302017003274e3c",
"zh:2c0a180f6d1fc2ba6e03f7dfc5f73b617e45408681f75bca75aa82f3796df0e4",
"zh:4b92ae44c6baef4c4952c47be00541055cb5280dd3bc8031dba5a1b2ee982387",
"zh:5641694d5daf3893d7ea90be03b6fa575211a08814ffe70998d5adb8b59cdc0a",
"zh:5bd55a2be8a1c20d732ac9c604b839e1cadc8c49006315dffa4d709b6874df32",
"zh:6e0ef5d11e1597202424b7d69b9da7b881494c9b13a3d4026fc47012dc651c79",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:9e19f89fa25004d3b926a8d15ea630b4bde62f1fa4ed5e11a3d27aabddb77353",
"zh:b763efdd69fd097616b4a4c89cf333b4cee9699ac6432d73d2756f8335d1213f",
"zh:e3b561efdee510b2b445f76a52a902c52bee8e13095e7f4bed7c80f10f8d294a",
"zh:fe660bb8781ee043a093b9a20e53069974475dcaa5791a1f45fd03c61a26478a",
]
}
provider "registry.terraform.io/hashicorp/nomad" {
version = "1.4.19"
hashes = [

View File

@ -108,83 +108,190 @@ resource "consul_config_entry" "nzbget_intents" {
# }
# }
module "photoprism" {
source = "IamTheFij/levant/nomad"
version = "0.1.0"
# resource "nomad_job" "photoprism" {
# jobspec = templatefile("services_tf.nomad", {
# vars = {
# name = "photoprism"
# image = "photoprism/photoprism:latest"
# ingress = true
# service_port = 2342
# sticky_disk = true
# healthcheck = "/library/login"
# env = {
# PHOTOPRISM_SITE_CAPTION = "AI-Powered Photos App"
# PHOTOPRISM_SITE_DESCRIPTION = "Fijolek home photos"
# PHOTOPRISM_SITE_TITLE = "PhotoPrism"
# PHOTOPRISM_SITE_URL = "https://photoprism.thefij.rocks:2342/" # server URL in the format "http(s)://domain.name(:port)/(path)"
# PHOTOPRISM_SPONSOR = "true"
# # Paths
# PHOTOPRISM_ORIGINALS_PATH = "/photoprism-media/Library"
# PHOTOPRISM_IMPORT_PATH = "/photoprism-media/Import"
# PHOTOPRISM_STORAGE_PATH = "$${NOMAD_TASK_DIR}/storage" # Storage PATH for generated files like cache and index
# # Unix permissions
# PHOTOPRISM_UID = 500
# PHOTOPRISM_GID = 100
# PHOTOPRISM_UMASK = 0000
# }
# mysql = true
# vault = true
# resources = {
# cpu = 100
# memory = 500
# memory_max = 1000
# }
# host_volumes = [
# {
# name = "photoprism-media"
# dest = "/photoprism-media"
# read_only = false
# },
# ]
# mysql_bootstrap = {
# vault_key = "kv/data/photoprism"
# }
# templates = [
# {
# data = <<EOF
# {{ with secret "kv/data/photoprism" -}}
# PHOTOPRISM_ADMIN_USER={{ .Data.data.admin_user }}
# PHOTOPRISM_ADMIN_PASSWORD={{ .Data.data.admin_password }}
# PHOTOPRISM_DATABASE_DRIVER=mysql
# PHOTOPRISM_DATABASE_NAME={{ .Data.data.db_name }}
# PHOTOPRISM_DATABASE_USER={{ .Data.data.db_user }}
# PHOTOPRISM_DATABASE_PASSWORD={{ .Data.data.db_pass }}
# PHOTOPRISM_DATABASE_SERVER="{{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}"
# {{ end -}}
# EOF
# dest_prefix = "$${NOMAD_SECRETS_DIR}/"
# dest = "env"
# env = true
# mount = false
# },
# ]
# }
# })
# }
template_path = "service.nomad"
variables = {
name = "photoprism"
image = "photoprism/photoprism:latest"
ingress = true
service_port = 2342
sticky_disk = true
healthcheck = "/library/login"
env = jsonencode({
PHOTOPRISM_DETECT_NSFW = "false" # automatically flags photos as private that MAY be offensive (requires TensorFlow)
PHOTOPRISM_DISABLE_CHOWN = "false" # disables updating storage permissions via chmod and chown on startup
PHOTOPRISM_DISABLE_CLASSIFICATION = "false" # disables image classification (requires TensorFlow)
PHOTOPRISM_DISABLE_FACES = "false" # disables face detection and recognition (requires TensorFlow)
PHOTOPRISM_DISABLE_RAW = "false" # disables indexing and conversion of RAW files
PHOTOPRISM_DISABLE_SETTINGS = "false" # disables settings UI and API
PHOTOPRISM_DISABLE_TENSORFLOW = "false" # disables all features depending on TensorFlow
PHOTOPRISM_DISABLE_WEBDAV = "false" # disables built-in WebDAV server
PHOTOPRISM_EXPERIMENTAL = "false" # enables experimental features
PHOTOPRISM_HTTP_COMPRESSION = "gzip" # improves transfer speed and bandwidth utilization (none or gzip)
PHOTOPRISM_JPEG_QUALITY = 85 # a higher value increases the quality and file size of JPEG images and thumbnails (25-100)
PHOTOPRISM_LOG_LEVEL = "info" # log level: trace, debug, info, warning, error, fatal, or panic
PHOTOPRISM_ORIGINALS_LIMIT = 5000 # file size limit for originals in MB (increase for high-res video)
PHOTOPRISM_RAW_PRESETS = "false" # enables applying user presets when converting RAW files (reduces performance)
PHOTOPRISM_READONLY = "false" # do not modify originals directory (reduced functionality)
PHOTOPRISM_SITE_AUTHOR = "" # meta site author
PHOTOPRISM_SITE_CAPTION = "AI-Powered Photos App"
PHOTOPRISM_SITE_DESCRIPTION = "" # meta site description
PHOTOPRISM_SITE_TITLE = "PhotoPrism"
PHOTOPRISM_SITE_URL = "http://localhost:2342/" # server URL in the format "http(s)://domain.name(:port)/(path)"
PHOTOPRISM_UPLOAD_NSFW = "true" # allows uploads that MAY be offensive (no effect without TensorFlow)
PHOTOPRISM_SPONSOR = "true"
# Paths
PHOTOPRISM_ORIGINALS_PATH = "/photoprism-media/Library"
PHOTOPRISM_IMPORT_PATH = "/photoprism-media/Import"
PHOTOPRISM_STORAGE_PATH = "$${NOMAD_TASK_DIR}/storage" # Storage PATH for generated files like cache and index
# Unix permissions
PHOTOPRISM_UID = 500
PHOTOPRISM_GID = 100
PHOTOPRISM_UMASK = 0000
})
mysql = true
vault = true
host_volumes = jsonencode([
{
name = "photoprism-media"
dest = "/photoprism-media"
read_only = false
},
])
mysql_bootstrap = jsonencode({
vault_key = "kv/data/photoprism"
db_name_key = "db_name"
db_user_key = "db_user"
db_pass_key = "db_pass"
})
templates = jsonencode([
{
data = <<EOF
{{ with secret "kv/data/photoprism" -}}
PHOTOPRISM_ADMIN_USER={{ .Data.data.admin_user }}
PHOTOPRISM_ADMIN_PASSWORD={{ .Data.data.admin_password }}
PHOTOPRISM_DATABASE_DRIVER=mysql
PHOTOPRISM_DATABASE_NAME={{ .Data.data.db_name }}
PHOTOPRISM_DATABASE_USER={{ .Data.data.db_user }}
PHOTOPRISM_DATABASE_PASSWORD={{ .Data.data.db_pass }}
PHOTOPRISM_DATABASE_SERVER="{{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}"
{{ end -}}
EOF
dest_prefix = "$${NOMAD_SECRETS_DIR}/"
dest = "env"
env = true
mount = false
},
])
# module "photoprism" {
# source = "IamTheFij/levant/nomad"
# version = "0.1.0"
#
# template_path = "service.nomad"
# variables = {
# name = "photoprism"
# image = "photoprism/photoprism:latest"
# ingress = true
# service_port = 2342
# sticky_disk = true
# healthcheck = "/library/login"
# env = jsonencode({
# PHOTOPRISM_SITE_CAPTION = "AI-Powered Photos App"
# PHOTOPRISM_SITE_DESCRIPTION = "Fijolek home photos"
# PHOTOPRISM_SITE_TITLE = "PhotoPrism"
# PHOTOPRISM_SITE_URL = "https://photoprism.thefij.rocks:2342/" # server URL in the format "http(s)://domain.name(:port)/(path)"
# PHOTOPRISM_SPONSOR = "true"
# # Paths
# PHOTOPRISM_ORIGINALS_PATH = "/photoprism-media/Library"
# PHOTOPRISM_IMPORT_PATH = "/photoprism-media/Import"
# PHOTOPRISM_STORAGE_PATH = "$${NOMAD_TASK_DIR}/storage" # Storage PATH for generated files like cache and index
# # Unix permissions
# PHOTOPRISM_UID = 500
# PHOTOPRISM_GID = 100
# PHOTOPRISM_UMASK = 0000
# })
# mysql = true
# vault = true
# host_volumes = jsonencode([
# {
# name = "photoprism-media"
# dest = "/photoprism-media"
# read_only = false
# },
# ])
# mysql_bootstrap = jsonencode({
# vault_key = "kv/data/photoprism"
# })
# templates = jsonencode([
# {
# data = <<EOF
# {{ with secret "kv/data/photoprism" -}}
# PHOTOPRISM_ADMIN_USER={{ .Data.data.admin_user }}
# PHOTOPRISM_ADMIN_PASSWORD={{ .Data.data.admin_password }}
# PHOTOPRISM_DATABASE_DRIVER=mysql
# PHOTOPRISM_DATABASE_NAME={{ .Data.data.db_name }}
# PHOTOPRISM_DATABASE_USER={{ .Data.data.db_user }}
# PHOTOPRISM_DATABASE_PASSWORD={{ .Data.data.db_pass }}
# PHOTOPRISM_DATABASE_SERVER="{{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}"
# {{ end -}}
# EOF
# dest_prefix = "$${NOMAD_SECRETS_DIR}/"
# dest = "env"
# env = true
# mount = false
# },
# ])
# }
# }
module "photoprism_module" {
source = "./service"
name = "photoprism"
image = "photoprism/photoprism:latest"
ingress = true
service_port = 2342
sticky_disk = true
healthcheck_path = "/library/login"
env = {
PHOTOPRISM_SITE_CAPTION = "AI-Powered Photos App"
PHOTOPRISM_SITE_DESCRIPTION = "Fijolek home photos"
PHOTOPRISM_SITE_TITLE = "PhotoPrism"
PHOTOPRISM_SITE_URL = "https://photoprism.thefij.rocks:2342/" # server URL in the format "http(s)://domain.name(:port)/(path)"
PHOTOPRISM_SPONSOR = "true"
# Paths
PHOTOPRISM_ORIGINALS_PATH = "/photoprism-media/Library"
PHOTOPRISM_IMPORT_PATH = "/photoprism-media/Import"
PHOTOPRISM_STORAGE_PATH = "$${NOMAD_TASK_DIR}/storage" # Storage PATH for generated files like cache and index
# Unix permissions
PHOTOPRISM_UID = 500
PHOTOPRISM_GID = 100
PHOTOPRISM_UMASK = 0000
}
resources = {
cpu = 100
memory = 500
memory_max = 1000
}
use_mysql = true
use_vault = true
host_volumes = [
{
name = "photoprism-media"
dest = "/photoprism-media"
read_only = false
},
]
mysql_bootstrap = {
vault_key = "kv/data/photoprism"
}
templates = [
{
data = <<EOF
{{ with secret "kv/data/photoprism" -}}
PHOTOPRISM_ADMIN_USER={{ .Data.data.admin_user }}
PHOTOPRISM_ADMIN_PASSWORD={{ .Data.data.admin_password }}
PHOTOPRISM_DATABASE_DRIVER=mysql
PHOTOPRISM_DATABASE_NAME={{ .Data.data.db_name }}
PHOTOPRISM_DATABASE_USER={{ .Data.data.db_user }}
PHOTOPRISM_DATABASE_PASSWORD={{ .Data.data.db_pass }}
PHOTOPRISM_DATABASE_SERVER="{{ env "NOMAD_UPSTREAM_ADDR_mysql_server" }}"
{{ end -}}
EOF
dest_prefix = "$${NOMAD_SECRETS_DIR}/"
dest = "env"
env = true
mount = false
},
]
}

View File

@ -0,0 +1,21 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/nomad" {
version = "1.4.19"
hashes = [
"h1:EdBny2gaLr/IE+l+6csyCKeIGFMYZ/4tHKpcbS7ArgE=",
"zh:2f3ceeb3318a6304026035b0ac9ee3e52df04913bb9ee78827e58c5398b41254",
"zh:3fbe76c7d957d20dfe3c8c0528b33084651f22a95be9e0452b658e0922916e2a",
"zh:595671a05828cfe6c42ef73aac894ac39f81a52cc662a76f37eb74ebe04ddf75",
"zh:5d76e8788d2af3e60daf8076babf763ec887480bbb9734baccccd8fcddf4f03e",
"zh:676985afeaca6e67b22d60d43fd0ed7055763029ffebc3026089fe2fd3b4a288",
"zh:69152ce6164ac999a640cff962ece45208270e1ac37c10dac484eeea5cf47275",
"zh:6da0b15c05b81f947ec8e139bd81eeeb05c0d36eb5a967b985d0625c60998b40",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:822c0a3bbada5e38099a379db8b2e339526843699627c3be3664cc3b3752bab7",
"zh:af23af2f98a84695b25c8eba7028a81ad4aad63c44aefb79e01bbe2dc82e7f78",
"zh:e36cac9960b7506d92925b667254322520966b9c3feb3ca6102e57a1fb9b1761",
"zh:ffd1e096c1cc35de879c740a91918e9f06b627818a3cb4b1d87b829b54a6985f",
]
}

28
services/service/main.tf Normal file
View File

@ -0,0 +1,28 @@
resource "nomad_job" "service" {
jobspec = templatefile("${path.module}/service_template.nomad", {
name = var.name
image = var.image
args = var.args
env = var.env
service_port = var.service_port
sticky_disk = var.sticky_disk
resources = var.resources
ingress = var.ingress
ingress_rule = var.ingress_rule
ingress_middlewares = var.ingress_middlewares
healthcheck_path = var.healthcheck_path
templates = var.templates
host_volumes = var.host_volumes
upstreams = var.upstreams
use_mysql = var.use_mysql
use_redis = var.use_redis
use_ldap = var.use_ldap
use_vault = var.use_vault
mysql_bootstrap = var.mysql_bootstrap
})
}

View File

@ -0,0 +1,239 @@
job "${name}" {
region = "global"
datacenters = ["dc1"]
type = "service"
group "${name}" {
network {
mode = "bridge"
%{ if service_port != null ~}
port "main" {
%{ if ingress ~}
host_network = "loopback"
%{ endif ~}
to = ${service_port}
}
%{ endif ~}
}
%{ if sticky_disk ~}
ephemeral_disk {
migrate = true
sticky = true
}
%{ endif ~}
%{ for host_volume in host_volumes ~}
volume "${host_volume.name}" {
type = "host"
read_only = ${host_volume.read_only}
source = "${host_volume.name}"
}
%{ endfor ~}
%{ if service_port != null ~}
service {
name = "${replace(name, "_", "-")}"
port = "main"
%{ if anytrue([ingress, use_mysql, use_redis, use_ldap, length(upstreams) > 0]) }
connect {
sidecar_service {
proxy {
local_service_port = ${service_port}
%{ if use_mysql }
upstreams {
destination_name = "mysql-server"
local_bind_port = 4040
}
%{ endif ~}
%{ if use_redis }
upstreams {
destination_name = "redis"
local_bind_port = 6379
}
%{ endif ~}
%{ if use_ldap }
upstreams {
destination_name = "lldap"
local_bind_port = 3890
}
%{ endif ~}
%{ for upstream in upstreams ~}
upstreams {
destination_name = "${upstream.destination_name}"
local_bind_port = ${upstream.local_bind_port}
}
%{ endfor }
}
}
sidecar_task {
resources {
cpu = 50
memory = 20
memory_max = 50
}
}
}
%{ endif ~}
%{ if healthcheck_path != null ~}
check {
type = "http"
path = "${healthcheck_path}"
port = "main"
interval = "10s"
timeout = "10s"
}
%{ endif ~}
tags = [
%{ if ingress ~}
"traefik.enable=true",
"traefik.http.routers.${name}.entryPoints=websecure",
%{ if try(ingress_rule, null) != null ~}
"traefik.http.routers.${name}.rule=${ingress_rule}",
%{ endif ~}
%{ for middleware in ingress_middlewares ~}
"traefik.http.routers.${name}.middlewares=${middleware}",
%{ endfor ~}
%{ endif ~}
]
}
%{ endif ~}
task "${name}" {
driver = "docker"
config {
image = "${image}"
%{ if service_port != null ~}
ports = ["main"]
%{ endif ~}
%{ if length(try(args, [])) > 0 ~}
args = ${jsonencode(args)}
%{ endif ~}
%{ for template in templates ~}
%{ if template.mount && !template.env ~}
mount {
type = "bind"
target = "${template.dest}"
source = "${template.dest_prefix}/${template.dest}"
}
%{ endif ~}
%{ endfor ~}
}
%{ if use_vault ~}
vault {
policies = [
"access-tables",
"nomad-task",
]
}
%{ endif ~}
%{ if length(env) > 0 ~}
env = {
%{ for k, v in env ~}
"${k}" = "${v}"
%{ endfor }
}
%{ endif ~}
%{ for volume in host_volumes ~}
volume_mount {
volume = "${volume.name}"
destination = "${volume.dest}"
read_only = ${volume.read_only}
}
%{ endfor ~}
%{ for template in templates ~}
template {
data = <<EOF
${template.data}
EOF
destination = "${coalesce(template.dest_prefix, "$${NOMAD_TASK_DIR}")}/${template.dest}"
%{ if template.left_delimiter != null }left_delimiter = "${template.left_delimiter}"%{ endif }
%{ if template.right_delimiter != null }right_delimiter = "${template.right_delimiter}"%{ endif }
%{ if template.change_mode != null }change_mode = "${template.change_mode}"%{ endif }
%{ if template.change_signal != null }change_signal = "${template.change_signal}"%{ endif }
%{ if template.env != null }env = ${template.env}%{ endif }
}
%{ endfor ~}
%{ if resources != null }
resources {
cpu = ${resources.cpu}
memory = ${resources.memory}
%{ if resources.memory_max != null }memory_max = ${resources.memory_max}%{ endif }
}
%{~ endif }
}
%{ if mysql_bootstrap != null }
task "${name}-bootstrap" {
driver = "docker"
lifecycle {
hook = "prestart"
sidecar = false
}
config {
image = "mariadb:10"
args = [
"/bin/bash",
"-c",
"/usr/bin/mysql --defaults-extra-file=$${NOMAD_SECRETS_DIR}/my.cnf < $${NOMAD_SECRETS_DIR}/bootstrap.sql",
]
}
vault {
policies = [
"access-tables",
"nomad-task",
]
}
template {
data = <<EOF
[client]
host={{ env "NOMAD_UPSTREAM_IP_mysql_server" }}
port={{ env "NOMAD_UPSTREAM_PORT_mysql_server" }}
user=root
{{ with secret "kv/data/mysql" -}}
password={{ .Data.data.root_password }}
{{ end -}}
EOF
destination = "$${NOMAD_SECRETS_DIR}/my.cnf"
}
template {
data = <<EOF
{{ with secret "${mysql_bootstrap.vault_key}" -}}
CREATE DATABASE IF NOT EXISTS `{{ .Data.data.${mysql_bootstrap.db_name_key} }}`
CHARACTER SET = 'utf8mb4'
COLLATE = 'utf8mb4_unicode_ci';
CREATE USER IF NOT EXISTS '{{ .Data.data.${mysql_bootstrap.db_user_key} }}'@'%'
IDENTIFIED BY '{{ .Data.data.${mysql_bootstrap.db_pass_key} }}';
GRANT ALL ON `{{ .Data.data.${mysql_bootstrap.db_name_key} }}`.*
TO '{{ .Data.data.${mysql_bootstrap.db_user_key} }}'@'%';
{{ else -}}
SELECT 'NOOP';
{{ end -}}
EOF
destination = "$${NOMAD_SECRETS_DIR}/bootstrap.sql"
}
resources {
cpu = 50
memory = 50
}
}
%{ endif }
}
}

133
services/service/vars.tf Normal file
View File

@ -0,0 +1,133 @@
variable "name" {
type = string
description = "Name of the service"
}
variable "image" {
type = string
description = "Image that should be run"
}
variable "service_port" {
type = number
default = null
description = "Port used by the service for ingress"
}
variable "ingress" {
type = bool
default = false
}
variable "sticky_disk" {
type = bool
default = false
}
variable "args" {
type = list(string)
default = []
description = "Arguments passed to the Docker container"
}
variable "resources" {
type = object({
cpu = number
memory = number
memory_max = optional(number)
})
default = {
cpu = 50
memory = 100
memory_max = null
}
description = "Resources to be assigned to the main task"
}
variable "env" {
type = map(string)
default = {}
description = "Env variables for the main task"
}
variable "ingress_rule" {
type = string
default = null
description = "Routing rule for ingress"
}
variable "ingress_middlewares" {
type = list(string)
default = []
description = "Traefik middlewares that should be used"
}
variable "templates" {
type = list(object({
data = string
dest = string
dest_prefix = optional(string, "$${NOMAD_TASK_DIR}")
change_mode = optional(string)
change_signal = optional(string)
left_delimiter = optional(string)
right_delimiter = optional(string)
mount = optional(bool)
env = optional(bool)
}))
default = []
description = "Templates to be used"
}
variable "host_volumes" {
type = list(object({
name = string
dest = string
read_only = optional(bool)
}))
default = []
}
variable "healthcheck_path" {
type = string
default = "/"
}
variable "upstreams" {
type = list(object({
destination_name = string
local_bind_port = number
}))
default = []
}
variable "use_mysql" {
type = bool
default = false
}
variable "use_redis" {
type = bool
default = false
}
variable "use_vault" {
type = bool
default = false
}
variable "use_ldap" {
type = bool
default = false
}
variable "mysql_bootstrap" {
type = object({
vault_key = string
db_name_key = optional(string, "db_name")
db_user_key = optional(string, "db_user")
db_pass_key = optional(string, "db_pass")
})
default = null
}