From d21d0b8b38a5c403eb170756992e15d5bc6f4901 Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Mon, 26 Sep 2022 16:42:25 -0700
Subject: [PATCH] WIP: Deploy immich with postgres

---
 nomad/ansible_hosts.yml        |  12 ++
 nomad/databases/adminer.nomad  |   5 +
 nomad/databases/postgres.nomad |  98 ++++++++++
 nomad/databases/postgres.tf    |  39 ++++
 nomad/immich.nomad             | 344 +++++++++++++++++++++++++++++++++
 nomad/services.tf              |   9 +
 6 files changed, 507 insertions(+)
 create mode 100644 nomad/databases/postgres.nomad
 create mode 100644 nomad/databases/postgres.tf
 create mode 100644 nomad/immich.nomad

diff --git a/nomad/ansible_hosts.yml b/nomad/ansible_hosts.yml
index 267e3ac..5f7e7bc 100644
--- a/nomad/ansible_hosts.yml
+++ b/nomad/ansible_hosts.yml
@@ -13,6 +13,12 @@ all:
               group: "bin"
               mode: "0755"
               read_only: false
+            - name: postgres-data
+              path: /srv/volumes/postgres
+              owner: "root"
+              group: "bin"
+              mode: "0755"
+              read_only: false
             - name: lldap-data
               path: /srv/volumes/lldap
               owner: "root"
@@ -41,6 +47,12 @@ all:
               group: "bin"
               mode: "0755"
               read_only: false
+            - name: immich-upload  # TODO: Use NFS instead
+              path: /srv/volumes/immich-upload
+              owner: "root"
+              group: "bin"
+              mode: "0755"
+              read_only: false
         # n3.thefij:
         #   nomad_node_class: ingress
         #   nomad_node_role: both
diff --git a/nomad/databases/adminer.nomad b/nomad/databases/adminer.nomad
index fa33339..b0ff03b 100644
--- a/nomad/databases/adminer.nomad
+++ b/nomad/databases/adminer.nomad
@@ -28,6 +28,11 @@ job "adminer" {
               local_bind_port = 4040
             }
 
+            upstreams {
+              destination_name = "postgres"
+              local_bind_port = 5432
+            }
+
             config {
               protocol = "tcp"
             }
diff --git a/nomad/databases/postgres.nomad b/nomad/databases/postgres.nomad
new file mode 100644
index 0000000..ae0fb12
--- /dev/null
+++ b/nomad/databases/postgres.nomad
@@ -0,0 +1,98 @@
+job "postgres" {
+  datacenters = ["dc1"]
+  type = "service"
+  priority = 80
+
+  group "postgres" {
+    count = 1
+
+    restart {
+      attempts = 10
+      interval = "5m"
+      delay = "25s"
+      mode = "delay"
+    }
+
+    network {
+      mode = "bridge"
+      port "db" {
+        host_network = "loopback"
+        to = 5432
+      }
+    }
+
+    volume "postgres-data" {
+      type = "host"
+      read_only = false
+      source = "postgres-data"
+    }
+
+    service {
+      name = "postgres"
+      port = "db"
+
+      connect {
+        sidecar_service {
+          proxy {
+            local_service_port = 5432
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu    = 50
+            memory = 50
+          }
+        }
+      }
+
+      # Can't use a tcp check with bridge network or proxy
+      # check {
+      #   type = "tcp"
+      #   interval = "10s"
+      #   timeout = "2s"
+      # }
+    }
+
+    task "postgres" {
+      driver = "docker"
+
+      config {
+        image = "postgres:14"
+        ports = ["db"]
+      }
+
+      vault {
+        policies = [
+          "access-tables",
+          "nomad-task",
+        ]
+      }
+
+      volume_mount {
+        volume = "postgres-data"
+        destination = "/var/lib/postgresql/data"
+        read_only = false
+      }
+
+      env = {
+        PGDATA = "/var/lib/postgresql/data/pgdata"
+      }
+
+      template {
+        data = <<EOH
+{{ with secret "kv/data/postgres" }}
+POSTGRES_PASSWORD={{ .Data.data.superuser_password }}
+{{ end }}
+        EOH
+        destination = "secrets/db.env"
+        env = true
+      }
+
+      resources {
+        cpu = 300
+        memory = 1024
+      }
+    }
+  }
+}
diff --git a/nomad/databases/postgres.tf b/nomad/databases/postgres.tf
new file mode 100644
index 0000000..ced4b13
--- /dev/null
+++ b/nomad/databases/postgres.tf
@@ -0,0 +1,39 @@
+resource "nomad_job" "postgres-server" {
+  hcl2 {
+    enabled = true
+  }
+
+  jobspec = file("${path.module}/postgres.nomad")
+
+  # Block until deployed as there are servics dependent on this one
+  detach = false
+}
+
+# NOTE: This may need to be moved to after the services are created
+resource "consul_config_entry" "postgres_intents" {
+  name = "postgres-server"
+  kind = "service-intentions"
+
+  config_json = jsonencode({
+    Sources = [
+      {
+        Action     = "allow"
+        Name       = "adminer"
+        Precedence = 9
+        Type       = "consul"
+      },
+      {
+        Action     = "allow"
+        Name       = "backups"
+        Precedence = 9
+        Type       = "consul"
+      },
+      {
+        Action     = "allow"
+        Name       = "immich"
+        Precedence = 9
+        Type       = "consul"
+      },
+    ]
+  })
+}
diff --git a/nomad/immich.nomad b/nomad/immich.nomad
new file mode 100644
index 0000000..18de5f8
--- /dev/null
+++ b/nomad/immich.nomad
@@ -0,0 +1,344 @@
+variable "postgres_image" {
+  type = string
+  default = "postgres:14"
+}
+
+variable "immich_tag" {
+  type = string
+  default = "release"
+}
+
+job "immich" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "immich" {
+    count = 1
+
+    network {
+      mode = "bridge"
+
+      port "server" {
+        host_network = "loopback"
+        to = 3001
+      }
+
+      port "microservices" {
+        host_network = "loopback"
+        to = 3001
+      }
+
+      port "web" {
+        host_network = "loopback"
+        to = 3000
+      }
+
+      port "proxy" {
+        host_network = "loopback"
+        to = 80
+      }
+    }
+
+    volume "immich-upload" {
+      type = "host"
+      read_only = false
+      source = "immich-upload"
+    }
+
+    service {
+      name = "immich"
+      port = "proxy"
+
+      connect {
+        sidecar_service {
+          proxy {
+            local_service_port = 80
+
+            upstreams {
+              destination_name = "redis"
+              local_bind_port = 6379
+            }
+
+            upstreams {
+              destination_name = "postgres"
+              local_bind_port = 5432
+            }
+          }
+        }
+
+        sidecar_task {
+          resources {
+            cpu = 50
+            memory = 50
+          }
+        }
+      }
+
+      tags = [
+        "traefik.enable=true",
+      ]
+    }
+
+    task "immich-bootstrap" {
+      driver = "docker"
+
+      config {
+        image = "${var.postgres_image}"
+        args = [
+          "/bin/bash",
+          "-c",
+          "/usr/bin/psql --no-password -f ${NOMAD_SECRETS_DIR}/bootstrap.sql",
+        ]
+      }
+
+      resources {
+        cpu = 50
+        memory = 20
+        memory_max = 100
+      }
+
+      vault {
+        policies = [
+          "access-tables",
+          "nomad-task",
+        ]
+      }
+
+      env {
+        PGHOST = "${NOMAD_UPSTREAM_IP_postgres}"
+        PGPORT = "${NOMAD_UPSTREAM_PORT_postgres}"
+        PGUSER = "root"
+      }
+
+      template {
+        data = <<EOH
+        {{ with secret "kv/data/postgres" }}
+        PGPASSWORD={{ .Data.data.superuser_password }}
+        {{ end }}
+        EOH
+        destination = "secrets/pgpass.env"
+        env = true
+      }
+
+      template {
+        data = <<EOF
+        {{ with secret "kv/data/immich" }}
+        DB_DATABASE_NAME={{ .Data.data.db_name }}
+        DB_USERNAME={{ .Data.data.db_user }}
+        DB_PASSWORD={{ .Data.data.db_pass }}
+        {{ end }}
+        EOF
+        destination = "secrets/immich-db.env"
+        env = true
+      }
+
+      template {
+        data = <<EOH
+        {{ with secret "kv/data/immich" }}
+        DO
+        $do$
+        BEGIN
+        IF EXISTS (
+          SELECT FROM pg_catalog.pg_roles
+          WHERE  rolname = '{{ .Data.data.db_user }}') THEN
+
+          RAISE NOTICE 'Role "{{ .Data.data.db_user }}" already exists. Skipping.';
+          ELSE
+          CREATE ROLE {{ .Data.data.db_user }} LOGIN PASSWORD '{{ .Data.data.db_pass }}';
+          END IF;
+
+          IF EXISTS (SELECT FROM pg_database WHERE datname = '{{ .Data.data.db_name }}') THEN
+          RAISE NOTICE 'Database already exists';  -- optional
+          ELSE
+          PERFORM dblink_exec('dbname=' || current_database()  -- current db
+          , 'CREATE DATABASE {{ .Data.data.db_name }}');
+          REVOKE ALL ON DATABASE {{ .Data.data.db_name }} FROM public;
+          GRANT ALL PRIVILEGES ON DATABASE {{ .Data.data.db_name }} TO {{ .Data.data.db_user }};
+          END IF;
+          END
+          $do$;
+          {{ end }}
+          EOH
+          destination = "secrets/bootstrap.sql"
+        }
+      }
+
+      task "immich-server" {
+        driver = "docker"
+
+        volume_mount {
+          volume = "immich-upload"
+          destination = "/usr/src/app/upload"
+          read_only = false
+        }
+
+        config {
+          image = "altran1502/immich-server:${var.immich_tag}"
+          entrypoint = ["/bin/sh", "./start-server.sh"]
+          ports = ["server"]
+        }
+
+        resources {
+          cpu = 100
+          memory = 200
+        }
+
+        env {
+          NODE_ENV = "production"
+          REDIS_HOSTNAME = "${NOMAD_UPSTREAM_IP_redis}"
+          REDIS_PORT = "${NOMAD_UPSTREAM_PORT_redis}"
+          # REDIS_DBINDEX=0
+          # REDIS_PASSWORD=
+          # REDIS_SOCKET=
+        }
+
+        vault {
+          policies = [
+            "access-tables",
+            "nomad-task",
+          ]
+        }
+
+        template {
+          data = <<EOF
+          DB_HOSTNAME="${NOMAD_UPSTREAM_IP_postgres}"
+          DB_PORT="${NOMAD_UPSTREAM_PORT_postgres}"
+          {{ with secret "kv/data/immich" }}
+          DB_DATABASE_NAME={{ .Data.data.db_name }}
+          DB_USERNAME={{ .Data.data.db_user }}
+          DB_PASSWORD={{ .Data.data.db_pass }}
+          {{ end }}
+          EOF
+          destination = "secrets/db.env"
+          env = true
+        }
+      }
+
+      task "immich-microservices" {
+        driver = "docker"
+
+        volume_mount {
+          volume = "immich-upload"
+          destination = "/usr/src/app/upload"
+          read_only = false
+        }
+
+        config {
+          image = "altran1502/immich-server:${var.immich_tag}"
+          entrypoint = ["/bin/sh", "./start-microservices.sh"]
+          ports = ["microservices"]
+        }
+
+        resources {
+          cpu = 100
+          memory = 50
+          memory_max = 200
+        }
+
+        env {
+          NODE_ENV = "production"
+          REDIS_HOSTNAME = "${NOMAD_UPSTREAM_IP_redis}"
+          REDIS_PORT = "${NOMAD_UPSTREAM_PORT_redis}"
+          # REDIS_DBINDEX=0
+          # REDIS_PASSWORD=
+          # REDIS_SOCKET=
+        }
+
+        vault {
+          policies = [
+            "access-tables",
+            "nomad-task",
+          ]
+        }
+
+        template {
+          data = <<EOF
+          DB_HOSTNAME="${NOMAD_UPSTREAM_IP_postgres}"
+          DB_PORT="${NOMAD_UPSTREAM_PORT_postgres}"
+          {{ with secret "kv/data/immich" }}
+          DB_DATABASE_NAME={{ .Data.data.db_name }}
+          DB_USERNAME={{ .Data.data.db_user }}
+          DB_PASSWORD={{ .Data.data.db_pass }}
+          {{ end }}
+          EOF
+          destination = "secrets/db.env"
+          env = true
+        }
+      }
+
+      task "immich-machine-learning" {
+        driver = "docker"
+
+        volume_mount {
+          volume = "immich-upload"
+          destination = "/usr/src/app/upload"
+          read_only = false
+        }
+
+        config {
+          image = "altran1502/immich-machine-learning:${var.immich_tag}"
+          entrypoint = ["/bin/sh", "./entrypoint.sh"]
+        }
+
+        resources {
+          cpu = 500
+          memory = 100
+          memory_max = 500
+        }
+
+        env {
+          NODE_ENV = "production"
+        }
+
+        vault {
+          policies = [
+            "access-tables",
+            "nomad-task",
+          ]
+        }
+
+        template {
+          data = <<EOF
+          DB_HOSTNAME="${NOMAD_UPSTREAM_IP_postgres}"
+          DB_PORT="${NOMAD_UPSTREAM_PORT_postgres}"
+          {{ with secret "kv/data/immich" }}
+          DB_DATABASE_NAME={{ .Data.data.db_name }}
+          DB_USERNAME={{ .Data.data.db_user }}
+          DB_PASSWORD={{ .Data.data.db_pass }}
+          {{ end }}
+          EOF
+          destination = "secrets/db.env"
+          env = true
+        }
+      }
+
+      task "immich-web" {
+        driver = "docker"
+
+        config {
+          image = "altran1502/immich-web:${var.immich_tag}"
+          entrypoint = ["/bin/sh", "./entrypoint.sh"]
+          ports = ["web"]
+        }
+
+        resources {
+          cpu = 50
+          memory = 50
+        }
+      }
+
+      task "immich-proxy" {
+        driver = "docker"
+
+        config {
+          ports = ["proxy"]
+          image = "altran1502/immich-proxy:${var.immich_tag}"
+        }
+
+        resources {
+          cpu = 50
+          memory = 50
+        }
+      }
+    }
+  }
diff --git a/nomad/services.tf b/nomad/services.tf
index 6f51e7a..fa1fb06 100644
--- a/nomad/services.tf
+++ b/nomad/services.tf
@@ -14,6 +14,15 @@ module "media" {
   source = "./media"
 }
 
+resource "nomad_job" "immich" {
+  hcl2 {
+    enabled = true
+  }
+
+  # depends_on = [nomad_job.postgres-server, nomad_job.redis]
+  jobspec = file("${path.module}/immich.nomad")
+}
+
 resource "nomad_job" "whoami" {
   hcl2 {
     enabled = true