Add some basic Nomad and k8s tests

k8s-test/traefik-chart-values.yml

@@ -0,0 +1,35 @@
+---
+additionalArguments:
+  - --providers.file.filename=/data/traefik-config.yaml
+  - --entrypoints.websecure.http.tls.certresolver=cloudflare
+  - --entrypoints.websecure.http.tls.domains[0].main=example.com
+  - --entrypoints.websecure.http.tls.domains[0].sans=*.example.com
+  - --certificatesresolvers.cloudflare.acme.dnschallenge.provider=cloudflare
+  - --certificatesresolvers.cloudflare.acme.email=mail@example.com
+  - --certificatesresolvers.cloudflare.acme.dnschallenge.resolvers=1.1.1.1
+  - --certificatesresolvers.cloudflare.acme.storage=/certs/acme.json
+ports:
+  web:
+    redirectTo: websecure
+env:
+  - name: CF_API_EMAIL
+    valueFrom:
+      secretKeyRef:
+        key: email
+        name: cloudflare-api-credentials
+  - name: CF_API_KEY
+    valueFrom:
+      secretKeyRef:
+        key: apiKey
+        name: cloudflare-api-credentials
+ingressRoute:
+  dashboard:
+    enabled: false
+persistence:
+  enabled: true
+  path: /certs
+  size: 128Mi
+volumes:
+  - mountPath: /data
+    name: traefik-config
+    type: configMap

k8s-test/whoami/deployment.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy-whoami
+  labels:
+    app: whoami
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: whoami
+  template:
+    metadata:
+      labels:
+        app: whoami
+    spec:
+      containers:
+        - name: whoami
+          image: containous/whoami:latest
+          ports:
+            - containerPort: 80

k8s-test/whoami/ingress.yml

@@ -0,0 +1,16 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress-whoami
+spec:
+  rules:
+    - http:
+        paths:
+          - path: /whoami
+            pathType: Prefix
+            backend:
+              service:
+                name: whoami
+                port:
+                  number: 80

nomad/.gitignore

@@ -0,0 +1 @@
+roles/

nomad/.terraform.lock.hcl

@@ -0,0 +1,38 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/consul" {
+  version = "2.14.0"
+  hashes = [
+    "h1:xRwktNwLL3Vo43F7v73tfcgbcnjCE2KgCzcNrsQJ1cc=",
+    "zh:06dcca1f76b839af8f86c7b6f65b944003a7a35b30b865b3884f48e2c42f9aee",
+    "zh:16111df6a485e21cee6ca33cb863434baa1ca360c819c8e2af85e465c1361d2b",
+    "zh:26b59c82ac2861b2651c1fa31955c3e7790e3c2d5d097f22aa34d3c294da63cf",
+    "zh:70fd6853099126a602d5ac26caa80214a4a8a38f0cad8a5e3b7bef49923419d3",
+    "zh:7d4f0061d6fb86e0a5639ed02381063b868245082ec4e3a461bcda964ed00fcc",
+    "zh:a48cbf57d6511922362d5b0f76f449fba7a550c9d0702635fabb43b4f0a09fc0",
+    "zh:bb54994a53dd8e1ff84ca50742ce893863dc166fd41b91d951f4cb89fe6a6bc0",
+    "zh:bc61b19ee3c8d55a9915a3ad84203c87bfd0d57eca8eec788524b14e8b67f090",
+    "zh:cbe3238e756ada23c1e7c97c42a5c72bf810dc5bd1265c9f074c3e739d1090b0",
+    "zh:e30198054239eab46493e59956b9cd8c376c3bbd9515ac102a96d1fbd32e423f",
+    "zh:e74365dba529a0676107e413986d7be81c2125c197754ce69e3e89d8daa53153",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/nomad" {
+  version = "1.4.16"
+  hashes = [
+    "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
+    "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
+    "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
+    "zh:0df88393271078533a217654b96f0672c60eb59570d72e6aefcb839eea87a7a0",
+    "zh:2883b335bb6044b0db6a00e602d6926c047c7f330294a73a90d089f98b24d084",
+    "zh:390158d928009a041b3a182bdd82376b50530805ae92be2b84ed7c3b0fa902a0",
+    "zh:7169b8f8df4b8e9659c49043848fd5f7f8473d0471f67815e8b04980f827f5ef",
+    "zh:9417ee1383b1edd137024882d7035be4dca51fb4f725ca00ed87729086ec1755",
+    "zh:a22910b5a29eeab5610350700b4899267c1b09b66cf21f7e4d06afc61d425800",
+    "zh:a6185c9cd7aa458cd81861058ba568b6411fbac344373a20155e20256f4a7557",
+    "zh:b6260ca9f034df1b47905b4e2a9c33b67dbf77224a694d5b10fb09ae92ffad4c",
+    "zh:d87c12a6a7768f2b6c2a59495c7dc00f9ecc52b1b868331d4c284f791e278a1e",
+  ]
+}

nomad/Makefile

@@ -0,0 +1,60 @@
+SERVER ?= "192.168.2.41"
+SSH_USER = iamthefij
+SSH_KEY = ~/.ssh/id_ed25519
+
+.PHONY: rm-nomad
+rm-nomad:
+	hashi-up nomad uninstall \
+		--ssh-target-addr $(SERVER) \
+		--ssh-target-key $(SSH_KEY) \
+		--ssh-target-user $(SSH_USER) \
+		--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS)
+
+.PHONY: nomad
+nomad:
+	hashi-up nomad install \
+		--ssh-target-addr $(SERVER) \
+		--ssh-target-key $(SSH_KEY) \
+		--ssh-target-user $(SSH_USER) \
+		--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \
+		--server --client
+
+.PHONY: rm-consul
+rm-consul:
+	hashi-up consul uninstall \
+		--ssh-target-addr $(SERVER) \
+		--ssh-target-key $(SSH_KEY) \
+		--ssh-target-user $(SSH_USER) \
+		--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS)
+
+.PHONY: consul
+consul:
+	hashi-up consul install \
+		--ssh-target-addr $(SERVER) \
+		--advertise-addr $(SERVER) \
+		--client-addr 0.0.0.0 \
+		--http-addr 0.0.0.0 \
+		--ssh-target-key $(SSH_KEY) \
+		--ssh-target-user $(SSH_USER) \
+		--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \
+		--connect \
+		--server
+
+.PHONY: cluster
+cluster:
+	ansible-galaxy install -p roles -r roles/requirements.yml
+	ansible-playbook -K -vv -i ansible_hosts -M roles/ ./setup-cluster.yml
+
+.PHONY: plan
+plan:
+	terraform plan
+
+.PHONY: apply
+apply:
+	terraform apply
+
+# Install CNI on hosts?
+# curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-$( [ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)"-v1.0.0.tgz
+# sudo mkdir -p /opt/cni/bin
+# sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz
+

nomad/adminer.nomad

@@ -0,0 +1,68 @@
+variable "base_hostname" {
+  type = string
+  description = "Base hostname to serve content from"
+  default = "dev.homelab"
+}
+
+job "adminer" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "adminer" {
+    count = 1
+    # Some affinity to stateful hosts?
+
+    network {
+      mode = "bridge"
+      port "adminer" {
+        static = 8080
+        to = 8080
+      }
+    }
+
+    service {
+      name = "adminer"
+      port = "adminer"
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "mysql-server"
+              # TODO: how do I get these to not bind to the host eth0 address
+              local_bind_port = 4040
+            }
+            config {
+              protocol = "tcp"
+            }
+          }
+        }
+      }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.adminer.entrypoints=web,websecure",
+        "traefik.http.routers.adminer.rule=Host(`adminer.${var.base_hostname}`)",
+        "traefik.http.routers.adminer.tls=true",
+      ]
+    }
+
+    task "adminer" {
+      driver = "docker"
+
+      config {
+        image = "adminer"
+        ports = ["adminer"]
+      }
+
+      env = {
+        "ADMINER_DEFAULT_SERVER" = "${NOMAD_UPSTREAM_ADDR_mysql_server}"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
+  }
+}

nomad/ansible_hosts

@@ -0,0 +1,8 @@
+[servers]
+services.thefij
+
+[consul_instances]
+services.thefij consul_node_role=bootstrap
+
+[nomad_instances]
+services.thefij nomad_node_role=both

nomad/hashi-up.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+export VERIFY_CHECKSUM=0
+export ALIAS_NAME=
+export OWNER=jsiebens
+export REPO=hashi-up
+export SUCCESS_CMD="$REPO version"
+export BINLOCATION="~/bin"
+
+###############################
+# Content common across repos #
+###############################
+
+version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r')
+if [ ! $version ]; then
+    echo "Failed while attempting to install $REPO. Please manually install:"
+    echo ""
+    echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases"
+    echo "2. Download the latest release for your platform. Call it '$REPO'."
+    echo "3. chmod +x ./$REPO"
+    echo "4. mv ./$REPO $BINLOCATION"
+    if [ -n "$ALIAS_NAME" ]; then
+        echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME"
+    fi
+    exit 1
+fi
+
+getPackage() {
+    uname=$(uname)
+    userid=$(id -u)
+
+    suffix=""
+    case $uname in
+    "Darwin")
+    suffix="-darwin"
+    ;;
+    "MINGW"*)
+    suffix=".exe"
+    BINLOCATION="$HOME/bin"
+    mkdir -p $BINLOCATION
+
+    ;;
+    "Linux")
+        arch=$(uname -m)
+        case $arch in
+        "aarch64")
+        suffix="-arm64"
+        ;;
+        esac
+        case $arch in
+        "armv6l" | "armv7l")
+        suffix="-armhf"
+        ;;
+        esac
+    ;;
+    esac
+
+    targetFile="/tmp/$REPO$suffix"
+
+    if [ "$userid" != "0" ]; then
+        targetFile="$(pwd)/$REPO$suffix"
+    fi
+
+    if [ -e "$targetFile" ]; then
+        rm "$targetFile"
+    fi
+
+    url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix
+    echo "Downloading package $url as $targetFile"
+
+    curl -sSL $url --output "$targetFile"
+
+    if [ "$?" = "0" ]; then
+
+        if [ "$VERIFY_CHECKSUM" = "1" ]; then
+            checkHash
+        fi
+
+    chmod +x "$targetFile"
+
+    echo "Download complete."
+
+    if [ ! -w "$BINLOCATION" ]; then
+
+            echo
+            echo "============================================================"
+            echo "  The script was run as a user who is unable to write"
+            echo "  to $BINLOCATION. To complete the installation the"
+            echo "  following commands may need to be run manually."
+            echo "============================================================"
+            echo
+            echo "  sudo cp $REPO$suffix $BINLOCATION/$REPO"
+
+            if [ -n "$ALIAS_NAME" ]; then
+                echo "  sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME"
+            fi
+
+            echo
+
+        else
+
+            echo
+            echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION"
+
+            if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then
+
+            echo
+            echo "================================================================"
+            echo "  $BINLOCATION/$REPO already exists and is not writeable"
+            echo "  by the current user.  Please adjust the binary ownership"
+            echo "  or run sh/bash with sudo."
+            echo "================================================================"
+            echo
+            exit 1
+
+            fi
+
+            mv "$targetFile" $BINLOCATION/$REPO
+
+            if [ "$?" = "0" ]; then
+                echo "New version of $REPO installed to $BINLOCATION"
+            fi
+
+            if [ -e "$targetFile" ]; then
+                rm "$targetFile"
+            fi
+
+            if [ $(which $ALIAS_NAME) ]; then
+                echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias"
+            else
+                if [ -n "$ALIAS_NAME" ]; then
+                    if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then
+                        ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME
+                        echo "Creating alias '$ALIAS_NAME' for '$REPO'."
+                    fi
+                fi
+            fi
+
+            ${SUCCESS_CMD}
+        fi
+    fi
+}
+
+getPackage

nomad/mysql.nomad

@@ -0,0 +1,69 @@
+job "mysql-server" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "mysql-server" {
+    count = 1
+    # Some affinity to stateful hosts?
+
+    restart {
+      attempts = 10
+      interval = "5m"
+      delay = "25s"
+      mode = "delay"
+    }
+
+    network {
+      mode = "bridge"
+      port "db" {
+        static = 3306
+      }
+    }
+
+    volume "mysql-data" {
+      type = "host"
+      read_only = false
+      source = "mysql-data"
+    }
+
+    service {
+      name = "mysql-server"
+      port = "db"
+
+      connect {
+        sidecar_service {}
+      }
+
+      # check {
+      #   type = "tcp"
+      #   interval = "10s"
+      #   timeout = "2s"
+      # }
+    }
+
+    task "mysql-server" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "mysql-data"
+        destination = "/var/lib/mysql"
+        read_only = false
+      }
+
+      env = {
+        "MYSQL_ROOT_PASSWORD" = "supersecretpassword"
+        "MYSQL_ROOT_HOST" = "%"
+      }
+
+      config {
+        image = "mysql:8"
+        ports = ["db"]
+      }
+
+      resources {
+        cpu = 500
+        memory = 1024
+      }
+    }
+  }
+}

nomad/services.tf

@@ -0,0 +1,66 @@
+# Configure Consul provider
+variable "consul_address" {
+  type = string
+  default = "http://192.168.2.41:8500"
+}
+
+provider "consul" {
+  address    = "${var.consul_address}"
+}
+
+# Get Nomad client from Consul
+data "consul_service" "read-nomad-cluster" {
+    name = "nomad-client"
+    # name = "nomad-clients"
+}
+
+locals {
+  nomad_node = "${data.consul_service.read-nomad-cluster.service[0]}"
+  nomad_node_address = "http://${local.nomad_node.node_address}:${local.nomad_node.port}"
+}
+
+# Configure the Consul provider
+provider "nomad" {
+  # address = "http://services.thefij:4646"
+  address = "${local.nomad_node_address}"
+  region  = "global"
+}
+
+# Create mysql server
+resource "nomad_job" "mysql-server" {
+  hcl2 {
+    enabled = true
+  }
+
+  jobspec = file("${path.module}/mysql.nomad")
+}
+
+# Create mysql server
+resource "nomad_job" "adminer" {
+  hcl2 {
+    enabled = true
+  }
+
+  jobspec = file("${path.module}/adminer.nomad")
+}
+
+# Create Traefik
+resource "nomad_job" "traefik" {
+  hcl2 {
+    enabled = true
+    vars = {
+      "consul_address" = "${var.consul_address}",
+    }
+  }
+
+  jobspec = file("${path.module}/traefik.nomad")
+}
+
+# Create a sample host
+resource "nomad_job" "whoami" {
+  hcl2 {
+    enabled = true
+  }
+
+  jobspec = file("${path.module}/whoami.nomad")
+}

nomad/setup-cluster.yml

@@ -0,0 +1,50 @@
+---
+- name: Build Consul cluster
+  hosts: consul_instances
+  any_errors_fatal: true
+  become: true
+
+  roles:
+    - name: ansible-consul
+      consul_version: "1.11.3"
+      consul_install_upgrade: true
+      # consul_tls_enable: true
+      consul_connect_enabled: true
+      consul_ports_grpc: 8502
+      consul_client_address: "0.0.0.0"
+      consul_auto_encrypt:
+        enabled: true
+        dns_san: ["services.thefij"]
+        ip_san: ["192.168.2.41", "127.0.0.1"]
+
+  # tasks:
+  # # Limit to consul host
+  #   - name: Add a value to Consul
+  #     consul_kv:
+  #       key: ansible_test
+  #       value: Hello from Ansible!
+  #     execute_once: true
+
+- name: Build Consul cluster
+  hosts: nomad_instances
+  any_errors_fatal: true
+  become: true
+
+  roles:
+    - name: ansible-nomad
+      nomad_version: "1.2.6"
+      nomad_install_upgrade: true
+      nomad_allow_purge_config: true
+      nomad_encrypt_enable: true
+      nomad_cni_enable: true
+      nomad_docker_enable: true
+      # nomad_use_consul: true
+
+      # TODO: this should probably be based on host
+      nomad_host_volumes:
+        - name: mysql-data
+          path: /srv/volumes/mysql-data
+          owner: "nomad"
+          group: "bin"
+          mode: "0755"
+          read_only: false

nomad/traefik.nomad

@@ -0,0 +1,99 @@
+variable "consul_address" {
+  type = string
+  description = "Full address of Consul instance to get catalog from"
+  default = "http://127.0.0.1:5400"
+}
+
+variable "base_hostname" {
+  type = string
+  description = "Base hostname to serve content from"
+  default = "dev.homelab"
+}
+
+job "traefik" {
+  region = "global"
+  datacenters = ["dc1"]
+
+  type = "service"
+
+  group "traefik" {
+    count = 1
+
+    network  {
+      port "web" {
+        static = 80
+      }
+      port "websecure" {
+        static = 443
+      }
+    }
+
+    service {
+      name = "traefik"
+      port = "web"
+
+      check {
+        type = "http"
+        path = "/ping"
+        port = "web"
+        interval = "10s"
+        timeout = "2s"
+      }
+
+      connect {
+        native = true
+      }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.traefik_dashboard.entrypoints=web,websecure",
+        "traefik.http.routers.traefik_dashboard.rule=Host(`traefik.${var.base_hostname}`)",
+        "traefik.http.routers.traefik_dashboard.service=api@internal",
+        "traefik.http.routers.traefik_dashboard.tls=true",
+      ]
+    }
+
+    task "traefik" {
+      driver = "docker"
+
+      config {
+        image = "traefik:2.6"
+        args = [
+          "--log.level=DEBUG",
+          "--entryPoints.web.address=:80",
+          "--entryPoints.websecure.address=:443",
+          # "--entryPoints.websecure.tls=true",
+          # "--entrypoints.web.http.redirections.entryPoint.to=websecure",
+          # "--entryPoints.admin.address=:8080",
+          "--accesslog=true",
+          "--api=true",
+          "--api.dashboard=true",
+          # "--metrics=true",
+          # "--metrics.prometheus=true",
+          # "--metrics.prometheus.entryPoint=admin",
+          # "--metrics.prometheus.manualrouting=true",
+          "--ping=true",
+          "--ping.entryPoint=web",
+          "--providers.consulcatalog=true",
+          "--providers.consulcatalog.connectaware=true",
+          "--providers.consulcatalog.connectbydefault=true",
+          "--providers.consulcatalog.exposedbydefault=false",
+          "--providers.consulcatalog.endpoint.address=${var.consul_address}",
+          "--providers.consulcatalog.servicename=traefik",
+          "--providers.consulcatalog.prefix=traefik",
+          "--providers.consulcatalog.defaultrule=Host(`{{normalize .Name}}.${var.base_hostname}`)",
+        ]
+
+        ports = ["web", "websecure"]
+        network_mode = "host"
+
+        volumes = []
+      }
+
+      resources {
+        cpu = 500
+        memory = 100
+      }
+    }
+  }
+}

nomad/whoami.nomad

@@ -0,0 +1,62 @@
+variable "base_hostname" {
+  type = string
+  description = "Base hostname to serve content from"
+  default = "dev.homelab"
+}
+
+job "whoami" {
+  region = "global"
+  datacenters = ["dc1"]
+
+  type = "service"
+
+  group "whoami" {
+    count = 2
+
+    network {
+      mode = "bridge"
+      port "web" {
+        # to = 80
+      }
+    }
+
+    service {
+      name = "whoami"
+      port = "web"
+
+      connect {
+        sidecar_service {}
+      }
+
+      check {
+        type = "http"
+        path = "/health"
+        port = "web"
+        interval = "10s"
+        timeout = "10s"
+      }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.whoami.entrypoints=web,websecure",
+        "traefik.http.routers.whoami.rule=Host(`whoami.${var.base_hostname}`)",
+        "traefik.http.routers.whoami.tls=true",
+      ]
+    }
+
+    task "whoami" {
+      driver = "docker"
+
+      config {
+        image = "containous/whoami:latest"
+        ports = ["web"]
+        args = ["--port", "${NOMAD_PORT_web}"]
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+    }
+  }
+}