From 87dfd449c4b280733ace1dee9a42bfbcb8e6f2f6 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Wed, 16 Feb 2022 09:56:18 -0800 Subject: [PATCH] Add some basic Nomad and k8s tests --- k8s-test/traefik-chart-values.yml | 35 ++++++++ k8s-test/whoami/deployment.yaml | 22 +++++ k8s-test/whoami/ingress.yml | 16 ++++ nomad/.gitignore | 1 + nomad/.terraform.lock.hcl | 38 ++++++++ nomad/Makefile | 60 +++++++++++++ nomad/adminer.nomad | 68 ++++++++++++++ nomad/ansible_hosts | 8 ++ nomad/hashi-up.sh | 144 ++++++++++++++++++++++++++++++ nomad/mysql.nomad | 69 ++++++++++++++ nomad/root.tf | 0 nomad/services.tf | 66 ++++++++++++++ nomad/setup-cluster.yml | 50 +++++++++++ nomad/traefik.nomad | 99 ++++++++++++++++++++ nomad/whoami.nomad | 62 +++++++++++++ 15 files changed, 738 insertions(+) create mode 100644 k8s-test/traefik-chart-values.yml create mode 100644 k8s-test/whoami/deployment.yaml create mode 100644 k8s-test/whoami/ingress.yml create mode 100644 nomad/.gitignore create mode 100644 nomad/.terraform.lock.hcl create mode 100644 nomad/Makefile create mode 100644 nomad/adminer.nomad create mode 100644 nomad/ansible_hosts create mode 100644 nomad/hashi-up.sh create mode 100644 nomad/mysql.nomad create mode 100644 nomad/root.tf create mode 100644 nomad/services.tf create mode 100644 nomad/setup-cluster.yml create mode 100644 nomad/traefik.nomad create mode 100644 nomad/whoami.nomad diff --git a/k8s-test/traefik-chart-values.yml b/k8s-test/traefik-chart-values.yml new file mode 100644 index 0000000..92ea970 --- /dev/null +++ b/k8s-test/traefik-chart-values.yml @@ -0,0 +1,35 @@ +--- +additionalArguments: + - --providers.file.filename=/data/traefik-config.yaml + - --entrypoints.websecure.http.tls.certresolver=cloudflare + - --entrypoints.websecure.http.tls.domains[0].main=example.com + - --entrypoints.websecure.http.tls.domains[0].sans=*.example.com + - --certificatesresolvers.cloudflare.acme.dnschallenge.provider=cloudflare + - --certificatesresolvers.cloudflare.acme.email=mail@example.com + - --certificatesresolvers.cloudflare.acme.dnschallenge.resolvers=1.1.1.1 + - --certificatesresolvers.cloudflare.acme.storage=/certs/acme.json +ports: + web: + redirectTo: websecure +env: + - name: CF_API_EMAIL + valueFrom: + secretKeyRef: + key: email + name: cloudflare-api-credentials + - name: CF_API_KEY + valueFrom: + secretKeyRef: + key: apiKey + name: cloudflare-api-credentials +ingressRoute: + dashboard: + enabled: false +persistence: + enabled: true + path: /certs + size: 128Mi +volumes: + - mountPath: /data + name: traefik-config + type: configMap diff --git a/k8s-test/whoami/deployment.yaml b/k8s-test/whoami/deployment.yaml new file mode 100644 index 0000000..ffe25ed --- /dev/null +++ b/k8s-test/whoami/deployment.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deploy-whoami + labels: + app: whoami +spec: + replicas: 1 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: containous/whoami:latest + ports: + - containerPort: 80 diff --git a/k8s-test/whoami/ingress.yml b/k8s-test/whoami/ingress.yml new file mode 100644 index 0000000..5dd0dcf --- /dev/null +++ b/k8s-test/whoami/ingress.yml @@ -0,0 +1,16 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-whoami +spec: + rules: + - http: + paths: + - path: /whoami + pathType: Prefix + backend: + service: + name: whoami + port: + number: 80 diff --git a/nomad/.gitignore b/nomad/.gitignore new file mode 100644 index 0000000..755189b --- /dev/null +++ b/nomad/.gitignore @@ -0,0 +1 @@ +roles/ diff --git a/nomad/.terraform.lock.hcl b/nomad/.terraform.lock.hcl new file mode 100644 index 0000000..083e649 --- /dev/null +++ b/nomad/.terraform.lock.hcl @@ -0,0 +1,38 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/consul" { + version = "2.14.0" + hashes = [ + "h1:xRwktNwLL3Vo43F7v73tfcgbcnjCE2KgCzcNrsQJ1cc=", + "zh:06dcca1f76b839af8f86c7b6f65b944003a7a35b30b865b3884f48e2c42f9aee", + "zh:16111df6a485e21cee6ca33cb863434baa1ca360c819c8e2af85e465c1361d2b", + "zh:26b59c82ac2861b2651c1fa31955c3e7790e3c2d5d097f22aa34d3c294da63cf", + "zh:70fd6853099126a602d5ac26caa80214a4a8a38f0cad8a5e3b7bef49923419d3", + "zh:7d4f0061d6fb86e0a5639ed02381063b868245082ec4e3a461bcda964ed00fcc", + "zh:a48cbf57d6511922362d5b0f76f449fba7a550c9d0702635fabb43b4f0a09fc0", + "zh:bb54994a53dd8e1ff84ca50742ce893863dc166fd41b91d951f4cb89fe6a6bc0", + "zh:bc61b19ee3c8d55a9915a3ad84203c87bfd0d57eca8eec788524b14e8b67f090", + "zh:cbe3238e756ada23c1e7c97c42a5c72bf810dc5bd1265c9f074c3e739d1090b0", + "zh:e30198054239eab46493e59956b9cd8c376c3bbd9515ac102a96d1fbd32e423f", + "zh:e74365dba529a0676107e413986d7be81c2125c197754ce69e3e89d8daa53153", + ] +} + +provider "registry.terraform.io/hashicorp/nomad" { + version = "1.4.16" + hashes = [ + "h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=", + "zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e", + "zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572", + "zh:0df88393271078533a217654b96f0672c60eb59570d72e6aefcb839eea87a7a0", + "zh:2883b335bb6044b0db6a00e602d6926c047c7f330294a73a90d089f98b24d084", + "zh:390158d928009a041b3a182bdd82376b50530805ae92be2b84ed7c3b0fa902a0", + "zh:7169b8f8df4b8e9659c49043848fd5f7f8473d0471f67815e8b04980f827f5ef", + "zh:9417ee1383b1edd137024882d7035be4dca51fb4f725ca00ed87729086ec1755", + "zh:a22910b5a29eeab5610350700b4899267c1b09b66cf21f7e4d06afc61d425800", + "zh:a6185c9cd7aa458cd81861058ba568b6411fbac344373a20155e20256f4a7557", + "zh:b6260ca9f034df1b47905b4e2a9c33b67dbf77224a694d5b10fb09ae92ffad4c", + "zh:d87c12a6a7768f2b6c2a59495c7dc00f9ecc52b1b868331d4c284f791e278a1e", + ] +} diff --git a/nomad/Makefile b/nomad/Makefile new file mode 100644 index 0000000..cabeadc --- /dev/null +++ b/nomad/Makefile @@ -0,0 +1,60 @@ +SERVER ?= "192.168.2.41" +SSH_USER = iamthefij +SSH_KEY = ~/.ssh/id_ed25519 + +.PHONY: rm-nomad +rm-nomad: + hashi-up nomad uninstall \ + --ssh-target-addr $(SERVER) \ + --ssh-target-key $(SSH_KEY) \ + --ssh-target-user $(SSH_USER) \ + --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) + +.PHONY: nomad +nomad: + hashi-up nomad install \ + --ssh-target-addr $(SERVER) \ + --ssh-target-key $(SSH_KEY) \ + --ssh-target-user $(SSH_USER) \ + --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \ + --server --client + +.PHONY: rm-consul +rm-consul: + hashi-up consul uninstall \ + --ssh-target-addr $(SERVER) \ + --ssh-target-key $(SSH_KEY) \ + --ssh-target-user $(SSH_USER) \ + --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) + +.PHONY: consul +consul: + hashi-up consul install \ + --ssh-target-addr $(SERVER) \ + --advertise-addr $(SERVER) \ + --client-addr 0.0.0.0 \ + --http-addr 0.0.0.0 \ + --ssh-target-key $(SSH_KEY) \ + --ssh-target-user $(SSH_USER) \ + --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \ + --connect \ + --server + +.PHONY: cluster +cluster: + ansible-galaxy install -p roles -r roles/requirements.yml + ansible-playbook -K -vv -i ansible_hosts -M roles/ ./setup-cluster.yml + +.PHONY: plan +plan: + terraform plan + +.PHONY: apply +apply: + terraform apply + +# Install CNI on hosts? +# curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-$( [ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)"-v1.0.0.tgz +# sudo mkdir -p /opt/cni/bin +# sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz + diff --git a/nomad/adminer.nomad b/nomad/adminer.nomad new file mode 100644 index 0000000..e53af41 --- /dev/null +++ b/nomad/adminer.nomad @@ -0,0 +1,68 @@ +variable "base_hostname" { + type = string + description = "Base hostname to serve content from" + default = "dev.homelab" +} + +job "adminer" { + datacenters = ["dc1"] + type = "service" + + group "adminer" { + count = 1 + # Some affinity to stateful hosts? + + network { + mode = "bridge" + port "adminer" { + static = 8080 + to = 8080 + } + } + + service { + name = "adminer" + port = "adminer" + + connect { + sidecar_service { + proxy { + upstreams { + destination_name = "mysql-server" + # TODO: how do I get these to not bind to the host eth0 address + local_bind_port = 4040 + } + config { + protocol = "tcp" + } + } + } + } + + tags = [ + "traefik.enable=true", + "traefik.http.routers.adminer.entrypoints=web,websecure", + "traefik.http.routers.adminer.rule=Host(`adminer.${var.base_hostname}`)", + "traefik.http.routers.adminer.tls=true", + ] + } + + task "adminer" { + driver = "docker" + + config { + image = "adminer" + ports = ["adminer"] + } + + env = { + "ADMINER_DEFAULT_SERVER" = "${NOMAD_UPSTREAM_ADDR_mysql_server}" + } + + resources { + cpu = 50 + memory = 50 + } + } + } +} diff --git a/nomad/ansible_hosts b/nomad/ansible_hosts new file mode 100644 index 0000000..145b5f4 --- /dev/null +++ b/nomad/ansible_hosts @@ -0,0 +1,8 @@ +[servers] +services.thefij + +[consul_instances] +services.thefij consul_node_role=bootstrap + +[nomad_instances] +services.thefij nomad_node_role=both diff --git a/nomad/hashi-up.sh b/nomad/hashi-up.sh new file mode 100644 index 0000000..82cf5db --- /dev/null +++ b/nomad/hashi-up.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +export VERIFY_CHECKSUM=0 +export ALIAS_NAME= +export OWNER=jsiebens +export REPO=hashi-up +export SUCCESS_CMD="$REPO version" +export BINLOCATION="~/bin" + +############################### +# Content common across repos # +############################### + +version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r') +if [ ! $version ]; then + echo "Failed while attempting to install $REPO. Please manually install:" + echo "" + echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases" + echo "2. Download the latest release for your platform. Call it '$REPO'." + echo "3. chmod +x ./$REPO" + echo "4. mv ./$REPO $BINLOCATION" + if [ -n "$ALIAS_NAME" ]; then + echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME" + fi + exit 1 +fi + +getPackage() { + uname=$(uname) + userid=$(id -u) + + suffix="" + case $uname in + "Darwin") + suffix="-darwin" + ;; + "MINGW"*) + suffix=".exe" + BINLOCATION="$HOME/bin" + mkdir -p $BINLOCATION + + ;; + "Linux") + arch=$(uname -m) + case $arch in + "aarch64") + suffix="-arm64" + ;; + esac + case $arch in + "armv6l" | "armv7l") + suffix="-armhf" + ;; + esac + ;; + esac + + targetFile="/tmp/$REPO$suffix" + + if [ "$userid" != "0" ]; then + targetFile="$(pwd)/$REPO$suffix" + fi + + if [ -e "$targetFile" ]; then + rm "$targetFile" + fi + + url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix + echo "Downloading package $url as $targetFile" + + curl -sSL $url --output "$targetFile" + + if [ "$?" = "0" ]; then + + if [ "$VERIFY_CHECKSUM" = "1" ]; then + checkHash + fi + + chmod +x "$targetFile" + + echo "Download complete." + + if [ ! -w "$BINLOCATION" ]; then + + echo + echo "============================================================" + echo " The script was run as a user who is unable to write" + echo " to $BINLOCATION. To complete the installation the" + echo " following commands may need to be run manually." + echo "============================================================" + echo + echo " sudo cp $REPO$suffix $BINLOCATION/$REPO" + + if [ -n "$ALIAS_NAME" ]; then + echo " sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME" + fi + + echo + + else + + echo + echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION" + + if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then + + echo + echo "================================================================" + echo " $BINLOCATION/$REPO already exists and is not writeable" + echo " by the current user. Please adjust the binary ownership" + echo " or run sh/bash with sudo." + echo "================================================================" + echo + exit 1 + + fi + + mv "$targetFile" $BINLOCATION/$REPO + + if [ "$?" = "0" ]; then + echo "New version of $REPO installed to $BINLOCATION" + fi + + if [ -e "$targetFile" ]; then + rm "$targetFile" + fi + + if [ $(which $ALIAS_NAME) ]; then + echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias" + else + if [ -n "$ALIAS_NAME" ]; then + if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then + ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME + echo "Creating alias '$ALIAS_NAME' for '$REPO'." + fi + fi + fi + + ${SUCCESS_CMD} + fi + fi +} + +getPackage diff --git a/nomad/mysql.nomad b/nomad/mysql.nomad new file mode 100644 index 0000000..530af96 --- /dev/null +++ b/nomad/mysql.nomad @@ -0,0 +1,69 @@ +job "mysql-server" { + datacenters = ["dc1"] + type = "service" + + group "mysql-server" { + count = 1 + # Some affinity to stateful hosts? + + restart { + attempts = 10 + interval = "5m" + delay = "25s" + mode = "delay" + } + + network { + mode = "bridge" + port "db" { + static = 3306 + } + } + + volume "mysql-data" { + type = "host" + read_only = false + source = "mysql-data" + } + + service { + name = "mysql-server" + port = "db" + + connect { + sidecar_service {} + } + + # check { + # type = "tcp" + # interval = "10s" + # timeout = "2s" + # } + } + + task "mysql-server" { + driver = "docker" + + volume_mount { + volume = "mysql-data" + destination = "/var/lib/mysql" + read_only = false + } + + env = { + "MYSQL_ROOT_PASSWORD" = "supersecretpassword" + "MYSQL_ROOT_HOST" = "%" + } + + config { + image = "mysql:8" + ports = ["db"] + } + + resources { + cpu = 500 + memory = 1024 + } + } + } +} diff --git a/nomad/root.tf b/nomad/root.tf new file mode 100644 index 0000000..e69de29 diff --git a/nomad/services.tf b/nomad/services.tf new file mode 100644 index 0000000..0a25db1 --- /dev/null +++ b/nomad/services.tf @@ -0,0 +1,66 @@ +# Configure Consul provider +variable "consul_address" { + type = string + default = "http://192.168.2.41:8500" +} + +provider "consul" { + address = "${var.consul_address}" +} + +# Get Nomad client from Consul +data "consul_service" "read-nomad-cluster" { + name = "nomad-client" + # name = "nomad-clients" +} + +locals { + nomad_node = "${data.consul_service.read-nomad-cluster.service[0]}" + nomad_node_address = "http://${local.nomad_node.node_address}:${local.nomad_node.port}" +} + +# Configure the Consul provider +provider "nomad" { + # address = "http://services.thefij:4646" + address = "${local.nomad_node_address}" + region = "global" +} + +# Create mysql server +resource "nomad_job" "mysql-server" { + hcl2 { + enabled = true + } + + jobspec = file("${path.module}/mysql.nomad") +} + +# Create mysql server +resource "nomad_job" "adminer" { + hcl2 { + enabled = true + } + + jobspec = file("${path.module}/adminer.nomad") +} + +# Create Traefik +resource "nomad_job" "traefik" { + hcl2 { + enabled = true + vars = { + "consul_address" = "${var.consul_address}", + } + } + + jobspec = file("${path.module}/traefik.nomad") +} + +# Create a sample host +resource "nomad_job" "whoami" { + hcl2 { + enabled = true + } + + jobspec = file("${path.module}/whoami.nomad") +} diff --git a/nomad/setup-cluster.yml b/nomad/setup-cluster.yml new file mode 100644 index 0000000..cecf335 --- /dev/null +++ b/nomad/setup-cluster.yml @@ -0,0 +1,50 @@ +--- +- name: Build Consul cluster + hosts: consul_instances + any_errors_fatal: true + become: true + + roles: + - name: ansible-consul + consul_version: "1.11.3" + consul_install_upgrade: true + # consul_tls_enable: true + consul_connect_enabled: true + consul_ports_grpc: 8502 + consul_client_address: "0.0.0.0" + consul_auto_encrypt: + enabled: true + dns_san: ["services.thefij"] + ip_san: ["192.168.2.41", "127.0.0.1"] + + # tasks: + # # Limit to consul host + # - name: Add a value to Consul + # consul_kv: + # key: ansible_test + # value: Hello from Ansible! + # execute_once: true + +- name: Build Consul cluster + hosts: nomad_instances + any_errors_fatal: true + become: true + + roles: + - name: ansible-nomad + nomad_version: "1.2.6" + nomad_install_upgrade: true + nomad_allow_purge_config: true + nomad_encrypt_enable: true + nomad_cni_enable: true + nomad_docker_enable: true + # nomad_use_consul: true + + # TODO: this should probably be based on host + nomad_host_volumes: + - name: mysql-data + path: /srv/volumes/mysql-data + owner: "nomad" + group: "bin" + mode: "0755" + read_only: false diff --git a/nomad/traefik.nomad b/nomad/traefik.nomad new file mode 100644 index 0000000..68ee242 --- /dev/null +++ b/nomad/traefik.nomad @@ -0,0 +1,99 @@ +variable "consul_address" { + type = string + description = "Full address of Consul instance to get catalog from" + default = "http://127.0.0.1:5400" +} + +variable "base_hostname" { + type = string + description = "Base hostname to serve content from" + default = "dev.homelab" +} + +job "traefik" { + region = "global" + datacenters = ["dc1"] + + type = "service" + + group "traefik" { + count = 1 + + network { + port "web" { + static = 80 + } + port "websecure" { + static = 443 + } + } + + service { + name = "traefik" + port = "web" + + check { + type = "http" + path = "/ping" + port = "web" + interval = "10s" + timeout = "2s" + } + + connect { + native = true + } + + tags = [ + "traefik.enable=true", + "traefik.http.routers.traefik_dashboard.entrypoints=web,websecure", + "traefik.http.routers.traefik_dashboard.rule=Host(`traefik.${var.base_hostname}`)", + "traefik.http.routers.traefik_dashboard.service=api@internal", + "traefik.http.routers.traefik_dashboard.tls=true", + ] + } + + task "traefik" { + driver = "docker" + + config { + image = "traefik:2.6" + args = [ + "--log.level=DEBUG", + "--entryPoints.web.address=:80", + "--entryPoints.websecure.address=:443", + # "--entryPoints.websecure.tls=true", + # "--entrypoints.web.http.redirections.entryPoint.to=websecure", + # "--entryPoints.admin.address=:8080", + "--accesslog=true", + "--api=true", + "--api.dashboard=true", + # "--metrics=true", + # "--metrics.prometheus=true", + # "--metrics.prometheus.entryPoint=admin", + # "--metrics.prometheus.manualrouting=true", + "--ping=true", + "--ping.entryPoint=web", + "--providers.consulcatalog=true", + "--providers.consulcatalog.connectaware=true", + "--providers.consulcatalog.connectbydefault=true", + "--providers.consulcatalog.exposedbydefault=false", + "--providers.consulcatalog.endpoint.address=${var.consul_address}", + "--providers.consulcatalog.servicename=traefik", + "--providers.consulcatalog.prefix=traefik", + "--providers.consulcatalog.defaultrule=Host(`{{normalize .Name}}.${var.base_hostname}`)", + ] + + ports = ["web", "websecure"] + network_mode = "host" + + volumes = [] + } + + resources { + cpu = 500 + memory = 100 + } + } + } +} diff --git a/nomad/whoami.nomad b/nomad/whoami.nomad new file mode 100644 index 0000000..fd3667e --- /dev/null +++ b/nomad/whoami.nomad @@ -0,0 +1,62 @@ +variable "base_hostname" { + type = string + description = "Base hostname to serve content from" + default = "dev.homelab" +} + +job "whoami" { + region = "global" + datacenters = ["dc1"] + + type = "service" + + group "whoami" { + count = 2 + + network { + mode = "bridge" + port "web" { + # to = 80 + } + } + + service { + name = "whoami" + port = "web" + + connect { + sidecar_service {} + } + + check { + type = "http" + path = "/health" + port = "web" + interval = "10s" + timeout = "10s" + } + + tags = [ + "traefik.enable=true", + "traefik.http.routers.whoami.entrypoints=web,websecure", + "traefik.http.routers.whoami.rule=Host(`whoami.${var.base_hostname}`)", + "traefik.http.routers.whoami.tls=true", + ] + } + + task "whoami" { + driver = "docker" + + config { + image = "containous/whoami:latest" + ports = ["web"] + args = ["--port", "${NOMAD_PORT_web}"] + } + + resources { + cpu = 50 + memory = 50 + } + } + } +}