iamthefij/orchestration-tests
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add some basic Nomad and k8s tests

Browse Source
This commit is contained in:
IamTheFij 2022-02-16 09:56:18 -08:00
parent 4ba33e266a
commit 87dfd449c4
15 changed files with 738 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
k8s-test/traefik-chart-values.yml Normal file
View File

@ -0,0 +1,35 @@
---
additionalArguments:
- --providers.file.filename=/data/traefik-config.yaml
- --entrypoints.websecure.http.tls.certresolver=cloudflare
- --entrypoints.websecure.http.tls.domains[0].main=example.com
- --entrypoints.websecure.http.tls.domains[0].sans=*.example.com
- --certificatesresolvers.cloudflare.acme.dnschallenge.provider=cloudflare
- --certificatesresolvers.cloudflare.acme.email=mail@example.com
- --certificatesresolvers.cloudflare.acme.dnschallenge.resolvers=1.1.1.1
- --certificatesresolvers.cloudflare.acme.storage=/certs/acme.json
ports:
web:
redirectTo: websecure
env:
- name: CF_API_EMAIL
valueFrom:
secretKeyRef:
key: email
name: cloudflare-api-credentials
- name: CF_API_KEY
valueFrom:
secretKeyRef:
key: apiKey
name: cloudflare-api-credentials
ingressRoute:
dashboard:
enabled: false
persistence:
enabled: true
path: /certs
size: 128Mi
volumes:
- mountPath: /data
name: traefik-config
type: configMap

22
k8s-test/whoami/deployment.yaml Normal file
View File

@ -0,0 +1,22 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-whoami
labels:
app: whoami
spec:
replicas: 1
selector:
matchLabels:
app: whoami
template:
metadata:
labels:
app: whoami
spec:
containers:
- name: whoami
image: containous/whoami:latest
ports:
- containerPort: 80

16
k8s-test/whoami/ingress.yml Normal file
View File

@ -0,0 +1,16 @@
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-whoami
spec:
rules:
- http:
paths:
- path: /whoami
pathType: Prefix
backend:
service:
name: whoami
port:
number: 80

1
nomad/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
roles/

38
nomad/.terraform.lock.hcl Normal file
View File

@ -0,0 +1,38 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/consul" {
version = "2.14.0"
hashes = [
"h1:xRwktNwLL3Vo43F7v73tfcgbcnjCE2KgCzcNrsQJ1cc=",
"zh:06dcca1f76b839af8f86c7b6f65b944003a7a35b30b865b3884f48e2c42f9aee",
"zh:16111df6a485e21cee6ca33cb863434baa1ca360c819c8e2af85e465c1361d2b",
"zh:26b59c82ac2861b2651c1fa31955c3e7790e3c2d5d097f22aa34d3c294da63cf",
"zh:70fd6853099126a602d5ac26caa80214a4a8a38f0cad8a5e3b7bef49923419d3",
"zh:7d4f0061d6fb86e0a5639ed02381063b868245082ec4e3a461bcda964ed00fcc",
"zh:a48cbf57d6511922362d5b0f76f449fba7a550c9d0702635fabb43b4f0a09fc0",
"zh:bb54994a53dd8e1ff84ca50742ce893863dc166fd41b91d951f4cb89fe6a6bc0",
"zh:bc61b19ee3c8d55a9915a3ad84203c87bfd0d57eca8eec788524b14e8b67f090",
"zh:cbe3238e756ada23c1e7c97c42a5c72bf810dc5bd1265c9f074c3e739d1090b0",
"zh:e30198054239eab46493e59956b9cd8c376c3bbd9515ac102a96d1fbd32e423f",
"zh:e74365dba529a0676107e413986d7be81c2125c197754ce69e3e89d8daa53153",
]
}
provider "registry.terraform.io/hashicorp/nomad" {
version = "1.4.16"
hashes = [
"h1:tyfjD/maKzb0RxxD9KWgLnkJu9lnYziYsQgGw85Giz8=",
"zh:0d4fbb7030d9caac3b123e60afa44f50c83cc2a983e1866aec7f30414abe7b0e",
"zh:0db080228e07c72d6d8ca8c45249d6f97cd0189fce82a77abbdcd49a52e57572",
"zh:0df88393271078533a217654b96f0672c60eb59570d72e6aefcb839eea87a7a0",
"zh:2883b335bb6044b0db6a00e602d6926c047c7f330294a73a90d089f98b24d084",
"zh:390158d928009a041b3a182bdd82376b50530805ae92be2b84ed7c3b0fa902a0",
"zh:7169b8f8df4b8e9659c49043848fd5f7f8473d0471f67815e8b04980f827f5ef",
"zh:9417ee1383b1edd137024882d7035be4dca51fb4f725ca00ed87729086ec1755",
"zh:a22910b5a29eeab5610350700b4899267c1b09b66cf21f7e4d06afc61d425800",
"zh:a6185c9cd7aa458cd81861058ba568b6411fbac344373a20155e20256f4a7557",
"zh:b6260ca9f034df1b47905b4e2a9c33b67dbf77224a694d5b10fb09ae92ffad4c",
"zh:d87c12a6a7768f2b6c2a59495c7dc00f9ecc52b1b868331d4c284f791e278a1e",
]
}

60
nomad/Makefile Normal file
View File

@ -0,0 +1,60 @@
SERVER ?= "192.168.2.41"
SSH_USER = iamthefij
SSH_KEY = ~/.ssh/id_ed25519
.PHONY: rm-nomad
rm-nomad:
hashi-up nomad uninstall \
--ssh-target-addr $(SERVER) \
--ssh-target-key $(SSH_KEY) \
--ssh-target-user $(SSH_USER) \
--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS)
.PHONY: nomad
nomad:
hashi-up nomad install \
--ssh-target-addr $(SERVER) \
--ssh-target-key $(SSH_KEY) \
--ssh-target-user $(SSH_USER) \
--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \
--server --client
.PHONY: rm-consul
rm-consul:
hashi-up consul uninstall \
--ssh-target-addr $(SERVER) \
--ssh-target-key $(SSH_KEY) \
--ssh-target-user $(SSH_USER) \
--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS)
.PHONY: consul
consul:
hashi-up consul install \
--ssh-target-addr $(SERVER) \
--advertise-addr $(SERVER) \
--client-addr 0.0.0.0 \
--http-addr 0.0.0.0 \
--ssh-target-key $(SSH_KEY) \
--ssh-target-user $(SSH_USER) \
--ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \
--connect \
--server
.PHONY: cluster
cluster:
ansible-galaxy install -p roles -r roles/requirements.yml
ansible-playbook -K -vv -i ansible_hosts -M roles/ ./setup-cluster.yml
.PHONY: plan
plan:
terraform plan
.PHONY: apply
apply:
terraform apply
# Install CNI on hosts?
# curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-$( [ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)"-v1.0.0.tgz
# sudo mkdir -p /opt/cni/bin
# sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz

68
nomad/adminer.nomad Normal file
View File

@ -0,0 +1,68 @@
variable "base_hostname" {
type = string
description = "Base hostname to serve content from"
default = "dev.homelab"
}
job "adminer" {
datacenters = ["dc1"]
type = "service"
group "adminer" {
count = 1
# Some affinity to stateful hosts?
network {
mode = "bridge"
port "adminer" {
static = 8080
to = 8080
}
}
service {
name = "adminer"
port = "adminer"
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "mysql-server"
# TODO: how do I get these to not bind to the host eth0 address
local_bind_port = 4040
}
config {
protocol = "tcp"
}
}
}
}
tags = [
"traefik.enable=true",
"traefik.http.routers.adminer.entrypoints=web,websecure",
"traefik.http.routers.adminer.rule=Host(`adminer.${var.base_hostname}`)",
"traefik.http.routers.adminer.tls=true",
]
}
task "adminer" {
driver = "docker"
config {
image = "adminer"
ports = ["adminer"]
}
env = {
"ADMINER_DEFAULT_SERVER" = "${NOMAD_UPSTREAM_ADDR_mysql_server}"
}
resources {
cpu = 50
memory = 50
}
}
}
}

8
nomad/ansible_hosts Normal file
View File

@ -0,0 +1,8 @@
[servers]
services.thefij
[consul_instances]
services.thefij consul_node_role=bootstrap
[nomad_instances]
services.thefij nomad_node_role=both

144
nomad/hashi-up.sh Normal file
View File

@ -0,0 +1,144 @@
#!/usr/bin/env bash
export VERIFY_CHECKSUM=0
export ALIAS_NAME=
export OWNER=jsiebens
export REPO=hashi-up
export SUCCESS_CMD="$REPO version"
export BINLOCATION="~/bin"
###############################
# Content common across repos #
###############################
version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r')
if [ ! $version ]; then
echo "Failed while attempting to install $REPO. Please manually install:"
echo ""
echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases"
echo "2. Download the latest release for your platform. Call it '$REPO'."
echo "3. chmod +x ./$REPO"
echo "4. mv ./$REPO $BINLOCATION"
if [ -n "$ALIAS_NAME" ]; then
echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME"
fi
exit 1
fi
getPackage() {
uname=$(uname)
userid=$(id -u)
suffix=""
case $uname in
"Darwin")
suffix="-darwin"
;;
"MINGW"*)
suffix=".exe"
BINLOCATION="$HOME/bin"
mkdir -p $BINLOCATION
;;
"Linux")
arch=$(uname -m)
case $arch in
"aarch64")
suffix="-arm64"
;;
esac
case $arch in
"armv6l" | "armv7l")
suffix="-armhf"
;;
esac
;;
esac
targetFile="/tmp/$REPO$suffix"
if [ "$userid" != "0" ]; then
targetFile="$(pwd)/$REPO$suffix"
fi
if [ -e "$targetFile" ]; then
rm "$targetFile"
fi
url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix
echo "Downloading package $url as $targetFile"
curl -sSL $url --output "$targetFile"
if [ "$?" = "0" ]; then
if [ "$VERIFY_CHECKSUM" = "1" ]; then
checkHash
fi
chmod +x "$targetFile"
echo "Download complete."
if [ ! -w "$BINLOCATION" ]; then
echo
echo "============================================================"
echo " The script was run as a user who is unable to write"
echo " to $BINLOCATION. To complete the installation the"
echo " following commands may need to be run manually."
echo "============================================================"
echo
echo " sudo cp $REPO$suffix $BINLOCATION/$REPO"
if [ -n "$ALIAS_NAME" ]; then
echo " sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME"
fi
echo
else
echo
echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION"
if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then
echo
echo "================================================================"
echo " $BINLOCATION/$REPO already exists and is not writeable"
echo " by the current user. Please adjust the binary ownership"
echo " or run sh/bash with sudo."
echo "================================================================"
echo
exit 1
fi
mv "$targetFile" $BINLOCATION/$REPO
if [ "$?" = "0" ]; then
echo "New version of $REPO installed to $BINLOCATION"
fi
if [ -e "$targetFile" ]; then
rm "$targetFile"
fi
if [ $(which $ALIAS_NAME) ]; then
echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias"
else
if [ -n "$ALIAS_NAME" ]; then
if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then
ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME
echo "Creating alias '$ALIAS_NAME' for '$REPO'."
fi
fi
fi
${SUCCESS_CMD}
fi
fi
}
getPackage

69
nomad/mysql.nomad Normal file
View File

@ -0,0 +1,69 @@
job "mysql-server" {
datacenters = ["dc1"]
type = "service"
group "mysql-server" {
count = 1
# Some affinity to stateful hosts?
restart {
attempts = 10
interval = "5m"
delay = "25s"
mode = "delay"
}
network {
mode = "bridge"
port "db" {
static = 3306
}
}
volume "mysql-data" {
type = "host"
read_only = false
source = "mysql-data"
}
service {
name = "mysql-server"
port = "db"
connect {
sidecar_service {}
}
# check {
# type = "tcp"
# interval = "10s"
# timeout = "2s"
# }
}
task "mysql-server" {
driver = "docker"
volume_mount {
volume = "mysql-data"
destination = "/var/lib/mysql"
read_only = false
}
env = {
"MYSQL_ROOT_PASSWORD" = "supersecretpassword"
"MYSQL_ROOT_HOST" = "%"
}
config {
image = "mysql:8"
ports = ["db"]
}
resources {
cpu = 500
memory = 1024
}
}
}
}

0
nomad/root.tf Normal file
View File

66
nomad/services.tf Normal file
View File

@ -0,0 +1,66 @@
# Configure Consul provider
variable "consul_address" {
type = string
default = "http://192.168.2.41:8500"
}
provider "consul" {
address = "${var.consul_address}"
}
# Get Nomad client from Consul
data "consul_service" "read-nomad-cluster" {
name = "nomad-client"
# name = "nomad-clients"
}
locals {
nomad_node = "${data.consul_service.read-nomad-cluster.service[0]}"
nomad_node_address = "http://${local.nomad_node.node_address}:${local.nomad_node.port}"
}
# Configure the Consul provider
provider "nomad" {
# address = "http://services.thefij:4646"
address = "${local.nomad_node_address}"
region = "global"
}
# Create mysql server
resource "nomad_job" "mysql-server" {
hcl2 {
enabled = true
}
jobspec = file("${path.module}/mysql.nomad")
}
# Create mysql server
resource "nomad_job" "adminer" {
hcl2 {
enabled = true
}
jobspec = file("${path.module}/adminer.nomad")
}
# Create Traefik
resource "nomad_job" "traefik" {
hcl2 {
enabled = true
vars = {
"consul_address" = "${var.consul_address}",
}
}
jobspec = file("${path.module}/traefik.nomad")
}
# Create a sample host
resource "nomad_job" "whoami" {
hcl2 {
enabled = true
}
jobspec = file("${path.module}/whoami.nomad")
}

50
nomad/setup-cluster.yml Normal file
View File

@ -0,0 +1,50 @@
---
- name: Build Consul cluster
hosts: consul_instances
any_errors_fatal: true
become: true
roles:
- name: ansible-consul
consul_version: "1.11.3"
consul_install_upgrade: true
# consul_tls_enable: true
consul_connect_enabled: true
consul_ports_grpc: 8502
consul_client_address: "0.0.0.0"
consul_auto_encrypt:
enabled: true
dns_san: ["services.thefij"]
ip_san: ["192.168.2.41", "127.0.0.1"]
# tasks:
# # Limit to consul host
# - name: Add a value to Consul
# consul_kv:
# key: ansible_test
# value: Hello from Ansible!
# execute_once: true
- name: Build Consul cluster
hosts: nomad_instances
any_errors_fatal: true
become: true
roles:
- name: ansible-nomad
nomad_version: "1.2.6"
nomad_install_upgrade: true
nomad_allow_purge_config: true
nomad_encrypt_enable: true
nomad_cni_enable: true
nomad_docker_enable: true
# nomad_use_consul: true
# TODO: this should probably be based on host
nomad_host_volumes:
- name: mysql-data
path: /srv/volumes/mysql-data
owner: "nomad"
group: "bin"
mode: "0755"
read_only: false

99
nomad/traefik.nomad Normal file
View File

@ -0,0 +1,99 @@
variable "consul_address" {
type = string
description = "Full address of Consul instance to get catalog from"
default = "http://127.0.0.1:5400"
}
variable "base_hostname" {
type = string
description = "Base hostname to serve content from"
default = "dev.homelab"
}
job "traefik" {
region = "global"
datacenters = ["dc1"]
type = "service"
group "traefik" {
count = 1
network {
port "web" {
static = 80
}
port "websecure" {
static = 443
}
}
service {
name = "traefik"
port = "web"
check {
type = "http"
path = "/ping"
port = "web"
interval = "10s"
timeout = "2s"
}
connect {
native = true
}
tags = [
"traefik.enable=true",
"traefik.http.routers.traefik_dashboard.entrypoints=web,websecure",
"traefik.http.routers.traefik_dashboard.rule=Host(`traefik.${var.base_hostname}`)",
"traefik.http.routers.traefik_dashboard.service=api@internal",
"traefik.http.routers.traefik_dashboard.tls=true",
]
}
task "traefik" {
driver = "docker"
config {
image = "traefik:2.6"
args = [
"--log.level=DEBUG",
"--entryPoints.web.address=:80",
"--entryPoints.websecure.address=:443",
# "--entryPoints.websecure.tls=true",
# "--entrypoints.web.http.redirections.entryPoint.to=websecure",
# "--entryPoints.admin.address=:8080",
"--accesslog=true",
"--api=true",
"--api.dashboard=true",
# "--metrics=true",
# "--metrics.prometheus=true",
# "--metrics.prometheus.entryPoint=admin",
# "--metrics.prometheus.manualrouting=true",
"--ping=true",
"--ping.entryPoint=web",
"--providers.consulcatalog=true",
"--providers.consulcatalog.connectaware=true",
"--providers.consulcatalog.connectbydefault=true",
"--providers.consulcatalog.exposedbydefault=false",
"--providers.consulcatalog.endpoint.address=${var.consul_address}",
"--providers.consulcatalog.servicename=traefik",
"--providers.consulcatalog.prefix=traefik",
"--providers.consulcatalog.defaultrule=Host(`{{normalize .Name}}.${var.base_hostname}`)",
]
ports = ["web", "websecure"]
network_mode = "host"
volumes = []
}
resources {
cpu = 500
memory = 100
}
}
}
}

62
nomad/whoami.nomad Normal file
View File

@ -0,0 +1,62 @@
variable "base_hostname" {
type = string
description = "Base hostname to serve content from"
default = "dev.homelab"
}
job "whoami" {
region = "global"
datacenters = ["dc1"]
type = "service"
group "whoami" {
count = 2
network {
mode = "bridge"
port "web" {
# to = 80
}
}
service {
name = "whoami"
port = "web"
connect {
sidecar_service {}
}
check {
type = "http"
path = "/health"
port = "web"
interval = "10s"
timeout = "10s"
}
tags = [
"traefik.enable=true",
"traefik.http.routers.whoami.entrypoints=web,websecure",
"traefik.http.routers.whoami.rule=Host(`whoami.${var.base_hostname}`)",
"traefik.http.routers.whoami.tls=true",
]
}
task "whoami" {
driver = "docker"
config {
image = "containous/whoami:latest"
ports = ["web"]
args = ["--port", "${NOMAD_PORT_web}"]
}
resources {
cpu = 50
memory = 50
}
}
}
}