Refactor ansible to clean root dir

.gitignore

@@ -37,8 +37,8 @@ terraform.rc
 
 # ---> Ansible
 *.retry
-collections/ansible_collections/
+ansible_galaxy/ansible_collections/
-roles/
+ansible_galaxy/roles/
 
 # Repo specific
 venv/

.secrets-baseline

@@ -20,6 +20,9 @@
     {
       "name": "CloudantDetector"
     },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
     {
       "name": "GitHubTokenDetector"
     },
@@ -115,6 +118,32 @@
     }
   ],
   "results": {
+    "ansible_playbooks/vars/vault_hashi_vault_values.example.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
+        "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
+        "is_verified": false,
+        "line_number": 8,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
+        "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
+        "is_verified": false,
+        "line_number": 10,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
+        "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
+        "is_verified": false,
+        "line_number": 22,
+        "is_secret": false
+      }
+    ],
     "core/metrics/grafana/grafana.ini": [
       {
         "type": "Basic Auth Credentials",
@@ -182,33 +211,7 @@
         "line_number": 159,
         "is_secret": false
       }
-    ],
-    "vault_hashi_vault_values.example.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": "vault_hashi_vault_values.example.yml",
-        "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
-        "is_verified": false,
-        "line_number": 8,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "vault_hashi_vault_values.example.yml",
-        "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
-        "is_verified": false,
-        "line_number": 10,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "vault_hashi_vault_values.example.yml",
-        "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
-        "is_verified": false,
-        "line_number": 22,
-        "is_secret": false
-      }
     ]
   },
-  "generated_at": "2022-11-01T23:43:19Z"
+  "generated_at": "2022-11-02T21:19:52Z"
 }

Makefile

@@ -35,35 +35,33 @@ secrets-audit: $(VENV) .secrets-baseline
 secrets-update: $(VENV) .secrets-baseline
 	$(VENV)/bin/detect-secrets scan --baseline .secrets-baseline
 
-.PHONY: galaxy
+.PHONY: ansible_galaxy
-galaxy: $(VENV)
+ansible_galaxy: ansible_galaxy/ansible_collections ansible_galaxy/roles
-	$(VENV)/bin/ansible-galaxy install -p roles -r roles/requirements.yml
+
-	$(VENV)/bin/ansible-galaxy collection install -r collections/requirements.yml
+ansible_galaxy/ansible_collections: $(VENV) ./ansible_galaxy/requirements.yml
+	$(VENV)/bin/ansible-galaxy collection install -p ./ansible_galaxy -r ./ansible_collections/requirements.yml
+
+ansible_galaxy/roles: $(VENV) ./ansible_galaxy/requirements.yml
+	$(VENV)/bin/ansible-galaxy install -p ./ansible_galaxy/roles -r ./ansible_roles/requirements.yml
 
 
 .PHONY: ansible-cluster
-ansible-cluster: $(VENV) galaxy
+ansible-cluster: $(VENV) ansible_galaxy
 	env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
 		$(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \
-		-i ansible_hosts.yml \
+		./ansible_playbooks/setup-cluster.yml
-		-M ./roles \
-		./setup-cluster.yml
 
 .PHONY: bootstrap-values
 bootstrap-values: $(VENV) galaxy
 	env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \
 		-e "@vault-keys.json" \
-		-i ansible_hosts.yml \
+		./ansible_playbooks/bootstrap-values.yml
-		-M ./roles \
-		./bootstrap-values.yml
 
 .PHONY: unseal-vault
 unseal-vault: $(VENV) galaxy
 	env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
 		-e "@vault-keys.json" \
-		-i ansible_hosts.yml \
+		./ansible_playbooks/unseal-vault.yml
-		-M ./roles \
-		./unseal-vault.yml
 
 .PHONY: init
 init:

ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+inventory=ansible_playbooks/ansible_hosts.yml
+collections_paths=ansible_galaxy
+roles_path=ansible_galaxy/roles
+
+[inventory]
+enable_plugins=yaml

ansible_galaxy/requirements.yml

@@ -1,4 +1,7 @@
 ---
+collections:
+  - name: community.hashi_vault
+    version: 3.0.0
 roles:
   - src: https://github.com/IamTheFij/ansible-consul.git
     name: ansible-consul

ansible_playbooks/bootstrap-values.yml

@@ -4,7 +4,7 @@
   gather_facts: false
 
   vars_files:
-    - consul_values.yml
+    - vars/consul_values.yml
 
   tasks:
     - name: Add values
@@ -28,7 +28,7 @@
   gather_facts: false
 
   vars_files:
-    - ./vault_hashi_vault_values.yml
+    - vars/vault_hashi_vault_values.yml
 
   tasks:
     - name: Bootstrap Vault secrets

ansible_playbooks/setup-cluster.yml

@@ -3,9 +3,6 @@
   hosts: consul_instances
   any_errors_fatal: true
 
-  vars_files:
-    - consul_values.yml
-
   roles:
     - role: ansible-consul
       vars:
@@ -70,15 +67,14 @@
 - name: Setup Vault cluster
   hosts: vault_instances
 
-  vars_files:
-    - ./vault_hashi_vault_values.yml
-
   roles:
     - name: ansible-vault
       vars:
         vault_version: 1.12.0-1
         vault_install_hashi_repo: true
         vault_harden_file_perms: true
+        # Maybe this should be restricted
+        vault_group: bin
         vault_bin_path: /usr/bin
         vault_address: 0.0.0.0
 
@@ -114,7 +110,7 @@
         - name: Save initialize result
           copy:
             content: "{{ vault_init.stdout }}"
-            dest: "./vault-keys.json"
+            dest: "../vault-keys.json"
           when: vault_init is succeeded
           delegate_to: localhost
           run_once: true
@@ -409,7 +405,7 @@
     - name: Save bootstrap result
       copy:
         content: "{{ bootstrap_result.stdout }}"
-        dest: "./nomad_bootstrap.json"
+        dest: "../nomad_bootstrap.json"
       when: bootstrap_result is succeeded
       delegate_to: localhost
       run_once: true
@@ -420,7 +416,7 @@
           - jq
           - -r
           - .SecretID
-          - nomad_bootstrap.json
+          - ../nomad_bootstrap.json
       delegate_to: localhost
       run_once: true
       no_log: true
@@ -455,7 +451,7 @@
 
     - name: Copy policy
       copy:
-        src: ./acls/nomad-anon-policy.hcl
+        src: ../acls/nomad-anon-policy.hcl
         dest: /tmp/anonymous.policy.hcl
       delegate_to: "{{ play_hosts[0] }}"
       register: anon_policy
@@ -479,7 +475,7 @@
 
     - name: Set up Nomad backend and roles in Vault
       community.general.terraform:
-        project_path: ./acls
+        project_path: ../acls
         force_init: true
         variables:
           consul_address: "{{ play_hosts[0] }}:8500"

collections/requirements.yml

@@ -1,4 +0,0 @@
----
-collections:
-  - name: community.hashi_vault
-    version: 3.0.0

core/main.tf

@@ -14,7 +14,8 @@ module "traefik" {
 }
 
 module "nomad_login" {
-  source = "../levant"
+  source = "IamTheFij/levant/nomad"
+  version = "0.1.0"
 
   template_path = "service.nomad"
   variables = {
@@ -36,7 +37,8 @@ module "metrics" {
 }
 
 module "loki" {
-  source = "../levant"
+  source = "IamTheFij/levant/nomad"
+  version = "0.1.0"
 
   template_path = "service.nomad"
   variables = {

hashi-up.sh

@@ -1,144 +0,0 @@
-#!/usr/bin/env bash
-
-export VERIFY_CHECKSUM=0
-export ALIAS_NAME=
-export OWNER=jsiebens
-export REPO=hashi-up
-export SUCCESS_CMD="$REPO version"
-export BINLOCATION="~/bin"
-
-###############################
-# Content common across repos #
-###############################
-
-version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r')
-if [ ! $version ]; then
-    echo "Failed while attempting to install $REPO. Please manually install:"
-    echo ""
-    echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases"
-    echo "2. Download the latest release for your platform. Call it '$REPO'."
-    echo "3. chmod +x ./$REPO"
-    echo "4. mv ./$REPO $BINLOCATION"
-    if [ -n "$ALIAS_NAME" ]; then
-        echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME"
-    fi
-    exit 1
-fi
-
-getPackage() {
-    uname=$(uname)
-    userid=$(id -u)
-
-    suffix=""
-    case $uname in
-    "Darwin")
-    suffix="-darwin"
-    ;;
-    "MINGW"*)
-    suffix=".exe"
-    BINLOCATION="$HOME/bin"
-    mkdir -p $BINLOCATION
-
-    ;;
-    "Linux")
-        arch=$(uname -m)
-        case $arch in
-        "aarch64")
-        suffix="-arm64"
-        ;;
-        esac
-        case $arch in
-        "armv6l" | "armv7l")
-        suffix="-armhf"
-        ;;
-        esac
-    ;;
-    esac
-
-    targetFile="/tmp/$REPO$suffix"
-
-    if [ "$userid" != "0" ]; then
-        targetFile="$(pwd)/$REPO$suffix"
-    fi
-
-    if [ -e "$targetFile" ]; then
-        rm "$targetFile"
-    fi
-
-    url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix
-    echo "Downloading package $url as $targetFile"
-
-    curl -sSL $url --output "$targetFile"
-
-    if [ "$?" = "0" ]; then
-
-        if [ "$VERIFY_CHECKSUM" = "1" ]; then
-            checkHash
-        fi
-
-    chmod +x "$targetFile"
-
-    echo "Download complete."
-
-    if [ ! -w "$BINLOCATION" ]; then
-
-            echo
-            echo "============================================================"
-            echo "  The script was run as a user who is unable to write"
-            echo "  to $BINLOCATION. To complete the installation the"
-            echo "  following commands may need to be run manually."
-            echo "============================================================"
-            echo
-            echo "  sudo cp $REPO$suffix $BINLOCATION/$REPO"
-
-            if [ -n "$ALIAS_NAME" ]; then
-                echo "  sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME"
-            fi
-
-            echo
-
-        else
-
-            echo
-            echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION"
-
-            if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then
-
-            echo
-            echo "================================================================"
-            echo "  $BINLOCATION/$REPO already exists and is not writeable"
-            echo "  by the current user.  Please adjust the binary ownership"
-            echo "  or run sh/bash with sudo."
-            echo "================================================================"
-            echo
-            exit 1
-
-            fi
-
-            mv "$targetFile" $BINLOCATION/$REPO
-
-            if [ "$?" = "0" ]; then
-                echo "New version of $REPO installed to $BINLOCATION"
-            fi
-
-            if [ -e "$targetFile" ]; then
-                rm "$targetFile"
-            fi
-
-            if [ $(which $ALIAS_NAME) ]; then
-                echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias"
-            else
-                if [ -n "$ALIAS_NAME" ]; then
-                    if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then
-                        ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME
-                        echo "Creating alias '$ALIAS_NAME' for '$REPO'."
-                    fi
-                fi
-            fi
-
-            ${SUCCESS_CMD}
-        fi
-    fi
-}
-
-getPackage

levant/.terraform.lock.hcl

@@ -1,42 +0,0 @@
-# This file is maintained automatically by "terraform init".
-# Manual edits may be lost in future updates.
-
-provider "registry.terraform.io/hashicorp/external" {
-  version = "2.2.2"
-  hashes = [
-    "h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=",
-    "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
-    "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
-    "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
-    "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
-    "zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39",
-    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327",
-    "zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955",
-    "zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb",
-    "zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0",
-    "zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a",
-    "zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372",
-    "zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/nomad" {
-  version = "1.4.17"
-  hashes = [
-    "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
-    "h1:oWV3VXZhqPZ8Ia07nlIZLeXDBqVULMg9lP3dVMczDCo=",
-    "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
-    "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
-    "zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
-    "zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
-    "zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
-    "zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
-    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
-    "zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
-    "zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
-    "zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
-    "zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
-  ]
-}

levant/README.md

@@ -1,7 +0,0 @@
-# Terraform Levant
-
-This module renders a levant template and then creates a Nomad job based on that template.
-
-It only covers a subset of levant capabilities because much else can be done with Terraform already.
-
-required:

levant/levant.py

@@ -1,63 +0,0 @@
-#! /usr/bin/env python3
-import json
-import sys
-from subprocess import check_output
-from typing import Optional
-from typing import overload
-from typing import TypeVar
-
-
-T = TypeVar("T")
-
-
-@overload
-def get_json(d: dict[str, str], key: str, default: None = None) -> None:
-    ...
-
-
-@overload
-def get_json(d: dict[str, str], key: str, default: T = None) -> T:
-    ...
-
-
-def get_json(d: dict[str, str], key: str, default: Optional[T] = None) -> Optional[T]:
-    if key not in d:
-        return default
-
-    return json.loads(d[key])
-
-
-query = json.load(sys.stdin)
-
-# Required
-template_path = query["template_path"]
-
-# Optional
-consul_address = query.get("consul_address")
-if consul_address is not None:
-    consul_address = f"-consul-address={consul_address}"
-
-# Need to parse JSON back
-variables = [
-    f'--var={key}={value}' for key, value in get_json(query, "variables", {}).items()
-]
-variable_files = [
-    f'--var-file={value}' for value in get_json(query, "var_files", [])
-]
-
-args: list[str] = list(
-    filter(
-        None,
-        ["levant", "render", consul_address]
-        + variables
-        + variable_files
-        + [template_path],
-    )
-)
-
-# print(" ".join(args), file=sys.stderr)
-# exit(1)
-
-template = check_output(args, stderr=sys.stderr)
-
-print(json.dumps({"template": template.decode()}))

levant/main.tf

@@ -1,37 +0,0 @@
-variable "template_path" {
-  type     = string
-  nullable = false
-}
-variable "consul_address" {
-  type        = string
-  default     = null
-  nullable    = true
-  description = "Consul host and port for making KeyValue lookups"
-}
-
-variable "variables" {
-  type        = map(string)
-  description = "Variables to be passed into nomad-pack with values in JSON form"
-  default     = {}
-}
-
-variable "var_files" {
-  type        = list(string)
-  description = "HCL files containing variables to be used by nomad-pack"
-  default     = []
-}
-
-data "external" "levant" {
-  program = ["${path.module}/levant.py"]
-
-  query = {
-    template_path  = var.template_path
-    consul_address = var.consul_address
-    variables      = jsonencode(var.variables)
-    var_files      = jsonencode(var.var_files)
-  }
-}
-
-resource "nomad_job" "levant" {
-  jobspec = data.external.levant.result.template
-}

levant/test.nomad

@@ -1,2 +0,0 @@
-job {
-}