From 37e80980bcd416f3aa5ff8679d0ddec4cd8c32e5 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Wed, 2 Nov 2022 14:20:09 -0700 Subject: [PATCH] Refactor ansible to clean root dir --- .gitignore | 4 +- .secrets-baseline | 57 +++---- Makefile | 26 ++-- ansible.cfg | 7 + {roles => ansible_galaxy}/requirements.yml | 3 + .../ansible_hosts.yml | 0 .../bootstrap-values.yml | 4 +- .../clear-data.yml | 0 .../recover-consul.yaml | 0 .../setup-cluster.yml | 18 +-- .../unseal-vault.yml | 0 .../vars/consul_values.example.yml | 0 .../vars/vault_hashi_vault_values.example.yml | 0 collections/requirements.yml | 4 - core/main.tf | 6 +- hashi-up.sh | 144 ------------------ levant/.terraform.lock.hcl | 42 ----- levant/README.md | 7 - levant/levant.py | 63 -------- levant/main.tf | 37 ----- levant/test.nomad | 2 - 21 files changed, 67 insertions(+), 357 deletions(-) create mode 100644 ansible.cfg rename {roles => ansible_galaxy}/requirements.yml (87%) rename ansible_hosts.yml => ansible_playbooks/ansible_hosts.yml (100%) rename bootstrap-values.yml => ansible_playbooks/bootstrap-values.yml (97%) rename clear-data.yml => ansible_playbooks/clear-data.yml (100%) rename recover-consul.yaml => ansible_playbooks/recover-consul.yaml (100%) rename setup-cluster.yml => ansible_playbooks/setup-cluster.yml (98%) rename unseal-vault.yml => ansible_playbooks/unseal-vault.yml (100%) rename consul_values.example.yml => ansible_playbooks/vars/consul_values.example.yml (100%) rename vault_hashi_vault_values.example.yml => ansible_playbooks/vars/vault_hashi_vault_values.example.yml (100%) delete mode 100644 collections/requirements.yml delete mode 100644 hashi-up.sh delete mode 100644 levant/.terraform.lock.hcl delete mode 100644 levant/README.md delete mode 100755 levant/levant.py delete mode 100644 levant/main.tf delete mode 100644 levant/test.nomad diff --git a/.gitignore b/.gitignore index dee4eb9..695571b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,8 +37,8 @@ terraform.rc # ---> Ansible *.retry -collections/ansible_collections/ -roles/ +ansible_galaxy/ansible_collections/ +ansible_galaxy/roles/ # Repo specific venv/ diff --git a/.secrets-baseline b/.secrets-baseline index 1f28b5e..36b2c85 100644 --- a/.secrets-baseline +++ b/.secrets-baseline @@ -20,6 +20,9 @@ { "name": "CloudantDetector" }, + { + "name": "DiscordBotTokenDetector" + }, { "name": "GitHubTokenDetector" }, @@ -115,6 +118,32 @@ } ], "results": { + "ansible_playbooks/vars/vault_hashi_vault_values.example.yml": [ + { + "type": "Secret Keyword", + "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml", + "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3", + "is_verified": false, + "line_number": 8, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml", + "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9", + "is_verified": false, + "line_number": 10, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml", + "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", + "is_verified": false, + "line_number": 22, + "is_secret": false + } + ], "core/metrics/grafana/grafana.ini": [ { "type": "Basic Auth Credentials", @@ -182,33 +211,7 @@ "line_number": 159, "is_secret": false } - ], - "vault_hashi_vault_values.example.yml": [ - { - "type": "Secret Keyword", - "filename": "vault_hashi_vault_values.example.yml", - "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3", - "is_verified": false, - "line_number": 8, - "is_secret": false - }, - { - "type": "Secret Keyword", - "filename": "vault_hashi_vault_values.example.yml", - "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9", - "is_verified": false, - "line_number": 10, - "is_secret": false - }, - { - "type": "Secret Keyword", - "filename": "vault_hashi_vault_values.example.yml", - "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", - "is_verified": false, - "line_number": 22, - "is_secret": false - } ] }, - "generated_at": "2022-11-01T23:43:19Z" + "generated_at": "2022-11-02T21:19:52Z" } diff --git a/Makefile b/Makefile index 078b050..db47bf1 100644 --- a/Makefile +++ b/Makefile @@ -35,35 +35,33 @@ secrets-audit: $(VENV) .secrets-baseline secrets-update: $(VENV) .secrets-baseline $(VENV)/bin/detect-secrets scan --baseline .secrets-baseline -.PHONY: galaxy -galaxy: $(VENV) - $(VENV)/bin/ansible-galaxy install -p roles -r roles/requirements.yml - $(VENV)/bin/ansible-galaxy collection install -r collections/requirements.yml +.PHONY: ansible_galaxy +ansible_galaxy: ansible_galaxy/ansible_collections ansible_galaxy/roles + +ansible_galaxy/ansible_collections: $(VENV) ./ansible_galaxy/requirements.yml + $(VENV)/bin/ansible-galaxy collection install -p ./ansible_galaxy -r ./ansible_collections/requirements.yml + +ansible_galaxy/roles: $(VENV) ./ansible_galaxy/requirements.yml + $(VENV)/bin/ansible-galaxy install -p ./ansible_galaxy/roles -r ./ansible_roles/requirements.yml .PHONY: ansible-cluster -ansible-cluster: $(VENV) galaxy +ansible-cluster: $(VENV) ansible_galaxy env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ $(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \ - -i ansible_hosts.yml \ - -M ./roles \ - ./setup-cluster.yml + ./ansible_playbooks/setup-cluster.yml .PHONY: bootstrap-values bootstrap-values: $(VENV) galaxy env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \ -e "@vault-keys.json" \ - -i ansible_hosts.yml \ - -M ./roles \ - ./bootstrap-values.yml + ./ansible_playbooks/bootstrap-values.yml .PHONY: unseal-vault unseal-vault: $(VENV) galaxy env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ -e "@vault-keys.json" \ - -i ansible_hosts.yml \ - -M ./roles \ - ./unseal-vault.yml + ./ansible_playbooks/unseal-vault.yml .PHONY: init init: diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..9212a68 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +inventory=ansible_playbooks/ansible_hosts.yml +collections_paths=ansible_galaxy +roles_path=ansible_galaxy/roles + +[inventory] +enable_plugins=yaml diff --git a/roles/requirements.yml b/ansible_galaxy/requirements.yml similarity index 87% rename from roles/requirements.yml rename to ansible_galaxy/requirements.yml index da9cbba..d5939de 100644 --- a/roles/requirements.yml +++ b/ansible_galaxy/requirements.yml @@ -1,4 +1,7 @@ --- +collections: + - name: community.hashi_vault + version: 3.0.0 roles: - src: https://github.com/IamTheFij/ansible-consul.git name: ansible-consul diff --git a/ansible_hosts.yml b/ansible_playbooks/ansible_hosts.yml similarity index 100% rename from ansible_hosts.yml rename to ansible_playbooks/ansible_hosts.yml diff --git a/bootstrap-values.yml b/ansible_playbooks/bootstrap-values.yml similarity index 97% rename from bootstrap-values.yml rename to ansible_playbooks/bootstrap-values.yml index 1a3237c..d271901 100644 --- a/bootstrap-values.yml +++ b/ansible_playbooks/bootstrap-values.yml @@ -4,7 +4,7 @@ gather_facts: false vars_files: - - consul_values.yml + - vars/consul_values.yml tasks: - name: Add values @@ -28,7 +28,7 @@ gather_facts: false vars_files: - - ./vault_hashi_vault_values.yml + - vars/vault_hashi_vault_values.yml tasks: - name: Bootstrap Vault secrets diff --git a/clear-data.yml b/ansible_playbooks/clear-data.yml similarity index 100% rename from clear-data.yml rename to ansible_playbooks/clear-data.yml diff --git a/recover-consul.yaml b/ansible_playbooks/recover-consul.yaml similarity index 100% rename from recover-consul.yaml rename to ansible_playbooks/recover-consul.yaml diff --git a/setup-cluster.yml b/ansible_playbooks/setup-cluster.yml similarity index 98% rename from setup-cluster.yml rename to ansible_playbooks/setup-cluster.yml index a9b5d66..caf4181 100644 --- a/setup-cluster.yml +++ b/ansible_playbooks/setup-cluster.yml @@ -3,9 +3,6 @@ hosts: consul_instances any_errors_fatal: true - vars_files: - - consul_values.yml - roles: - role: ansible-consul vars: @@ -70,15 +67,14 @@ - name: Setup Vault cluster hosts: vault_instances - vars_files: - - ./vault_hashi_vault_values.yml - roles: - name: ansible-vault vars: vault_version: 1.12.0-1 vault_install_hashi_repo: true vault_harden_file_perms: true + # Maybe this should be restricted + vault_group: bin vault_bin_path: /usr/bin vault_address: 0.0.0.0 @@ -114,7 +110,7 @@ - name: Save initialize result copy: content: "{{ vault_init.stdout }}" - dest: "./vault-keys.json" + dest: "../vault-keys.json" when: vault_init is succeeded delegate_to: localhost run_once: true @@ -409,7 +405,7 @@ - name: Save bootstrap result copy: content: "{{ bootstrap_result.stdout }}" - dest: "./nomad_bootstrap.json" + dest: "../nomad_bootstrap.json" when: bootstrap_result is succeeded delegate_to: localhost run_once: true @@ -420,7 +416,7 @@ - jq - -r - .SecretID - - nomad_bootstrap.json + - ../nomad_bootstrap.json delegate_to: localhost run_once: true no_log: true @@ -455,7 +451,7 @@ - name: Copy policy copy: - src: ./acls/nomad-anon-policy.hcl + src: ../acls/nomad-anon-policy.hcl dest: /tmp/anonymous.policy.hcl delegate_to: "{{ play_hosts[0] }}" register: anon_policy @@ -479,7 +475,7 @@ - name: Set up Nomad backend and roles in Vault community.general.terraform: - project_path: ./acls + project_path: ../acls force_init: true variables: consul_address: "{{ play_hosts[0] }}:8500" diff --git a/unseal-vault.yml b/ansible_playbooks/unseal-vault.yml similarity index 100% rename from unseal-vault.yml rename to ansible_playbooks/unseal-vault.yml diff --git a/consul_values.example.yml b/ansible_playbooks/vars/consul_values.example.yml similarity index 100% rename from consul_values.example.yml rename to ansible_playbooks/vars/consul_values.example.yml diff --git a/vault_hashi_vault_values.example.yml b/ansible_playbooks/vars/vault_hashi_vault_values.example.yml similarity index 100% rename from vault_hashi_vault_values.example.yml rename to ansible_playbooks/vars/vault_hashi_vault_values.example.yml diff --git a/collections/requirements.yml b/collections/requirements.yml deleted file mode 100644 index c9f27c4..0000000 --- a/collections/requirements.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -collections: - - name: community.hashi_vault - version: 3.0.0 diff --git a/core/main.tf b/core/main.tf index a78a4ce..773a64e 100644 --- a/core/main.tf +++ b/core/main.tf @@ -14,7 +14,8 @@ module "traefik" { } module "nomad_login" { - source = "../levant" + source = "IamTheFij/levant/nomad" + version = "0.1.0" template_path = "service.nomad" variables = { @@ -36,7 +37,8 @@ module "metrics" { } module "loki" { - source = "../levant" + source = "IamTheFij/levant/nomad" + version = "0.1.0" template_path = "service.nomad" variables = { diff --git a/hashi-up.sh b/hashi-up.sh deleted file mode 100644 index 82cf5db..0000000 --- a/hashi-up.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -export VERIFY_CHECKSUM=0 -export ALIAS_NAME= -export OWNER=jsiebens -export REPO=hashi-up -export SUCCESS_CMD="$REPO version" -export BINLOCATION="~/bin" - -############################### -# Content common across repos # -############################### - -version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r') -if [ ! $version ]; then - echo "Failed while attempting to install $REPO. Please manually install:" - echo "" - echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases" - echo "2. Download the latest release for your platform. Call it '$REPO'." - echo "3. chmod +x ./$REPO" - echo "4. mv ./$REPO $BINLOCATION" - if [ -n "$ALIAS_NAME" ]; then - echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME" - fi - exit 1 -fi - -getPackage() { - uname=$(uname) - userid=$(id -u) - - suffix="" - case $uname in - "Darwin") - suffix="-darwin" - ;; - "MINGW"*) - suffix=".exe" - BINLOCATION="$HOME/bin" - mkdir -p $BINLOCATION - - ;; - "Linux") - arch=$(uname -m) - case $arch in - "aarch64") - suffix="-arm64" - ;; - esac - case $arch in - "armv6l" | "armv7l") - suffix="-armhf" - ;; - esac - ;; - esac - - targetFile="/tmp/$REPO$suffix" - - if [ "$userid" != "0" ]; then - targetFile="$(pwd)/$REPO$suffix" - fi - - if [ -e "$targetFile" ]; then - rm "$targetFile" - fi - - url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix - echo "Downloading package $url as $targetFile" - - curl -sSL $url --output "$targetFile" - - if [ "$?" = "0" ]; then - - if [ "$VERIFY_CHECKSUM" = "1" ]; then - checkHash - fi - - chmod +x "$targetFile" - - echo "Download complete." - - if [ ! -w "$BINLOCATION" ]; then - - echo - echo "============================================================" - echo " The script was run as a user who is unable to write" - echo " to $BINLOCATION. To complete the installation the" - echo " following commands may need to be run manually." - echo "============================================================" - echo - echo " sudo cp $REPO$suffix $BINLOCATION/$REPO" - - if [ -n "$ALIAS_NAME" ]; then - echo " sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME" - fi - - echo - - else - - echo - echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION" - - if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then - - echo - echo "================================================================" - echo " $BINLOCATION/$REPO already exists and is not writeable" - echo " by the current user. Please adjust the binary ownership" - echo " or run sh/bash with sudo." - echo "================================================================" - echo - exit 1 - - fi - - mv "$targetFile" $BINLOCATION/$REPO - - if [ "$?" = "0" ]; then - echo "New version of $REPO installed to $BINLOCATION" - fi - - if [ -e "$targetFile" ]; then - rm "$targetFile" - fi - - if [ $(which $ALIAS_NAME) ]; then - echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias" - else - if [ -n "$ALIAS_NAME" ]; then - if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then - ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME - echo "Creating alias '$ALIAS_NAME' for '$REPO'." - fi - fi - fi - - ${SUCCESS_CMD} - fi - fi -} - -getPackage diff --git a/levant/.terraform.lock.hcl b/levant/.terraform.lock.hcl deleted file mode 100644 index 49c77b9..0000000 --- a/levant/.terraform.lock.hcl +++ /dev/null @@ -1,42 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/external" { - version = "2.2.2" - hashes = [ - "h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=", - "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=", - "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca", - "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28", - "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b", - "zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327", - "zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955", - "zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb", - "zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0", - "zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a", - "zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372", - "zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809", - ] -} - -provider "registry.terraform.io/hashicorp/nomad" { - version = "1.4.17" - hashes = [ - "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=", - "h1:oWV3VXZhqPZ8Ia07nlIZLeXDBqVULMg9lP3dVMczDCo=", - "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a", - "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f", - "zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1", - "zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792", - "zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916", - "zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85", - "zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328", - "zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65", - "zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d", - "zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6", - ] -} diff --git a/levant/README.md b/levant/README.md deleted file mode 100644 index e5c863e..0000000 --- a/levant/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Terraform Levant - -This module renders a levant template and then creates a Nomad job based on that template. - -It only covers a subset of levant capabilities because much else can be done with Terraform already. - -required: diff --git a/levant/levant.py b/levant/levant.py deleted file mode 100755 index 023d6be..0000000 --- a/levant/levant.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env python3 -import json -import sys -from subprocess import check_output -from typing import Optional -from typing import overload -from typing import TypeVar - - -T = TypeVar("T") - - -@overload -def get_json(d: dict[str, str], key: str, default: None = None) -> None: - ... - - -@overload -def get_json(d: dict[str, str], key: str, default: T = None) -> T: - ... - - -def get_json(d: dict[str, str], key: str, default: Optional[T] = None) -> Optional[T]: - if key not in d: - return default - - return json.loads(d[key]) - - -query = json.load(sys.stdin) - -# Required -template_path = query["template_path"] - -# Optional -consul_address = query.get("consul_address") -if consul_address is not None: - consul_address = f"-consul-address={consul_address}" - -# Need to parse JSON back -variables = [ - f'--var={key}={value}' for key, value in get_json(query, "variables", {}).items() -] -variable_files = [ - f'--var-file={value}' for value in get_json(query, "var_files", []) -] - -args: list[str] = list( - filter( - None, - ["levant", "render", consul_address] - + variables - + variable_files - + [template_path], - ) -) - -# print(" ".join(args), file=sys.stderr) -# exit(1) - -template = check_output(args, stderr=sys.stderr) - -print(json.dumps({"template": template.decode()})) diff --git a/levant/main.tf b/levant/main.tf deleted file mode 100644 index b7a9a7b..0000000 --- a/levant/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -variable "template_path" { - type = string - nullable = false -} -variable "consul_address" { - type = string - default = null - nullable = true - description = "Consul host and port for making KeyValue lookups" -} - -variable "variables" { - type = map(string) - description = "Variables to be passed into nomad-pack with values in JSON form" - default = {} -} - -variable "var_files" { - type = list(string) - description = "HCL files containing variables to be used by nomad-pack" - default = [] -} - -data "external" "levant" { - program = ["${path.module}/levant.py"] - - query = { - template_path = var.template_path - consul_address = var.consul_address - variables = jsonencode(var.variables) - var_files = jsonencode(var.var_files) - } -} - -resource "nomad_job" "levant" { - jobspec = data.external.levant.result.template -} diff --git a/levant/test.nomad b/levant/test.nomad deleted file mode 100644 index beffd62..0000000 --- a/levant/test.nomad +++ /dev/null @@ -1,2 +0,0 @@ -job { -}