Refactor ansible to clean root dir

This commit is contained in:
IamTheFij 2022-11-02 14:20:09 -07:00
parent 0996cfbf67
commit 37e80980bc
21 changed files with 67 additions and 357 deletions

4
.gitignore vendored
View File

@ -37,8 +37,8 @@ terraform.rc
# ---> Ansible # ---> Ansible
*.retry *.retry
collections/ansible_collections/ ansible_galaxy/ansible_collections/
roles/ ansible_galaxy/roles/
# Repo specific # Repo specific
venv/ venv/

View File

@ -20,6 +20,9 @@
{ {
"name": "CloudantDetector" "name": "CloudantDetector"
}, },
{
"name": "DiscordBotTokenDetector"
},
{ {
"name": "GitHubTokenDetector" "name": "GitHubTokenDetector"
}, },
@ -115,6 +118,32 @@
} }
], ],
"results": { "results": {
"ansible_playbooks/vars/vault_hashi_vault_values.example.yml": [
{
"type": "Secret Keyword",
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
"hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
"is_verified": false,
"line_number": 8,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
"hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
"is_verified": false,
"line_number": 10,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 22,
"is_secret": false
}
],
"core/metrics/grafana/grafana.ini": [ "core/metrics/grafana/grafana.ini": [
{ {
"type": "Basic Auth Credentials", "type": "Basic Auth Credentials",
@ -182,33 +211,7 @@
"line_number": 159, "line_number": 159,
"is_secret": false "is_secret": false
} }
],
"vault_hashi_vault_values.example.yml": [
{
"type": "Secret Keyword",
"filename": "vault_hashi_vault_values.example.yml",
"hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
"is_verified": false,
"line_number": 8,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "vault_hashi_vault_values.example.yml",
"hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
"is_verified": false,
"line_number": 10,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "vault_hashi_vault_values.example.yml",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 22,
"is_secret": false
}
] ]
}, },
"generated_at": "2022-11-01T23:43:19Z" "generated_at": "2022-11-02T21:19:52Z"
} }

View File

@ -35,35 +35,33 @@ secrets-audit: $(VENV) .secrets-baseline
secrets-update: $(VENV) .secrets-baseline secrets-update: $(VENV) .secrets-baseline
$(VENV)/bin/detect-secrets scan --baseline .secrets-baseline $(VENV)/bin/detect-secrets scan --baseline .secrets-baseline
.PHONY: galaxy .PHONY: ansible_galaxy
galaxy: $(VENV) ansible_galaxy: ansible_galaxy/ansible_collections ansible_galaxy/roles
$(VENV)/bin/ansible-galaxy install -p roles -r roles/requirements.yml
$(VENV)/bin/ansible-galaxy collection install -r collections/requirements.yml ansible_galaxy/ansible_collections: $(VENV) ./ansible_galaxy/requirements.yml
$(VENV)/bin/ansible-galaxy collection install -p ./ansible_galaxy -r ./ansible_collections/requirements.yml
ansible_galaxy/roles: $(VENV) ./ansible_galaxy/requirements.yml
$(VENV)/bin/ansible-galaxy install -p ./ansible_galaxy/roles -r ./ansible_roles/requirements.yml
.PHONY: ansible-cluster .PHONY: ansible-cluster
ansible-cluster: $(VENV) galaxy ansible-cluster: $(VENV) ansible_galaxy
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
$(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \ $(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \
-i ansible_hosts.yml \ ./ansible_playbooks/setup-cluster.yml
-M ./roles \
./setup-cluster.yml
.PHONY: bootstrap-values .PHONY: bootstrap-values
bootstrap-values: $(VENV) galaxy bootstrap-values: $(VENV) galaxy
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \ env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \
-e "@vault-keys.json" \ -e "@vault-keys.json" \
-i ansible_hosts.yml \ ./ansible_playbooks/bootstrap-values.yml
-M ./roles \
./bootstrap-values.yml
.PHONY: unseal-vault .PHONY: unseal-vault
unseal-vault: $(VENV) galaxy unseal-vault: $(VENV) galaxy
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
-e "@vault-keys.json" \ -e "@vault-keys.json" \
-i ansible_hosts.yml \ ./ansible_playbooks/unseal-vault.yml
-M ./roles \
./unseal-vault.yml
.PHONY: init .PHONY: init
init: init:

7
ansible.cfg Normal file
View File

@ -0,0 +1,7 @@
[defaults]
inventory=ansible_playbooks/ansible_hosts.yml
collections_paths=ansible_galaxy
roles_path=ansible_galaxy/roles
[inventory]
enable_plugins=yaml

View File

@ -1,4 +1,7 @@
--- ---
collections:
- name: community.hashi_vault
version: 3.0.0
roles: roles:
- src: https://github.com/IamTheFij/ansible-consul.git - src: https://github.com/IamTheFij/ansible-consul.git
name: ansible-consul name: ansible-consul

View File

@ -4,7 +4,7 @@
gather_facts: false gather_facts: false
vars_files: vars_files:
- consul_values.yml - vars/consul_values.yml
tasks: tasks:
- name: Add values - name: Add values
@ -28,7 +28,7 @@
gather_facts: false gather_facts: false
vars_files: vars_files:
- ./vault_hashi_vault_values.yml - vars/vault_hashi_vault_values.yml
tasks: tasks:
- name: Bootstrap Vault secrets - name: Bootstrap Vault secrets

View File

@ -3,9 +3,6 @@
hosts: consul_instances hosts: consul_instances
any_errors_fatal: true any_errors_fatal: true
vars_files:
- consul_values.yml
roles: roles:
- role: ansible-consul - role: ansible-consul
vars: vars:
@ -70,15 +67,14 @@
- name: Setup Vault cluster - name: Setup Vault cluster
hosts: vault_instances hosts: vault_instances
vars_files:
- ./vault_hashi_vault_values.yml
roles: roles:
- name: ansible-vault - name: ansible-vault
vars: vars:
vault_version: 1.12.0-1 vault_version: 1.12.0-1
vault_install_hashi_repo: true vault_install_hashi_repo: true
vault_harden_file_perms: true vault_harden_file_perms: true
# Maybe this should be restricted
vault_group: bin
vault_bin_path: /usr/bin vault_bin_path: /usr/bin
vault_address: 0.0.0.0 vault_address: 0.0.0.0
@ -114,7 +110,7 @@
- name: Save initialize result - name: Save initialize result
copy: copy:
content: "{{ vault_init.stdout }}" content: "{{ vault_init.stdout }}"
dest: "./vault-keys.json" dest: "../vault-keys.json"
when: vault_init is succeeded when: vault_init is succeeded
delegate_to: localhost delegate_to: localhost
run_once: true run_once: true
@ -409,7 +405,7 @@
- name: Save bootstrap result - name: Save bootstrap result
copy: copy:
content: "{{ bootstrap_result.stdout }}" content: "{{ bootstrap_result.stdout }}"
dest: "./nomad_bootstrap.json" dest: "../nomad_bootstrap.json"
when: bootstrap_result is succeeded when: bootstrap_result is succeeded
delegate_to: localhost delegate_to: localhost
run_once: true run_once: true
@ -420,7 +416,7 @@
- jq - jq
- -r - -r
- .SecretID - .SecretID
- nomad_bootstrap.json - ../nomad_bootstrap.json
delegate_to: localhost delegate_to: localhost
run_once: true run_once: true
no_log: true no_log: true
@ -455,7 +451,7 @@
- name: Copy policy - name: Copy policy
copy: copy:
src: ./acls/nomad-anon-policy.hcl src: ../acls/nomad-anon-policy.hcl
dest: /tmp/anonymous.policy.hcl dest: /tmp/anonymous.policy.hcl
delegate_to: "{{ play_hosts[0] }}" delegate_to: "{{ play_hosts[0] }}"
register: anon_policy register: anon_policy
@ -479,7 +475,7 @@
- name: Set up Nomad backend and roles in Vault - name: Set up Nomad backend and roles in Vault
community.general.terraform: community.general.terraform:
project_path: ./acls project_path: ../acls
force_init: true force_init: true
variables: variables:
consul_address: "{{ play_hosts[0] }}:8500" consul_address: "{{ play_hosts[0] }}:8500"

View File

@ -1,4 +0,0 @@
---
collections:
- name: community.hashi_vault
version: 3.0.0

View File

@ -14,7 +14,8 @@ module "traefik" {
} }
module "nomad_login" { module "nomad_login" {
source = "../levant" source = "IamTheFij/levant/nomad"
version = "0.1.0"
template_path = "service.nomad" template_path = "service.nomad"
variables = { variables = {
@ -36,7 +37,8 @@ module "metrics" {
} }
module "loki" { module "loki" {
source = "../levant" source = "IamTheFij/levant/nomad"
version = "0.1.0"
template_path = "service.nomad" template_path = "service.nomad"
variables = { variables = {

View File

@ -1,144 +0,0 @@
#!/usr/bin/env bash
export VERIFY_CHECKSUM=0
export ALIAS_NAME=
export OWNER=jsiebens
export REPO=hashi-up
export SUCCESS_CMD="$REPO version"
export BINLOCATION="~/bin"
###############################
# Content common across repos #
###############################
version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r')
if [ ! $version ]; then
echo "Failed while attempting to install $REPO. Please manually install:"
echo ""
echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases"
echo "2. Download the latest release for your platform. Call it '$REPO'."
echo "3. chmod +x ./$REPO"
echo "4. mv ./$REPO $BINLOCATION"
if [ -n "$ALIAS_NAME" ]; then
echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME"
fi
exit 1
fi
getPackage() {
uname=$(uname)
userid=$(id -u)
suffix=""
case $uname in
"Darwin")
suffix="-darwin"
;;
"MINGW"*)
suffix=".exe"
BINLOCATION="$HOME/bin"
mkdir -p $BINLOCATION
;;
"Linux")
arch=$(uname -m)
case $arch in
"aarch64")
suffix="-arm64"
;;
esac
case $arch in
"armv6l" | "armv7l")
suffix="-armhf"
;;
esac
;;
esac
targetFile="/tmp/$REPO$suffix"
if [ "$userid" != "0" ]; then
targetFile="$(pwd)/$REPO$suffix"
fi
if [ -e "$targetFile" ]; then
rm "$targetFile"
fi
url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix
echo "Downloading package $url as $targetFile"
curl -sSL $url --output "$targetFile"
if [ "$?" = "0" ]; then
if [ "$VERIFY_CHECKSUM" = "1" ]; then
checkHash
fi
chmod +x "$targetFile"
echo "Download complete."
if [ ! -w "$BINLOCATION" ]; then
echo
echo "============================================================"
echo " The script was run as a user who is unable to write"
echo " to $BINLOCATION. To complete the installation the"
echo " following commands may need to be run manually."
echo "============================================================"
echo
echo " sudo cp $REPO$suffix $BINLOCATION/$REPO"
if [ -n "$ALIAS_NAME" ]; then
echo " sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME"
fi
echo
else
echo
echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION"
if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then
echo
echo "================================================================"
echo " $BINLOCATION/$REPO already exists and is not writeable"
echo " by the current user. Please adjust the binary ownership"
echo " or run sh/bash with sudo."
echo "================================================================"
echo
exit 1
fi
mv "$targetFile" $BINLOCATION/$REPO
if [ "$?" = "0" ]; then
echo "New version of $REPO installed to $BINLOCATION"
fi
if [ -e "$targetFile" ]; then
rm "$targetFile"
fi
if [ $(which $ALIAS_NAME) ]; then
echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias"
else
if [ -n "$ALIAS_NAME" ]; then
if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then
ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME
echo "Creating alias '$ALIAS_NAME' for '$REPO'."
fi
fi
fi
${SUCCESS_CMD}
fi
fi
}
getPackage

View File

@ -1,42 +0,0 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/external" {
version = "2.2.2"
hashes = [
"h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=",
"h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
"zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
"zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
"zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
"zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327",
"zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955",
"zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb",
"zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0",
"zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a",
"zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372",
"zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809",
]
}
provider "registry.terraform.io/hashicorp/nomad" {
version = "1.4.17"
hashes = [
"h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
"h1:oWV3VXZhqPZ8Ia07nlIZLeXDBqVULMg9lP3dVMczDCo=",
"zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
"zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
"zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
"zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
"zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
"zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
"zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
"zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
"zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
"zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
]
}

View File

@ -1,7 +0,0 @@
# Terraform Levant
This module renders a levant template and then creates a Nomad job based on that template.
It only covers a subset of levant capabilities because much else can be done with Terraform already.
required:

View File

@ -1,63 +0,0 @@
#! /usr/bin/env python3
import json
import sys
from subprocess import check_output
from typing import Optional
from typing import overload
from typing import TypeVar
T = TypeVar("T")
@overload
def get_json(d: dict[str, str], key: str, default: None = None) -> None:
...
@overload
def get_json(d: dict[str, str], key: str, default: T = None) -> T:
...
def get_json(d: dict[str, str], key: str, default: Optional[T] = None) -> Optional[T]:
if key not in d:
return default
return json.loads(d[key])
query = json.load(sys.stdin)
# Required
template_path = query["template_path"]
# Optional
consul_address = query.get("consul_address")
if consul_address is not None:
consul_address = f"-consul-address={consul_address}"
# Need to parse JSON back
variables = [
f'--var={key}={value}' for key, value in get_json(query, "variables", {}).items()
]
variable_files = [
f'--var-file={value}' for value in get_json(query, "var_files", [])
]
args: list[str] = list(
filter(
None,
["levant", "render", consul_address]
+ variables
+ variable_files
+ [template_path],
)
)
# print(" ".join(args), file=sys.stderr)
# exit(1)
template = check_output(args, stderr=sys.stderr)
print(json.dumps({"template": template.decode()}))

View File

@ -1,37 +0,0 @@
variable "template_path" {
type = string
nullable = false
}
variable "consul_address" {
type = string
default = null
nullable = true
description = "Consul host and port for making KeyValue lookups"
}
variable "variables" {
type = map(string)
description = "Variables to be passed into nomad-pack with values in JSON form"
default = {}
}
variable "var_files" {
type = list(string)
description = "HCL files containing variables to be used by nomad-pack"
default = []
}
data "external" "levant" {
program = ["${path.module}/levant.py"]
query = {
template_path = var.template_path
consul_address = var.consul_address
variables = jsonencode(var.variables)
var_files = jsonencode(var.var_files)
}
}
resource "nomad_job" "levant" {
jobspec = data.external.levant.result.template
}

View File

@ -1,2 +0,0 @@
job {
}