Refactor ansible to clean root dir
This commit is contained in:
parent
0996cfbf67
commit
37e80980bc
|
|
@ -37,8 +37,8 @@ terraform.rc
|
|||
|
||||
# ---> Ansible
|
||||
*.retry
|
||||
collections/ansible_collections/
|
||||
roles/
|
||||
ansible_galaxy/ansible_collections/
|
||||
ansible_galaxy/roles/
|
||||
|
||||
# Repo specific
|
||||
venv/
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@
|
|||
{
|
||||
"name": "CloudantDetector"
|
||||
},
|
||||
{
|
||||
"name": "DiscordBotTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "GitHubTokenDetector"
|
||||
},
|
||||
|
|
@ -115,6 +118,32 @@
|
|||
}
|
||||
],
|
||||
"results": {
|
||||
"ansible_playbooks/vars/vault_hashi_vault_values.example.yml": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
|
||||
"is_verified": false,
|
||||
"line_number": 8,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
|
||||
"is_verified": false,
|
||||
"line_number": 10,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "ansible_playbooks/vars/vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
|
||||
"is_verified": false,
|
||||
"line_number": 22,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"core/metrics/grafana/grafana.ini": [
|
||||
{
|
||||
"type": "Basic Auth Credentials",
|
||||
|
|
@ -182,33 +211,7 @@
|
|||
"line_number": 159,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"vault_hashi_vault_values.example.yml": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
|
||||
"is_verified": false,
|
||||
"line_number": 8,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
|
||||
"is_verified": false,
|
||||
"line_number": 10,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "vault_hashi_vault_values.example.yml",
|
||||
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
|
||||
"is_verified": false,
|
||||
"line_number": 22,
|
||||
"is_secret": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"generated_at": "2022-11-01T23:43:19Z"
|
||||
"generated_at": "2022-11-02T21:19:52Z"
|
||||
}
|
||||
|
|
|
|||
26
Makefile
26
Makefile
|
|
@ -35,35 +35,33 @@ secrets-audit: $(VENV) .secrets-baseline
|
|||
secrets-update: $(VENV) .secrets-baseline
|
||||
$(VENV)/bin/detect-secrets scan --baseline .secrets-baseline
|
||||
|
||||
.PHONY: galaxy
|
||||
galaxy: $(VENV)
|
||||
$(VENV)/bin/ansible-galaxy install -p roles -r roles/requirements.yml
|
||||
$(VENV)/bin/ansible-galaxy collection install -r collections/requirements.yml
|
||||
.PHONY: ansible_galaxy
|
||||
ansible_galaxy: ansible_galaxy/ansible_collections ansible_galaxy/roles
|
||||
|
||||
ansible_galaxy/ansible_collections: $(VENV) ./ansible_galaxy/requirements.yml
|
||||
$(VENV)/bin/ansible-galaxy collection install -p ./ansible_galaxy -r ./ansible_collections/requirements.yml
|
||||
|
||||
ansible_galaxy/roles: $(VENV) ./ansible_galaxy/requirements.yml
|
||||
$(VENV)/bin/ansible-galaxy install -p ./ansible_galaxy/roles -r ./ansible_roles/requirements.yml
|
||||
|
||||
|
||||
.PHONY: ansible-cluster
|
||||
ansible-cluster: $(VENV) galaxy
|
||||
ansible-cluster: $(VENV) ansible_galaxy
|
||||
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
|
||||
$(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \
|
||||
-i ansible_hosts.yml \
|
||||
-M ./roles \
|
||||
./setup-cluster.yml
|
||||
./ansible_playbooks/setup-cluster.yml
|
||||
|
||||
.PHONY: bootstrap-values
|
||||
bootstrap-values: $(VENV) galaxy
|
||||
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \
|
||||
-e "@vault-keys.json" \
|
||||
-i ansible_hosts.yml \
|
||||
-M ./roles \
|
||||
./bootstrap-values.yml
|
||||
./ansible_playbooks/bootstrap-values.yml
|
||||
|
||||
.PHONY: unseal-vault
|
||||
unseal-vault: $(VENV) galaxy
|
||||
env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \
|
||||
-e "@vault-keys.json" \
|
||||
-i ansible_hosts.yml \
|
||||
-M ./roles \
|
||||
./unseal-vault.yml
|
||||
./ansible_playbooks/unseal-vault.yml
|
||||
|
||||
.PHONY: init
|
||||
init:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
[defaults]
|
||||
inventory=ansible_playbooks/ansible_hosts.yml
|
||||
collections_paths=ansible_galaxy
|
||||
roles_path=ansible_galaxy/roles
|
||||
|
||||
[inventory]
|
||||
enable_plugins=yaml
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
---
|
||||
collections:
|
||||
- name: community.hashi_vault
|
||||
version: 3.0.0
|
||||
roles:
|
||||
- src: https://github.com/IamTheFij/ansible-consul.git
|
||||
name: ansible-consul
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
gather_facts: false
|
||||
|
||||
vars_files:
|
||||
- consul_values.yml
|
||||
- vars/consul_values.yml
|
||||
|
||||
tasks:
|
||||
- name: Add values
|
||||
|
|
@ -28,7 +28,7 @@
|
|||
gather_facts: false
|
||||
|
||||
vars_files:
|
||||
- ./vault_hashi_vault_values.yml
|
||||
- vars/vault_hashi_vault_values.yml
|
||||
|
||||
tasks:
|
||||
- name: Bootstrap Vault secrets
|
||||
|
|
@ -3,9 +3,6 @@
|
|||
hosts: consul_instances
|
||||
any_errors_fatal: true
|
||||
|
||||
vars_files:
|
||||
- consul_values.yml
|
||||
|
||||
roles:
|
||||
- role: ansible-consul
|
||||
vars:
|
||||
|
|
@ -70,15 +67,14 @@
|
|||
- name: Setup Vault cluster
|
||||
hosts: vault_instances
|
||||
|
||||
vars_files:
|
||||
- ./vault_hashi_vault_values.yml
|
||||
|
||||
roles:
|
||||
- name: ansible-vault
|
||||
vars:
|
||||
vault_version: 1.12.0-1
|
||||
vault_install_hashi_repo: true
|
||||
vault_harden_file_perms: true
|
||||
# Maybe this should be restricted
|
||||
vault_group: bin
|
||||
vault_bin_path: /usr/bin
|
||||
vault_address: 0.0.0.0
|
||||
|
||||
|
|
@ -114,7 +110,7 @@
|
|||
- name: Save initialize result
|
||||
copy:
|
||||
content: "{{ vault_init.stdout }}"
|
||||
dest: "./vault-keys.json"
|
||||
dest: "../vault-keys.json"
|
||||
when: vault_init is succeeded
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
|
|
@ -409,7 +405,7 @@
|
|||
- name: Save bootstrap result
|
||||
copy:
|
||||
content: "{{ bootstrap_result.stdout }}"
|
||||
dest: "./nomad_bootstrap.json"
|
||||
dest: "../nomad_bootstrap.json"
|
||||
when: bootstrap_result is succeeded
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
|
|
@ -420,7 +416,7 @@
|
|||
- jq
|
||||
- -r
|
||||
- .SecretID
|
||||
- nomad_bootstrap.json
|
||||
- ../nomad_bootstrap.json
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
no_log: true
|
||||
|
|
@ -455,7 +451,7 @@
|
|||
|
||||
- name: Copy policy
|
||||
copy:
|
||||
src: ./acls/nomad-anon-policy.hcl
|
||||
src: ../acls/nomad-anon-policy.hcl
|
||||
dest: /tmp/anonymous.policy.hcl
|
||||
delegate_to: "{{ play_hosts[0] }}"
|
||||
register: anon_policy
|
||||
|
|
@ -479,7 +475,7 @@
|
|||
|
||||
- name: Set up Nomad backend and roles in Vault
|
||||
community.general.terraform:
|
||||
project_path: ./acls
|
||||
project_path: ../acls
|
||||
force_init: true
|
||||
variables:
|
||||
consul_address: "{{ play_hosts[0] }}:8500"
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
---
|
||||
collections:
|
||||
- name: community.hashi_vault
|
||||
version: 3.0.0
|
||||
|
|
@ -14,7 +14,8 @@ module "traefik" {
|
|||
}
|
||||
|
||||
module "nomad_login" {
|
||||
source = "../levant"
|
||||
source = "IamTheFij/levant/nomad"
|
||||
version = "0.1.0"
|
||||
|
||||
template_path = "service.nomad"
|
||||
variables = {
|
||||
|
|
@ -36,7 +37,8 @@ module "metrics" {
|
|||
}
|
||||
|
||||
module "loki" {
|
||||
source = "../levant"
|
||||
source = "IamTheFij/levant/nomad"
|
||||
version = "0.1.0"
|
||||
|
||||
template_path = "service.nomad"
|
||||
variables = {
|
||||
|
|
|
|||
144
hashi-up.sh
144
hashi-up.sh
|
|
@ -1,144 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
export VERIFY_CHECKSUM=0
|
||||
export ALIAS_NAME=
|
||||
export OWNER=jsiebens
|
||||
export REPO=hashi-up
|
||||
export SUCCESS_CMD="$REPO version"
|
||||
export BINLOCATION="~/bin"
|
||||
|
||||
###############################
|
||||
# Content common across repos #
|
||||
###############################
|
||||
|
||||
version=$(curl -sI https://github.com/$OWNER/$REPO/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d '\r')
|
||||
if [ ! $version ]; then
|
||||
echo "Failed while attempting to install $REPO. Please manually install:"
|
||||
echo ""
|
||||
echo "1. Open your web browser and go to https://github.com/$OWNER/$REPO/releases"
|
||||
echo "2. Download the latest release for your platform. Call it '$REPO'."
|
||||
echo "3. chmod +x ./$REPO"
|
||||
echo "4. mv ./$REPO $BINLOCATION"
|
||||
if [ -n "$ALIAS_NAME" ]; then
|
||||
echo "5. ln -sf $BINLOCATION/$REPO /usr/local/bin/$ALIAS_NAME"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
getPackage() {
|
||||
uname=$(uname)
|
||||
userid=$(id -u)
|
||||
|
||||
suffix=""
|
||||
case $uname in
|
||||
"Darwin")
|
||||
suffix="-darwin"
|
||||
;;
|
||||
"MINGW"*)
|
||||
suffix=".exe"
|
||||
BINLOCATION="$HOME/bin"
|
||||
mkdir -p $BINLOCATION
|
||||
|
||||
;;
|
||||
"Linux")
|
||||
arch=$(uname -m)
|
||||
case $arch in
|
||||
"aarch64")
|
||||
suffix="-arm64"
|
||||
;;
|
||||
esac
|
||||
case $arch in
|
||||
"armv6l" | "armv7l")
|
||||
suffix="-armhf"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
|
||||
targetFile="/tmp/$REPO$suffix"
|
||||
|
||||
if [ "$userid" != "0" ]; then
|
||||
targetFile="$(pwd)/$REPO$suffix"
|
||||
fi
|
||||
|
||||
if [ -e "$targetFile" ]; then
|
||||
rm "$targetFile"
|
||||
fi
|
||||
|
||||
url=https://github.com/$OWNER/$REPO/releases/download/$version/$REPO$suffix
|
||||
echo "Downloading package $url as $targetFile"
|
||||
|
||||
curl -sSL $url --output "$targetFile"
|
||||
|
||||
if [ "$?" = "0" ]; then
|
||||
|
||||
if [ "$VERIFY_CHECKSUM" = "1" ]; then
|
||||
checkHash
|
||||
fi
|
||||
|
||||
chmod +x "$targetFile"
|
||||
|
||||
echo "Download complete."
|
||||
|
||||
if [ ! -w "$BINLOCATION" ]; then
|
||||
|
||||
echo
|
||||
echo "============================================================"
|
||||
echo " The script was run as a user who is unable to write"
|
||||
echo " to $BINLOCATION. To complete the installation the"
|
||||
echo " following commands may need to be run manually."
|
||||
echo "============================================================"
|
||||
echo
|
||||
echo " sudo cp $REPO$suffix $BINLOCATION/$REPO"
|
||||
|
||||
if [ -n "$ALIAS_NAME" ]; then
|
||||
echo " sudo ln -sf $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
else
|
||||
|
||||
echo
|
||||
echo "Running with sufficient permissions to attempt to move $REPO to $BINLOCATION"
|
||||
|
||||
if [ ! -w "$BINLOCATION/$REPO" ] && [ -f "$BINLOCATION/$REPO" ]; then
|
||||
|
||||
echo
|
||||
echo "================================================================"
|
||||
echo " $BINLOCATION/$REPO already exists and is not writeable"
|
||||
echo " by the current user. Please adjust the binary ownership"
|
||||
echo " or run sh/bash with sudo."
|
||||
echo "================================================================"
|
||||
echo
|
||||
exit 1
|
||||
|
||||
fi
|
||||
|
||||
mv "$targetFile" $BINLOCATION/$REPO
|
||||
|
||||
if [ "$?" = "0" ]; then
|
||||
echo "New version of $REPO installed to $BINLOCATION"
|
||||
fi
|
||||
|
||||
if [ -e "$targetFile" ]; then
|
||||
rm "$targetFile"
|
||||
fi
|
||||
|
||||
if [ $(which $ALIAS_NAME) ]; then
|
||||
echo "There is already a command '$ALIAS_NAME' in the path, NOT creating alias"
|
||||
else
|
||||
if [ -n "$ALIAS_NAME" ]; then
|
||||
if [ ! -L $BINLOCATION/$ALIAS_NAME ]; then
|
||||
ln -s $BINLOCATION/$REPO $BINLOCATION/$ALIAS_NAME
|
||||
echo "Creating alias '$ALIAS_NAME' for '$REPO'."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
${SUCCESS_CMD}
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
getPackage
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/external" {
|
||||
version = "2.2.2"
|
||||
hashes = [
|
||||
"h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=",
|
||||
"h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
|
||||
"zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
|
||||
"zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
|
||||
"zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
|
||||
"zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39",
|
||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||
"zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327",
|
||||
"zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955",
|
||||
"zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb",
|
||||
"zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0",
|
||||
"zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a",
|
||||
"zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372",
|
||||
"zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809",
|
||||
]
|
||||
}
|
||||
|
||||
provider "registry.terraform.io/hashicorp/nomad" {
|
||||
version = "1.4.17"
|
||||
hashes = [
|
||||
"h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=",
|
||||
"h1:oWV3VXZhqPZ8Ia07nlIZLeXDBqVULMg9lP3dVMczDCo=",
|
||||
"zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a",
|
||||
"zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f",
|
||||
"zh:3d0f971f79b615c1014c75e2f99f34bd4b4da542ca9f31d5ea7fadc4e9de39c1",
|
||||
"zh:46099a750c752ce05aa14d663a86478a5ad66d95aff3d69367f1d3628aac7792",
|
||||
"zh:71e56006b013dcfe1e4e059b2b07148b44fcd79351ae2c357e0d97e27ae0d916",
|
||||
"zh:74febd25d776688f0558178c2f5a0e6818bbf4cdaa2e160d7049da04103940f0",
|
||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||
"zh:af18c064a5f0dd5422d6771939274841f635b619ab392c73d5bf9720945fdb85",
|
||||
"zh:c133d7a862079da9f06e301c530eacbd70e9288fa2276ec0704df907270ee328",
|
||||
"zh:c894cf98d239b9f5a4b7cde9f5c836face0b5b93099048ee817b0380ea439c65",
|
||||
"zh:c918642870f0cafdbe4d7dd07c909701fc3ddb47cac8357bdcde1327bf78c11d",
|
||||
"zh:f8f5655099a57b4b9c0018a2d49133771e24c7ff8262efb1ceb140fd224aa9b6",
|
||||
]
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
# Terraform Levant
|
||||
|
||||
This module renders a levant template and then creates a Nomad job based on that template.
|
||||
|
||||
It only covers a subset of levant capabilities because much else can be done with Terraform already.
|
||||
|
||||
required:
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
#! /usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
from subprocess import check_output
|
||||
from typing import Optional
|
||||
from typing import overload
|
||||
from typing import TypeVar
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@overload
|
||||
def get_json(d: dict[str, str], key: str, default: None = None) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def get_json(d: dict[str, str], key: str, default: T = None) -> T:
|
||||
...
|
||||
|
||||
|
||||
def get_json(d: dict[str, str], key: str, default: Optional[T] = None) -> Optional[T]:
|
||||
if key not in d:
|
||||
return default
|
||||
|
||||
return json.loads(d[key])
|
||||
|
||||
|
||||
query = json.load(sys.stdin)
|
||||
|
||||
# Required
|
||||
template_path = query["template_path"]
|
||||
|
||||
# Optional
|
||||
consul_address = query.get("consul_address")
|
||||
if consul_address is not None:
|
||||
consul_address = f"-consul-address={consul_address}"
|
||||
|
||||
# Need to parse JSON back
|
||||
variables = [
|
||||
f'--var={key}={value}' for key, value in get_json(query, "variables", {}).items()
|
||||
]
|
||||
variable_files = [
|
||||
f'--var-file={value}' for value in get_json(query, "var_files", [])
|
||||
]
|
||||
|
||||
args: list[str] = list(
|
||||
filter(
|
||||
None,
|
||||
["levant", "render", consul_address]
|
||||
+ variables
|
||||
+ variable_files
|
||||
+ [template_path],
|
||||
)
|
||||
)
|
||||
|
||||
# print(" ".join(args), file=sys.stderr)
|
||||
# exit(1)
|
||||
|
||||
template = check_output(args, stderr=sys.stderr)
|
||||
|
||||
print(json.dumps({"template": template.decode()}))
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
variable "template_path" {
|
||||
type = string
|
||||
nullable = false
|
||||
}
|
||||
variable "consul_address" {
|
||||
type = string
|
||||
default = null
|
||||
nullable = true
|
||||
description = "Consul host and port for making KeyValue lookups"
|
||||
}
|
||||
|
||||
variable "variables" {
|
||||
type = map(string)
|
||||
description = "Variables to be passed into nomad-pack with values in JSON form"
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "var_files" {
|
||||
type = list(string)
|
||||
description = "HCL files containing variables to be used by nomad-pack"
|
||||
default = []
|
||||
}
|
||||
|
||||
data "external" "levant" {
|
||||
program = ["${path.module}/levant.py"]
|
||||
|
||||
query = {
|
||||
template_path = var.template_path
|
||||
consul_address = var.consul_address
|
||||
variables = jsonencode(var.variables)
|
||||
var_files = jsonencode(var.var_files)
|
||||
}
|
||||
}
|
||||
|
||||
resource "nomad_job" "levant" {
|
||||
jobspec = data.external.levant.result.template
|
||||
}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
job {
|
||||
}
|
||||
Loading…
Reference in New Issue