From a2030674006bfcf8aeb4538078ef4612d49d2c2f Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Wed, 2 Nov 2022 11:26:52 -0700 Subject: [PATCH] Migrate pre-commits from parent repo up to this one --- .gitignore | 47 +++++- .pre-commit-config.yaml | 22 +++ .secrets-baseline | 214 +++++++++++++++++++++++++++ Makefile | 122 +++++++-------- levant/.terraform.lock.hcl | 2 + requirements.txt | 5 + services/backups/.terraform.lock.hcl | 19 --- storage_plugins/.terraform.lock.hcl | 21 +++ 8 files changed, 360 insertions(+), 92 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets-baseline create mode 100644 requirements.txt create mode 100644 storage_plugins/.terraform.lock.hcl diff --git a/.gitignore b/.gitignore index fb66e6b..dee4eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,51 @@ +# ---> Terraform +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# ---> Ansible +*.retry +collections/ansible_collections/ roles/ + +# Repo specific venv/ +ca/ + +# Non-public bootstrap values vault-keys.json nomad_bootstrap.json -ca/ -collections/ansible_collections/ consul_values.yml vault_hashi_vault_values.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f069db3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +--- +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.64.1 + hooks: + - id: terraform_fmt + - id: terraform_validate + - id: terraform_providers_lock + # - id: terraform_tflint + # - id: terraform_tfsec + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/Yelp/detect-secrets + rev: v1.2.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets-baseline'] diff --git a/.secrets-baseline b/.secrets-baseline new file mode 100644 index 0000000..400bce6 --- /dev/null +++ b/.secrets-baseline @@ -0,0 +1,214 @@ +{ + "version": "1.2.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets-baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_secret", + "pattern": [ + "(\\${.*}|from_env|fake|!secret)" + ] + } + ], + "results": { + "core/metrics/grafana/grafana.ini": [ + { + "type": "Basic Auth Credentials", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "e5e9fa1ba31ecd1ae84f75caaa474f3a663f05f4", + "is_verified": false, + "line_number": 78, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "55ebda65c08313526e7ba08ad733e5ebea9900bd", + "is_verified": false, + "line_number": 109, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", + "is_verified": false, + "line_number": 151, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "10bea62ff1e1a7540dc7a6bc10f5fa992349023f", + "is_verified": false, + "line_number": 154, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "5718bce97710e6be87ea160b36eaefb5032857d3", + "is_verified": false, + "line_number": 239, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "core/metrics/grafana/grafana.ini", + "hashed_secret": "10aed9d7ebef778a9b3033dba3f7813b639e0d50", + "is_verified": false, + "line_number": 252, + "is_secret": false + } + ], + "core/syslogng.nomad": [ + { + "type": "Base64 High Entropy String", + "filename": "core/syslogng.nomad", + "hashed_secret": "298b5925fe7c7458cb8a12a74621fdedafea5ad6", + "is_verified": false, + "line_number": 159, + "is_secret": false + }, + { + "type": "Base64 High Entropy String", + "filename": "core/syslogng.nomad", + "hashed_secret": "3a1cec2d3c3de7e4da4d99c6731ca696c24b72b4", + "is_verified": false, + "line_number": 159, + "is_secret": false + } + ], + "vault_hashi_vault_values.example.yml": [ + { + "type": "Secret Keyword", + "filename": "vault_hashi_vault_values.example.yml", + "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3", + "is_verified": false, + "line_number": 8, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "vault_hashi_vault_values.example.yml", + "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9", + "is_verified": false, + "line_number": 10, + "is_secret": false + }, + { + "type": "Secret Keyword", + "filename": "vault_hashi_vault_values.example.yml", + "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", + "is_verified": false, + "line_number": 22, + "is_secret": false + } + ] + }, + "generated_at": "2022-11-01T23:43:19Z" +} diff --git a/Makefile b/Makefile index e4d1f47..078b050 100644 --- a/Makefile +++ b/Makefile @@ -1,84 +1,69 @@ -SERVER ?= "192.168.2.41" -SSH_USER = iamthefij -SSH_KEY = ~/.ssh/id_ed25519 +VENV ?= venv -.PHONY: rm-nomad -rm-nomad: - hashi-up nomad uninstall \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) - -.PHONY: nomad-up -nomad-up: - hashi-up nomad install \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \ - --server --client - hashi-up nomad start \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) - -.PHONY: rm-consul -rm-consul: - hashi-up consul uninstall \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) - -.PHONY: consul-up -consul-up: - hashi-up consul install \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) \ - --advertise-addr $(SERVER) \ - --client-addr 0.0.0.0 \ - --http-addr 0.0.0.0 \ - --connect \ - --server - hashi-up consul start \ - --ssh-target-addr $(SERVER) \ - --ssh-target-key $(SSH_KEY) \ - --ssh-target-user $(SSH_USER) \ - --ssh-target-sudo-pass $(SSH_TARGET_SUDO_PASS) +.PHONY: default +default: check .PHONY: cluster cluster: ansible-cluster -venv/bin/ansible: - python3 -m venv venv - ./venv/bin/pip install ansible python-consul hvac +# Ensures virtualenv is present +$(VENV): + python3 -m venv $(VENV) + $(VENV)/bin/pip install -r requirements.txt + +# Installs pre-commit hooks +.PHONY: install-hooks +install-hooks: $(VENV) + $(VENV)/bin/pre-commit install --install-hooks + +# Checks files for encryption +.PHONY: check +check: $(VENV) + $(VENV)/bin/pre-commit run --all-files + +# Creates a new secrets baseline +.secrets-baseline: $(VENV) + $(VENV)/bin/detect-secrets scan --exclude-secrets '(\$${.*}|from_env|fake|!secret)' > .secrets-baseline + +# Audits secrets against baseline +.PHONY: secrets-audit +secrets-audit: $(VENV) .secrets-baseline + $(VENV)/bin/detect-secrets audit .secrets-baseline + +# Updates secrets baseline +.PHONY: secrets-update +secrets-update: $(VENV) .secrets-baseline + $(VENV)/bin/detect-secrets scan --baseline .secrets-baseline .PHONY: galaxy -galaxy: venv/bin/ansible - ./venv/bin/ansible-galaxy install -p roles -r roles/requirements.yml - ./venv/bin/ansible-galaxy collection install -r collections/requirements.yml +galaxy: $(VENV) + $(VENV)/bin/ansible-galaxy install -p roles -r roles/requirements.yml + $(VENV)/bin/ansible-galaxy collection install -r collections/requirements.yml .PHONY: ansible-cluster -ansible-cluster: venv/bin/ansible galaxy - env VIRTUAL_ENV=/Users/ifij/workspace/iamthefij/orchestration-tests/nomad/venv ./venv/bin/ansible-playbook -K -vv \ +ansible-cluster: $(VENV) galaxy + env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ $(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \ - -i ansible_hosts.yml -M ./roles ./setup-cluster.yml + -i ansible_hosts.yml \ + -M ./roles \ + ./setup-cluster.yml .PHONY: bootstrap-values -bootstrap-values: venv/bin/ansible galaxy - env VIRTUAL_ENV=/Users/ifij/workspace/iamthefij/orchestration-tests/nomad/venv ./venv/bin/ansible-playbook -vv \ - $(shell test -f vault-keys.json && echo '-e "@vault-keys.json"') \ - -i ansible_hosts.yml -M ./roles ./bootstrap-values.yml +bootstrap-values: $(VENV) galaxy + env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -vv \ + -e "@vault-keys.json" \ + -i ansible_hosts.yml \ + -M ./roles \ + ./bootstrap-values.yml .PHONY: unseal-vault -unseal-vault: venv/bin/ansible galaxy - env VIRTUAL_ENV=/Users/ifij/workspace/iamthefij/orchestration-tests/nomad/venv ./venv/bin/ansible-playbook -K -vv \ - -e "@vault-keys.json" -i ansible_hosts.yml -M ./roles ./unseal-vault.yml +unseal-vault: $(VENV) galaxy + env VIRTUAL_ENV=$(VENV) $(VENV)/bin/ansible-playbook -K -vv \ + -e "@vault-keys.json" \ + -i ansible_hosts.yml \ + -M ./roles \ + ./unseal-vault.yml .PHONY: init init: @@ -95,8 +80,3 @@ apply: @terraform apply \ -var "nomad_secret_id=$(shell jq -r .SecretID nomad_bootstrap.json)" \ -var "vault_token=$(shell jq -r .root_token vault-keys.json)" - -# Install CNI on hosts? -# curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-$( [ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)"-v1.0.0.tgz -# sudo mkdir -p /opt/cni/bin -# sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz diff --git a/levant/.terraform.lock.hcl b/levant/.terraform.lock.hcl index 8bbed29..49c77b9 100644 --- a/levant/.terraform.lock.hcl +++ b/levant/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/external" { version = "2.2.2" hashes = [ "h1:BKQ5f5ijzeyBSnUr+j0wUi+bYv6KBQVQNDXNRVEcfJE=", + "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=", "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca", "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28", "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b", @@ -23,6 +24,7 @@ provider "registry.terraform.io/hashicorp/external" { provider "registry.terraform.io/hashicorp/nomad" { version = "1.4.17" hashes = [ + "h1:iPylWr144mqXvM8NBVMTm+MS6JRhqIihlpJG91GYDyA=", "h1:oWV3VXZhqPZ8Ia07nlIZLeXDBqVULMg9lP3dVMczDCo=", "zh:146f97eacd9a0c78b357a6cfd2cb12765d4b18e9660a75500ee3e748c6eba41a", "zh:2eb89a6e5cee9aea03a96ea9f141096fe3baf219b2700ce30229d2d882f5015f", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9b9c138 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pre-commit +detect-secrets==1.2.0 +ansible +python-consul +hvac diff --git a/services/backups/.terraform.lock.hcl b/services/backups/.terraform.lock.hcl index 429dbc2..70d1fae 100644 --- a/services/backups/.terraform.lock.hcl +++ b/services/backups/.terraform.lock.hcl @@ -1,25 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/consul" { - version = "2.16.2" - hashes = [ - "h1:epldE7sZPBTQHnWEA4WlNJIOVT1UEX+/02SMg5nniaE=", - "zh:0a2e11ca2ba650954951a087a1daec95eee2f3000456b295409a9880c4a10b1a", - "zh:34f6bda06a0d1c213fa8d87d4313687681e67bc8c40c4cbaa7dbe59ce24a4f7e", - "zh:5b85cf93db11ee890f720c317a38158927071feb634855786a0c0cd65825a43c", - "zh:75ef915f3d087e6045751a66fbb7066a852a0944ec8c97200d1134dd84df7ffc", - "zh:8a4a95697bd91ad51a581c12fe50ac61a114afba27895d027f77ac4154a7ea15", - "zh:973d538c8d72793861a1ac9718249a9493f417a2b5096846367560054fd843b9", - "zh:9feb2bdc06fdc2d8370cc9aad9a0c69e7e5ae38aac43f315c3f57507c57be030", - "zh:c5709672d0afecbbe298bf519741ebcb9d04f02a73b5ee0c186dfa241aa5a524", - "zh:c65c60570de6da7190e1e7762577655a463caeb59bc5d38e33034821ed0cbcb9", - "zh:c958d6282650fc472aade61d5df4300936033f43cfb898293ef86aceccdfdf1d", - "zh:cdd3632c81e1d11d3becd193aaa061688840f39147950c45c4301d042743ae6a", - "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5", - ] -} - provider "registry.terraform.io/hashicorp/nomad" { version = "1.4.16" hashes = [ diff --git a/storage_plugins/.terraform.lock.hcl b/storage_plugins/.terraform.lock.hcl new file mode 100644 index 0000000..04ee26b --- /dev/null +++ b/storage_plugins/.terraform.lock.hcl @@ -0,0 +1,21 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/nomad" { + version = "1.4.19" + hashes = [ + "h1:EdBny2gaLr/IE+l+6csyCKeIGFMYZ/4tHKpcbS7ArgE=", + "zh:2f3ceeb3318a6304026035b0ac9ee3e52df04913bb9ee78827e58c5398b41254", + "zh:3fbe76c7d957d20dfe3c8c0528b33084651f22a95be9e0452b658e0922916e2a", + "zh:595671a05828cfe6c42ef73aac894ac39f81a52cc662a76f37eb74ebe04ddf75", + "zh:5d76e8788d2af3e60daf8076babf763ec887480bbb9734baccccd8fcddf4f03e", + "zh:676985afeaca6e67b22d60d43fd0ed7055763029ffebc3026089fe2fd3b4a288", + "zh:69152ce6164ac999a640cff962ece45208270e1ac37c10dac484eeea5cf47275", + "zh:6da0b15c05b81f947ec8e139bd81eeeb05c0d36eb5a967b985d0625c60998b40", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:822c0a3bbada5e38099a379db8b2e339526843699627c3be3664cc3b3752bab7", + "zh:af23af2f98a84695b25c8eba7028a81ad4aad63c44aefb79e01bbe2dc82e7f78", + "zh:e36cac9960b7506d92925b667254322520966b9c3feb3ca6102e57a1fb9b1761", + "zh:ffd1e096c1cc35de879c740a91918e9f06b627818a3cb4b1d87b829b54a6985f", + ] +}