Add detect-secrets (there are a lot of false positives right now)

.pre-commit-config.yaml

@@ -15,8 +15,8 @@ repos:
       - id: check-merge-conflict
       - id: end-of-file-fixer
       - id: trailing-whitespace
-  # - repo: https://github.com/Yelp/detect-secrets
+  - repo: https://github.com/Yelp/detect-secrets
-  #   rev: v1.0.3
+    rev: v1.2.0
-  #   hooks:
+    hooks:
-  #     - id: detect-secrets
+      - id: detect-secrets
-  #       args: ['--baseline', '.secrets-baseline']
+        args: ['--baseline', '.secrets-baseline']

.secrets-baseline

@@ -0,0 +1,574 @@
+{
+  "version": "1.2.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets-baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    },
+    {
+      "path": "detect_secrets.filters.regex.should_exclude_secret",
+      "pattern": [
+        "(\\${.*}|from_env|fake|!secret)"
+      ]
+    }
+  ],
+  "results": {
+    "nomad/backups/backup.nomad": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/backups/backup.nomad",
+        "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
+        "is_verified": false,
+        "line_number": 94,
+        "is_secret": false
+      }
+    ],
+    "nomad/backups/oneoff.nomad": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/backups/oneoff.nomad",
+        "hashed_secret": "f2baa52d02ca888455ce47823f47bf372d5eecb3",
+        "is_verified": false,
+        "line_number": 114,
+        "is_secret": false
+      }
+    ],
+    "nomad/databases/mysql.nomad": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/databases/mysql.nomad",
+        "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
+        "is_verified": false,
+        "line_number": 66,
+        "is_secret": false
+      }
+    ],
+    "nomad/metrics/grafana.nomad": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/metrics/grafana.nomad",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 75,
+        "is_secret": false
+      }
+    ],
+    "nomad/packer/cloud-config": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "9ef2b7de7d9cb43de75586aa57c8325a46639ac9",
+        "is_verified": false,
+        "line_number": 26,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "2bb3f24183094c8ff5d5ac381a411fc4ab7a35da",
+        "is_verified": false,
+        "line_number": 27,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "67d96cf75c8d2edca3bdd2614003c4d1fc62055c",
+        "is_verified": false,
+        "line_number": 28,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "2f86f87d3ecf5a696afa6d8f61d0c9a13f2f6304",
+        "is_verified": false,
+        "line_number": 29,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "0462eefb3a04a6e4b97137d7682d9730d433efef",
+        "is_verified": false,
+        "line_number": 30,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "2bc96fb643b5c5149711f1a6630e92a0a40b5b52",
+        "is_verified": false,
+        "line_number": 31,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "3219ab282e5f68beb580dd3b7de2c8f171e0490d",
+        "is_verified": false,
+        "line_number": 32,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "5d167ddff0f00dce98abf89c8a924b5930d7ad83",
+        "is_verified": false,
+        "line_number": 33,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "d2a685cccdd672ec626c079d449e99cc094077b0",
+        "is_verified": false,
+        "line_number": 34,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "05a42fe5f719093045673ce08eeab08ecb019923",
+        "is_verified": false,
+        "line_number": 35,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "67cb7a776194efdd644961546be659b2c9167560",
+        "is_verified": false,
+        "line_number": 36,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "9a696a465a523fa4658747f902443af71329d5b1",
+        "is_verified": false,
+        "line_number": 37,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "1b3b4a544abe1482fb00cb1cdcd6b2a8164be8a3",
+        "is_verified": false,
+        "line_number": 38,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "d63d3ee4601ae418a9fafb284f6f57e7caa3372f",
+        "is_verified": false,
+        "line_number": 39,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "f1b6163dfe3e65a418a5d76dc2c3c730df79456d",
+        "is_verified": false,
+        "line_number": 40,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "bbc7610266af9f573207f340beaa494ea1e95ed7",
+        "is_verified": false,
+        "line_number": 41,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "e3f1c5b2b28515fd232629f226227d014a0a6870",
+        "is_verified": false,
+        "line_number": 42,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "7346f3b1b1e953966a71f35a83fab1351ca21510",
+        "is_verified": false,
+        "line_number": 43,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "c178ec42fc63c81c594d2320c01b2d618fd6256b",
+        "is_verified": false,
+        "line_number": 44,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "9a9c57ad4c90af8557c4abea07e156d288c435c8",
+        "is_verified": false,
+        "line_number": 45,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "41d969550bd78c1c4ba03eac7e7196f9507489d4",
+        "is_verified": false,
+        "line_number": 46,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "aa837393fc553576af61b2c3b00d51c356790070",
+        "is_verified": false,
+        "line_number": 47,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "b12573ed44f9ced804f4b67cb3decdaf950aa118",
+        "is_verified": false,
+        "line_number": 48,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "78663a675e5480881bf74645cd34a4a532cc6251",
+        "is_verified": false,
+        "line_number": 49,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "fccc316b54ab46ccadf00e94252e813ea59aca44",
+        "is_verified": false,
+        "line_number": 50,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "efce3378a7e2e3c4cf7e987049b89c2f90a472e8",
+        "is_verified": false,
+        "line_number": 51,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "94c80e1690072d1f88b21a0252d973fb7ee4beb7",
+        "is_verified": false,
+        "line_number": 52,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "8842e7efc9473b354d140170dbf6381208046b9c",
+        "is_verified": false,
+        "line_number": 53,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "bbeca400bf38dcf4b1a9243a6e026bdf86a1e0b4",
+        "is_verified": false,
+        "line_number": 54,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "d82b9a8fe372666d26021efd1ca9f8509d8d17ac",
+        "is_verified": false,
+        "line_number": 55,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "014dac6cb8f4a13bb0c7411261a386a95a7b693d",
+        "is_verified": false,
+        "line_number": 56,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "8645a12846d5ff41bf134336620a75fa56df87a6",
+        "is_verified": false,
+        "line_number": 57,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "e99e046a926b00dc114ae0372cfa841202d72409",
+        "is_verified": false,
+        "line_number": 58,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "587aeadfd3e6cff1e79ebd7218e7d7eb205039d2",
+        "is_verified": false,
+        "line_number": 59,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "b109b6e5c12a0801f8ee3625f83ce88d338c6bbb",
+        "is_verified": false,
+        "line_number": 60,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "fc9a86e095e968baebdc6f0f3a8c1fe7cc0680a5",
+        "is_verified": false,
+        "line_number": 61,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "fcab48515cfe5b2611fa6240d1f43bb6832734f4",
+        "is_verified": false,
+        "line_number": 62,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "16cb0e2482414d7b0dfce595ae782c437b0113ae",
+        "is_verified": false,
+        "line_number": 63,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "46b706d44f86eab95c68353b4e766afba43d3cf7",
+        "is_verified": false,
+        "line_number": 64,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "bc766ecc3c4300e5898db57ac69aa6daaf41183a",
+        "is_verified": false,
+        "line_number": 65,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "edf48876ce85b3041038d38ea21ca254826383e0",
+        "is_verified": false,
+        "line_number": 66,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "d0b110105dac510d2795c2b0d55f72e574311c5a",
+        "is_verified": false,
+        "line_number": 67,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "243997353c494328938298dd999ea751a85572a8",
+        "is_verified": false,
+        "line_number": 68,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "a00ed23fe8d7e981a4e39159cf2a9cb9d9a473f0",
+        "is_verified": false,
+        "line_number": 69,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "8d4b327f0feab6ee6088a19b44798b129f3dde27",
+        "is_verified": false,
+        "line_number": 70,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "0215562638f2418de7c39d85628f529b455fc46b",
+        "is_verified": false,
+        "line_number": 71,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "f152eebec4ed5168d64c48d34c5e574884c70992",
+        "is_verified": false,
+        "line_number": 72,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "nomad/packer/cloud-config",
+        "hashed_secret": "084f9e7b38bf21a62094d4eff295373125f5d1b8",
+        "is_verified": false,
+        "line_number": 73,
+        "is_secret": false
+      }
+    ],
+    "nomad/packer/ubuntu-cloud-init.pkr.hcl": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/packer/ubuntu-cloud-init.pkr.hcl",
+        "hashed_secret": "cbd2e782c0b1331013ac63de0b8d3b6f6a2ab5af",
+        "is_verified": false,
+        "line_number": 27,
+        "is_secret": false
+      }
+    ],
+    "nomad/vault_hashi_vault_values.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/vault_hashi_vault_values.yml",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 6,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "nomad/vault_hashi_vault_values.yml",
+        "hashed_secret": "18960546905b75c869e7de63961dc185f9a0a7c9",
+        "is_verified": false,
+        "line_number": 9,
+        "is_secret": false
+      }
+    ]
+  },
+  "generated_at": "2022-07-21T23:01:40Z"
+}

Makefile

@@ -0,0 +1,35 @@
+.PHONY: default
+default: check
+
+# Ensures virtualenv is present
+virtualenv_run:
+	virtualenv --python python3 virtualenv_run
+	./virtualenv_run/bin/pip install -r requirements.txt
+
+# Alias for virtualenv_run
+.PHONY: virtualenv
+virtualenv: virtualenv_run
+
+# Installs pre-commit hooks
+.PHONY: install-hooks
+install-hooks: virtualenv_run
+	./virtualenv_run/bin/pre-commit install --install-hooks
+
+# Checks files for encryption
+.PHONY: check
+check: virtualenv_run
+	./virtualenv_run/bin/pre-commit run --all-files
+
+# Creates a new secrets baseline
+.secrets-baseline: virtualenv_run
+	./virtualenv_run/bin/detect-secrets scan --exclude-secrets '(\$${.*}|from_env|fake|!secret)' > .secrets-baseline
+
+# Audits secrets against baseline
+.PHONY: secrets-audit
+secrets-audit: virtualenv_run .secrets-baseline
+	./virtualenv_run/bin/detect-secrets audit .secrets-baseline
+
+# Updates secrets baseline
+.PHONY: secrets-update
+secrets-update: virtualenv_run .secrets-baseline
+	./virtualenv_run/bin/detect-secrets scan --baseline .secrets-baseline

requirements.txt

@@ -0,0 +1,2 @@
+pre-commit
+detect-secrets==1.2.0