--- - name: Build Consul cluster hosts: consul_instances any_errors_fatal: true roles: - role: ansible-consul vars: consul_version: "1.13.3-1" consul_install_upgrade: true consul_install_from_repo: true consul_os_repo_prerequisites: [] consul_node_role: server consul_raft_protocol: 3 consul_bootstrap_expect: true consul_bootstrap_expect_value: "{{ [(play_hosts | length), 3] | min }}" consul_user: consul consul_manage_user: true consul_group: bin consul_manage_group: true # consul_tls_enable: true consul_connect_enabled: true consul_ports_grpc: 8502 consul_client_address: "0.0.0.0" # Autopilot consul_autopilot_enable: true consul_autopilot_cleanup_dead_Servers: true # Enable metrics consul_config_custom: telemetry: prometheus_retention_time: "2h" # DNS forwarding consul_dnsmasq_enable: true consul_dnsmasq_servers: # TODO: use addresses of other nomad nodes? # Maybe this can be [] to get the values from dhcp - 1.1.1.1 - 1.0.0.1 consul_dnsmasq_bind_interfaces: true consul_dnsmasq_listen_addresses: # Listen only to loopback interface - 127.0.0.1 become: true tasks: - name: Start Consul systemd: state: started name: consul become: true # If DNS is broken after dnsmasq, then need to set /etc/resolv.conf to something # pointing to 127.0.0.1 and possibly restart Docker and Nomad - name: Update resolv.conf lineinfile: dest: /etc/resolv.conf create: true line: "nameserver 127.0.0.1" become: true - name: Setup Vault cluster hosts: vault_instances roles: - name: ansible-vault vars: vault_version: 1.12.0-1 vault_install_hashi_repo: true vault_harden_file_perms: true # Maybe this should be restricted vault_group: bin vault_bin_path: /usr/bin vault_address: 0.0.0.0 vault_backend: consul become: true tasks: - name: Get Vault status uri: url: http://127.0.0.1:8200/v1/sys/health method: GET status_code: 200, 429, 472, 473, 501, 503 body_format: json return_content: true register: vault_status - name: Initialize Vault when: not vault_status.json["initialized"] block: - name: Initialize Vault command: argv: - "vault" - "operator" - "init" - "-format=json" - "-address=http://127.0.0.1:8200/" - "-key-shares={{ vault_init_key_shares|default(3) }}" - "-key-threshold={{ vault_init_key_threshold|default(2) }}" run_once: true register: vault_init - name: Save initialize result copy: content: "{{ vault_init.stdout }}" dest: "../vault-keys.json" when: vault_init is succeeded delegate_to: localhost run_once: true - name: Unseal from init no_log: true command: argv: - "vault" - "operator" - "unseal" - "-address=http://127.0.0.1:8200/" - "{{ item }}" loop: "{{ (vault_init.stdout | from_json)['unseal_keys_hex'] }}" when: vault_init is succeeded - name: Unseal Vault no_log: true command: argv: - "vault" - "operator" - "unseal" - "-address=http://127.0.0.1:8200/" - "{{ item }}" loop: "{{ unseal_keys_hex }}" when: - unseal_keys_hex is defined - vault_status.json["sealed"] - name: Install Docker hosts: nomad_instances become: true vars: docker_architecture_map: x86_64: amd64 armv7l: armhf aarch64: arm64 docker_apt_arch: "{{ docker_architecture_map[ansible_architecture] }}" docker_compose_arch: "{{ (ansible_architecture == 'armv7l') | ternary('armv7', ansible_architecture) }}" roles: - geerlingguy.docker tasks: - name: Remove snapd package: name: snapd state: absent # Not on Ubuntu 20.04 # - name: Install Podman # hosts: nomad_instances # become: true # # tasks: # - name: Install Podman # package: # name: podman # state: present - name: Create NFS mounts hosts: nomad_instances become: true tasks: - name: Install nfs package: name: nfs-common state: present - name: Create Motioneye NFS mount ansible.posix.mount: src: 192.168.2.10:/Recordings/Motioneye path: /srv/volumes/motioneye-recordings opts: proto=tcp,port=2049,rw state: mounted fstype: nfs4 - name: Create Media Library RO NFS mount ansible.posix.mount: src: 192.168.2.10:/Multimedia path: /srv/volumes/media-read opts: proto=tcp,port=2049,ro state: mounted fstype: nfs4 - name: Create Media Library RW NFS mount ansible.posix.mount: src: 192.168.2.10:/Multimedia path: /srv/volumes/media-write opts: proto=tcp,port=2049,rw state: mounted fstype: nfs4 - name: Create Download RW NFS mount ansible.posix.mount: src: 192.168.2.10:/Download path: /srv/volumes/download opts: proto=tcp,port=2049,rw state: mounted fstype: nfs4 - name: Create Container NAS RW NFS mount ansible.posix.mount: src: 192.168.2.10:/Container path: /srv/volumes/container opts: proto=tcp,port=2049,rw state: mounted fstype: nfs4 - name: Build Nomad cluster hosts: nomad_instances any_errors_fatal: true become: true vars: shared_host_volumes: - name: motioneye-recordings path: /srv/volumes/motioneye-recordings owner: "root" group: "root" mode: "0755" read_only: false - name: media-read path: /srv/volumes/media-write read_only: true - name: media-write path: /srv/volumes/media-write owner: "root" group: "root" mode: "0755" read_only: false - name: tv-sonarr path: "/srv/volumes/media-write/TV Shows" owner: 1001 group: 100 mode: "0755" read_only: false - name: download path: /srv/volumes/download owner: 1001 group: 100 mode: "0755" read_only: false - name: nzbget-data path: /srv/volumes/container/nzbget/config read_only: false - name: gitea-data path: /srv/volumes/container/gitea read_only: false - name: photoprism-media path: /srv/volumes/media-write/Photoprism read_only: false - name: all-volumes path: /srv/volumes owner: "root" group: "root" mode: "0755" read_only: false roles: - name: ansible-nomad vars: nomad_version: "1.4.2-1" nomad_install_upgrade: true nomad_allow_purge_config: true nomad_meta: # There are issues with v1.23.0 on arm64 connect.sidecar_image: envoyproxy/envoy:v1.23.1 # Where nomad gets installed to nomad_bin_dir: /usr/bin nomad_install_from_repo: true nomad_bootstrap_expect: "{{ [(play_hosts | length), 3] | min }}" nomad_raft_protocol: 3 nomad_autopilot: true nomad_encrypt_enable: true # nomad_use_consul: true # Metrics nomad_telemetry: true nomad_telemetry_prometheus_metrics: true nomad_telemetry_publish_allocation_metrics: true nomad_telemetry_publish_node_metrics: true # Enable container plugins nomad_cni_enable: true nomad_cni_version: 1.0.1 nomad_docker_enable: true nomad_docker_dmsetup: false # nomad_podman_enable: true # Merge shared host volumes with node volumes nomad_host_volumes: "{{ shared_host_volumes + (nomad_unique_host_volumes | default([])) }}" # Customize docker plugin nomad_plugins: docker: config: allow_privileged: true volumes: enabled: true selinuxlabel: "z" # Send logs to journald so we can scrape them for Loki logging: type: journald extra_labels: - "job_name" - "job_id" - "task_group_name" - "task_name" - "namespace" - "node_name" - "node_id" # Bind nomad nomad_bind_address: 0.0.0.0 # Default interface for binding tasks # nomad_network_interface: lo # Create networks for binding task ports nomad_host_networks: - name: nomad-bridge interface: nomad reserved_ports: "22" - name: loopback interface: lo reserved_ports: "22" # Enable ACLs nomad_acl_enabled: true # Enable vault integration # HACK: Only talk to local Vault for now because it doesn't have HTTPS # TODO: Would be really great to have this over https and point to vault.consul.service # nomad_vault_address: "https://vault.service.consul:8200" # Right now, each node only talks to it's local Vault, so if that node is rebooted and # that vault is sealed, it will not have access to vault. This is a problem if a node # must reboot. nomad_vault_address: "http://127.0.0.1:8200" # TODO: This fails on first run because the Nomad-Vault integration can't be set up # until Nomad has started. Could maybe figure out if ACLs have been set up and leave # these out until the later play, maybe just bootstrap the nomad-cluster role in Vault # befor Nomad is set up nomad_vault_create_from_role: "nomad-cluster" # TODO: (security) Probably want to restict this to a narrower scoped token nomad_vault_enabled: "{{ root_token is defined }}" nomad_vault_token: "{{ root_token | default('') }}" nomad_config_custom: ui: enabled: true consul: ui_url: "https://consul.thefij.rocks/ui" vault: ui_url: "https://vault.thefij.rocks/ui" consul: tags: - "traefik.enable=true" - "traefik.consulcatalog.connect=true" - "traefik.http.routers.nomadclient.entrypoints=websecure" - name: Bootstrap Nomad ACLs and scheduler hosts: nomad_instances tasks: - name: Start Nomad systemd: state: started name: nomad - name: Nomad API reachable? uri: url: "http://127.0.0.1:4646/v1/status/leader" method: GET status_code: 200 register: nomad_check_result retries: 6 until: nomad_check_result is succeeded delay: 10 changed_when: false run_once: true - name: Bootstrap ACLs command: argv: - "nomad" - "acl" - "bootstrap" - "-json" run_once: true ignore_errors: true register: bootstrap_result - name: Save bootstrap result copy: content: "{{ bootstrap_result.stdout }}" dest: "../nomad_bootstrap.json" when: bootstrap_result is succeeded delegate_to: localhost run_once: true - name: Read secret command: argv: - jq - -r - .SecretID - ../nomad_bootstrap.json delegate_to: localhost run_once: true no_log: true changed_when: false register: read_secretid - name: Enable service scheduler preemption command: argv: - nomad - operator - scheduler - set-config - -preempt-system-scheduler=true - -preempt-service-scheduler=true environment: NOMAD_TOKEN: "{{ read_secretid.stdout }}" delegate_to: "{{ play_hosts[0] }}" run_once: true - name: Look for policy command: argv: - nomad - acl - policy - list environment: NOMAD_TOKEN: "{{ read_secretid.stdout }}" run_once: true register: policies - name: Copy policy copy: src: ../acls/nomad-anon-policy.hcl dest: /tmp/anonymous.policy.hcl delegate_to: "{{ play_hosts[0] }}" register: anon_policy run_once: true - name: Create anon-policy command: argv: - nomad - acl - policy - apply - -description="Anon read only" - anonymous - /tmp/anonymous.policy.hcl environment: NOMAD_TOKEN: "{{ read_secretid.stdout }}" when: policies.stdout == "No policies found" or anon_policy.changed delegate_to: "{{ play_hosts[0] }}" run_once: true - name: Set up Nomad backend and roles in Vault community.general.terraform: project_path: ../acls force_init: true variables: consul_address: "{{ play_hosts[0] }}:8500" vault_token: "{{ root_token }}" nomad_secret_id: "{{ read_secretid.stdout }}" delegate_to: localhost run_once: true notify: - Restart Nomad handlers: - name: Restart Nomad systemd: state: restarted name: nomad retries: 6 delay: 5