From 16b9440e12f60b53141079573760cad349dd4953 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Fri, 17 Jun 2022 15:19:19 -0700 Subject: [PATCH] WIP: Add democratic-csi storage plugin --- nomad/root.tf | 4 ++ nomad/setup-cluster.yml | 1 + .../democratic-csi-nfs-controller.nomad | 55 +++++++++++++++ .../democratic-csi-nfs-node.nomad | 69 +++++++++++++++++++ nomad/storage_plugins/democratic-csi.tf | 51 ++++++++++++++ .../storage_plugins/democratic-nfs-config.yml | 10 +++ nomad/storage_plugins/democratic-nfs.nomad | 67 ++++++++++++++++++ 7 files changed, 257 insertions(+) create mode 100644 nomad/storage_plugins/democratic-csi-nfs-controller.nomad create mode 100644 nomad/storage_plugins/democratic-csi-nfs-node.nomad create mode 100644 nomad/storage_plugins/democratic-csi.tf create mode 100644 nomad/storage_plugins/democratic-nfs-config.yml create mode 100644 nomad/storage_plugins/democratic-nfs.nomad diff --git a/nomad/root.tf b/nomad/root.tf index 74a4b88..e4ac533 100644 --- a/nomad/root.tf +++ b/nomad/root.tf @@ -1,3 +1,7 @@ module "acls" { source = "./acls" } + +module "storage_plugins" { + source = "./storage_plugins" +} diff --git a/nomad/setup-cluster.yml b/nomad/setup-cluster.yml index c1c2480..2c3d81d 100644 --- a/nomad/setup-cluster.yml +++ b/nomad/setup-cluster.yml @@ -241,6 +241,7 @@ nomad_plugins: docker: config: + allow_privileged: true volumes: enabled: true selinuxlabel: "z" diff --git a/nomad/storage_plugins/democratic-csi-nfs-controller.nomad b/nomad/storage_plugins/democratic-csi-nfs-controller.nomad new file mode 100644 index 0000000..fa56d83 --- /dev/null +++ b/nomad/storage_plugins/democratic-csi-nfs-controller.nomad @@ -0,0 +1,55 @@ +variable "image_name" { + type = string + default = "docker.io/democraticcsi/democratic-csi:latest" +} + +variable "csi_version" { + type = string + default = "1.5.0" +} + +variable "config_data" { + type = string +} + +job "democratic-csi-nfs-controller" { + datacenters = ["dc1"] + + group "controller" { + task "plugin" { + driver = "docker" + + config { + image = var.image_name + + args = [ + "--csi-version=${var.csi_version}", + # must match the csi_plugin.id attribute below + "--csi-name=org.democratic-csi.nfs", + "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml", + "--log-level=info", + "--csi-mode=controller", + "--server-socket=/csi/csi.sock", + ] + } + + template { + destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml" + + data = var.config_data + } + + csi_plugin { + # must match --csi-name arg + id = "org.democratic-csi.nfs" + type = "controller" + mount_dir = "/csi" + } + + resources { + cpu = 250 + memory = 128 + } + } + } +} diff --git a/nomad/storage_plugins/democratic-csi-nfs-node.nomad b/nomad/storage_plugins/democratic-csi-nfs-node.nomad new file mode 100644 index 0000000..0d5fae3 --- /dev/null +++ b/nomad/storage_plugins/democratic-csi-nfs-node.nomad @@ -0,0 +1,69 @@ +variable "image_name" { + type = string + default = "docker.io/democraticcsi/democratic-csi:latest" +} + +variable "csi_version" { + type = string + default = "1.5.0" +} + +variable "config_data" { + type = string +} + +job "democratic-csi-nfs-node" { + datacenters = ["dc1"] + + # you can run node plugins as service jobs as well, but this ensures + # that all nodes in the DC have a copy. + type = "system" + + group "nodes" { + task "plugin" { + driver = "docker" + + env { + CSI_NODE_ID = "${attr.unique.hostname}" + } + + config { + image = var.image_name + + args = [ + "--csi-version=${var.csi_version}", + # must match the csi_plugin.id attribute below + "--csi-name=org.democratic-csi.nfs", + "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml", + "--log-level=info", + "--csi-mode=node", + "--server-socket=/csi/csi.sock", + ] + + # node plugins must run as privileged jobs because they + # mount disks to the host + privileged = true + ipc_mode = "host" + network_mode = "host" + } + + template { + destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml" + + data = var.config_data + } + + csi_plugin { + # must match --csi-name arg + id = "org.democratic-csi.nfs" + type = "node" + mount_dir = "/csi" + } + + resources { + cpu = 250 + memory = 128 + } + } + } +} diff --git a/nomad/storage_plugins/democratic-csi.tf b/nomad/storage_plugins/democratic-csi.tf new file mode 100644 index 0000000..6dba3b5 --- /dev/null +++ b/nomad/storage_plugins/democratic-csi.tf @@ -0,0 +1,51 @@ +variable "image_name" { + type = string + default = "docker.io/democraticcsi/democratic-csi:latest" +} + +variable "csi_version" { + type = string + default = "1.5.0" +} + +locals { + democratic_nfs_config = file("${path.module}/democratic-nfs-config.yml") +} + +# resource "nomad_job" "nfs-monolith" { +# hcl2 { +# enabled = true +# vars = { +# "image_name" = "${var.image_name}", +# "csi_version" = "${var.csi_version}", +# } +# } +# +# jobspec = file("${path.module}/democratic-nfs.nomad") +# } + +resource "nomad_job" "storage-controller" { + hcl2 { + enabled = true + vars = { + "image_name" = "${var.image_name}", + "csi_version" = "${var.csi_version}", + "config_data" = "${local.democratic_nfs_config}" + } + } + + jobspec = file("${path.module}/democratic-csi-nfs-controller.nomad") +} + +resource "nomad_job" "storage-node" { + hcl2 { + enabled = true + vars = { + "image_name" = "${var.image_name}", + "csi_version" = "${var.csi_version}", + "config_data" = "${local.democratic_nfs_config}" + } + } + + jobspec = file("${path.module}/democratic-csi-nfs-node.nomad") +} diff --git a/nomad/storage_plugins/democratic-nfs-config.yml b/nomad/storage_plugins/democratic-nfs-config.yml new file mode 100644 index 0000000..0b8b412 --- /dev/null +++ b/nomad/storage_plugins/democratic-nfs-config.yml @@ -0,0 +1,10 @@ +driver: nfs-client +instance_id: ${attr.unique.hostname} +nfs: + shareHost: 192.168.2.10 + shareBasePath: "/Containers/nomad-csi" + # shareHost:shareBasePath should be mounted at this location in the controller container + controllerBasePath: "/storage" + dirPermissionsMode: "0777" + # dirPermissionsUser: root + # dirPermissionsGroup: wheel diff --git a/nomad/storage_plugins/democratic-nfs.nomad b/nomad/storage_plugins/democratic-nfs.nomad new file mode 100644 index 0000000..7367646 --- /dev/null +++ b/nomad/storage_plugins/democratic-nfs.nomad @@ -0,0 +1,67 @@ +variable "image_name" { + type = string + default = "docker.io/democraticcsi/democratic-csi:latest" +} + +variable "csi_version" { + type = string + default = "1.5.0" +} + +job "democratic-csi-nfs" { + datacenters = ["dc1"] + + # you can run node plugins as service jobs as well, but this ensures + # that all nodes in the DC have a copy. + type = "system" + + group "monolith" { + task "plugin" { + driver = "docker" + + config { + image = var.image_name + + args = [ + "--csi-version=${var.csi_version}", + # must match the csi_plugin.id attribute below + "--csi-name=org.democratic-csi.nfs", + "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml", + "--log-level=info", + "--csi-mode=node", + "--csi-mode=controller", + "--server-socket=/csi/csi.sock", + ] + } + + template { + destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml" + + data = <