WIP: Add democratic-csi storage plugin

nomad/root.tf

@@ -1,3 +1,7 @@
 module "acls" {
   source = "./acls"
 }
+
+module "storage_plugins" {
+  source = "./storage_plugins"
+}

nomad/setup-cluster.yml

@@ -241,6 +241,7 @@
         nomad_plugins:
           docker:
             config:
+              allow_privileged: true
               volumes:
                 enabled: true
                 selinuxlabel: "z"

nomad/storage_plugins/democratic-csi-nfs-controller.nomad

@@ -0,0 +1,55 @@
+variable "image_name" {
+  type = string
+  default = "docker.io/democraticcsi/democratic-csi:latest"
+}
+
+variable "csi_version" {
+  type = string
+  default = "1.5.0"
+}
+
+variable "config_data" {
+  type = string
+}
+
+job "democratic-csi-nfs-controller" {
+  datacenters = ["dc1"]
+
+  group "controller" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = var.image_name
+
+        args = [
+          "--csi-version=${var.csi_version}",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock",
+        ]
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = var.config_data
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 250
+        memory = 128
+      }
+    }
+  }
+}

nomad/storage_plugins/democratic-csi-nfs-node.nomad

@@ -0,0 +1,69 @@
+variable "image_name" {
+  type = string
+  default = "docker.io/democraticcsi/democratic-csi:latest"
+}
+
+variable "csi_version" {
+  type = string
+  default = "1.5.0"
+}
+
+variable "config_data" {
+  type = string
+}
+
+job "democratic-csi-nfs-node" {
+  datacenters = ["dc1"]
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+      }
+
+      config {
+        image = var.image_name
+
+        args = [
+          "--csi-version=${var.csi_version}",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged = true
+        ipc_mode = "host"
+        network_mode = "host"
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = var.config_data
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 250
+        memory = 128
+      }
+    }
+  }
+}

nomad/storage_plugins/democratic-csi.tf

@@ -0,0 +1,51 @@
+variable "image_name" {
+  type    = string
+  default = "docker.io/democraticcsi/democratic-csi:latest"
+}
+
+variable "csi_version" {
+  type    = string
+  default = "1.5.0"
+}
+
+locals {
+  democratic_nfs_config = file("${path.module}/democratic-nfs-config.yml")
+}
+
+# resource "nomad_job" "nfs-monolith" {
+#   hcl2 {
+#     enabled = true
+#     vars = {
+#       "image_name" = "${var.image_name}",
+#       "csi_version" = "${var.csi_version}",
+#     }
+#   }
+#
+#   jobspec = file("${path.module}/democratic-nfs.nomad")
+# }
+
+resource "nomad_job" "storage-controller" {
+  hcl2 {
+    enabled = true
+    vars = {
+      "image_name"  = "${var.image_name}",
+      "csi_version" = "${var.csi_version}",
+      "config_data" = "${local.democratic_nfs_config}"
+    }
+  }
+
+  jobspec = file("${path.module}/democratic-csi-nfs-controller.nomad")
+}
+
+resource "nomad_job" "storage-node" {
+  hcl2 {
+    enabled = true
+    vars = {
+      "image_name"  = "${var.image_name}",
+      "csi_version" = "${var.csi_version}",
+      "config_data" = "${local.democratic_nfs_config}"
+    }
+  }
+
+  jobspec = file("${path.module}/democratic-csi-nfs-node.nomad")
+}

nomad/storage_plugins/democratic-nfs-config.yml

@@ -0,0 +1,10 @@
+driver: nfs-client
+instance_id: ${attr.unique.hostname}
+nfs:
+  shareHost: 192.168.2.10
+  shareBasePath: "/Containers/nomad-csi"
+  # shareHost:shareBasePath should be mounted at this location in the controller container
+  controllerBasePath: "/storage"
+  dirPermissionsMode: "0777"
+  # dirPermissionsUser: root
+  # dirPermissionsGroup: wheel

nomad/storage_plugins/democratic-nfs.nomad

@@ -0,0 +1,67 @@
+variable "image_name" {
+  type = string
+  default = "docker.io/democraticcsi/democratic-csi:latest"
+}
+
+variable "csi_version" {
+  type = string
+  default = "1.5.0"
+}
+
+job "democratic-csi-nfs" {
+  datacenters = ["dc1"]
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  group "monolith" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = var.image_name
+
+        args = [
+          "--csi-version=${var.csi_version}",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock",
+        ]
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+driver: nfs-client
+instance_id: ${attr.unique.hostname}
+nfs:
+  shareHost: 192.168.2.10
+  shareBasePath: "/Containers/nomad-csi"
+  # shareHost:shareBasePath should be mounted at this location in the controller container
+  controllerBasePath: "/storage"
+  dirPermissionsMode: "0777"
+  # dirPermissionsUser: root
+  # dirPermissionsGroup: wheel
+EOH
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "monolith"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}