iamthefij/homelab-nomad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP: Add democratic-csi storage plugin

Browse Source
This commit is contained in:
IamTheFij 2022-06-17 15:19:19 -07:00
parent 18dbc89b2a
commit 2f65105592
7 changed files with 257 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
root.tf
View File

@ -1,3 +1,7 @@
module "acls" {
source = "./acls"
}
module "storage_plugins" {
source = "./storage_plugins"
}

1
setup-cluster.yml
View File

@ -241,6 +241,7 @@
nomad_plugins:
docker:
config:
allow_privileged: true
volumes:
enabled: true
selinuxlabel: "z"

55
storage_plugins/democratic-csi-nfs-controller.nomad Normal file
View File

@ -0,0 +1,55 @@
variable "image_name" {
type = string
default = "docker.io/democraticcsi/democratic-csi:latest"
}
variable "csi_version" {
type = string
default = "1.5.0"
}
variable "config_data" {
type = string
}
job "democratic-csi-nfs-controller" {
datacenters = ["dc1"]
group "controller" {
task "plugin" {
driver = "docker"
config {
image = var.image_name
args = [
"--csi-version=${var.csi_version}",
# must match the csi_plugin.id attribute below
"--csi-name=org.democratic-csi.nfs",
"--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=controller",
"--server-socket=/csi/csi.sock",
]
}
template {
destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
data = var.config_data
}
csi_plugin {
# must match --csi-name arg
id = "org.democratic-csi.nfs"
type = "controller"
mount_dir = "/csi"
}
resources {
cpu = 250
memory = 128
}
}
}
}

69
storage_plugins/democratic-csi-nfs-node.nomad Normal file
View File

@ -0,0 +1,69 @@
variable "image_name" {
type = string
default = "docker.io/democraticcsi/democratic-csi:latest"
}
variable "csi_version" {
type = string
default = "1.5.0"
}
variable "config_data" {
type = string
}
job "democratic-csi-nfs-node" {
datacenters = ["dc1"]
# you can run node plugins as service jobs as well, but this ensures
# that all nodes in the DC have a copy.
type = "system"
group "nodes" {
task "plugin" {
driver = "docker"
env {
CSI_NODE_ID = "${attr.unique.hostname}"
}
config {
image = var.image_name
args = [
"--csi-version=${var.csi_version}",
# must match the csi_plugin.id attribute below
"--csi-name=org.democratic-csi.nfs",
"--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=node",
"--server-socket=/csi/csi.sock",
]
# node plugins must run as privileged jobs because they
# mount disks to the host
privileged = true
ipc_mode = "host"
network_mode = "host"
}
template {
destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
data = var.config_data
}
csi_plugin {
# must match --csi-name arg
id = "org.democratic-csi.nfs"
type = "node"
mount_dir = "/csi"
}
resources {
cpu = 250
memory = 128
}
}
}
}

51
storage_plugins/democratic-csi.tf Normal file
View File

@ -0,0 +1,51 @@
variable "image_name" {
type = string
default = "docker.io/democraticcsi/democratic-csi:latest"
}
variable "csi_version" {
type = string
default = "1.5.0"
}
locals {
democratic_nfs_config = file("${path.module}/democratic-nfs-config.yml")
}
# resource "nomad_job" "nfs-monolith" {
# hcl2 {
# enabled = true
# vars = {
# "image_name" = "${var.image_name}",
# "csi_version" = "${var.csi_version}",
# }
# }
#
# jobspec = file("${path.module}/democratic-nfs.nomad")
# }
resource "nomad_job" "storage-controller" {
hcl2 {
enabled = true
vars = {
"image_name" = "${var.image_name}",
"csi_version" = "${var.csi_version}",
"config_data" = "${local.democratic_nfs_config}"
}
}
jobspec = file("${path.module}/democratic-csi-nfs-controller.nomad")
}
resource "nomad_job" "storage-node" {
hcl2 {
enabled = true
vars = {
"image_name" = "${var.image_name}",
"csi_version" = "${var.csi_version}",
"config_data" = "${local.democratic_nfs_config}"
}
}
jobspec = file("${path.module}/democratic-csi-nfs-node.nomad")
}

10
storage_plugins/democratic-nfs-config.yml Normal file
View File

@ -0,0 +1,10 @@
driver: nfs-client
instance_id: ${attr.unique.hostname}
nfs:
shareHost: 192.168.2.10
shareBasePath: "/Containers/nomad-csi"
# shareHost:shareBasePath should be mounted at this location in the controller container
controllerBasePath: "/storage"
dirPermissionsMode: "0777"
# dirPermissionsUser: root
# dirPermissionsGroup: wheel

67
storage_plugins/democratic-nfs.nomad Normal file
View File

@ -0,0 +1,67 @@
variable "image_name" {
type = string
default = "docker.io/democraticcsi/democratic-csi:latest"
}
variable "csi_version" {
type = string
default = "1.5.0"
}
job "democratic-csi-nfs" {
datacenters = ["dc1"]
# you can run node plugins as service jobs as well, but this ensures
# that all nodes in the DC have a copy.
type = "system"
group "monolith" {
task "plugin" {
driver = "docker"
config {
image = var.image_name
args = [
"--csi-version=${var.csi_version}",
# must match the csi_plugin.id attribute below
"--csi-name=org.democratic-csi.nfs",
"--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=node",
"--csi-mode=controller",
"--server-socket=/csi/csi.sock",
]
}
template {
destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
data = <<EOH
driver: nfs-client
instance_id: ${attr.unique.hostname}
nfs:
shareHost: 192.168.2.10
shareBasePath: "/Containers/nomad-csi"
# shareHost:shareBasePath should be mounted at this location in the controller container
controllerBasePath: "/storage"
dirPermissionsMode: "0777"
# dirPermissionsUser: root
# dirPermissionsGroup: wheel
EOH
}
csi_plugin {
# must match --csi-name arg
id = "org.democratic-csi.nfs"
type = "monolith"
mount_dir = "/csi"
}
resources {
cpu = 500
memory = 256
}
}
}
}