From 766e15b326d71e6eeadea954043ccf3b244f6e3e Mon Sep 17 00:00:00 2001
From: main <magic_rb@redalder.org>
Date: Thu, 27 Oct 2022 11:38:38 +0200
Subject: [PATCH] Switch to democratic CSI driver

Signed-off-by: main <magic_rb@redalder.org>
---
 nomad/modules/nfs/controller.hcl          | 36 +++++++++++------
 nomad/modules/nfs/node.hcl                | 48 ++++++++++++++++-------
 nomad/regions/do-1/ingress.tf             |  4 +-
 nomad/regions/homelab-1/home-assistant.tf | 18 ++++++---
 nomad/regions/homelab-1/jellyfin.tf       | 18 ++++++---
 nomad/regions/homelab-1/syncthing.tf      | 18 ++++++---
 6 files changed, 98 insertions(+), 44 deletions(-)

diff --git a/nomad/modules/nfs/controller.hcl b/nomad/modules/nfs/controller.hcl
index 87b5bb4..a19d36e 100644
--- a/nomad/modules/nfs/controller.hcl
+++ b/nomad/modules/nfs/controller.hcl
@@ -6,7 +6,7 @@ variable "datacenters" {
   type = list(string)
 }
 
-job "nfs-controller" {
+job "democratic-csi-nfs-controller" {
   datacenters = var.datacenters
   region = var.region
 
@@ -15,23 +15,37 @@ job "nfs-controller" {
       driver = "docker"
 
       config {
-	image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
+        image = "docker.io/democraticcsi/democratic-csi:latest"
 
-	args = [
-	  "--endpoint=unix://csi/csi.sock",
-	  "-v=5",
-	]
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock",
+        ]
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+driver: node-manual
+EOH
       }
 
       csi_plugin {
-	id = "nfs"
-	type = "controller"
-	mount_dir = "/csi"
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "controller"
+        mount_dir = "/csi"
       }
 
       resources {
-	cpu = 250
-	memory = 128
+        cpu    = 500
+        memory = 256
       }
     }
   }
diff --git a/nomad/modules/nfs/node.hcl b/nomad/modules/nfs/node.hcl
index e3ba9d6..9171f22 100644
--- a/nomad/modules/nfs/node.hcl
+++ b/nomad/modules/nfs/node.hcl
@@ -6,7 +6,7 @@ variable "datacenters" {
   type = list(string)
 }
 
-job "nfs-node" {
+job "democratic-csi-nfs-node" {
   datacenters = var.datacenters
   region = var.region
 
@@ -16,28 +16,48 @@ job "nfs-node" {
     task "plugin" {
       driver = "docker"
 
-      config {
-	image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+      }
 
-	args = [
-	  "--v=5",
-          "--nodeid=${attr.unique.hostname}",
-          "--endpoint=unix:///csi/csi.sock",
-          "--drivername=nfs.csi.k8s.io"
+      config {
+        image = "docker.io/democraticcsi/democratic-csi:latest"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
         ]
 
-	privileged = true
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged = true
+        ipc_mode = "host"
+        network_mode = "host"
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+driver: node-manual
+EOH
       }
 
       csi_plugin {
-	id = "nfs"
-	type = "node"
-	mount_dir = "/csi"
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "node"
+        mount_dir = "/csi"
       }
 
       resources {
-	cpu = 250
-	memory = 128
+        cpu    = 500
+        memory = 256
       }
     }
   }
diff --git a/nomad/regions/do-1/ingress.tf b/nomad/regions/do-1/ingress.tf
index 395863a..6c88b14 100644
--- a/nomad/regions/do-1/ingress.tf
+++ b/nomad/regions/do-1/ingress.tf
@@ -4,7 +4,7 @@ data "local_file" "ingress-upstreams" {
 
 resource "nomad_volume" "ingress-letsencrypt" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "ingress-letsencrypt"
   name        = "ingress-letsencrypt"
   external_id = "ingress-letsencrypt"
@@ -17,6 +17,8 @@ resource "nomad_volume" "ingress-letsencrypt" {
   context = {
     server = "10.64.2.1"
     share = "/var/nfs/ingress-letsencrypt"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
diff --git a/nomad/regions/homelab-1/home-assistant.tf b/nomad/regions/homelab-1/home-assistant.tf
index 6c5390e..66da7f3 100644
--- a/nomad/regions/homelab-1/home-assistant.tf
+++ b/nomad/regions/homelab-1/home-assistant.tf
@@ -1,6 +1,6 @@
 resource "nomad_volume" "home-assistant_hass" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "home-assistant_hass"
   name        = "home-assistant_hass"
   external_id = "home-assistant_hass"
@@ -13,17 +13,19 @@ resource "nomad_volume" "home-assistant_hass" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/home-assistant_hass"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "home-assistant_zigbee2mqtt" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "home-assistant_zigbee2mqtt"
   name        = "home-assistant_zigbee2mqtt"
   external_id = "home-assistant_zigbee2mqtt"
@@ -36,17 +38,19 @@ resource "nomad_volume" "home-assistant_zigbee2mqtt" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/home-assistant_zigbee2mqtt"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "home-assistant_mosquitto" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "home-assistant_mosquitto"
   name        = "home-assistant_mosquitto"
   external_id = "home-assistant_mosquitto"
@@ -59,11 +63,13 @@ resource "nomad_volume" "home-assistant_mosquitto" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/home-assistant_mosquitto"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
diff --git a/nomad/regions/homelab-1/jellyfin.tf b/nomad/regions/homelab-1/jellyfin.tf
index 83e4a7d..5a47210 100644
--- a/nomad/regions/homelab-1/jellyfin.tf
+++ b/nomad/regions/homelab-1/jellyfin.tf
@@ -1,6 +1,6 @@
 resource "nomad_volume" "jellyfin-cache" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "jellyfin-cache"
   name        = "jellyfin-cache"
   external_id = "jellyfin-cache"
@@ -13,17 +13,19 @@ resource "nomad_volume" "jellyfin-cache" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/jellyfin/cache"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "jellyfin-config" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "jellyfin-config"
   name        = "jellyfin-config"
   external_id = "jellyfin-config"
@@ -36,17 +38,19 @@ resource "nomad_volume" "jellyfin-config" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/jellyfin/config"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "jellyfin-media" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "jellyfin-media"
   name        = "jellyfin-media"
   external_id = "jellyfin-media"
@@ -59,11 +63,13 @@ resource "nomad_volume" "jellyfin-media" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/jellyfin/media"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
diff --git a/nomad/regions/homelab-1/syncthing.tf b/nomad/regions/homelab-1/syncthing.tf
index e34160e..a10cb4b 100644
--- a/nomad/regions/homelab-1/syncthing.tf
+++ b/nomad/regions/homelab-1/syncthing.tf
@@ -1,6 +1,6 @@
 resource "nomad_volume" "syncthing-data" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "syncthing-data"
   name        = "syncthing-data"
   external_id = "syncthing-data"
@@ -13,17 +13,19 @@ resource "nomad_volume" "syncthing-data" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/syncthing/data"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "syncthing-storage" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "syncthing-storage"
   name        = "syncthing-storage"
   external_id = "syncthing-storage"
@@ -36,17 +38,19 @@ resource "nomad_volume" "syncthing-storage" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/syncthing/storage"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }
 
 resource "nomad_volume" "syncthing-config" {
   type        = "csi"
-  plugin_id   = "nfs"
+  plugin_id   = "org.democratic-csi.nfs"
   volume_id   = "syncthing-config"
   name        = "syncthing-config"
   external_id = "syncthing-config"
@@ -59,11 +63,13 @@ resource "nomad_volume" "syncthing-config" {
   context = {
     server = "blowhole.hosts.in.redalder.org"
     share = "/var/nfs/syncthing/config"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
   }
 
   mount_options {
     fs_type = "nfs"
-    mount_flags = [ "nolock", "hard" ]
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
   }
 }