magic_rb/cluster
1
0
Fork 0
mirror of https://git.sr.ht/~magic_rb/cluster synced 2024-11-22 08:04:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse source 
Signed-off-by: Magic_RB <magic_rb@redalder.org>
This commit is contained in:
Magic_RB 2022-09-20 00:36:50 +02:00
parent e7216e601a
commit a27a4c4e78
No known key found for this signature in database
GPG key ID: 08D5287CC5DDCA0E
13 changed files with 412 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
nomad/modules/nfs/controller.hcl
View file

@ -6,7 +6,7 @@ variable "datacenters" {
type = list(string)
}
job "nfs-controller" {
job "democratic-csi-nfs-controller" {
datacenters = var.datacenters
region = var.region
@ -15,23 +15,37 @@ job "nfs-controller" {
driver = "docker"
config {
image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
image = "docker.io/democraticcsi/democratic-csi:latest"
args = [
"--endpoint=unix://csi/csi.sock",
"-v=5",
]
args = [
"--csi-version=1.5.0",
# must match the csi_plugin.id attribute below
"--csi-name=org.democratic-csi.nfs",
"--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=controller",
"--server-socket=/csi/csi.sock",
]
}
template {
destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
data = <<EOH
driver: node-manual
EOH
}
csi_plugin {
id = "nfs"
type = "controller"
mount_dir = "/csi"
# must match --csi-name arg
id = "org.democratic-csi.nfs"
type = "controller"
mount_dir = "/csi"
}
resources {
cpu = 250
memory = 128
cpu = 500
memory = 256
}
}
}

48
nomad/modules/nfs/node.hcl
View file

@ -6,7 +6,7 @@ variable "datacenters" {
type = list(string)
}
job "nfs-node" {
job "democratic-csi-nfs-node" {
datacenters = var.datacenters
region = var.region
@ -16,28 +16,48 @@ job "nfs-node" {
task "plugin" {
driver = "docker"
config {
image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
env {
CSI_NODE_ID = "${attr.unique.hostname}"
}
args = [
"--v=5",
"--nodeid=${attr.unique.hostname}",
"--endpoint=unix:///csi/csi.sock",
"--drivername=nfs.csi.k8s.io"
config {
image = "docker.io/democraticcsi/democratic-csi:latest"
args = [
"--csi-version=1.5.0",
# must match the csi_plugin.id attribute below
"--csi-name=org.democratic-csi.nfs",
"--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=node",
"--server-socket=/csi/csi.sock",
]
privileged = true
# node plugins must run as privileged jobs because they
# mount disks to the host
privileged = true
ipc_mode = "host"
network_mode = "host"
}
template {
destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
data = <<EOH
driver: node-manual
EOH
}
csi_plugin {
id = "nfs"
type = "node"
mount_dir = "/csi"
# must match --csi-name arg
id = "org.democratic-csi.nfs"
type = "node"
mount_dir = "/csi"
}
resources {
cpu = 250
memory = 128
cpu = 500
memory = 256
}
}
}

43
nomad/regions/do-1/ingress-upstreams.conf
View file

@ -18,8 +18,16 @@ upstream reicio {
server {{ env "NOMAD_UPSTREAM_ADDR_reicio" }};
}
upstream matrix-synapse {
server {{ env "NOMAD_UPSTREAM_ADDR_matrix_synapse" }};
}
upstream matrix-mautrix-facebook {
server {{ env "NOMAD_UPSTREAM_ADDR_matrix-mautrix-facebook" }};
}
server {
listen 443 ssl;
listen 443 ssl http2;
server_name _;
@ -29,7 +37,7 @@ server {
}
server {
listen 443 ssl;
listen 443 ssl http2;
server_name gitea.redalder.org;
@ -45,7 +53,7 @@ server {
}
server {
listen 443 ssl;
listen 443 ssl http2;
server_name hydra.redalder.org;
@ -59,7 +67,7 @@ server {
}
server {
listen 443 ssl;
listen 443 ssl http2;
server_name redalder.org nixng.org;
@ -81,3 +89,30 @@ server {
proxy_pass http://website;
}
}
server {
listen 443 ssl http2;
listen 8448 ssl http2;
server_name matrix.redalder.org;
merge_slashes off;
location /_matrix/ {
proxy_pass http://matrix-synapse$request_uri;
proxy_set_header Host $http_host;
proxy_buffering off;
}
location /mufb/ {
proxy_pass http://matrix-mautrix-facebook$request_uri;
proxy_set_header Host $http_host;
proxy_buffering off;
}
location / {
return 404;
}
include /local/security.conf;
include /local/ssl.conf;
}

42
nomad/regions/do-1/ingress.hcl
View file

@ -40,6 +40,12 @@ job "ingress" {
host_network = "public"
}
port "http-matrix" {
static = 8448
to = 8448
host_network = "public"
}
port "https" {
static = 443
to = 443
@ -119,13 +125,33 @@ job "ingress" {
mode = "local"
}
}
upstreams {
destination_name = "matrix-synapse"
local_bind_port = 6167
datacenter = "homelab-1"
mesh_gateway {
mode = "local"
}
}
upstreams {
destination_name = "matrix-mautrix-facebook"
local_bind_port = 29319
datacenter = "homelab-1"
mesh_gateway {
mode = "local"
}
}
}
}
}
}
task "nginx" {
driver = "docker"
driver = "containerd-driver"
volume_mount {
volume = "ingress-letsencrypt"
@ -133,21 +159,15 @@ job "ingress" {
read_only = false
}
# artifact {
# source = "http://hydra/build/99/download/1/image.tar.gz"
# }
config {
# load = "nixng-ingress.tar.gz"
image = "nixng-ingress:local"
ports = ["http", "https", "minecraft"]
memory_hard_limit = 128
flake_ref = "${var.flake_ref}#nixngSystems.ingressToothpick.config.system.build.toplevel"
flake_sha = var.flake_sha
entrypoint = [ "init" ]
}
resources {
cpu = 200
memory = 32
memory = 128
}
template {

4
nomad/regions/do-1/ingress.tf
View file

@ -4,7 +4,7 @@ data "local_file" "ingress-upstreams" {
resource "nomad_volume" "ingress-letsencrypt" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "ingress-letsencrypt"
name = "ingress-letsencrypt"
external_id = "ingress-letsencrypt"
@ -17,6 +17,8 @@ resource "nomad_volume" "ingress-letsencrypt" {
context = {
server = "10.64.2.1"
share = "/var/nfs/ingress-letsencrypt"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {

86
nomad/regions/homelab-1/conduit.tf
View file

@ -1,9 +1,9 @@
resource "nomad_volume" "conduit-data" {
resource "nomad_volume" "matrix-synapse" {
type = "csi"
plugin_id = "nfs"
volume_id = "conduit-data"
name = "conduit-data"
external_id = "conduit-data"
plugin_id = "org.democratic-csi.nfs"
volume_id = "matrix-synapse"
name = "matrix-synapse"
external_id = "matrix-synapse"
capability {
access_mode = "single-node-writer"
@ -12,22 +12,92 @@ resource "nomad_volume" "conduit-data" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/conduit-data"
share = "/var/nfs/matrix/synapse"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "matrix-registrations" {
type = "csi"
plugin_id = "org.democratic-csi.nfs"
volume_id = "matrix-registrations"
name = "matrix-registrations"
external_id = "matrix-registrations"
capability {
access_mode = "multi-node-multi-writer"
attachment_mode = "file-system"
}
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/matrix/registrations"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "matrix-mautrix-facebook" {
type = "csi"
plugin_id = "org.democratic-csi.nfs"
volume_id = "matrix-mautrix-facebook"
name = "matrix-mautrix-facebook"
external_id = "matrix-mautrix-facebook"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/matrix/mautrix-facebook"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "vault_policy" "matrix-mautrix-facebook-policy" {
name = "matrix-mautrix-facebook-policy"
policy = <<EOF
path "kv/data/matrix/mautrix-facebook" {
capabilities = ["read"]
}
EOF
}
resource "vault_policy" "matrix-synapse-policy" {
name = "matrix-synapse-policy"
policy = <<EOF
path "kv/data/matrix/synapse" {
capabilities = ["read"]
}
EOF
}
resource "nomad_job" "conduit" {
jobspec = file("${path.module}/job/conduit.hcl")
hcl2 {
enabled = true
vars = {
flake_ref = "${var.flake_host_alt}?rev=${var.flake_rev}"
flake_ref = "${var.flake_host}?ref=${var.flake_ref}&rev=${var.flake_rev}"
flake_sha = var.flake_sha
}
}

18
nomad/regions/homelab-1/home-assistant.tf
View file

@ -1,6 +1,6 @@
resource "nomad_volume" "home-assistant_hass" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "home-assistant_hass"
name = "home-assistant_hass"
external_id = "home-assistant_hass"
@ -13,17 +13,19 @@ resource "nomad_volume" "home-assistant_hass" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/home-assistant_hass"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "home-assistant_zigbee2mqtt" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "home-assistant_zigbee2mqtt"
name = "home-assistant_zigbee2mqtt"
external_id = "home-assistant_zigbee2mqtt"
@ -36,17 +38,19 @@ resource "nomad_volume" "home-assistant_zigbee2mqtt" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/home-assistant_zigbee2mqtt"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "home-assistant_mosquitto" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "home-assistant_mosquitto"
name = "home-assistant_mosquitto"
external_id = "home-assistant_mosquitto"
@ -59,11 +63,13 @@ resource "nomad_volume" "home-assistant_mosquitto" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/home-assistant_mosquitto"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}

18
nomad/regions/homelab-1/jellyfin.tf
View file

@ -1,6 +1,6 @@
resource "nomad_volume" "jellyfin-cache" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "jellyfin-cache"
name = "jellyfin-cache"
external_id = "jellyfin-cache"
@ -13,17 +13,19 @@ resource "nomad_volume" "jellyfin-cache" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/jellyfin/cache"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "jellyfin-config" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "jellyfin-config"
name = "jellyfin-config"
external_id = "jellyfin-config"
@ -36,17 +38,19 @@ resource "nomad_volume" "jellyfin-config" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/jellyfin/config"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "jellyfin-media" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "jellyfin-media"
name = "jellyfin-media"
external_id = "jellyfin-media"
@ -59,11 +63,13 @@ resource "nomad_volume" "jellyfin-media" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/jellyfin/media"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}

166
nomad/regions/homelab-1/job/conduit.hcl
View file

@ -6,22 +6,31 @@ variable "flake_sha" {
type = string
}
job "conduit" {
job "matrix" {
datacenters = [ "homelab-1" ]
type = "service"
group "svc" {
group "mautrix-facebook" {
count = 1
volume "conduit-data" {
volume "matrix-mautrix-facebook" {
type = "csi"
source = "conduit-data"
source = "matrix-mautrix-facebook"
read_only = false
attachment_mode = "file-system"
access_mode = "single-node-writer"
}
volume "matrix-registrations" {
type = "csi"
source = "matrix-registrations"
read_only = false
attachment_mode = "file-system"
access_mode = "multi-node-multi-writer"
}
restart {
attempts = 5
delay = "5s"
@ -32,29 +41,133 @@ job "conduit" {
}
service {
name = "conduit"
port = "6167"
name = "matrix-mautrix-facebook"
port = "29319"
check {
type = "http"
address_mode = "alloc"
path = "/"
port = "6167"
interval = "2s"
timeout = "2s"
}
# check {
# type = "http"
# address_mode = "alloc"
# path = "/public"
# port = "29319"
# interval = "2s"
# timeout = "2s"
# }
connect {
sidecar_service {}
}
}
task "app" {
task "mautrix-facebook" {
driver = "containerd-driver"
volume_mount {
volume = "conduit-data"
destination = "/var/lib/matrix-conduit"
volume = "matrix-mautrix-facebook"
destination = "/var/lib/mautrix-facebook"
read_only = false
}
volume_mount {
volume = "matrix-registrations"
destination = "/var/lib/registrations"
read_only = false
}
config {
flake_ref = "${var.flake_ref}#nixngSystems.mautrix-facebook.config.system.build.toplevel"
flake_sha = var.flake_sha
entrypoint = [ "init" ]
}
vault {
policies = ["matrix-mautrix-facebook-policy"]
}
template {
data = <<EOF
{{ with secret "kv/data/matrix/mautrix-facebook" }}
MAUTRIX_FACEBOOK_APPSERVICE_AS_TOKEN={{ .Data.data.as_token }}
MAUTRIX_FACEBOOK_APPSERVICE_HS_TOKEN={{ .Data.data.hs_token }}
{{ end }}
EOF
destination = "secrets/environment"
env = true
}
resources {
cpu = 500
memory = 1024
}
}
}
group "synapse" {
count = 1
volume "matrix-synapse" {
type = "csi"
source = "matrix-synapse"
read_only = false
attachment_mode = "file-system"
access_mode = "single-node-writer"
}
volume "matrix-registrations" {
type = "csi"
source = "matrix-registrations"
read_only = false
attachment_mode = "file-system"
access_mode = "multi-node-multi-writer"
}
restart {
attempts = 5
delay = "5s"
}
network {
mode = "bridge"
}
service {
name = "matrix-synapse"
port = "6167"
check {
type = "http"
address_mode = "alloc"
path = "/_matrix/client/versions"
port = "6167"
interval = "2s"
timeout = "2s"
}
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "matrix-mautrix-facebook"
local_bind_port = 29319
}
}
}
}
}
task "synapse" {
driver = "containerd-driver"
volume_mount {
volume = "matrix-synapse"
destination = "/var/lib/synapse"
read_only = false
}
volume_mount {
volume = "matrix-registrations"
destination = "/var/lib/registrations"
read_only = false
}
@ -65,8 +178,23 @@ job "conduit" {
}
resources {
cpu = 500
memory = 1024
cpu = 2048
memory = 4096
}
vault {
policies = ["matrix-synapse-policy"]
}
template {
data = <<EOF
{{ with secret "kv/data/matrix/synapse" }}
registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
form_secret: "{{ .Data.data.form_secret }}"
{{ end }}
EOF
destination = "/secrets/extra.yaml"
}
}
}

4
nomad/regions/homelab-1/job/home-assistant.hcl
View file

@ -229,8 +229,8 @@ EOF
resources {
cpu = 128
memory = 128
memory_max = 256
memory = 256
memory_max = 512
}
volume_mount {

20
nomad/regions/homelab-1/job/jellyfin.hcl
View file

@ -86,16 +86,16 @@ job "jellyfin" {
config {
image = "jellyfin/jellyfin@sha256:73501b70b0e884e5815d8f03d22973513ae7cadbcd8dba95da60e1d7c82dac7b"
devices = [
{
host_path = "/dev/dri/renderD128"
container_path = "/dev/dri/renderD128"
},
{
host_path = "/dev/dri/card0"
container_path = "/dev/dri/card0"
}
]
# devices = [
# {
# host_path = "/dev/dri/renderD128"
# container_path = "/dev/dri/renderD128"
# },
# {
# host_path = "/dev/dri/card0"
# container_path = "/dev/dri/card0"
# }
# ]
}
resources {

18
nomad/regions/homelab-1/syncthing.tf
View file

@ -1,6 +1,6 @@
resource "nomad_volume" "syncthing-data" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "syncthing-data"
name = "syncthing-data"
external_id = "syncthing-data"
@ -13,17 +13,19 @@ resource "nomad_volume" "syncthing-data" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/syncthing/data"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "syncthing-storage" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "syncthing-storage"
name = "syncthing-storage"
external_id = "syncthing-storage"
@ -36,17 +38,19 @@ resource "nomad_volume" "syncthing-storage" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/syncthing/storage"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}
resource "nomad_volume" "syncthing-config" {
type = "csi"
plugin_id = "nfs"
plugin_id = "org.democratic-csi.nfs"
volume_id = "syncthing-config"
name = "syncthing-config"
external_id = "syncthing-config"
@ -59,11 +63,13 @@ resource "nomad_volume" "syncthing-config" {
context = {
server = "blowhole.hosts.in.redalder.org"
share = "/var/nfs/syncthing/config"
node_attach_driver = "nfs"
provisioner_driver = "node-manual"
}
mount_options {
fs_type = "nfs"
mount_flags = [ "nolock", "hard" ]
mount_flags = [ "nfsvers=3", "hard", "async" ]
}
}

12
terraform.tfvars
View file

@ -1,6 +1,6 @@
flake_rev = "1de7117dca94932945dab6cb92315a74edcf01d5"
flake_ref = "home-assistant"
# flake_host = "git+https://gitea.redalder.org/RedAlder/systems"
flake_host = "git+https://git.irunx.org/MagicRB/ra-systems"
flake_host_alt = "git+https://git.irunx.org/MagicRB/ra-systems"
flake_sha = "sha256-QhVT7sAEzMNyYeInRNdI30JK6E7x1ksTZc1DSPNHNZ0="
flake_rev = "e7216e601a70f595893971d8b6f2296f3e867269"
flake_ref = "conduit-wip"
flake_host = "git+https://git.sr.ht/~magic_rb/cluster"
flake_host_alt = "git+https://git.sr.ht/~magic_rb/cluster"
flake_sha = "sha256-RJDAmsZJjBVvbYBG1vP5Se+i3n2IO3E2Y5fWFFmQ7lQ="