From a9c3b31b2cfa45abc6c85dc16c7e24f6348eb8a5 Mon Sep 17 00:00:00 2001 From: Magic_RB Date: Thu, 29 Jun 2023 19:11:39 +0200 Subject: [PATCH] Tell Nomad about the expected store path to potentially skip build Signed-off-by: Magic_RB --- flake.nix | 1 + patches/0001-Add-Nix-integration.patch | 100 ++++++++++++------ .../containers/ingress-blowhole/default.nix | 3 +- terranix/containers/ingress-blowhole/job.hcl | 5 + terranix/containers/matrix/default.nix | 3 +- .../containers/matrix/matrix-heisenbridge.hcl | 5 + terranix/containers/syncthing/default.nix | 3 +- terranix/containers/syncthing/job.hcl | 5 + 8 files changed, 89 insertions(+), 36 deletions(-) diff --git a/flake.nix b/flake.nix index 32cac6a..2cce5f0 100644 --- a/flake.nix +++ b/flake.nix @@ -120,6 +120,7 @@ flake_ref = "master"; flake_host = "git+https://git.sr.ht/~magic_rb/dotfiles-2.0"; }; + _module.args.config' = config; } ]; diff --git a/patches/0001-Add-Nix-integration.patch b/patches/0001-Add-Nix-integration.patch index 442287d..01922c9 100644 --- a/patches/0001-Add-Nix-integration.patch +++ b/patches/0001-Add-Nix-integration.patch @@ -47,22 +47,24 @@ index 40d98966a..9aa26458f 100644 "advertise_ipv6_address": hclspec.NewAttr("advertise_ipv6_address", "bool", false), "args": hclspec.NewAttr("args", "list(string)", false), "auth": hclspec.NewBlock("auth", false, hclspec.NewObject(map[string]*hclspec.Spec{ -@@ -402,6 +411,9 @@ var ( +@@ -402,6 +411,10 @@ var ( "volumes": hclspec.NewAttr("volumes", "list(string)", false), "volume_driver": hclspec.NewAttr("volume_driver", "string", false), "work_dir": hclspec.NewAttr("work_dir", "string", false), + -+ "nix_flake_ref": hclspec.NewAttr("nix_flake_ref", "string", false), -+ "nix_flake_sha": hclspec.NewAttr("nix_flake_sha", "string", false), ++ "nix_flake_ref": hclspec.NewAttr("nix_flake_ref", "string", false), ++ "nix_flake_sha": hclspec.NewAttr("nix_flake_sha", "string", false), ++ "nix_flake_store_path": hclspec.NewAttr("nix_flake_store_path", "string", false), }) // driverCapabilities represents the RPC response for what features are -@@ -474,6 +486,9 @@ type TaskConfig struct { +@@ -474,6 +486,10 @@ type TaskConfig struct { VolumeDriver string `codec:"volume_driver"` WorkDir string `codec:"work_dir"` + NixFlakeRef string `codec:"nix_flake_ref"` + NixFlakeSha string `codec:"nix_flake_sha"` ++ NixFlakeStorePath string `codec:"nix_flake_store_path"` + // MountsList supports the pre-1.0 mounts array syntax MountsList []DockerMount `codec:"mounts"` @@ -93,11 +95,11 @@ index 0aa993845..812952fad 100644 } - if driverConfig.Image == "" { -+ if driverConfig.Image == "" && !(driverConfig.NixFlakeRef != "" && driverConfig.NixFlakeSha != "") { ++ if driverConfig.Image == "" && !(driverConfig.NixFlakeRef != "" && driverConfig.NixFlakeSha != "" && driverConfig.NixFlakeStorePath != "") { return nil, nil, fmt.Errorf("image name required for docker driver") } -@@ -269,6 +270,47 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive +@@ -269,6 +270,79 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive return nil, nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) } @@ -109,34 +111,66 @@ index 0aa993845..812952fad 100644 + return nil, nil, fmt.Errorf("failed to find `nix` executable") + } + -+ err = NixBuildFlake(nixExecutable, driverConfig.NixFlakeRef, driverConfig.NixFlakeSha) -+ if err != nil { -+ return nil, nil, err ++ if _, err := os.Stat(driverConfig.NixFlakeStorePath); err != nil { ++ err = NixBuildFlake(nixExecutable, driverConfig.NixFlakeRef, driverConfig.NixFlakeSha) ++ if err != nil { ++ return nil, nil, err ++ } ++ ++ ++ deps, err := NixGetDeps(nixExecutable, driverConfig.NixFlakeRef) ++ if err != nil { ++ return nil, nil, err ++ } ++ ++ for _, dep := range deps { ++ var mount DockerMount ++ mount.Type = "bind" ++ mount.Target = dep; ++ mount.Source = dep; ++ mount.ReadOnly = true; ++ ++ driverConfig.Mounts = append(driverConfig.Mounts, mount); ++ } ++ ++ storePath, err := NixGetStorePath(nixExecutable, driverConfig.NixFlakeRef) ++ if err != nil { ++ return nil, nil, err ++ } ++ ++ driverConfig.Entrypoint[0] = storePath + "/" + driverConfig.Entrypoint[0] ++ ++ os.Symlink(storePath, GetGCRoot(d.config.GCRootsDir, cfg.Name, cfg.AllocID)) ++ } else { ++ d.eventer.EmitEvent(&drivers.TaskEvent{ ++ TaskID: cfg.ID, ++ AllocID: cfg.AllocID, ++ TaskName: cfg.Name, ++ Timestamp: time.Now(), ++ Message: "Skipping nix build as store path exists", ++ Annotations: map[string]string{ ++ "store_path": driverConfig.NixFlakeStorePath, ++ }, ++ }) ++ deps, err := NixGetDeps(nixExecutable, driverConfig.NixFlakeStorePath) ++ if err != nil { ++ return nil, nil, err ++ } ++ ++ for _, dep := range deps { ++ var mount DockerMount ++ mount.Type = "bind" ++ mount.Target = dep; ++ mount.Source = dep; ++ mount.ReadOnly = true; ++ ++ driverConfig.Mounts = append(driverConfig.Mounts, mount); ++ } ++ ++ driverConfig.Entrypoint[0] = driverConfig.NixFlakeStorePath + "/" + driverConfig.Entrypoint[0] ++ ++ os.Symlink(driverConfig.NixFlakeStorePath, GetGCRoot(d.config.GCRootsDir, cfg.Name, cfg.AllocID)) + } -+ -+ deps, err := NixGetDeps(nixExecutable, driverConfig.NixFlakeRef) -+ if err != nil { -+ return nil, nil, err -+ } -+ -+ for _, dep := range deps { -+ var mount DockerMount -+ mount.Type = "bind" -+ mount.Target = dep; -+ mount.Source = dep; -+ mount.ReadOnly = true; -+ -+ driverConfig.Mounts = append(driverConfig.Mounts, mount); -+ } -+ -+ storePath, err := NixGetStorePath(nixExecutable, driverConfig.NixFlakeRef) -+ if err != nil { -+ return nil, nil, err -+ } -+ -+ driverConfig.Entrypoint[0] = storePath + "/" + driverConfig.Entrypoint[0] -+ -+ os.Symlink(storePath, GetGCRoot(d.config.GCRootsDir, cfg.Name, cfg.AllocID)) + } + if (driverConfig.NixFlakeRef != "") != (driverConfig.NixFlakeSha != "") { + d.logger.Warn("one of either nix_flake_ref or nix_flake_sha is not set", "container_id", cfg.ID, "nix_flake_ref", driverConfig.NixFlakeRef, "nix_flake_sha", driverConfig.NixFlakeSha) diff --git a/terranix/containers/ingress-blowhole/default.nix b/terranix/containers/ingress-blowhole/default.nix index c004baf..867dac9 100644 --- a/terranix/containers/ingress-blowhole/default.nix +++ b/terranix/containers/ingress-blowhole/default.nix @@ -1,4 +1,4 @@ -{ elib, vars, ... }: +{ config', elib, vars, ... }: let inherit (elib) nomadJob; @@ -10,6 +10,7 @@ in vars = { flake_ref = "${vars.flake_host}?rev=${vars.flake_rev}&ref=${vars.flake_ref}"; flake_sha = vars.flake_sha; + store_path = config'.flake.nixngConfigurations.ingressBlowhole.config.system.build.toplevel; upstreams = "\${file(\"${./upstreams.conf}\")}"; }; }; diff --git a/terranix/containers/ingress-blowhole/job.hcl b/terranix/containers/ingress-blowhole/job.hcl index ffeabf2..5dc41f2 100644 --- a/terranix/containers/ingress-blowhole/job.hcl +++ b/terranix/containers/ingress-blowhole/job.hcl @@ -6,6 +6,10 @@ variable "flake_sha" { type = string } +variable "store_path" { + type = string +} + variable "upstreams" { type = string } @@ -202,6 +206,7 @@ job "ingress" { config { nix_flake_ref = "${var.flake_ref}#nixngConfigurations.ingressBlowhole.config.system.build.toplevel" nix_flake_sha = var.flake_sha + nix_flake_store_path = var.store_path entrypoint = [ "init" ] } diff --git a/terranix/containers/matrix/default.nix b/terranix/containers/matrix/default.nix index fd4654c..b815bea 100644 --- a/terranix/containers/matrix/default.nix +++ b/terranix/containers/matrix/default.nix @@ -1,4 +1,4 @@ -{ config, elib, vars, ... }: +{ config', config, elib, vars, ... }: let inherit (elib) nfsVolume @@ -173,6 +173,7 @@ in vars = { flake_ref = "${vars.flake_host}?ref=${vars.flake_ref}&rev=${vars.flake_rev}"; flake_sha = vars.flake_sha; + store_path = config'.flake.nixngConfigurations.heisenbridge.config.system.build.toplevel; }; }; } diff --git a/terranix/containers/matrix/matrix-heisenbridge.hcl b/terranix/containers/matrix/matrix-heisenbridge.hcl index ba31f9f..4b5394b 100644 --- a/terranix/containers/matrix/matrix-heisenbridge.hcl +++ b/terranix/containers/matrix/matrix-heisenbridge.hcl @@ -6,6 +6,10 @@ variable "flake_sha" { type = string } +variable "store_path" { + type = string +} + job "matrix-heisenbridge" { datacenters = [ "homelab-1" ] type = "service" @@ -52,6 +56,7 @@ job "matrix-heisenbridge" { config { nix_flake_ref = "${var.flake_ref}#nixngConfigurations.heisenbridge.config.system.build.toplevel" nix_flake_sha = var.flake_sha + nix_flake_store_path = var.store_path entrypoint = [ "init" ] } diff --git a/terranix/containers/syncthing/default.nix b/terranix/containers/syncthing/default.nix index 465ce7f..e3c52fa 100644 --- a/terranix/containers/syncthing/default.nix +++ b/terranix/containers/syncthing/default.nix @@ -1,4 +1,4 @@ -{ pkgs, elib, vars, ... }: +{ config', pkgs, elib, vars, ... }: let inherit (elib) nfsVolume @@ -36,6 +36,7 @@ in vars = { flake_ref = "${vars.flake_host}?rev=${vars.flake_rev}&ref=${vars.flake_ref}"; flake_sha = vars.flake_sha; + store_path = config'.flake.nixngConfigurations.syncthing.config.system.build.toplevel; }; }; } diff --git a/terranix/containers/syncthing/job.hcl b/terranix/containers/syncthing/job.hcl index e7c5912..7163bde 100644 --- a/terranix/containers/syncthing/job.hcl +++ b/terranix/containers/syncthing/job.hcl @@ -6,6 +6,10 @@ variable "flake_sha" { type = string } +variable "store_path" { + type = string +} + job "syncthing" { datacenters = [ "homelab-1" ] type = "service" @@ -78,6 +82,7 @@ job "syncthing" { config { nix_flake_ref = "${var.flake_ref}#nixngConfigurations.syncthing.config.system.build.toplevel" nix_flake_sha = var.flake_sha + nix_flake_store_path = var.store_path entrypoint = [ "init" ] }