Tell Nomad about the expected store path to potentially skip build

Signed-off-by: Magic_RB <magic_rb@redalder.org>

flake.nix

@@ -120,6 +120,7 @@
             flake_ref = "master";
             flake_host = "git+https://git.sr.ht/~magic_rb/dotfiles-2.0";
           };
+          _module.args.config' = config;
         }
       ];
 

patches/0001-Add-Nix-integration.patch

@@ -47,22 +47,24 @@ index 40d98966a..9aa26458f 100644
  		"advertise_ipv6_address": hclspec.NewAttr("advertise_ipv6_address", "bool", false),
  		"args":                   hclspec.NewAttr("args", "list(string)", false),
  		"auth": hclspec.NewBlock("auth", false, hclspec.NewObject(map[string]*hclspec.Spec{
-@@ -402,6 +411,9 @@ var (
+@@ -402,6 +411,10 @@ var (
  		"volumes":         hclspec.NewAttr("volumes", "list(string)", false),
  		"volume_driver":   hclspec.NewAttr("volume_driver", "string", false),
  		"work_dir":        hclspec.NewAttr("work_dir", "string", false),
 +
 +		"nix_flake_ref":        hclspec.NewAttr("nix_flake_ref", "string", false),
 +		"nix_flake_sha":        hclspec.NewAttr("nix_flake_sha", "string", false),
++		"nix_flake_store_path": hclspec.NewAttr("nix_flake_store_path", "string", false),
  	})
  
  	// driverCapabilities represents the RPC response for what features are
-@@ -474,6 +486,9 @@ type TaskConfig struct {
+@@ -474,6 +486,10 @@ type TaskConfig struct {
  	VolumeDriver      string             `codec:"volume_driver"`
  	WorkDir           string             `codec:"work_dir"`
  
 +	NixFlakeRef       string             `codec:"nix_flake_ref"`
 +	NixFlakeSha       string             `codec:"nix_flake_sha"`
++	NixFlakeStorePath string             `codec:"nix_flake_store_path"`
 +
  	// MountsList supports the pre-1.0 mounts array syntax
  	MountsList []DockerMount `codec:"mounts"`
@@ -93,11 +95,11 @@ index 0aa993845..812952fad 100644
  	}
  
 -	if driverConfig.Image == "" {
-+	if driverConfig.Image == "" && !(driverConfig.NixFlakeRef != "" && driverConfig.NixFlakeSha != "") {
++	if driverConfig.Image == "" && !(driverConfig.NixFlakeRef != "" && driverConfig.NixFlakeSha != "" && driverConfig.NixFlakeStorePath != "") {
  		return nil, nil, fmt.Errorf("image name required for docker driver")
  	}
  
-@@ -269,6 +270,47 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
+@@ -269,6 +270,79 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
  		return nil, nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
  	}
  
@@ -109,11 +111,13 @@ index 0aa993845..812952fad 100644
 +			return nil, nil, fmt.Errorf("failed to find `nix` executable")
 +		}
 +
++		if _, err := os.Stat(driverConfig.NixFlakeStorePath); err != nil {
 +			err = NixBuildFlake(nixExecutable, driverConfig.NixFlakeRef, driverConfig.NixFlakeSha)
 +			if err != nil {
 +				return nil, nil, err
 +			}
 +
++
 +			deps, err := NixGetDeps(nixExecutable, driverConfig.NixFlakeRef)
 +			if err != nil {
 +				return nil, nil, err
@@ -137,6 +141,36 @@ index 0aa993845..812952fad 100644
 +			driverConfig.Entrypoint[0] = storePath + "/" + driverConfig.Entrypoint[0]
 +
 +			os.Symlink(storePath, GetGCRoot(d.config.GCRootsDir, cfg.Name, cfg.AllocID))
++		} else {
++			d.eventer.EmitEvent(&drivers.TaskEvent{
++				TaskID:    cfg.ID,
++				AllocID:   cfg.AllocID,
++				TaskName:  cfg.Name,
++				Timestamp: time.Now(),
++				Message:   "Skipping nix build as store path exists",
++				Annotations: map[string]string{
++					"store_path": driverConfig.NixFlakeStorePath,
++				},
++			})
++			deps, err := NixGetDeps(nixExecutable, driverConfig.NixFlakeStorePath)
++			if err != nil {
++				return nil, nil, err
++			}
++
++			for _, dep := range deps {
++				var mount DockerMount
++				mount.Type = "bind"
++				mount.Target = dep;
++				mount.Source = dep;
++				mount.ReadOnly = true;
++
++				driverConfig.Mounts = append(driverConfig.Mounts, mount);
++			}
++
++			driverConfig.Entrypoint[0] = driverConfig.NixFlakeStorePath + "/" + driverConfig.Entrypoint[0]
++
++			os.Symlink(driverConfig.NixFlakeStorePath, GetGCRoot(d.config.GCRootsDir, cfg.Name, cfg.AllocID))
++		}
 +	}
 +	if (driverConfig.NixFlakeRef != "") != (driverConfig.NixFlakeSha != "") {
 +		d.logger.Warn("one of either nix_flake_ref or nix_flake_sha is not set", "container_id", cfg.ID, "nix_flake_ref", driverConfig.NixFlakeRef, "nix_flake_sha", driverConfig.NixFlakeSha)

terranix/containers/ingress-blowhole/default.nix

@@ -1,4 +1,4 @@
-{ elib, vars, ... }:
+{ config', elib, vars, ... }:
 let
   inherit (elib)
     nomadJob;
@@ -10,6 +10,7 @@ in
     vars = {
       flake_ref = "${vars.flake_host}?rev=${vars.flake_rev}&ref=${vars.flake_ref}";
       flake_sha = vars.flake_sha;
+      store_path = config'.flake.nixngConfigurations.ingressBlowhole.config.system.build.toplevel;
       upstreams = "\${file(\"${./upstreams.conf}\")}";
     };
   };

terranix/containers/ingress-blowhole/job.hcl

@@ -6,6 +6,10 @@ variable "flake_sha" {
   type = string
 }
 
+variable "store_path" {
+  type = string
+}
+
 variable "upstreams" {
   type = string
 }
@@ -202,6 +206,7 @@ job "ingress" {
       config {
         nix_flake_ref = "${var.flake_ref}#nixngConfigurations.ingressBlowhole.config.system.build.toplevel"
         nix_flake_sha = var.flake_sha
+        nix_flake_store_path = var.store_path
         entrypoint = [ "init" ]
       }
 

terranix/containers/matrix/default.nix

@@ -1,4 +1,4 @@
-{ config, elib, vars, ... }:
+{ config', config, elib, vars, ... }:
 let
   inherit (elib)
     nfsVolume
@@ -173,6 +173,7 @@ in
     vars = {
       flake_ref = "${vars.flake_host}?ref=${vars.flake_ref}&rev=${vars.flake_rev}";
       flake_sha = vars.flake_sha;
+      store_path = config'.flake.nixngConfigurations.heisenbridge.config.system.build.toplevel;
     };
   };
 }

terranix/containers/matrix/matrix-heisenbridge.hcl

@@ -6,6 +6,10 @@ variable "flake_sha" {
   type = string
 }
 
+variable "store_path" {
+  type = string
+}
+
 job "matrix-heisenbridge" {
   datacenters = [ "homelab-1" ]
   type = "service"
@@ -52,6 +56,7 @@ job "matrix-heisenbridge" {
       config {
         nix_flake_ref = "${var.flake_ref}#nixngConfigurations.heisenbridge.config.system.build.toplevel"
         nix_flake_sha = var.flake_sha
+        nix_flake_store_path = var.store_path
         entrypoint = [ "init" ]
       }
 

terranix/containers/syncthing/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, elib, vars, ... }:
+{ config', pkgs, elib, vars, ... }:
 let
   inherit (elib)
     nfsVolume
@@ -36,6 +36,7 @@ in
     vars = {
       flake_ref = "${vars.flake_host}?rev=${vars.flake_rev}&ref=${vars.flake_ref}";
       flake_sha = vars.flake_sha;
+      store_path = config'.flake.nixngConfigurations.syncthing.config.system.build.toplevel;
     };
   };
 }

terranix/containers/syncthing/job.hcl

@@ -6,6 +6,10 @@ variable "flake_sha" {
   type = string
 }
 
+variable "store_path" {
+  type = string
+}
+
 job "syncthing" {
   datacenters = [ "homelab-1" ]
   type = "service"
@@ -78,6 +82,7 @@ job "syncthing" {
         config {
           nix_flake_ref =  "${var.flake_ref}#nixngConfigurations.syncthing.config.system.build.toplevel"
           nix_flake_sha = var.flake_sha
+          nix_flake_store_path = var.store_path
           entrypoint = [ "init" ]
         }