dotfiles/nixos/systems/blowhole/nomad.nix

{
  inputs',
  lib,
  config,
  pkgs,
  secret,
  config',
  ...
}: let
  inherit
    (lib)
    singleton
    ;
in {
  environment.systemPackages = [pkgs.git];

  services.hashicorp.vault-agent = {
    settings.template = singleton {
      source = pkgs.writeText "nomad.json.vtmpl" ''
        {
          "server": {
            "encrypt": "{{ with secret "kv/data/homelab-1/blowhole/nomad/encryption_key" }}{{ or .Data.data.key "" }}{{ end }}"
          },
          "vault": {
            "token": "{{ with secret "kv/data/homelab-1/blowhole/nomad/vault_token" }}{{ or .Data.data.secret "" }}{{ end }}"
          },
          "consul": {
            "token": "{{ with secret "kv/data/homelab-1/blowhole/nomad/consul_token" }}{{ or .Data.data.secret "" }}{{ end }}"
          }
        }
      '';
      destination = "/run/secrets/nomad.json";
      command = pkgs.writeShellScript "nomad-command" ''
        sudo systemctl try-reload-or-restart hashicorp-nomad.service
      '';
    };
  };

  systemd.services."hashicorp-nomad" = {
    requires = ["vault-unsealed.service"];
    after = ["vault-unsealed.service"];
  };

  services.hashicorp.nomad = {
    enable = true;

    extraPackages = with pkgs; [
      coreutils
      iproute2
      iptables
      consul
      glibc
      config.nix.package
      git
    ];
    extraSettingsPaths = [
      "/run/secrets/nomad.json"
    ];
    package = pkgs.nomad_1_5.overrideAttrs (old: {
      patches = with config'.flake.patches; [
        hashicorp-nomad.revert-change-consul-si-tokens-to-be-local
        hashicorp-nomad.add-nix-integration
      ];
    });

    settings = {
      bind_addr = secret.network.ips.blowhole.ip or "";
      server.enabled = true;

      tls = {
        # http = false # true
        # rpc = true

        # ca_file   = "nomad-ca.pem"
        # cert_file = "client.pem"
        # key_file  = "client-key.pem"

        # verify_server_hostname = true
        # verify_https_client    = true
      };

      vault = {
        enabled = true;
        address = "https://${secret.network.ips.vault.dns or ""}:8200";
        allow_unauthenticated = true;
        create_from_role = "nomad-cluster";
      };

      consul = {
        address = "${secret.network.ips.blowhole.ip or ""}:8500";
        auto_advertise = true;
        server_auto_join = true;
        client_auto_join = true;
      };

      acl.enabled = true;

      telemetry = {
        publish_allocation_metrics = true;
        publish_node_metrics = true;
      };

      client = {
        cni_path = "${pkgs.cni-plugins}/bin";

        min_dynamic_port = 20000;
        max_dynamic_port = 32000;

        options = {
          "docker.privileged.enabled" = "true";
        };

        host_network."wan".cidr = secret.network.networks.home.wan or "";
        host_network."default".cidr = secret.network.networks.home.amsterdam or "";
        host_network."mesh".cidr = secret.network.networks.vpn or "";

        network_interface = "eno1";

        host_volume."jellyfin-media".path = "/mnt/kyle/infrastructure/jellyfin/media";
        host_volume."hydra-nix".path = "/var/nfs/hydra-nix";
        host_volume."cctv" = {
          path = "/mnt/cctv";
          read_only = false;
        };

        enabled = true;
      };

      plugin."docker" = {
        config = {
          allow_caps = [
            "CHOWN"
            "DAC_OVERRIDE"
            "FSETID"
            "FOWNER"
            "MKNOD"
            "NET_RAW"
            "SETGID"
            "SETUID"
            "SETFCAP"
            "SETPCAP"
            "NET_BIND_SERVICE"
            "SYS_CHROOT"
            "KILL"
            "AUDIT_WRITE"
            "SYS_ADMIN"
          ];
          allow_privileged = true;
          extra_labels = [
            "job_name"
            "job_id"
            "task_group_name"
            "task_name"
            "namespace"
            "node_name"
            "node_id"
          ];
        };
      };

      disable_update_check = true;
      data_dir = "/var/lib/nomad";

      datacenter = "homelab-1";
      region = "homelab-1";
    };
  };

  virtualisation.docker.enable = true;
  virtualisation.docker.package = pkgs.docker.override rec {
    version = "24.0.5";
    cliRev = "v${version}";
    cliHash = "sha256-u1quVGTx/p8BDyRn33vYyyuE5BOhWMnGQ5uVX0PZ5mg=";
    mobyRev = "v${version}";
    mobyHash = "sha256-JQjRz1fHZlQRkNw/R8WWLV8caN3/U3mrKKQXbZt2crU=";
    # version = "25.0.3";
    # cliRev = "v${version}";
    # cliHash = "sha256-Jvb0plV1O/UzrcpzN4zH5OulmTVF+p9UQQQ9xqkiObQ=";
    # mobyRev = "v${version}";
    # mobyHash = "sha256-cDlRVdQNzH/X2SJUYHK1QLUHlKQtSyRYCVbz3wPx1ZM=";
    runcRev = "v1.1.12";
    runcHash = "sha256-N77CU5XiGYIdwQNPFyluXjseTeaYuNJ//OsEUS0g/v0=";
    containerdRev = "v1.7.13";
    containerdHash = "sha256-y3CYDZbA2QjIn1vyq/p1F1pAVxQHi/0a6hGWZCRWzyk=";
    tiniRev = "v0.19.0";
    tiniHash = "sha256-ZDKu/8yE5G0RYFJdhgmCdN3obJNyRWv6K/Gd17zc1sI=";
  };

  virtualisation.docker.daemon.settings.dns = [
    (secret.network.ips.blowhole.ip or "")
  ];
}