dotfiles/nix/systems/toothpick.nix

inputs: {
  system = "x86_64-linux";

  modules = [
    ../nixos-modules/default.nix
    ({ lib, pkgs, config, secret, ... }:
      let
        HUPToConsul = pkgs.writeShellScriptBin "hup-to-consul"
          ''
            export PATH=${pkgs.procps}/bin:$PATH

            pkill -x -HUP consul
          '';
      in
        {
          magic_rb = {
            pins = inputs;
            overlays = inputs.self.overlays;

            hardware.toothpick = true;
            flakes.enable = true;
            sshdEmacs.enable = true;
            vpsRemoteAccess =
              { enable = true;
                trustedWheel = true;
              };
          };
          boot.loader.grub = {
            enable = true;
            version = 2;
            efiSupport = false;
          };

          environment.systemPackages =
            [ pkgs.git
              pkgs.envoy
            ];

          boot.kernel.sysctl = { "net.ipv4.ip_forward" = "1"; };

          services.nomad = {
            enable = true;
            enableDocker = false;
            dropPrivileges = false;

            package = config.magic_rb.pkgs.nixpkgs-master.nomad_1_1;
            extraPackages = with pkgs; [ consul glibc ];

            extraSettingsPaths = [ "/var/secrets/nomad.hcl" ];
          };

          # https://github.com/NixOS/nixpkgs/issues/76671
          # the rpc.statd daemon is not running when not mounting any nfs filesystems on boot
          # and can't be manually started...
          services.nfs.server.enable = true;

          # create default network with `podman -r network create default`
          # virtualisation.podman = {
          #  enable = true;
          # };

          virtualisation.docker = {
            enable = true;
          };

          services.consul = {
            enable = true;
            interface = {
              bind = "wg0";
              advertise = "wg0";
            };

            extraConfigFiles = [
              "/var/secrets/consul.hcl"
            ];
          };

          systemd.services.vault-agent = {
            restartTriggers = [ "/etc/vault-agent/vault.crt" "/etc/vault-agent/vault.key" ];
            serviceConfig = {
              ExecPreStart = "mkdir -p /var/secrets/ && chown -R vault-agent:secrets /var/secrets/";
            };
          };

          security.sudo.extraRules =
            [ { users = [ "vault-agent" ];
                runAs = "consul";
                commands =
                  [ { command = "${HUPToConsul}/bin/hup-to-consul";
                      options = [ "NOPASSWD" ];
                    }
                  ];
              }
            ];

          services.vault-agent = {
            enable = true;
            settings = {
              vault = {
                address = "https://${secret.network.ips.vault.dns}:8200";

                client_cert = "/etc/vault-agent/vault.crt";
                client_key = "/etc/vault-agent/vault.key";
              };

              auto_auth = {
                method = [
                  {
                    "cert" = {
                      name = "system-toothpick";
                    };
                  }
                ];

                sink = [
                  {
                    "file" = {
                      type = "file";
                      config = {
                        path = "/etc/vault-agent/vault.token";
                      };
                    };
                  }
                ];
              };

              template =
                let
                  refreshConsul = pkgs.writeShellScript "refresh-consul.sh" ''
                    export PATH=/run/wrappers/bin:$PATH

                    sudo -u consul ${HUPToConsul}/bin/hup-to-consul
                  '';
                in [
                  {
                    source = pkgs.writeText "vault.key.tpl" ''
                      {{ with secret "pki_dynra/issue/vault.toothpick" "common_name=vault.toothpick.dyn.redalder.org" "ttl=24h" "alt_names=localhost" "ip_sans=127.0.0.1" }}
                      {{ .Data.private_key }}
                      {{ end }}
                    '';
                    destination = "/etc/vault-agent/vault.key.new";
                  }
                  # create role with
                  #   vault write pki_dynra/roles/vault.toothpick \
                  #     allowed_domains=vault.toothpick.dyn.redalder.org \
                  #     allow_subdomains=false \
                  #     max_ttl=72h \
                  #     allow_bare_domains=true
                  #
                  # also requires bootstrapping by generating the key/crt
                  # the first time, look into the source template. Then the
                  # generated cert must once again be first added to the
                  # cert auth method, look into the command. Basically you
                  # must manually do what vault-agent will do on its own
                  # after bootstrap
                  {
                    source = pkgs.writeText "vault.crt.tpl" ''
                      {{ with secret "pki_dynra/issue/vault.toothpick" "common_name=vault.toothpick.dyn.redalder.org" "ttl=24h" "alt_names=localhost" "ip_sans=127.0.0.1" }}
                      {{ .Data.certificate }}
                      {{ end }}
                    '';
                    destination = "/etc/vault-agent/vault.crt.new";
                    command = pkgs.writeShellScript "vault.crt-renew" ''
                      export PATH=${pkgs.vault}/bin:${pkgs.procps}/bin:$PATH

                      set -ve

                      export VAULT_ADDR="https://${secret.network.ips.vault.dns}:8200/"
                      export VAULT_TOKEN="$(cat /etc/vault-agent/vault.token)"

                      vault write auth/cert/certs/vault.toothpick \
                        display_name=vault.toothpick \
                        policies=vault.toothpick \
                        certificate=@/etc/vault-agent/vault.crt.new \
                        ttl=24h

                      echo "$VAULT_TOKEN" > /var/secrets/vault.token

                      cp /etc/vault-agent/vault.crt.new /etc/vault-agent/vault.crt
                      cp /etc/vault-agent/vault.key.new /etc/vault-agent/vault.key

                      pkill sleep || true
                      ( sleep "$((12 * 60 * 60))" && pkill -HUP vault ) &
                      disown
                    '';
                  }
                  # create role with
                  #   vault write pki_dynra/roles/consul.toothpick \
                  #     allowed_domains=consul.toothpick.dyn.redalder.org \
                  #     allow_subdomains=false \
                  #     max_ttl=72h \
                  #     allow_bare_domains=true
                  #
                  {
                    source = pkgs.writeText "consul.crt.tpl" ''
                      {{ with secret "pki_dynra/issue/consul.toothpick" "common_name=consul.toothpick.dyn.redalder.org" "ttl=24h" "alt_names=localhost" "ip_sans=127.0.0.1" }}
                      {{ .Data.certificate }}
                      {{ end }}
                    '';
                    destination = "/var/secrets/consul.crt";
                    command = refreshConsul;
                  }
                  {
                    source = pkgs.writeText "consul.key.tpl" ''
                      {{ with secret "pki_dynra/issue/consul.toothpick" "common_name=consul.toothpick.dyn.redalder.org" "ttl=24h" "alt_names=localhost" "ip_sans=127.0.0.1" }}
                      {{ .Data.private_key }}
                      {{ end }}
                    '';
                    destination = "/var/secrets/consul.key";
                    command = refreshConsul;
                  }
                  {
                    source = pkgs.writeText "ca.crt.tpl" ''
                      {{ with secret "pki_dynra/cert/ca" }}
                      {{ .Data.certificate }}
                      {{ end }}
                    '';
                    destination = "/var/secrets/ca.crt";
                  }
                  {
                    source = pkgs.writeText "nomad.hcl.tpl" ''
                      client {
                        enabled = true

                        cni_path = "${pkgs.cni-plugins}/bin"

                        host_network "public" {
                          cidr = "64.225.104.221/32"
                          reserved_ports = ""
                        }

                        host_network "vpn" {
                          cidr = "${secret.network.networks.vpn}"
                          reserved_ports = ""
                        }
                      }

                      advertise {
                        http = "${secret.network.ips.toothpick}"
                        rpc  = "${secret.network.ips.toothpick}"
                        serf = "${secret.network.ips.toothpick}"
                      }

                      plugin "docker" {
                        config {
                          endpoint = "unix:///var/run/docker.sock"

                          allow_privileged = true
                        }
                      }

                      vault {
                        enabled = true
                        address = "https://${secret.network.ips.vault.dns}:8200"
                        allow_unauthenticated = false
                        create_from_role = "nomad-cluster"
                      }

                      {{ with secret "kv/data/systems/toothpick/nomad" }}
                      consul {
                        ssl = false
                        address = "127.0.0.1:8500"
                        # ca_file = "/var/secrets/ca.crt"
                        # key_file = "/var/secrets/consul.key"
                        # cert_file = "/var/secrets/consul.crt"

                        token = "{{ .Data.data.consul_token }}"
                        auto_advertise = true
                        server_auto_join = true
                        client_auto_join = true
                      }
                      {{ end }}

                      log_level = "DEBUG"
                      disable_update_check = true
                      datacenter = "do-1"
                      data_dir = "/var/lib/nomad"
                    '';
                    destination = "/var/secrets/nomad.hcl";
                    perms = "0644";
                  }
                  {
                    source = pkgs.writeText "consul.hcl.tpl" ''
                      datacenter = "do-1"
                      node_name = "toothpick"
                      data_dir = "/var/lib/consul"

                      retry_join_wan = [ "${secret.network.ips.blowhole.ip}" ]

                      server = true

                      primary_datacenter = "homelab-1"

                      acl {
                        enabled = true
                        default_policy = "deny"
                        enable_token_persistence = true
                        enable_token_replication = true

                        {{ with secret "kv/data/systems/toothpick/consul" }}
                        tokens {
                          "agent" = "{{ .Data.data.agent_token }}"
                          "replication" = "{{ .Data.data.replication_token }}"
                        }
                        {{ end }}
                      }

                      ui_config {
                        enabled = true
                      }

                      connect {
                        enabled = true
                      }

                      # ca_file = "/var/secrets/ca.crt"
                      # cert_file = "/var/secrets/consul.crt"
                      # key_file = "/var/secrets/consul.key"
                      verify_incoming = false
                      verify_outgoing = false
                      verify_server_hostname = false
                      log_level = "DEBUG"
                      ports {
                        http = 8500
                      #  https = 8501
                        grpc = 8502
                      }
                    '';

                    # ca_provider = "vault"
                    # ca_config {
                    #   address = "https://${secret.network.ips.vault.dns}:8200"
                    #   token = "{{ file "/var/secrets/vault.token" | trimSpace }}"
                    #   root_pki_path = "consul_root"
                    #   intermediate_pki_path = "consul_intermediate"
                    # }
                    destination = "/var/secrets/consul.hcl";
                    perms = "0644";
                  }
                ];
            };
          };

          networking = {
            hostName = "toothpick";

            nameservers =
              [ "${secret.network.ips.woodchip}"
                "93.184.77.2"
                "67.207.67.3"
              ];

            wireguard = {
              enable = true;
              interfaces."wg0" =
                {
                  postSetup = ''
                ${pkgs.iptables}/bin/iptables -I FORWARD -i wg0 -o wg0 -j ACCEPT
              '';

                  postShutdown = ''
                ${pkgs.iptables}/bin/iptables -D FORWARD -i wg0 -o wg0 -j ACCEPT
              '';
                } // config.magic_rb.secret.wireguard."toothpick";
            };

            defaultGateway = "64.225.96.1";
            defaultGateway6 = "";
            dhcpcd.enable = false;
            usePredictableInterfaceNames = lib.mkForce false;

            firewall = {
              extraCommands = ''
            iptables -P FORWARD DROP
          '';

              # extraStopCommands = ''
              # '';

              interfaces."eth0" = {
                allowedTCPPorts =
                  [ 80
                    443
                  ];
                allowedUDPPorts =
                  [ 6666
                  ];
              };

              interfaces."wg0" = {
                allowedTCPPorts =
                  [ 8501
                    8502
                    8301
                    8302
                    8300
                    10000
                  ];
                allowedTCPPortRanges =
                  [ {
                    from = 21000;
                    to = 21255;
                  }];
                allowedUDPPorts =
                  [ 8301
                    8302
                  ];
                allowedUDPPortRanges =
                  [ {
                    from = 21000;
                    to = 21255;
                  }];
              };
            };

            interfaces = {
              eth0 = {
                ipv4.addresses =
                  [ { address="64.225.104.221"; prefixLength=20; }
                    { address="10.19.0.6"; prefixLength=16; }
                  ];
                ipv6.addresses =
                  [ { address="fe80::8ce0:84ff:fefb:f981";
                      prefixLength=64;
                    }
                  ];
                ipv4.routes =
                  [ { address = "64.225.96.1"; prefixLength = 32; }
                  ];
              };
            };
          };

          security.pki.certificates =
            [ (builtins.readFile ../redalder.org.crt)
              (builtins.readFile ../dyn.redalder.org.crt)
            ];

          services.udev.extraRules = ''
        ATTR{address}=="8e:e0:84:fb:f9:81", NAME="eth0"
      '';

          time.timeZone = "Europe/Bratislava";
          system.stateVersion = "21.05";
        })
  ];
}