dotfiles/nixos/systems/blowhole/firewall.nix

{ pkgs, secret, config, lib, ... }:
let
  inherit (lib)
    mapAttrs
    const
    mkForce
    singleton;

  wlan = "wlp10s0";
  lan = "enp8s0f1";
  wan = "enp3s0";
  doVPN = "do_vpn0";

  nomad = mapAttrs (const toString) {
    inherit (config.services.hashicorp.nomad.settings.client)
      min_dynamic_port
      max_dynamic_port;
  };
in
{
  boot.kernel.sysctl = {
    # Enable forwarding on IPv4 but disable on IPv6
    "net.ipv4.conf.all.forwarding" = true;
    "net.ipv6.conf.all.forwarding" = false;

    # source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52
    # By default, not automatically configure any IPv6 addresses.
    "net.ipv6.conf.all.accept_ra" = 0;
    "net.ipv6.conf.all.autoconf" = 0;
    "net.ipv6.conf.all.use_tempaddr" = 0;

    # On WAN, allow IPv6 autoconfiguration and tempory address use.
    # "net.ipv6.conf.${name}.accept_ra" = 2;
    # "net.ipv6.conf.${name}.autoconf" = 1;
  };

  services.dnscrypt-proxy2 = {
    enable = true;
    upstreamDefaults = true;
    settings = {
      listen_addresses = singleton "127.0.0.1:5353";

      dnscrypt_servers = false;
      doh_servers = true;
      odoh_servers = false;

      block_ipv6 = true;

      static."mullvad".stamp = "sdns://AgcAAAAAAAAAAAAPZG9oLm11bGx2YWQubmV0Ci9kbnMtcXVlcnk";
      static."meganerd".stamp = "sdns://AQcAAAAAAAAADjEzNi4yNDQuOTcuMTE0ICif6V9M6EF_9Xo_MHwkDN4ZJjERopSJN8hBuUWg9YeMJTIuZG5zY3J5cHQtY2VydC5jaGV3YmFjY2EubWVnYW5lcmQubmw";
      sources = {};
    };
  };

  systemd.services.dnscrypt-proxy2 = {
    before = [ "network-online.target" ];
  };

  services.dhcpd4 = {
    enable = true;
    interfaces = [ "${lan}" "${wlan}" ];
    extraConfig = ''
      option domain-name-servers ${secret.network.ips.blowhole.ip or ""};
      option subnet-mask 255.255.255.0;

      ${secret.dhcp.blowhole.zones or (const "") { inherit wlan lan; }}
    '';
  };

  networking = {
    useDHCP = false;
    hostName = "blowhole";

    resolvconf.useLocalResolver = false;
    nameservers = singleton (secret.network.ips.blowhole.ip or "");

    # Disable the in-built iptable based firewall
    firewall.enable = mkForce false;

    localCommands = ''
      ip link add enp4s0 type dummy || true
      ip link set enp4s0 up || true
      ip addr add ${secret.network.ips.blowhole.ip}/24 dev enp4s0 || true
    '';

    interfaces = {
      # Don't do DHCP on the LAN interface
      "${lan}" = {
        useDHCP = false;
        ipv4.addresses = [{
          address = secret.network.ips.blowhole.ip or "";
          prefixLength = 24;
        }];
      };
      "${wlan}" = {
        useDHCP = false;
        ipv4.addresses = [{
          address = secret.network.ips.blowhole.wlan or "";
          prefixLength = 24;
        }];
      };
      # But do DHCP on the WAN interface
      "${wan}".useDHCP = true;
    };

    wireguard = {
      enable = true;
      interfaces."${doVPN}" =
        secret.wireguard."${config.networking.hostName}" or {} // {
          listenPort = 6666;
          privateKeyFile = "/var/secrets/${doVPN}.key";
        };
    };

    nftables = {
      enable = true;
      ruleset = ''
        table ip nf_filter {
          chain input_out {
            ct state { established, related } accept comment "Allow established traffic"
            icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
          }

          chain input_doVPN {
            tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic"
            tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic"
            tcp dport { 8200 } accept comment "Vault traffic"
            tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
            tcp dport { 53 } accept comment "DNS traffic"
            tcp dport { 80 } accept comment "HTTP traffic"

            udp dport { 8600, 8301, 8302 } comment "Consul traffic"
            udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
            udp dport { 53 } accept comment "DNS traffic"
          }

          chain input {
            type filter hook input priority 0; policy drop;

            tcp dport 22 accept comment "Accept SSH traffic always"
            iifname != "lo" tcp dport 5353 drop comment "Drop traffic to dnscrypt-proxy always except for localhost to localhost traffic"

            iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface"
            iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router"
            iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper" } jump input_out
            iifname { "${doVPN}" } jump input_doVPN

            # Allow containers to reach the DNS server
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } tcp dport 53 accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } udp dport 53 accept

            # Allow Nomad Containers to reach Nomad
            iifname { "nomad" } tcp dport 4646 accept

            # Allow proxies to reach consul
            iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept
            iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept

            # Allow containers to reach the NFS server
            iifname { "docker0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            iifname { "docker0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"

            meta nftrace set 1
          }

          chain output {
            type filter hook output priority 0; policy accept;

            # Drop all DNS traffic if leaving through "wan"
            # oifname { "${wan}" } tcp dport 53 drop
            # oifname { "${wan}" } udp dport 53 drop
            # Allow DoT traffic to leave through "wan" if it comes from "lo"
            # iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop
          }

          chain forward {
            type filter hook forward priority 10; policy drop;

            # Enable flow offloading for better throughput
            # ip protocol { tcp, udp } flow offload @f

            # Drop all DNS or DoT traffic if forwarded through "wan"
            oifname { "${wan}" } tcp dport 853 drop
            oifname { "${wan}" } tcp dport 53 drop
            oifname { "${wan}" } udp dport 53 drop

            # Allow trusted LAN to WAN"
            iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept
            iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept


            iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept
            iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept
            iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept
            iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept

            # Allow containers to reach WAN
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${wan}" } accept
            iifname { "${wan}" } oifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } ct state established, related accept

            # Allow containers to reach the DNS and NFS server
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept


            # Rules to make CNI happy
            meta mark and 0x01 == 0x01 accept

            meta nftrace set 1
          }
        }

        table ip nf_nat {
          chain postrouting {
            type nat hook postrouting priority 100; policy accept;
            oifname "${wan}" masquerade
          }

          chain prerouting {
            type nat hook prerouting priority 100; policy accept;
          }
        }

        table ip6 nf_filter {
          chain output {
            type filter hook output priority 0; policy drop;

            meta nftrace set 1

            oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
            oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject
          }
          chain input {
            type filter hook input priority 0; policy drop;

            meta nftrace set 1

            iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
          }
          chain forward {
            type filter hook forward priority 0; policy drop;
          }
        }
      '';
    };
  };

  systemd.services.nftables = {
    serviceConfig =
      let
        rulesScript = pkgs.writeShellScript "nftables-rules" ''
          set -ex
          export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH

          tmpfile="$(mktemp)"
          iptables-save -t filter >> $tmpfile
          iptables-save -t nat >> $tmpfile

          nft flush ruleset

          cat $tmpfile | iptables-restore
          nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}"
          rm $tmpfile

          iptables -D FORWARD -j MARK --set-mark 0x01 || true
          iptables -D FORWARD -j MARK --set-mark 0x00 || true

          iptables -I FORWARD -j MARK --set-mark 0x01
          iptables -A FORWARD -j MARK --set-mark 0x00
        '';
      in {
        ExecStart = mkForce rulesScript;
        ExecReload = mkForce rulesScript;
        ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" ''
          set -ex
          export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH

          tmpfile="$(mktemp)"
          iptables-save -t filter >> $tmpfile
          iptables-save -t nat >> $tmpfile

          nft flush ruleset

          cat $tmpfile | iptables-restore
          rm $tmpfile

          iptables -D FORWARD -j MARK --set-mark 0x01 || true
          iptables -D FORWARD -j MARK --set-mark 0x00 || true

          iptables -I FORWARD -j MARK --set-mark 0x01
          iptables -A FORWARD -j MARK --set-mark 0x00
        '');
      };
  };
}