dotfiles/nixos/systems/blowhole/firewall.nix

{
  pkgs,
  secret,
  config,
  lib,
  ...
}: let
  inherit
    (lib)
    mapAttrs
    const
    mkForce
    singleton
    ;

  wlan = "wlp10s0";
  lan = "eno1";
  wan = "eno3";
  doVPN = "do_vpn0";

  nomad = mapAttrs (const toString) {
    inherit
      (config.services.hashicorp.nomad.settings.client)
      min_dynamic_port
      max_dynamic_port
      ;
  };
in {
  boot.kernel.sysctl = {
    # Enable forwarding on IPv4 but disable on IPv6
    "net.ipv4.conf.all.forwarding" = true;
    "net.ipv6.conf.all.forwarding" = false;

    # source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52
    # By default, not automatically configure any IPv6 addresses.
    "net.ipv6.conf.all.accept_ra" = 0;
    "net.ipv6.conf.all.autoconf" = 0;
    "net.ipv6.conf.all.use_tempaddr" = 0;

    # On WAN, allow IPv6 autoconfiguration and tempory address use.
    # "net.ipv6.conf.${name}.accept_ra" = 2;
    # "net.ipv6.conf.${name}.autoconf" = 1;
  };

  services.dnscrypt-proxy2 = {
    enable = true;
    upstreamDefaults = true;
    settings = {
      listen_addresses = singleton "127.0.0.1:5353";

      dnscrypt_servers = false;
      doh_servers = true;
      odoh_servers = false;

      block_ipv6 = true;

      static."mullvad".stamp = "sdns://AgcAAAAAAAAACzE5NC4yNDIuMi41ABxleHRlbmRlZC5kbnMubXVsbHZhZC5uZXQ6NDQzCi9kbnMtcXVlcnk";
      sources = {};

      max_clients = 1024;

      cache_size = 32768;
    };
  };

  systemd.services.dnscrypt-proxy2 = {
    before = ["network-online.target"];
  };

  services.kea.dhcp4 = {
    enable = true;
    settings =
      {
        interfaces-config.interfaces = [
          "${lan}"
        ];
        lease-database = {
          name = "/var/lib/kea/dhcp4.leases";
          persist = true;
          type = "memfile";
        };
        rebind-timer = 2000;
        renew-timer = 1000;
      }
      // (secret.dhcp.blowhole.zones or (const {}) {inherit wlan lan;});
  };

  networking = {
    useDHCP = false;
    hostName = "blowhole";

    resolvconf.useLocalResolver = false;
    nameservers = singleton (secret.network.ips.blowhole.ip or "");

    # Disable the in-built iptable based firewall
    firewall.enable = mkForce false;

    interfaces = {
      # Don't do DHCP on the LAN interface
      "${lan}" = {
        useDHCP = false;
        ipv4.addresses = [
          {
            address = secret.network.ips.blowhole.ip or "";
            prefixLength = 24;
          }
        ];
      };
      # "${wlan}" = {
      #   useDHCP = false;
      #   ipv4.addresses = [{
      #     address = secret.network.ips.blowhole.wlan or "";
      #     prefixLength = 24;
      #   }];
      # };
      # But do DHCP on the WAN interface
      "${wan}".useDHCP = true;
    };

    wireguard = {
      enable = true;
      interfaces."${doVPN}" =
        secret.wireguard."${config.networking.hostName}"
        or {}
        // {
          listenPort = 6666;
          privateKeyFile = "/var/secrets/${doVPN}.key";
        };
    };

    nftables = {
      enable = true;
      ruleset = ''
        table ip nf_filter {
          chain input_out {
            ct state { established, related } accept comment "Allow established traffic"
            icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
          }

          chain input_doVPN {
            tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic"
            tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic"
            tcp dport { 8200 } accept comment "Vault traffic"
            tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
            tcp dport { 53 } accept comment "DNS traffic"
            tcp dport { 80 } accept comment "HTTP traffic"

            udp dport { 8600, 8301, 8302 } comment "Consul traffic"
            udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
            udp dport { 53 } accept comment "DNS traffic"
          }

          chain input {
            type filter hook input priority 0; policy drop;

            iifname != "${wan}" tcp dport 22 accept comment "Accept SSH traffic always"
            iifname != "lo" tcp dport 5353 drop comment "Drop traffic to dnscrypt-proxy always except for localhost to localhost traffic"

            # Accept WireGuard
            iifname "${wan}" udp dport 6666 accept;

            # Accept file-share
            iifname "${wan}" tcp dport 5666 accept;

            # Accept minecraft
            iifname "${wan}" tcp dport 25560 accept;

            # iifname "cni0" accept;
            iifgroup { 99 } tcp dport { 6443, 10250, 4244 } accept
            iifgroup { 99 } oifgroup { 99 } accept
            iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr 10.64.48.0/21 accept
            iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr != 10.0.0.0/8 ip daddr != 192.168.0.0/16 ip daddr != 172.0.0.0/12 accept

            iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept comment "Allow Nomad to do whatever it wants in its interface"
            iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router"
            iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper", "mvm0", "uk3s0", "ve-ywecur-anp" } jump input_out
            iifgroup 99 jump input_out
            # iifname { "${doVPN}", "${lan}" } tcp dport { 6443 } accept
            iifname { "${doVPN}", "${lan}" } ip daddr 10.64.2.1 tcp dport { 8833 } accept
            iifname { "${doVPN}" } jump input_doVPN

            # Allow containers to reach the DNS server
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } tcp dport 53 accept
            iifgroup { 99 } tcp dport 53 accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } udp dport 53 accept
            iifgroup { 99 } udp dport 53 accept

            # Allow Nomad Containers to reach Nomad
            iifname { "nomad" } tcp dport 4646 accept

            # Allow proxies to reach consul
            iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept
            iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept

            # Allow containers to reach the NFS server
            iifname { "docker0", "uk3s0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
            iifname { "docker0", "uk3s0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"

            iifname { "${wan}" } udp dport 2302-2304 ip daddr == 192.168.2.20 accept

            # drop pointless IoT discovery traffic explicitly
            iifname { "${wan}" } ip daddr == 255.255.255.255 udp dport == 6667 drop

            iifname { "uk3s0" } tcp dport 8200 ip daddr 10.64.2.1 accept

            log prefix "[drop] nf_filter.input: " queue-threshold 1 group 2 drop
          }

          chain output {
            type filter hook output priority 0; policy accept;

            # Drop all DNS traffic if leaving through "wan"
            # oifname { "${wan}" } tcp dport 53 drop
            # oifname { "${wan}" } udp dport 53 drop
            # Allow DoT traffic to leave through "wan" if it comes from "lo"
            # iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop

            # log prefix "[drop] nf_filter.output: " queue-threshold 1 group 2 drop
          }

          chain forward {
            type filter hook forward priority 10; policy drop;

            # Enable flow offloading for better throughput
            # ip protocol { tcp, udp } flow offload @f

            ip daddr 10.64.52.130 nftrace set 1
            ip saddr 10.64.52.130 nftrace set 1

            # Drop all DNS or DoT traffic if forwarded through "wan"
            oifname { "${wan}" } tcp dport 853 drop
            oifname { "${wan}" } tcp dport 53 drop
            oifname { "${wan}" } udp dport 53 drop

            # Allow trusted LAN to WAN"
            iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept
            iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept

            iifgroup { 99 } oifname { "${doVPN}", "${lan}", "${wlan}" } ct state established, related accept

            iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept
            iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept
            iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept
            iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept

            # Allow containers to reach WAN
            iifname { "ve-ywecur-anp", "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${wan}" } accept
            iifgroup { 99 } oifname { "${wan}" } accept
            iifname { "${wan}" } oifname { "ve-ywecur-anp", "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } ct state established, related accept
            iifname { "${wan}" } oifgroup { 99 } ct state established, related accept

            # Allow containers to reach the DNS and NFS server
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0"} oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
            iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept

            # allow communication between all container interfaces
            iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept
            iifgroup { 99 } oifgroup { 99 } accept
            iifname { "${lan}" } oifgroup { 99 } accept

            # allow portforwarding to lan?
            iifname { "${wan}" } oifname { "${lan}" } ip daddr 10.64.2.20 udp dport 2302-2304 accept

            # allow lan to access kubernetes http ingress
            iifname { "${lan}" } oifname { "uk3s0" } ip daddr 172.26.96.2 tcp dport 80 accept

            # allow vpn to access kubernetes http ingress
            iifname { "${doVPN}" } oifname { "uk3s0" } ip saddr 10.64.0.1 ip daddr 172.26.96.2 tcp dport 80 accept
            # allow vpn to access kubernetes forgejo ssh
            iifname { "${doVPN}" } oifname { "uk3s0" } ip saddr 10.64.0.1 ip daddr 172.26.96.2 tcp dport 19022 accept

            # allow kubernetes to respond to incoming traffic
            iifname { "uk3s0" } oifname { "${lan}", "${doVPN}" } jump input_out

            # Rules to make CNI happy
            meta mark and 0x01 == 0x01 accept

            iifname "${wan}" oifname "ve-ywecur-anp" ct status dnat accept

            log prefix "[drop] nf_filter.forward: " queue-threshold 1 group 2 drop
          }
        }

        table ip nf_nat {
          # TCP: 2344-2345, 27015, 27036
          # UDP: 2302-2306, 2344, 27015, 27031-27036

          chain postrouting {
            type nat hook postrouting priority 100; policy accept;
            oifname "${wan}" masquerade
            udp dport 2302-2304 ip daddr { 192.168.2.20 } masquerade
            iifname "${doVPN}" tcp dport 8344 ip daddr 172.26.96.2 masquerade
          }

          chain prerouting {
            type nat hook prerouting priority 100; policy accept;
            ip daddr { 192.168.2.20 } udp dport 2302-2304 dnat 10.64.2.20
            ip daddr { ${secret.network.ips.blowhole.ip or ""} } tcp dport 8344 dnat 172.26.96.2:80
            ip daddr { ${secret.network.ips.blowhole.ip or ""} } tcp dport 19022 dnat 172.26.96.2:19022
            # ywecur anp container
            ip daddr { 192.168.2.20 } tcp dport 2288 dnat 172.20.69.2:22
          }
        }

        table ip6 nf_filter {
          chain output {
            type filter hook output priority 0; policy drop;

            # meta nftrace set 1

            oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
            oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject
          }
          chain input {
            type filter hook input priority 0; policy drop;

            # meta nftrace set 1

            iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
          }
          chain forward {
            type filter hook forward priority 0; policy drop;
          }
        }
      '';
    };
  };

  services.ulogd = {
    enable = true;
    settings = {
      # This one for logging to local file in emulated syslog format.
      global.stack = "log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOGEMU";

      log2.group = 2;

      emu1 = {
        file = "/var/log/nft_drop.log";
        sync = 1;
      };
    };
  };

  systemd.services.nftables = {
    path = with pkgs; [
      nftables
      iptables
      bash
    ];
    serviceConfig = let
      rulesScript = pkgs.writeShellScript "nftables-rules" ''
        set -ex
        exit_code=0

        dynamic_iptables_rules="$(mktemp)"
        iptables-save -t filter >> $dynamic_iptables_rules
        iptables-save -t nat >> $dynamic_iptables_rules

        static_nftables_rules="$(mktemp)"
        nft list table ip nf_filter >> $static_nftables_rules || nft create table ip nf_filter
        nft list table ip nf_nat >> $static_nftables_rules || nft create table ip nf_nat
        nft list table ip6 nf_filter >> $static_nftables_rules || nft create table ip6 nf_filter

        nft flush ruleset

        iptables-restore < $dynamic_iptables_rules
        if nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}" ; then
          iptables -D FORWARD -j MARK --set-mark 0x01 || true
          iptables -D FORWARD -j MARK --set-mark 0x00 || true

          iptables -I FORWARD -j MARK --set-mark 0x01
          iptables -A FORWARD -j MARK --set-mark 0x00
        else
          echo "Apply failed, restoring previous rules!"
          nft -f $static_nftables_rules
          exit_code=1
        fi

        rm $dynamic_iptables_rules $static_nftables_rules

        exit $exit_code
      '';
    in {
      ExecStart = mkForce rulesScript;
      ExecReload = mkForce rulesScript;
      ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" ''
        set -ex

        tmpfile="$(mktemp)"
        iptables-save -t filter >> $tmpfile
        iptables-save -t nat >> $tmpfile

        nft flush ruleset

        cat $tmpfile | iptables-restore
        rm $tmpfile

        iptables -D FORWARD -j MARK --set-mark 0x01 || true
        iptables -D FORWARD -j MARK --set-mark 0x00 || true

        iptables -I FORWARD -j MARK --set-mark 0x01
        iptables -A FORWARD -j MARK --set-mark 0x00
      '');
    };
  };
}