{ pkgs, secret, config, lib, ... }: let inherit (lib) mapAttrs const mkForce singleton ; wlan = "wlp10s0"; lan = "eno1"; wan = "eno3"; doVPN = "do_vpn0"; nomad = mapAttrs (const toString) { inherit (config.services.hashicorp.nomad.settings.client) min_dynamic_port max_dynamic_port ; }; in { boot.kernel.sysctl = { # Enable forwarding on IPv4 but disable on IPv6 "net.ipv4.conf.all.forwarding" = true; "net.ipv6.conf.all.forwarding" = false; # source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52 # By default, not automatically configure any IPv6 addresses. "net.ipv6.conf.all.accept_ra" = 0; "net.ipv6.conf.all.autoconf" = 0; "net.ipv6.conf.all.use_tempaddr" = 0; # On WAN, allow IPv6 autoconfiguration and tempory address use. # "net.ipv6.conf.${name}.accept_ra" = 2; # "net.ipv6.conf.${name}.autoconf" = 1; }; services.dnscrypt-proxy2 = { enable = true; upstreamDefaults = true; settings = { listen_addresses = singleton "127.0.0.1:5353"; dnscrypt_servers = false; doh_servers = true; odoh_servers = false; block_ipv6 = true; static."mullvad".stamp = "sdns://AgcAAAAAAAAACzE5NC4yNDIuMi41ABxleHRlbmRlZC5kbnMubXVsbHZhZC5uZXQ6NDQzCi9kbnMtcXVlcnk"; sources = {}; max_clients = 1024; cache_size = 32768; }; }; systemd.services.dnscrypt-proxy2 = { before = ["network-online.target"]; }; services.kea.dhcp4 = { enable = true; settings = { interfaces-config.interfaces = [ "${lan}" ]; lease-database = { name = "/var/lib/kea/dhcp4.leases"; persist = true; type = "memfile"; }; rebind-timer = 2000; renew-timer = 1000; } // (secret.dhcp.blowhole.zones or (const {}) {inherit wlan lan;}); }; networking = { useDHCP = false; hostName = "blowhole"; resolvconf.useLocalResolver = false; nameservers = singleton (secret.network.ips.blowhole.ip or ""); # Disable the in-built iptable based firewall firewall.enable = mkForce false; interfaces = { # Don't do DHCP on the LAN interface "${lan}" = { useDHCP = false; ipv4.addresses = [ { address = secret.network.ips.blowhole.ip or ""; prefixLength = 24; } ]; }; # "${wlan}" = { # useDHCP = false; # ipv4.addresses = [{ # address = secret.network.ips.blowhole.wlan or ""; # prefixLength = 24; # }]; # }; # But do DHCP on the WAN interface "${wan}".useDHCP = true; }; wireguard = { enable = true; interfaces."${doVPN}" = secret.wireguard."${config.networking.hostName}" or {} // { listenPort = 6666; privateKeyFile = "/var/secrets/${doVPN}.key"; }; }; nftables = { enable = true; ruleset = '' table ip nf_filter { chain input_out { ct state { established, related } accept comment "Allow established traffic" icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain input_doVPN { tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic" tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic" tcp dport { 8200 } accept comment "Vault traffic" tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" tcp dport { 53 } accept comment "DNS traffic" tcp dport { 80 } accept comment "HTTP traffic" udp dport { 8600, 8301, 8302 } comment "Consul traffic" udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" udp dport { 53 } accept comment "DNS traffic" } chain input { type filter hook input priority 0; policy drop; iifname != "${wan}" tcp dport 22 accept comment "Accept SSH traffic always" iifname != "lo" tcp dport 5353 drop comment "Drop traffic to dnscrypt-proxy always except for localhost to localhost traffic" # Accept WireGuard iifname "${wan}" udp dport 6666 accept; # Accept file-share iifname "${wan}" tcp dport 5666 accept; # Accept minecraft iifname "${wan}" tcp dport 25560 accept; # iifname "cni0" accept; iifgroup { 99 } tcp dport { 6443, 10250, 4244 } accept iifgroup { 99 } oifgroup { 99 } accept iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr 10.64.48.0/21 accept iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr != 10.0.0.0/8 ip daddr != 192.168.0.0/16 ip daddr != 172.0.0.0/12 accept iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept comment "Allow Nomad to do whatever it wants in its interface" iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router" iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper", "mvm0", "uk3s0" } jump input_out iifgroup 99 jump input_out # iifname { "${doVPN}", "${lan}" } tcp dport { 6443 } accept iifname { "${doVPN}", "${lan}" } ip daddr 10.64.2.1 tcp dport { 8833 } accept iifname { "${doVPN}" } jump input_doVPN # Allow containers to reach the DNS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } tcp dport 53 accept iifgroup { 99 } tcp dport 53 accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } udp dport 53 accept iifgroup { 99 } udp dport 53 accept # Allow Nomad Containers to reach Nomad iifname { "nomad" } tcp dport 4646 accept # Allow proxies to reach consul iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept # Allow containers to reach the NFS server iifname { "docker0", "uk3s0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" iifname { "docker0", "uk3s0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" iifname { "${wan}" } udp dport 2302-2304 ip daddr == 192.168.2.20 accept # drop pointless IoT discovery traffic explicitly iifname { "${wan}" } ip daddr == 255.255.255.255 udp dport == 6667 drop log prefix "[drop] nf_filter.input: " queue-threshold 1 group 2 drop } chain output { type filter hook output priority 0; policy accept; # Drop all DNS traffic if leaving through "wan" # oifname { "${wan}" } tcp dport 53 drop # oifname { "${wan}" } udp dport 53 drop # Allow DoT traffic to leave through "wan" if it comes from "lo" # iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop # log prefix "[drop] nf_filter.output: " queue-threshold 1 group 2 drop } chain forward { type filter hook forward priority 10; policy drop; # Enable flow offloading for better throughput # ip protocol { tcp, udp } flow offload @f ip daddr 10.64.52.130 nftrace set 1 ip saddr 10.64.52.130 nftrace set 1 # Drop all DNS or DoT traffic if forwarded through "wan" oifname { "${wan}" } tcp dport 853 drop oifname { "${wan}" } tcp dport 53 drop oifname { "${wan}" } udp dport 53 drop # Allow trusted LAN to WAN" iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept iifgroup { 99 } oifname { "${doVPN}", "${lan}", "${wlan}" } ct state established, related accept iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept # Allow containers to reach WAN iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${wan}" } accept iifgroup { 99 } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } ct state established, related accept iifname { "${wan}" } oifgroup { 99 } ct state established, related accept # Allow containers to reach the DNS and NFS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0"} oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept # allow communication between all container interfaces iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept iifgroup { 99 } oifgroup { 99 } accept iifname { "${lan}" } oifgroup { 99 } accept # allow portforwarding to lan? iifname { "${wan}" } oifname { "${lan}" } ip daddr 10.64.2.20 udp dport 2302-2304 accept # allow lan to access kubernetes http ingress iifname { "${lan}" } oifname { "uk3s0" } ip daddr 172.26.96.2 tcp dport 80 accept # allow vpn to access kubernetes http ingress iifname { "${doVPN}" } oifname { "uk3s0" } ip saddr 10.64.0.1 ip daddr 172.26.96.2 tcp dport 80 accept # allow kubernetes to respond to incoming traffic iifname { "uk3s0" } oifname { "${lan}", "${doVPN}" } jump input_out # Rules to make CNI happy meta mark and 0x01 == 0x01 accept log prefix "[drop] nf_filter.forward: " queue-threshold 1 group 2 drop } } table ip nf_nat { # TCP: 2344-2345, 27015, 27036 # UDP: 2302-2306, 2344, 27015, 27031-27036 chain postrouting { type nat hook postrouting priority 100; policy accept; oifname "${wan}" masquerade udp dport 2302-2304 ip daddr { 192.168.2.20 } masquerade iifname "${doVPN}" tcp dport 8344 ip daddr 172.26.96.2 masquerade } chain prerouting { type nat hook prerouting priority 100; policy accept; ip daddr { 192.168.2.20 } udp dport 2302-2304 dnat 10.64.2.20 ip daddr { ${secret.network.ips.blowhole.ip or ""} } tcp dport 8344 dnat 172.26.96.2:80 } } table ip6 nf_filter { chain output { type filter hook output priority 0; policy drop; # meta nftrace set 1 oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject } chain input { type filter hook input priority 0; policy drop; # meta nftrace set 1 iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain forward { type filter hook forward priority 0; policy drop; } } ''; }; }; services.ulogd = { enable = true; settings = { # This one for logging to local file in emulated syslog format. global.stack = "log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOGEMU"; log2.group = 2; emu1 = { file = "/var/log/nft_drop.log"; sync = 1; }; }; }; systemd.services.nftables = { path = with pkgs; [ nftables iptables bash ]; serviceConfig = let rulesScript = pkgs.writeShellScript "nftables-rules" '' set -ex exit_code=0 dynamic_iptables_rules="$(mktemp)" iptables-save -t filter >> $dynamic_iptables_rules iptables-save -t nat >> $dynamic_iptables_rules static_nftables_rules="$(mktemp)" nft list table ip nf_filter >> $static_nftables_rules || nft create table ip nf_filter nft list table ip nf_nat >> $static_nftables_rules || nft create table ip nf_nat nft list table ip6 nf_filter >> $static_nftables_rules || nft create table ip6 nf_filter nft flush ruleset iptables-restore < $dynamic_iptables_rules if nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}" ; then iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 else echo "Apply failed, restoring previous rules!" nft -f $static_nftables_rules exit_code=1 fi rm $dynamic_iptables_rules $static_nftables_rules exit $exit_code ''; in { ExecStart = mkForce rulesScript; ExecReload = mkForce rulesScript; ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" '' set -ex tmpfile="$(mktemp)" iptables-save -t filter >> $tmpfile iptables-save -t nat >> $tmpfile nft flush ruleset cat $tmpfile | iptables-restore rm $tmpfile iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 ''); }; }; }