{ pkgs, secret, config, lib, ... }: let inherit (lib) mapAttrs const mkForce singleton; wlan = "wlp10s0"; lan = "enp8s0f1"; wan = "enp3s0"; doVPN = "do_vpn0"; nomad = mapAttrs (const toString) { inherit (config.services.hashicorp.nomad.settings.client) min_dynamic_port max_dynamic_port; }; in { boot.kernel.sysctl = { # Enable forwarding on IPv4 but disable on IPv6 "net.ipv4.conf.all.forwarding" = true; "net.ipv6.conf.all.forwarding" = false; # source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52 # By default, not automatically configure any IPv6 addresses. "net.ipv6.conf.all.accept_ra" = 0; "net.ipv6.conf.all.autoconf" = 0; "net.ipv6.conf.all.use_tempaddr" = 0; # On WAN, allow IPv6 autoconfiguration and tempory address use. # "net.ipv6.conf.${name}.accept_ra" = 2; # "net.ipv6.conf.${name}.autoconf" = 1; }; services.dnscrypt-proxy2 = { enable = true; upstreamDefaults = true; settings = { listen_addresses = singleton "127.0.0.1:5353"; dnscrypt_servers = false; doh_servers = true; odoh_servers = false; block_ipv6 = true; static."mullvad".stamp = "sdns://AgcAAAAAAAAAAAAPZG9oLm11bGx2YWQubmV0Ci9kbnMtcXVlcnk"; static."meganerd".stamp = "sdns://AQcAAAAAAAAADjEzNi4yNDQuOTcuMTE0ICif6V9M6EF_9Xo_MHwkDN4ZJjERopSJN8hBuUWg9YeMJTIuZG5zY3J5cHQtY2VydC5jaGV3YmFjY2EubWVnYW5lcmQubmw"; sources = {}; }; }; systemd.services.dnscrypt-proxy2 = { before = [ "network-online.target" ]; }; services.dhcpd4 = { enable = true; interfaces = [ "${lan}" "${wlan}" ]; extraConfig = '' option domain-name-servers ${secret.network.ips.blowhole.ip or ""}; option subnet-mask 255.255.255.0; ${secret.dhcp.blowhole.zones or (const "") { inherit wlan lan; }} ''; }; networking = { useDHCP = false; hostName = "blowhole"; resolvconf.useLocalResolver = false; nameservers = singleton (secret.network.ips.blowhole.ip or ""); # Disable the in-built iptable based firewall firewall.enable = mkForce false; localCommands = '' ip link add enp4s0 type dummy || true ip link set enp4s0 up || true ip addr add ${secret.network.ips.blowhole.ip or ""}/24 dev enp4s0 || true ''; interfaces = { # Don't do DHCP on the LAN interface "${lan}" = { useDHCP = false; ipv4.addresses = [{ address = secret.network.ips.blowhole.ip or ""; prefixLength = 24; }]; }; "${wlan}" = { useDHCP = false; ipv4.addresses = [{ address = secret.network.ips.blowhole.wlan or ""; prefixLength = 24; }]; }; # But do DHCP on the WAN interface "${wan}".useDHCP = true; }; wireguard = { enable = true; interfaces."${doVPN}" = secret.wireguard."${config.networking.hostName}" or {} // { listenPort = 6666; privateKeyFile = "/var/secrets/${doVPN}.key"; }; }; nftables = { enable = true; ruleset = '' table ip nf_filter { chain input_out { ct state { established, related } accept comment "Allow established traffic" icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain input_doVPN { tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic" tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic" tcp dport { 8200 } accept comment "Vault traffic" tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" tcp dport { 53 } accept comment "DNS traffic" tcp dport { 80 } accept comment "HTTP traffic" udp dport { 8600, 8301, 8302 } comment "Consul traffic" udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" udp dport { 53 } accept comment "DNS traffic" } chain input { type filter hook input priority 0; policy drop; tcp dport 22 accept comment "Accept SSH traffic always" iifname != "lo" tcp dport 5353 drop comment "Drop traffic to dnscrypt-proxy always except for localhost to localhost traffic" iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface" iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router" iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper" } jump input_out iifname { "${doVPN}" } jump input_doVPN # Allow containers to reach the DNS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } tcp dport 53 accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } udp dport 53 accept # Allow Nomad Containers to reach Nomad iifname { "nomad" } tcp dport 4646 accept # Allow proxies to reach consul iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept # Allow containers to reach the NFS server iifname { "docker0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" iifname { "docker0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" meta nftrace set 1 } chain output { type filter hook output priority 0; policy accept; # Drop all DNS traffic if leaving through "wan" # oifname { "${wan}" } tcp dport 53 drop # oifname { "${wan}" } udp dport 53 drop # Allow DoT traffic to leave through "wan" if it comes from "lo" # iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop } chain forward { type filter hook forward priority 10; policy drop; # Enable flow offloading for better throughput # ip protocol { tcp, udp } flow offload @f # Drop all DNS or DoT traffic if forwarded through "wan" oifname { "${wan}" } tcp dport 853 drop oifname { "${wan}" } tcp dport 53 drop oifname { "${wan}" } udp dport 53 drop # Allow trusted LAN to WAN" iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept # Allow containers to reach WAN iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } ct state established, related accept # Allow containers to reach the DNS and NFS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept # allow communication between all container interfaces iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept # Rules to make CNI happy meta mark and 0x01 == 0x01 accept meta nftrace set 1 } } table ip nf_nat { chain postrouting { type nat hook postrouting priority 100; policy accept; oifname "${wan}" masquerade } chain prerouting { type nat hook prerouting priority 100; policy accept; } } table ip6 nf_filter { chain output { type filter hook output priority 0; policy drop; meta nftrace set 1 oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject } chain input { type filter hook input priority 0; policy drop; meta nftrace set 1 iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain forward { type filter hook forward priority 0; policy drop; } } ''; }; }; systemd.services.nftables = { serviceConfig = let rulesScript = pkgs.writeShellScript "nftables-rules" '' set -ex export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH tmpfile="$(mktemp)" iptables-save -t filter >> $tmpfile iptables-save -t nat >> $tmpfile nft flush ruleset cat $tmpfile | iptables-restore nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}" rm $tmpfile iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 ''; in { ExecStart = mkForce rulesScript; ExecReload = mkForce rulesScript; ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" '' set -ex export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH tmpfile="$(mktemp)" iptables-save -t filter >> $tmpfile iptables-save -t nat >> $tmpfile nft flush ruleset cat $tmpfile | iptables-restore rm $tmpfile iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 ''); }; }; }