{ pkgs, secret, config, lib, ... }: with lib; let wlan = "wlp10s0"; lan = "enp8s0f1"; wan = "enp3s0"; doVPN = "do_vpn0"; nomad = mapAttrs (const toString) { inherit (config.services.hashicorp.nomad.settings.client) min_dynamic_port max_dynamic_port; }; in { boot.kernel.sysctl = { # Enable forwarding on IPv4 but disable on IPv6 "net.ipv4.conf.all.forwarding" = true; "net.ipv6.conf.all.forwarding" = false; # source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52 # By default, not automatically configure any IPv6 addresses. "net.ipv6.conf.all.accept_ra" = 0; "net.ipv6.conf.all.autoconf" = 0; "net.ipv6.conf.all.use_tempaddr" = 0; # On WAN, allow IPv6 autoconfiguration and tempory address use. # "net.ipv6.conf.${name}.accept_ra" = 2; # "net.ipv6.conf.${name}.autoconf" = 1; }; services.stubby = { enable = true; logLevel = 7; settings = { resolution_type = "GETDNS_RESOLUTION_STUB"; dns_transport_list = [ "GETDNS_TRANSPORT_TLS" ]; tls_authentication = "GETDNS_AUTHENTICATION_REQUIRED"; tls_query_padding_blocksize = 256; edns_client_subnet_private = 1; idle_timeout = 10000; listen_addresses = [ "127.0.0.1@5353" ]; dnssec_return_status = "GETDNS_EXTENSION_TRUE"; appdata_dir = "/var/cache/stubby"; round_robin_upstreams = 1; upstream_recursive_servers = [ { address_data = "9.9.9.9"; tls_auth_name = "dns.quad9.net"; } { address_data = "149.112.112.112"; tls_auth_name = "dns.quad9.net"; } ]; }; }; systemd.services.stubby = { before = [ "network-online.target" ]; }; services.dhcpd4 = { enable = true; interfaces = [ "${lan}" "${wlan}" ]; extraConfig = '' option domain-name-servers 10.64.2.1; option subnet-mask 255.255.255.0; subnet 10.64.2.0 netmask 255.255.255.0 { option broadcast-address 10.64.2.255; option routers 10.64.2.1; interface ${lan}; range 10.64.2.128 10.64.2.254; ${secret.network.dhcpd.blowhole-lan} } subnet 10.64.3.0 netmask 255.255.255.0 { option broadcast-address 10.64.3.255; option routers 10.64.3.1; interface ${wlan}; range 10.64.3.128 10.64.3.254; ${secret.network.dhcpd.blowhole-wlan} } ''; }; # systemd.services.dns-check = { # before = [ "network-online.target"]; # serviceConfig.Type = "oneshot"; # script = '' # ''; # }; networking = { useDHCP = false; hostName = "blowhole"; resolvconf.useLocalResolver = false; nameservers = [ "10.64.2.1" ]; # Disable the in-built iptable based firewall firewall.enable = mkForce false; localCommands = '' ip link add enp4s0 type dummy || true ip link set enp4s0 up || true ip addr add 10.64.2.1/24 dev enp4s0 || true ''; interfaces = { # Don't do DHCP on the LAN interface "${lan}" = { useDHCP = false; ipv4.addresses = [{ address = "10.64.2.1"; prefixLength = 24; }]; }; "${wlan}" = { useDHCP = false; ipv4.addresses = [{ address = "10.64.3.1"; prefixLength = 24; }]; }; # But do DHCP on the WAN interface "${wan}".useDHCP = true; }; wireguard = { enable = true; interfaces."${doVPN}" = config.magic_rb.secret.wireguard."${config.networking.hostName}" or {} // { listenPort = 6666; privateKeyFile = "/var/secrets/${doVPN}.key"; }; }; nftables = { enable = true; ruleset = '' table ip nf_filter { chain input_out { ct state { established, related } accept comment "Allow established traffic" icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain input_doVPN { tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic" tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic" tcp dport { 8200 } accept comment "Vault traffic" tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" tcp dport { 53 } accept comment "DNS traffic" tcp dport { 80 } accept comment "HTTP traffic" udp dport { 8600, 8301, 8302 } comment "Consul traffic" udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic" udp dport { 53 } accept comment "DNS traffic" } chain input { type filter hook input priority 0; policy drop; tcp dport 22 accept comment "Accept SSH traffic always" iifname != "lo" tcp dport 5353 drop comment "Drop traffic to stubby always except for localhost to localhost traffic" iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface" iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router" iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper" } jump input_out iifname { "${doVPN}" } jump input_doVPN # Allow containers to reach the DNS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } tcp dport 53 accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } udp dport 53 accept # Allow proxies to reach consul iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept # Allow containers to reach the NFS server iifname { "docker0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" iifname { "docker0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic" meta nftrace set 1 } chain output { type filter hook output priority 0; policy accept; # Drop all DNS traffic if leaving through "wan" oifname { "${wan}" } tcp dport 53 drop oifname { "${wan}" } udp dport 53 drop # Allow DoT traffic to leave through "wan" if it comes from "lo" iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop } chain forward { type filter hook forward priority 10; policy drop; # Enable flow offloading for better throughput # ip protocol { tcp, udp } flow offload @f # Drop all DNS or DoT traffic if forwarded through "wan" oifname { "${wan}" } tcp dport 853 drop oifname { "${wan}" } tcp dport 53 drop oifname { "${wan}" } udp dport 53 drop # Allow trusted LAN to WAN" iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept # Allow containers to reach WAN iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${wan}" } accept iifname { "${wan}" } oifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } ct state established, related accept # Allow containers to reach the DNS and NFS server iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept iifname { "nomad", "docker0", "ve-monitor", "ve-klipper" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept # Rules to make CNI happy meta mark and 0x01 == 0x01 accept meta nftrace set 1 } } table ip nf_nat { chain postrouting { type nat hook postrouting priority 100; policy accept; oifname "${wan}" masquerade } chain prerouting { type nat hook prerouting priority 100; policy accept; } } table ip6 nf_filter { chain output { type filter hook output priority 0; policy drop; meta nftrace set 1 oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject } chain input { type filter hook input priority 0; policy drop; meta nftrace set 1 iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP" } chain forward { type filter hook forward priority 0; policy drop; } } ''; }; }; systemd.services.nftables = { serviceConfig = let rulesScript = pkgs.writeShellScript "nftables-rules" '' set -ex export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH tmpfile="$(mktemp)" iptables-save -t filter >> $tmpfile iptables-save -t nat >> $tmpfile nft flush ruleset cat $tmpfile | iptables-restore nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}" rm $tmpfile iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 ''; in { ExecStart = mkForce rulesScript; ExecReload = mkForce rulesScript; ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" '' set -ex export PATH=${pkgs.nftables}/bin:${pkgs.iptables}/bin:${pkgs.bash}/bin:$PATH tmpfile="$(mktemp)" iptables-save -t filter >> $tmpfile iptables-save -t nat >> $tmpfile nft flush ruleset cat $tmpfile | iptables-restore rm $tmpfile iptables -D FORWARD -j MARK --set-mark 0x01 || true iptables -D FORWARD -j MARK --set-mark 0x00 || true iptables -I FORWARD -j MARK --set-mark 0x01 iptables -A FORWARD -j MARK --set-mark 0x00 ''); }; }; }