mirror of
https://git.sr.ht/~magic_rb/dotfiles
synced 2024-12-11 17:31:58 +01:00
414 lines
16 KiB
Nix
414 lines
16 KiB
Nix
{
|
|
pkgs,
|
|
secret,
|
|
config,
|
|
lib,
|
|
...
|
|
}: let
|
|
inherit
|
|
(lib)
|
|
mapAttrs
|
|
const
|
|
mkForce
|
|
singleton
|
|
;
|
|
|
|
wlan = "wlp10s0";
|
|
lan = "eno1";
|
|
wan = "eno3";
|
|
doVPN = "do_vpn0";
|
|
|
|
nomad = mapAttrs (const toString) {
|
|
inherit
|
|
(config.services.hashicorp.nomad.settings.client)
|
|
min_dynamic_port
|
|
max_dynamic_port
|
|
;
|
|
};
|
|
in {
|
|
boot.kernel.sysctl = {
|
|
# Enable forwarding on IPv4 but disable on IPv6
|
|
"net.ipv4.conf.all.forwarding" = true;
|
|
"net.ipv6.conf.all.forwarding" = false;
|
|
|
|
# source: https://github.com/mdlayher/homelab/blob/master/nixos/routnerr-2/configuration.nix#L52
|
|
# By default, not automatically configure any IPv6 addresses.
|
|
"net.ipv6.conf.all.accept_ra" = 0;
|
|
"net.ipv6.conf.all.autoconf" = 0;
|
|
"net.ipv6.conf.all.use_tempaddr" = 0;
|
|
|
|
# On WAN, allow IPv6 autoconfiguration and tempory address use.
|
|
# "net.ipv6.conf.${name}.accept_ra" = 2;
|
|
# "net.ipv6.conf.${name}.autoconf" = 1;
|
|
};
|
|
|
|
services.dnscrypt-proxy2 = {
|
|
enable = true;
|
|
upstreamDefaults = true;
|
|
settings = {
|
|
listen_addresses = singleton "127.0.0.1:5353";
|
|
|
|
dnscrypt_servers = false;
|
|
doh_servers = true;
|
|
odoh_servers = false;
|
|
|
|
block_ipv6 = true;
|
|
|
|
static."mullvad".stamp = "sdns://AgcAAAAAAAAACzE5NC4yNDIuMi41ABxleHRlbmRlZC5kbnMubXVsbHZhZC5uZXQ6NDQzCi9kbnMtcXVlcnk";
|
|
sources = {};
|
|
|
|
max_clients = 1024;
|
|
|
|
cache_size = 32768;
|
|
};
|
|
};
|
|
|
|
systemd.services.dnscrypt-proxy2 = {
|
|
before = ["network-online.target"];
|
|
};
|
|
|
|
services.kea.dhcp4 = {
|
|
enable = true;
|
|
settings =
|
|
{
|
|
interfaces-config.interfaces = [
|
|
"${lan}"
|
|
];
|
|
lease-database = {
|
|
name = "/var/lib/kea/dhcp4.leases";
|
|
persist = true;
|
|
type = "memfile";
|
|
};
|
|
rebind-timer = 2000;
|
|
renew-timer = 1000;
|
|
}
|
|
// (secret.dhcp.blowhole.zones or (const {}) {inherit wlan lan;});
|
|
};
|
|
|
|
networking = {
|
|
useDHCP = false;
|
|
hostName = "blowhole";
|
|
|
|
resolvconf.useLocalResolver = false;
|
|
nameservers = singleton (secret.network.ips.blowhole.ip or "");
|
|
|
|
# Disable the in-built iptable based firewall
|
|
firewall.enable = mkForce false;
|
|
|
|
interfaces = {
|
|
# Don't do DHCP on the LAN interface
|
|
"${lan}" = {
|
|
useDHCP = false;
|
|
ipv4.addresses = [
|
|
{
|
|
address = secret.network.ips.blowhole.ip or "";
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
};
|
|
# "${wlan}" = {
|
|
# useDHCP = false;
|
|
# ipv4.addresses = [{
|
|
# address = secret.network.ips.blowhole.wlan or "";
|
|
# prefixLength = 24;
|
|
# }];
|
|
# };
|
|
# But do DHCP on the WAN interface
|
|
"${wan}".useDHCP = true;
|
|
};
|
|
|
|
wireguard = {
|
|
enable = true;
|
|
interfaces."${doVPN}" =
|
|
secret.wireguard."${config.networking.hostName}"
|
|
or {}
|
|
// {
|
|
listenPort = 6666;
|
|
privateKeyFile = "/var/secrets/${doVPN}.key";
|
|
};
|
|
};
|
|
|
|
nftables = {
|
|
enable = true;
|
|
ruleset = ''
|
|
table ip nf_filter {
|
|
chain input_out {
|
|
ct state { established, related } accept comment "Allow established traffic"
|
|
icmp type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
|
|
}
|
|
|
|
chain input_doVPN {
|
|
tcp dport { 4646, 4647, 4648 } accept comment "Nomad traffic"
|
|
tcp dport { 8600, 8500, 8502, 8300, 8301, 8302 } accept comment "Consul traffic"
|
|
tcp dport { 8200 } accept comment "Vault traffic"
|
|
tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
|
|
tcp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
|
|
tcp dport { 53 } accept comment "DNS traffic"
|
|
tcp dport { 80 } accept comment "HTTP traffic"
|
|
|
|
udp dport { 8600, 8301, 8302 } comment "Consul traffic"
|
|
udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
|
|
udp dport ${nomad.min_dynamic_port}-${nomad.max_dynamic_port} accept comment "Consul Connect sidecar traffic"
|
|
udp dport { 53 } accept comment "DNS traffic"
|
|
}
|
|
|
|
chain input {
|
|
type filter hook input priority 0; policy drop;
|
|
|
|
iifname != "${wan}" tcp dport 22 accept comment "Accept SSH traffic always"
|
|
iifname != "lo" tcp dport 5353 drop comment "Drop traffic to dnscrypt-proxy always except for localhost to localhost traffic"
|
|
|
|
# Accept WireGuard
|
|
iifname "${wan}" udp dport 6666 accept;
|
|
|
|
# Accept file-share
|
|
iifname "${wan}" tcp dport 5666 accept;
|
|
|
|
# Accept minecraft
|
|
iifname "${wan}" tcp dport 25560 accept;
|
|
|
|
# iifname "cni0" accept;
|
|
iifgroup { 99 } tcp dport { 6443, 10250, 4244 } accept
|
|
iifgroup { 99 } oifgroup { 99 } accept
|
|
iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr 10.64.48.0/21 accept
|
|
iifgroup { 99 } ip saddr 10.64.48.0/21 ip daddr != 10.0.0.0/8 ip daddr != 192.168.0.0/16 ip daddr != 172.0.0.0/12 accept
|
|
|
|
iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept comment "Allow Nomad to do whatever it wants in its interface"
|
|
iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router"
|
|
iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper", "mvm0", "uk3s0", "ve-ywecur-anp" } jump input_out
|
|
iifgroup 99 jump input_out
|
|
# iifname { "${doVPN}", "${lan}" } tcp dport { 6443 } accept
|
|
iifname { "${doVPN}", "${lan}" } ip daddr 10.64.2.1 tcp dport { 8833 } accept
|
|
iifname { "${doVPN}" } jump input_doVPN
|
|
|
|
# Allow containers to reach the DNS server
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp", "monitor0" } tcp dport 53 accept
|
|
iifgroup { 99 } tcp dport 53 accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp", "monitor0" } udp dport 53 accept
|
|
iifgroup { 99 } udp dport 53 accept
|
|
|
|
# Allow Nomad Containers to reach Nomad
|
|
iifname { "nomad" } tcp dport 4646 accept
|
|
|
|
# Allow proxies to reach consul
|
|
iifname { "nomad", "ve-monitor", "ve-klipper" } tcp dport 8500 accept
|
|
iifname { "ve-monitor", "ve-klipper" } tcp dport 8502 accept
|
|
|
|
# Allow containers to reach the NFS server
|
|
iifname { "docker0", "uk3s0" } tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
|
|
iifname { "docker0", "uk3s0" } udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept comment "NFS traffic"
|
|
|
|
iifname { "${wan}" } udp dport 2302-2304 ip daddr == 192.168.2.20 accept
|
|
|
|
# drop pointless IoT discovery traffic explicitly
|
|
iifname { "${wan}" } ip daddr == 255.255.255.255 udp dport == 6667 drop
|
|
|
|
iifname { "uk3s0" } tcp dport 8200 ip daddr 10.64.2.1 accept
|
|
|
|
log prefix "[drop] nf_filter.input: " queue-threshold 1 group 2 drop
|
|
}
|
|
|
|
chain output {
|
|
type filter hook output priority 0; policy accept;
|
|
|
|
# Drop all DNS traffic if leaving through "wan"
|
|
# oifname { "${wan}" } tcp dport 53 drop
|
|
# oifname { "${wan}" } udp dport 53 drop
|
|
# Allow DoT traffic to leave through "wan" if it comes from "lo"
|
|
# iifname != { "lo" } oifname { "${wan}" } tcp dport 853 drop
|
|
|
|
# log prefix "[drop] nf_filter.output: " queue-threshold 1 group 2 drop
|
|
}
|
|
|
|
chain forward {
|
|
type filter hook forward priority 10; policy drop;
|
|
|
|
# Enable flow offloading for better throughput
|
|
# ip protocol { tcp, udp } flow offload @f
|
|
|
|
ip daddr 10.64.52.130 nftrace set 1
|
|
ip saddr 10.64.52.130 nftrace set 1
|
|
|
|
# Drop all DNS or DoT traffic if forwarded through "wan"
|
|
oifname { "${wan}" } tcp dport 853 drop
|
|
oifname { "${wan}" } tcp dport 53 drop
|
|
oifname { "${wan}" } udp dport 53 drop
|
|
|
|
# Allow trusted LAN to WAN"
|
|
iifname { "${lan}", "${wlan}" } oifname { "${wan}" } accept
|
|
iifname { "${wan}" } oifname { "${lan}", "${wlan}" } ct state established, related accept
|
|
|
|
iifgroup { 99 } oifname { "${doVPN}", "${lan}", "${wlan}" } ct state established, related accept
|
|
|
|
iifname { "nomad" } oifname { "${doVPN}", "${lan}", "${wlan}" } accept
|
|
iifname { "${doVPN}", "${lan}", "${wlan}" } oifname { "nomad" } accept
|
|
iifname { "${doVPN}" } oifname { "${lan}", "${wlan}" } accept
|
|
iifname { "${lan}", "${wlan}" } oifname { "${doVPN}" } accept
|
|
|
|
# Allow containers to reach WAN
|
|
iifname { "ve-ywecur-anp", "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${wan}" } accept
|
|
iifgroup { 99 } oifname { "${wan}" } accept
|
|
iifname { "${wan}" } oifname { "ve-ywecur-anp", "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } ct state established, related accept
|
|
iifname { "${wan}" } oifgroup { 99 } ct state established, related accept
|
|
|
|
# Allow containers to reach the DNS and NFS server
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 53 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 53 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip daddr 10.64.2.1 tcp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 tcp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 53 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0", "ve-ywecur-anp" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 53 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0"} oifname { "${lan}" } ip daddr 10.64.2.1 udp dport { 111, 2049, 4000, 4001, 4002, 20048 } accept
|
|
iifname { "nomad", "docker0", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "${lan}" } ip saddr 10.64.2.1 udp sport { 111, 2049, 4000, 4001, 4002, 20048 } accept
|
|
|
|
# allow communication between all container interfaces
|
|
iifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } oifname { "nomad", "ve-monitor", "ve-klipper", "uk3s0" } accept
|
|
iifgroup { 99 } oifgroup { 99 } accept
|
|
iifname { "${lan}" } oifgroup { 99 } accept
|
|
|
|
# allow portforwarding to lan?
|
|
iifname { "${wan}" } oifname { "${lan}" } ip daddr 10.64.2.20 udp dport 2302-2304 accept
|
|
|
|
# allow lan to access kubernetes http ingress
|
|
iifname { "${lan}" } oifname { "uk3s0" } ip daddr 172.26.96.2 tcp dport 80 accept
|
|
|
|
# allow vpn to access kubernetes http ingress
|
|
iifname { "${doVPN}" } oifname { "uk3s0" } ip saddr 10.64.0.1 ip daddr 172.26.96.2 tcp dport 80 accept
|
|
# allow vpn to access kubernetes forgejo ssh
|
|
iifname { "${doVPN}" } oifname { "uk3s0" } ip saddr 10.64.0.1 ip daddr 172.26.96.2 tcp dport 19022 accept
|
|
|
|
# allow kubernetes to respond to incoming traffic
|
|
iifname { "uk3s0" } oifname { "${lan}", "${doVPN}" } jump input_out
|
|
|
|
# Rules to make CNI happy
|
|
meta mark and 0x01 == 0x01 accept
|
|
|
|
iifname "${wan}" oifname "ve-ywecur-anp" ct status dnat accept
|
|
|
|
log prefix "[drop] nf_filter.forward: " queue-threshold 1 group 2 drop
|
|
}
|
|
}
|
|
|
|
table ip nf_nat {
|
|
# TCP: 2344-2345, 27015, 27036
|
|
# UDP: 2302-2306, 2344, 27015, 27031-27036
|
|
|
|
chain postrouting {
|
|
type nat hook postrouting priority 100; policy accept;
|
|
oifname "${wan}" masquerade
|
|
udp dport 2302-2304 ip daddr { 192.168.2.20 } masquerade
|
|
iifname "${doVPN}" tcp dport 8344 ip daddr 172.26.96.2 masquerade
|
|
}
|
|
|
|
chain prerouting {
|
|
type nat hook prerouting priority 100; policy accept;
|
|
ip daddr { 192.168.2.20 } udp dport 2302-2304 dnat 10.64.2.20
|
|
ip daddr { ${secret.network.ips.blowhole.ip or ""} } tcp dport 8344 dnat 172.26.96.2:80
|
|
ip daddr { ${secret.network.ips.blowhole.ip or ""} } tcp dport 19022 dnat 172.26.96.2:19022
|
|
# ywecur anp container
|
|
ip daddr { 192.168.2.20 } tcp dport 2288 dnat 172.20.69.2:22
|
|
}
|
|
}
|
|
|
|
table ip6 nf_filter {
|
|
chain output {
|
|
type filter hook output priority 0; policy drop;
|
|
|
|
# meta nftrace set 1
|
|
|
|
oifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
|
|
oifname "lo" ip6 saddr "::1" ip6 daddr "::1" reject
|
|
}
|
|
chain input {
|
|
type filter hook input priority 0; policy drop;
|
|
|
|
# meta nftrace set 1
|
|
|
|
iifname "lo" icmpv6 type { echo-request, destination-unreachable, time-exceeded } counter accept comment "Allow select ICMP"
|
|
}
|
|
chain forward {
|
|
type filter hook forward priority 0; policy drop;
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
};
|
|
|
|
services.ulogd = {
|
|
enable = true;
|
|
settings = {
|
|
# This one for logging to local file in emulated syslog format.
|
|
global.stack = "log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOGEMU";
|
|
|
|
log2.group = 2;
|
|
|
|
emu1 = {
|
|
file = "/var/log/nft_drop.log";
|
|
sync = 1;
|
|
};
|
|
};
|
|
};
|
|
|
|
systemd.services.nftables = {
|
|
path = with pkgs; [
|
|
nftables
|
|
iptables
|
|
bash
|
|
];
|
|
serviceConfig = let
|
|
rulesScript = pkgs.writeShellScript "nftables-rules" ''
|
|
set -ex
|
|
exit_code=0
|
|
|
|
dynamic_iptables_rules="$(mktemp)"
|
|
iptables-save -t filter >> $dynamic_iptables_rules
|
|
iptables-save -t nat >> $dynamic_iptables_rules
|
|
|
|
static_nftables_rules="$(mktemp)"
|
|
nft list table ip nf_filter >> $static_nftables_rules || nft create table ip nf_filter
|
|
nft list table ip nf_nat >> $static_nftables_rules || nft create table ip nf_nat
|
|
nft list table ip6 nf_filter >> $static_nftables_rules || nft create table ip6 nf_filter
|
|
|
|
nft flush ruleset
|
|
|
|
iptables-restore < $dynamic_iptables_rules
|
|
if nft -f "${pkgs.writeText "nftables-rules" config.networking.nftables.ruleset}" ; then
|
|
iptables -D FORWARD -j MARK --set-mark 0x01 || true
|
|
iptables -D FORWARD -j MARK --set-mark 0x00 || true
|
|
|
|
iptables -I FORWARD -j MARK --set-mark 0x01
|
|
iptables -A FORWARD -j MARK --set-mark 0x00
|
|
else
|
|
echo "Apply failed, restoring previous rules!"
|
|
nft -f $static_nftables_rules
|
|
exit_code=1
|
|
fi
|
|
|
|
rm $dynamic_iptables_rules $static_nftables_rules
|
|
|
|
exit $exit_code
|
|
'';
|
|
in {
|
|
ExecStart = mkForce rulesScript;
|
|
ExecReload = mkForce rulesScript;
|
|
ExecStop = mkForce (pkgs.writeShellScript "nftables-flush" ''
|
|
set -ex
|
|
|
|
tmpfile="$(mktemp)"
|
|
iptables-save -t filter >> $tmpfile
|
|
iptables-save -t nat >> $tmpfile
|
|
|
|
nft flush ruleset
|
|
|
|
cat $tmpfile | iptables-restore
|
|
rm $tmpfile
|
|
|
|
iptables -D FORWARD -j MARK --set-mark 0x01 || true
|
|
iptables -D FORWARD -j MARK --set-mark 0x00 || true
|
|
|
|
iptables -I FORWARD -j MARK --set-mark 0x01
|
|
iptables -A FORWARD -j MARK --set-mark 0x00
|
|
'');
|
|
};
|
|
};
|
|
}
|