From 34bb23c67febbee63136d4af74805281d08d3e97 Mon Sep 17 00:00:00 2001 From: Magic_RB Date: Thu, 28 Sep 2023 10:31:25 +0200 Subject: [PATCH] Initial implementation of microvm-extras Signed-off-by: Magic_RB --- nixos/modules/microvm-extras-host.nix | 209 ++++++++++++++++ nixos/modules/microvm-extras.nix | 348 ++++++++++++++++++++++++++ nixos/modules/notnft.nix | 122 +++++++++ nixos/systems/blowhole/default.nix | 7 + nixos/systems/blowhole/firewall.nix | 2 +- nixos/systems/blowhole/microvms.nix | 179 +++++++++++++ nixos/systems/omen/default.nix | 307 ++++++++++++++++++++++- nixos/systems/omen/networking.nix | 92 ++++++- 8 files changed, 1255 insertions(+), 11 deletions(-) create mode 100644 nixos/modules/microvm-extras-host.nix create mode 100644 nixos/modules/microvm-extras.nix create mode 100644 nixos/modules/notnft.nix create mode 100644 nixos/systems/blowhole/microvms.nix diff --git a/nixos/modules/microvm-extras-host.nix b/nixos/modules/microvm-extras-host.nix new file mode 100644 index 0000000..721125c --- /dev/null +++ b/nixos/modules/microvm-extras-host.nix @@ -0,0 +1,209 @@ +{ config, lib, notnft, ... }: +let + inherit (lib) + mapAttrsToList + mkOption + hasAttr + types + traceVal + flip + mapAttrs' + mapAttrs + nameValuePair; + + # a = [ + # [ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq ip.daddr "10.80.1.2") (is.eq th.dport "22") accept ] + # ]; + + cfg = config.microvm; + + protocolEnumToNft = f: proto: + f.${proto}; + + tcpUdpServiceOptions.options = { + hostName = mkOption { + type = types.str; + }; + + port = mkOption { + type = types.port; + }; + + protocol = mkOption { + type = types.listOf (types.enum [ "tcp" "udp" ]); + }; + }; + + httpServiceOptions.options = { + hostName = mkOption { + type = types.str; + }; + + port = mkOption { + type = types.port; + }; + }; + + icmpServiceOptions.options = { + hostName = mkOption { + type = types.str; + }; + }; + + tcpUdpConnectionOptions.options = { + target = mkOption { + type = types.str; + }; + }; + + icmpConnectionOptions.options = { + target = mkOption { + type = types.str; + }; + }; + + httpConnectionOptions.options = { + target = mkOption { + type = types.str; + }; + }; + + lookupService = name: type: context: + if hasAttr name cfg.services.${type} then + cfg.services.${type}.${name} + else + throw "Unknown ${type} service ${name} at ${context}"; + + lookupIds = hostName: context: + if hasAttr hostName subConfigurations then + { + inherit (subConfigurations.${hostName}.config.config.microvm) + groupId + taskId; + } + else + throw "Unknown hostName ${hostName} at ${context}"; + + subConfigurations = cfg.vms // config.containers; +in +{ + options.microvm = { + services = { + tcpUdp = mkOption { + type = with types; types.attrsOf (submodule tcpUdpServiceOptions); + }; + + icmp = mkOption { + type = with types; types.attrsOf (submodule icmpServiceOptions); + }; + + http = mkOption { + type = with types; types.attrsOf (submodule httpServiceOptions); + }; + }; + + connections = { + tcpUdp = mkOption { + type = with types; + listOf (submodule tcpUdpConnectionOptions); + default = []; + }; + + icmp = mkOption { + type = with types; + listOf (submodule icmpConnectionOptions); + default = []; + }; + + http = mkOption { + type = with types; + listOf (submodule httpConnectionOptions); + default = []; + }; + }; + }; + + config.microvm.services.tcpUdp = flip mapAttrs' cfg.services.http + (n: v: + nameValuePair + (n + "@http") + { + inherit (v) + hostName + port; + protocol = [ "tcp" ]; + } + ); + + config.microvm.connections.tcpUdp = flip map cfg.connections.http + (v: + { + target = v.target + "@http"; + } + ); + + config.networking.notnft.rules = + with notnft.dsl; with payload; ruleset { + bridge-t = add table { family = f: f.bridge; } { + output-body = lib.foldl (acc: x: acc x) (add chain) ((flip mapAttrsToList subConfigurations + (n: v: + let + microvmConfig = v.config.config.microvm; + tcpUdpRules = + flip map microvmConfig.connections.tcpUdp (connection: + let + service = lookupService connection.target "tcpUdp" n; + ids = lookupIds service.hostName n; + in + [ + (is.eq meta.oifname "mvm-${microvmConfig.hostName}") + (is.eq ip.protocol (f: with f; set (map (protocolEnumToNft f) service.protocol))) + (is.eq ip.saddr "10.80.${toString microvmConfig.groupId}.${toString microvmConfig.taskId}") + (is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}") + (is.eq th.dport service.port) + accept + ]); + icmpRules = + flip map microvmConfig.connections.icmp (connection: + let + service = lookupService connection.target "icmp" n; + ids = lookupIds service.hostName n; + in + [ + (is.eq meta.oifname "mvm-${microvmConfig.hostName}") + (is.eq ip.protocol (f: with f; icmp)) + (is.eq ip.saddr "10.80.${toString microvmConfig.groupId}.${toString microvmConfig.taskId}") + (is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}") + accept + ]); + in + tcpUdpRules ++ icmpRules + )) ++ (flip map cfg.connections.icmp (connection: + let + service = lookupService connection.target "icmp" "host"; + ids = lookupIds service.hostName "host"; + in + [ + (is.eq meta.oifname "mvm-${service.hostName}") + (is.eq ip.protocol (f: with f; icmp)) + (is.eq ip.saddr "10.80.${toString ids.groupId}.1") + (is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}") + accept + ] + )) ++ (flip map cfg.connections.tcpUdp (connection: + let + service = lookupService connection.target "tcpUdp" "host"; + ids = lookupIds service.hostName "host"; + in + [ + (is.eq meta.oifname "mvm-${service.hostName}") + (is.eq ip.protocol (f: with f; set (map (protocolEnumToNft f) service.protocol))) + (is.eq ip.saddr "10.80.${toString ids.groupId}.1") + (is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}") + (is.eq th.dport service.port) + accept + ] + ))); + }; + }; +} diff --git a/nixos/modules/microvm-extras.nix b/nixos/modules/microvm-extras.nix new file mode 100644 index 0000000..27a180a --- /dev/null +++ b/nixos/modules/microvm-extras.nix @@ -0,0 +1,348 @@ +{ config, lib, ... }: +let + inherit (lib) + mkOption + mkEnableOption + types; + + cfg = config.microvm; + + intToHex = int: + { + "0" = "00"; + "1" = "01"; + "2" = "02"; + "3" = "03"; + "4" = "04"; + "5" = "05"; + "6" = "06"; + "7" = "07"; + "8" = "08"; + "9" = "09"; + "10" = "0a"; + "11" = "0b"; + "12" = "0c"; + "13" = "0d"; + "14" = "0e"; + "15" = "0f"; + "16" = "10"; + "17" = "11"; + "18" = "12"; + "19" = "13"; + "20" = "14"; + "21" = "15"; + "22" = "16"; + "23" = "17"; + "24" = "18"; + "25" = "19"; + "26" = "1a"; + "27" = "1b"; + "28" = "1c"; + "29" = "1d"; + "30" = "1e"; + "31" = "1f"; + "32" = "20"; + "33" = "21"; + "34" = "22"; + "35" = "23"; + "36" = "24"; + "37" = "25"; + "38" = "26"; + "39" = "27"; + "40" = "28"; + "41" = "29"; + "42" = "2a"; + "43" = "2b"; + "44" = "2c"; + "45" = "2d"; + "46" = "2e"; + "47" = "2f"; + "48" = "30"; + "49" = "31"; + "50" = "32"; + "51" = "33"; + "52" = "34"; + "53" = "35"; + "54" = "36"; + "55" = "37"; + "56" = "38"; + "57" = "39"; + "58" = "3a"; + "59" = "3b"; + "60" = "3c"; + "61" = "3d"; + "62" = "3e"; + "63" = "3f"; + "64" = "40"; + "65" = "41"; + "66" = "42"; + "67" = "43"; + "68" = "44"; + "69" = "45"; + "70" = "46"; + "71" = "47"; + "72" = "48"; + "73" = "49"; + "74" = "4a"; + "75" = "4b"; + "76" = "4c"; + "77" = "4d"; + "78" = "4e"; + "79" = "4f"; + "80" = "50"; + "81" = "51"; + "82" = "52"; + "83" = "53"; + "84" = "54"; + "85" = "55"; + "86" = "56"; + "87" = "57"; + "88" = "58"; + "89" = "59"; + "90" = "5a"; + "91" = "5b"; + "92" = "5c"; + "93" = "5d"; + "94" = "5e"; + "95" = "5f"; + "96" = "60"; + "97" = "61"; + "98" = "62"; + "99" = "63"; + "100" = "64"; + "101" = "65"; + "102" = "66"; + "103" = "67"; + "104" = "68"; + "105" = "69"; + "106" = "6a"; + "107" = "6b"; + "108" = "6c"; + "109" = "6d"; + "110" = "6e"; + "111" = "6f"; + "112" = "70"; + "113" = "71"; + "114" = "72"; + "115" = "73"; + "116" = "74"; + "117" = "75"; + "118" = "76"; + "119" = "77"; + "120" = "78"; + "121" = "79"; + "122" = "7a"; + "123" = "7b"; + "124" = "7c"; + "125" = "7d"; + "126" = "7e"; + "127" = "7f"; + "128" = "80"; + "129" = "81"; + "130" = "82"; + "131" = "83"; + "132" = "84"; + "133" = "85"; + "134" = "86"; + "135" = "87"; + "136" = "88"; + "137" = "89"; + "138" = "8a"; + "139" = "8b"; + "140" = "8c"; + "141" = "8d"; + "142" = "8e"; + "143" = "8f"; + "144" = "90"; + "145" = "91"; + "146" = "92"; + "147" = "93"; + "148" = "94"; + "149" = "95"; + "150" = "96"; + "151" = "97"; + "152" = "98"; + "153" = "99"; + "154" = "9a"; + "155" = "9b"; + "156" = "9c"; + "157" = "9d"; + "158" = "9e"; + "159" = "9f"; + "160" = "a0"; + "161" = "a1"; + "162" = "a2"; + "163" = "a3"; + "164" = "a4"; + "165" = "a5"; + "166" = "a6"; + "167" = "a7"; + "168" = "a8"; + "169" = "a9"; + "170" = "aa"; + "171" = "ab"; + "172" = "ac"; + "173" = "ad"; + "174" = "ae"; + "175" = "af"; + "176" = "b0"; + "177" = "b1"; + "178" = "b2"; + "179" = "b3"; + "180" = "b4"; + "181" = "b5"; + "182" = "b6"; + "183" = "b7"; + "184" = "b8"; + "185" = "b9"; + "186" = "ba"; + "187" = "bb"; + "188" = "bc"; + "189" = "bd"; + "190" = "be"; + "191" = "bf"; + "192" = "c0"; + "193" = "c1"; + "194" = "c2"; + "195" = "c3"; + "196" = "c4"; + "197" = "c5"; + "198" = "c6"; + "199" = "c7"; + "200" = "c8"; + "201" = "c9"; + "202" = "ca"; + "203" = "cb"; + "204" = "cc"; + "205" = "cd"; + "206" = "ce"; + "207" = "cf"; + "208" = "d0"; + "209" = "d1"; + "210" = "d2"; + "211" = "d3"; + "212" = "d4"; + "213" = "d5"; + "214" = "d6"; + "215" = "d7"; + "216" = "d8"; + "217" = "d9"; + "218" = "da"; + "219" = "db"; + "220" = "dc"; + "221" = "dd"; + "222" = "de"; + "223" = "df"; + "224" = "e0"; + "225" = "e1"; + "226" = "e2"; + "227" = "e3"; + "228" = "e4"; + "229" = "e5"; + "230" = "e6"; + "231" = "e7"; + "232" = "e8"; + "233" = "e9"; + "234" = "ea"; + "235" = "eb"; + "236" = "ec"; + "237" = "ed"; + "238" = "ee"; + "239" = "ef"; + "240" = "f0"; + "241" = "f1"; + "242" = "f2"; + "243" = "f3"; + "244" = "f4"; + "245" = "f5"; + "246" = "f6"; + "247" = "f7"; + "248" = "f8"; + "249" = "f9"; + "250" = "fa"; + "251" = "fb"; + "252" = "fc"; + "253" = "fd"; + "254" = "fe"; + "255" = "ff"; + }.${toString int}; + + groupIdOption = mkOption { + type = types.int; + default = config.microvm.groupId; + }; + + taskIdOption = mkOption { + type = types.int; + }; + + tcpUdpConnectionOptions.options = { + target = mkOption { + type = types.str; + }; + }; + + icmpConnectionOptions.options = { + target = mkOption { + type = types.str; + }; + }; +in +{ + options.microvm = { + enableExtras = mkEnableOption "Extras"; + groupId = mkOption { + type = types.int; + }; + taskId = mkOption { + type = types.int; + }; + hostsHostName = mkOption { + type = types.str; + }; + hostName = mkOption { + type = types.str; + }; + + connections = { + tcpUdp = mkOption { + type = with types; + listOf (submodule tcpUdpConnectionOptions); + default = []; + }; + + icmp = mkOption { + type = with types; + listOf (submodule icmpConnectionOptions); + default = []; + }; + }; + }; + + config = { + networking.hostName = "${cfg.hostName}-${cfg.hostsHostName}"; + + microvm.interfaces = [ + { + type = "tap"; + + # interface name on the host + id = "mvm-${cfg.hostName}"; + + # Ethernet address of the MicroVM's interface, not the host's + # + # Locally administered have one of 2/6/A/E in the second nibble. + mac = "02:00:00:00:${intToHex cfg.groupId}:${intToHex cfg.taskId}"; + } + ]; + + networking.interfaces."eth0" = { + ipv4.addresses = [ + { + address = "10.80.${toString cfg.groupId}.${toString cfg.taskId}"; + prefixLength = 24; + } + ]; + }; + }; +} diff --git a/nixos/modules/notnft.nix b/nixos/modules/notnft.nix new file mode 100644 index 0000000..9eb68e1 --- /dev/null +++ b/nixos/modules/notnft.nix @@ -0,0 +1,122 @@ +{ pkgs, config, lib, notnft, ... }: +let + inherit (lib) + types + mkOption + mkDefault + mkEnableOption + flip + concatMapStringsSep + optionalAttrs + listToAttrs + optional + filter; + cfg = config.networking.notnft; + jsonFormat = (pkgs.formats.json {}); +in +{ + options.networking.notnft = { + enable = mkEnableOption "notnft"; + + preRules = mkOption { + type = types.listOf jsonFormat.type; + default = []; + }; + + rules = mkOption { + type = notnft.types.ruleset; + default = {}; + }; + + postRules = mkOption { + type = types.listOf jsonFormat.type; + default = []; + }; + + json = mkOption { + type = jsonFormat.type; + readOnly = true; + }; + + jsonFile = mkOption { + type = types.path; + readOnly = true; + }; + + flush = mkOption { + type = types.bool; + default = true; + }; + + chains = { + dnsDrop = { + enable = mkEnableOption "Add dns-drop chain"; + + rule = mkOption { + type = notnft.type.rule; + readOnly = true; + default = with notnft.dsl; with payload; + [ jump "dns-drop" ]; + }; + }; + }; + }; + + config = { + networking.notnft.rules = with notnft.dsl; with payload; ruleset { + filter = add table { family = f: f.inet; } + (listToAttrs (filter (x: x != {}) [ + (optionalAttrs cfg.chains.dnsDrop.enable { + name = "dns-drop"; + value = add chain + [ (is.ne ip.daddr "10.64.2.1") (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ]; + }) + ])); + }; + + networking.notnft.json = builtins.toJSON { + nftables = (optional cfg.flush { flush.ruleset = null; }) ++ cfg.preRules ++ cfg.rules.nftables ++ cfg.postRules; + }; + networking.notnft.jsonFile = pkgs.writeText "rules.json" cfg.json; + + boot.blacklistedKernelModules = [ "ip_tables" ]; + environment.systemPackages = [ pkgs.nftables ]; + networking.networkmanager.firewallBackend = mkDefault "nftables"; + systemd.services.notnftables = { + description = "notnftables firewall"; + before = [ "network-pre.target" ]; + wants = [ "network-pre.target" ]; + wantedBy = [ "multi-user.target" ]; + reloadIfChanged = true; + serviceConfig = let + startScript = pkgs.writeShellScript "start-nft.sh" '' + ${pkgs.buildPackages.nftables}/bin/nft -j -f ${cfg.jsonFile} + ''; + # rulesScript = pkgs.writeTextFile { + # name = "nftables-rules"; + # executable = true; + # text = '' + # #! ${pkgs.nftables}/bin/nft -f + # flush ruleset + # ${if cfg.rulesetFile != null then '' + # include "${cfg.rulesetFile}" + # '' else cfg.ruleset} + # ''; + # checkPhase = lib.optionalString cfg.checkRuleset '' + # cp $out ruleset.conf + # ${cfg.preCheckRuleset} + # export NIX_REDIRECTS=/etc/protocols=${pkgs.buildPackages.iana-etc}/etc/protocols:/etc/services=${pkgs.buildPackages.iana-etc}/etc/services + # LD_PRELOAD="${pkgs.buildPackages.libredirect}/lib/libredirect.so ${pkgs.buildPackages.lklWithFirewall.lib}/lib/liblkl-hijack.so" \ + # ${pkgs.buildPackages.nftables}/bin/nft --check -j < ${cfg.jsonFile} + # ''; + # }; + in { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = startScript; + ExecReload = startScript; + ExecStop = "${pkgs.nftables}/bin/nft flush ruleset"; + }; + }; + }; +} diff --git a/nixos/systems/blowhole/default.nix b/nixos/systems/blowhole/default.nix index f636a79..5a7aeda 100644 --- a/nixos/systems/blowhole/default.nix +++ b/nixos/systems/blowhole/default.nix @@ -46,7 +46,14 @@ in ./users.nix ./sol.nix ../../common/remote_access.nix + ./microvms.nix inputs.serokell-nix.nixosModules.acme-sh + + inputs.notnft.nixosModules.default + inputs.self.nixosModules.notnft + inputs.microvm.nixosModules.host + inputs.self.nixosModules.microvm-extras-host + config'.flake.nixosModules.hashicorp config'.flake.nixosModules.hashicorp-envoy config'.flake.nixosModules.telegraf diff --git a/nixos/systems/blowhole/firewall.nix b/nixos/systems/blowhole/firewall.nix index 9da74b2..4f2b1ea 100644 --- a/nixos/systems/blowhole/firewall.nix +++ b/nixos/systems/blowhole/firewall.nix @@ -145,7 +145,7 @@ in iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface" iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router" - iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper" } jump input_out + iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper", "mvm0" } jump input_out iifname { "${doVPN}" } jump input_doVPN # Allow containers to reach the DNS server diff --git a/nixos/systems/blowhole/microvms.nix b/nixos/systems/blowhole/microvms.nix new file mode 100644 index 0000000..6c63967 --- /dev/null +++ b/nixos/systems/blowhole/microvms.nix @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2022 Richard Brežák +# +# SPDX-License-Identifier: LGPL-3.0-or-later +{ notnft, inputs', lib, config, ... }: +let + inherit (lib) + mkBefore + flip + genAttrs; +in +{ + networking.notnft = { + enable = true; + flush = false; + }; + + networking.notnft.preRules = [ + { add.table = { family = "bridge"; name = "bridge-t"; }; } + { flush.table = { family = "bridge"; name = "bridge-t"; }; } + ]; + + networking.notnft.rules = + let + interfaces = [ "mvm-test" "mvm0" ]; + logRule = with notnft.dsl; with payload; prefix: + [ + (log { prefix = "${prefix} dropped: "; flags = (f: [ f.all ]); } ) + ]; + + dropRule = with notnft.dsl; with payload; + [ drop ]; + in + with notnft.dsl; with payload; ruleset { + bridge-t = add table { family = f: f.bridge; } { + input-body = add chain; + + input-mvm = add chain + [ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ] + + [ (is.eq meta.protocol (f: f.arp)) accept ] + + [ (jump "input-body") ] + + (logRule "Bridge input") + (dropRule); + + input = add chain + { type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.accept; } + [ (vmap meta.iifname (genAttrs interfaces (_: (goto "input-mvm")))) ] + [ (vmap meta.oifname (genAttrs interfaces (_: (goto "input-mvm")))) ]; + + output-body = add chain; + + output-mvm = add chain + [ (is.eq ether.type (f: f.arp)) accept ] + + [ (jump "output-body") ] + + (logRule "Bridge output") + (dropRule); + + output = add chain + { type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.accept; } + [ (vmap meta.iifname (genAttrs interfaces (_: (goto "output-mvm")))) ] + [ (vmap meta.oifname (genAttrs interfaces (_: (goto "output-mvm")))) ]; + + + forward-body = add chain; + + forward-mvm = add chain + [ (jump "forward-body") ] + + (logRule "Bridge forward") + (dropRule); + + forward = add chain + { type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.accept; } + [ (vmap meta.iifname (genAttrs interfaces (_: (goto "input-mvm")))) ] + [ (vmap meta.oifname (genAttrs interfaces (_: (goto "input-mvm")))) ]; + + # prerouting = add chain + # { type = f: f.filter; hook = f: f.prerouting; prio = -300; policy = f: f.accept; } + # ; + + # postrouting = add chain + # { type = f: f.filter; hook = f: f.postrouting; prio = -300; policy = f: f.accept; } + # ; + }; + }; + + systemd.services.notnftables = { + requires = [ "nftables.service" ]; + after = [ "nftables.service" ]; + }; + + networking.bridges.mvm0 = { + interfaces = []; + }; + + networking.interfaces.mvm0 = { + useDHCP = false; + ipv4.addresses = [ + { + address = "10.80.1.1"; + prefixLength = 24; + } + ]; + }; + + microvm.services.tcpUdp.test-ssh = { + hostName = "test"; + port = 22; + protocol = [ "tcp" ]; + }; + + microvm.services.tcpUdp.test-http = { + hostName = "test"; + port = 80; + protocol = [ "tcp" ]; + }; + + microvm.services.icmp.test = { + hostName = "test"; + }; + + microvm.connections.tcpUdp = [ + { + target = "test-ssh"; + } + { + target = "test-http"; + } + ]; + microvm.connections.icmp = [ + { + target = "test"; + } + ]; + + microvm.vms = { + test.config = { + imports = [ inputs'.self.nixosModules.microvm-extras ]; + + microvm = { + hostName = "test"; + hostsHostName = "omen"; + groupId = 1; + taskId = 2; + }; + + microvm.hypervisor = "cloud-hypervisor"; + microvm.shares = [{ + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + }]; + microvm.storeOnDisk = false; + + networking.firewall.allowedTCPPorts = [ 80 22 ]; + + services.nginx = { + enable = true; + virtualHosts."example.com" = { + root = "/var/www/blog"; + }; + }; + + users.users.root.password = ""; + services.getty.helpLine = '' + Log in as "root" with an empty password. + ''; + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; + }; + }; +} diff --git a/nixos/systems/omen/default.nix b/nixos/systems/omen/default.nix index a3c00f4..fc39cfe 100644 --- a/nixos/systems/omen/default.nix +++ b/nixos/systems/omen/default.nix @@ -7,7 +7,8 @@ let flip mapAttrs singleton - loadSecrets; + loadSecrets + mkAfter; config' = config; in @@ -23,7 +24,7 @@ in }; modules = singleton - ({ pkgs, config, ... }: + ({ pkgs, lib, config, ... }: { imports = [ ./xserver.nix @@ -35,13 +36,19 @@ in ./users.nix ./nixpkgs.nix ../../common/sound.nix + # ./test-vm.nix inputs.dwarffs.nixosModules.dwarffs + inputs.microvm.nixosModules.host + inputs.notnft.nixosModules.default + inputs.self.nixosModules.notnft + inputs.self.nixosModules.microvm-extras-host ]; _module.args.nixinate = { host = secret.network.ips.omen.vpn or ""; sshUser = "main"; + buildOn = "local"; substituteOnTarget = true; hermetic = false; @@ -71,6 +78,302 @@ in virtualisation.podman.enable = true; virtualisation.podman.dockerCompat = true; + + nixpkgs.overlays = [ + (final: prev: + let + nixpkgs = final.fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs"; + rev = "2ca2346b60f72fc75bcc570367e24e1d68d55b18"; + sha256 = "sha256-GNUI2MRZGe8rm27DgY503aLyXRm/Dfcao6McHrHkA7s="; + }; + pkgs = import nixpkgs { system = final.stdenv.system; }; + virtiofsd = pkgs.virtiofsd; + in + { + virtiofsd = final.writeShellScriptBin "virtiofsd" '' + ok_args=() + while [[ $# -gt 0 ]] ; do + case "$1" in + --posix-acl) + ;; + *) + ok_args+=("$1") + ;; + esac + shift 1 + done + + exec ${lib.getExe virtiofsd} "''${ok_args[@]}" + ''; + } + ) + ]; + + # networking.nftables = { + # enable = true; + # rulesetFile = + networking.notnft.rules = + let + notnft = (inputs.notnft.lib.${pkgs.stdenv.system}); + logRule = with notnft.dsl; with payload; prefix: + [ (log { prefix = "${prefix} dropped: "; flags = (f: [ f.all ]); } ) ]; + traceChain = with notnft.dsl; with payload; + add chain + [ (is.eq th.dport 53) (mangle meta.nftrace 1) ] + [ (is.eq th.dport 53) (mangle meta.nftrace 1) ] + [ (is.eq th.dport 22) (mangle meta.nftrace 1) ] + [ (is.eq th.sport 22) (mangle meta.nftrace 1) ] + [ (is.eq meta.oifname "mvm0") (mangle meta.nftrace 1) ] + [ (is.eq meta.iifname "mvm0") (mangle meta.nftrace 1) ]; + in + # pkgs.writeText "nftables.json" (builtins.toJSON (with notnft.dsl; with payload; ruleset + with notnft.dsl; with payload; ruleset { + filter = add table { family = f: f.inet; } { + trace = traceChain; + + ### lo + input-lo = add chain + [ (is.ne ip.saddr (set [ "127.0.0.1" "127.0.0.53" ])) drop ] + [ (is.ne ip.daddr (set [ "127.0.0.1" "127.0.0.53" ])) drop ] + [ accept ]; + + output-lo = add chain + [ accept ]; + ### + + ### mvm + input-mvm = add chain + [ (is.eq ip.protocol (f: f.icmp)) accept ]; + + output-mvm = add chain + [ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ] + [ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr "10.80.1.2") (is.eq ip.daddr "10.80.1.1") accept ] + + [ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport 22) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ] + [ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport 80) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ]; + ### + + ### wlan0 + input-wlan0 = add chain + [ drop ]; + + output-wlan0 = add chain + [ (is.ne ip.daddr (secret.network.ips.blowhole.ip or "")) (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ] + [ accept ]; + ### + + ### wlan0 + input-eth0 = add chain + [ drop ]; + + output-eth0 = add chain + [ (is.ne ip.daddr (secret.network.ips.blowhole.ip or "")) (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ] + [ accept ]; + ### + + ### wg0 + input-wg0 = add chain + # accept syncthing sharing + [ (is.eq ip.protocol (f: f.udp)) (is.eq th.sport "22000") (is.eq th.dport "22000") accept ] + [ (is.eq ip.protocol (f: f.tcp)) (is.eq th.dport "22000") accept ] + + [ (is.eq ip.protocol (f: f.icmp)) accept ]; + + output-wg0 = add chain + # TCP, UDP 53 to blowhole + [ (is.eq ip.protocol (f: with f; set [ udp tcp ])) (is.eq th.dport 53) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accep t] + + # TCP 22, 80, 4646, 8200, 8500, 2049 to blowhole + [ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport (set [ 22 80 4646 8200 8500 2049 ])) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accept ] + + # ICMP to blowhole, toothpick + [ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (set [ (secret.network.ips.toothpick or "") (secret.network.ips.blowhole.ip or "") ])) accept ] + + # accept syncthing sharing + [ (is.eq ip.protocol (f: f.udp)) (is.eq th.sport "22000") (is.eq th.dport "22000") accept ] + [ (is.eq ip.protocol (f: f.tcp)) (is.eq th.dport "22000") accept ] + ; + ### + + input = add chain { type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.drop; } + # accept related, established and drop invalid + [ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ] + # # accept icmp between the same IP + # [ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.daddr ip.saddr) accept ] + [ (jump "trace") ] + + [ (is.eq meta.iifname "wlan0") (jump "input-wlan0") ] + [ (is.eq meta.iifname "eth0") (jump "input-eth0") ] + [ (is.eq meta.iifname "mvm0") (jump "input-mvm") ] + [ (is.eq meta.iifname "lo") (jump "input-lo") ] + [ (is.eq meta.iifname "wg0") (jump "input-wg0") ] + + [ (is.eq ip.protocol (f: f.icmp)) accept ] + (logRule "Input"); + + output = add chain { type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.drop; } + [ (jump "trace") ] + + [ (is.eq meta.oifname "wlan0") (jump "output-wlan0") ] + [ (is.eq meta.oifname "eth0") (jump "output-eth0") ] + [ (is.eq meta.oifname "lo") (jump "output-lo") ] + [ (is.eq meta.oifname "mvm0") (jump "output-mvm") ] + [ (is.eq meta.oifname "wg0") (jump "output-wg0") ] + (logRule "Output"); + + forward = add chain { type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.drop; } + [ (jump "trace") ] + # accept masquaraded packets incoming from wg0 + [ (is.eq meta.iifname "wg0") (vmap ct.state { established = accept; related = accept; }) ] + # accept TCP, UDP 53 from 10.80.1.2 to blowhole + [ (is.eq meta.iifname "mvm0") (is.eq meta.oifname "wg0") (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) (is.eq ip.saddr "10.80.1.2") (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accept ] + (logRule "Forward"); + + prerouting = add chain { type = f: f.nat; hook = f: f.prerouting; prio = -199; policy = f: f.accept; } + ; + + postrouting = add chain { type = f: f.nat; hook = f: f.postrouting; prio = -199; policy = f: f.accept; } + # masquarade from 10.80.1.2 heading to wg0 + [ (is.eq meta.iifname "mvm0") (is.eq meta.oifname "wg0") (is.eq ip.saddr (set [ "10.80.1.2" ])) masquerade ]; + }; + + bridge-t = add table { family = f: f.bridge; } { + trace = traceChain; + + input-body = add chain; + + input = add chain { type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.drop; } + [ (jump "trace") ] + [ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ] + + [ (is.eq meta.protocol (f: f.arp)) accept ] + + [ (jump "input-body") ] + + (logRule "Bridge input"); + + output-body = add chain; + + output = add chain { type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.drop; } + [ (jump "trace") ] + + [ (is.eq ether.type (f: f.arp)) accept ] + + [ (jump "output-body") ] + + (logRule "Bridge output"); + + + forward-body = add chain; + + forward = add chain { type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.drop; } + [ (jump "trace") ] + + [ (jump "forward-body") ] + + (logRule "Bridge forward"); + + prerouting = add chain { type = f: f.filter; hook = f: f.prerouting; prio = -300; policy = f: f.accept; } + ; + + postrouting = add chain { type = f: f.filter; hook = f: f.postrouting; prio = -300; policy = f: f.accept; } + ; + }; + }; + # )); + # }; + + systemd.network.netdevs."mvm0" = { + netdevConfig = { + Name = "mvm0"; + Kind = "bridge"; + }; + }; + + systemd.network.networks."10-mvm0" = { + matchConfig.Name = "mvm0"; + networkConfig.Address = "10.80.1.1/24"; + linkConfig.RequiredForOnline = "yes"; + }; + + systemd.network.networks."11-mvm-test" = { + matchConfig.Name = "mvm-test"; + networkConfig.Bridge = "mvm0"; + linkConfig.RequiredForOnline = "no"; + }; + + microvm.services.tcpUdp.test-ssh = { + hostName = "test"; + port = 22; + protocol = [ "tcp" ]; + }; + + microvm.services.http.test = { + hostName = "test"; + port = 80; + }; + + microvm.services.icmp.test = { + hostName = "test"; + }; + + microvm.connections.http = [ + { + target = "test"; + } + ]; + microvm.connections.tcpUdp = [ + { + target = "test-ssh"; + } + ]; + microvm.connections.icmp = [ + { + target = "test"; + } + ]; + + microvm.vms = { + test.config = { + imports = [ inputs.self.nixosModules.microvm-extras ]; + + microvm = { + hostName = "test"; + hostsHostName = "omen"; + groupId = 1; + taskId = 2; + }; + + microvm.hypervisor = "cloud-hypervisor"; + microvm.shares = [{ + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + }]; + microvm.storeOnDisk = false; + + networking.firewall.allowedTCPPorts = [ 80 22 ]; + + services.nginx = { + enable = true; + virtualHosts."example.com" = { + root = "/var/www/blog"; + }; + }; + + users.users.root.password = ""; + services.getty.helpLine = '' + Log in as "root" with an empty password. + ''; + services.openssh = { + enable = true; + settings.PermitRootLogin = "yes"; + }; + }; + }; }); }; } diff --git a/nixos/systems/omen/networking.nix b/nixos/systems/omen/networking.nix index 2800237..2087d2f 100644 --- a/nixos/systems/omen/networking.nix +++ b/nixos/systems/omen/networking.nix @@ -1,24 +1,100 @@ -{ pkgs, lib, inputs', secret, ... }: +{ pkgs, lib, inputs', secret, notnft, ... }: let inherit (lib) concatStringsSep; in { + systemd.network.enable = true; networking = { hostName = "omen"; - useDHCP = false; - # interfaces.eno1.useDHCP = true; - hostId = "10c7ffc5"; - networkmanager.dns = "none"; - nameservers = [ "10.64.2.1" ]; - firewall.allowedTCPPorts = [22000]; + hostId = "10c7ffc5"; + + nameservers = [ secret.network.ips.blowhole.ip ]; + + firewall.enable = false; wireguard.interfaces."wg0" = secret.wireguard."omen" or { privateKey = ""; }; }; - networking.networkmanager.enable = true; + networking.notnft.rules = with notnft.dsl; with payload; ruleset { + filter = add table { family = f: f.inet; } { + trace = add chain + [ (is.eq ip.protocol (f: f.icmp)) (mangle meta.nftrace 1) ]; + }; + }; + + services.networkd-dispatcher = { + enable = true; + rules.wlan-eth-switch = { + onState = [ "no-carrier" "configured" ]; + script = '' + #!${pkgs.runtimeShell} + export PATH=$PATH:${pkgs.iwd}/bin + echo "entered state: '$STATE' on interface '$IFACE' with IPs '$IP_ADDRS'" + + case $IFACE in + eth0) + echo $IP_ADDRS | ${lib.getExe pkgs.grepcidr} ${secret.network.networks.home.amsterdam} > /dev/null + home_net=$? + + case $STATE in + no-carrier) + if [ "$(iwctl station wlan0 show | grep -i State | tr -s ' ' | cut -f 3 -d ' ')" == "disconnected" ] ; then + iwctl device wlan0 set-property Powered off + iwctl device wlan0 set-property Powered on + fi + ;; + configured) + if [ "$home_net" == "0" ] ; then + iwctl station wlan0 disconnect + fi + ;; + *) + ;; + esac + ;; + *) + ;; + esac + ''; + }; + }; + + systemd.network.links."50-eth0" = { + matchConfig.MACAddress = secret.network.mac.usbc-omen; + linkConfig.Name = "eth0"; + }; + + systemd.network.networks."50-eth0" = { + matchConfig.Name = "eth0"; + networkConfig.DHCP = "ipv4"; + linkConfig.RequiredForOnline = "no"; + }; + + systemd.network.networks."50-wlan0" = { + matchConfig.Name = "wlan0"; + linkConfig.RequiredForOnline = "no"; + # networkConfig.DHCP = "ipv4"; + + # networkConfig.DNS = "${secret.network.ips.blowhole.ip}"; + # dhcpV4Config.UseDNS = false; + # dhcpV6Config.UseDNS = false; + }; + + services.resolved.enable = false; + environment.etc."resolv.conf".text = '' + nameserver ${secret.network.ips.blowhole.ip} + ''; + + services.resolved.extraConfig = '' + [Resolve] + DNS=${secret.network.ips.blowhole.ip} + FallbackDNS= + ''; + + networking.wireless.iwd.enable = true; hardware.bluetooth = { enable = true; settings = {