Initial implementation of microvm-extras

Signed-off-by: Magic_RB <magic_rb@redalder.org>
This commit is contained in:
Magic_RB 2023-09-28 10:31:25 +02:00
parent 20c8f05224
commit 34bb23c67f
No known key found for this signature in database
GPG key ID: 08D5287CC5DDCA0E
8 changed files with 1255 additions and 11 deletions

View file

@ -0,0 +1,209 @@
{ config, lib, notnft, ... }:
let
inherit (lib)
mapAttrsToList
mkOption
hasAttr
types
traceVal
flip
mapAttrs'
mapAttrs
nameValuePair;
# a = [
# [ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq ip.daddr "10.80.1.2") (is.eq th.dport "22") accept ]
# ];
cfg = config.microvm;
protocolEnumToNft = f: proto:
f.${proto};
tcpUdpServiceOptions.options = {
hostName = mkOption {
type = types.str;
};
port = mkOption {
type = types.port;
};
protocol = mkOption {
type = types.listOf (types.enum [ "tcp" "udp" ]);
};
};
httpServiceOptions.options = {
hostName = mkOption {
type = types.str;
};
port = mkOption {
type = types.port;
};
};
icmpServiceOptions.options = {
hostName = mkOption {
type = types.str;
};
};
tcpUdpConnectionOptions.options = {
target = mkOption {
type = types.str;
};
};
icmpConnectionOptions.options = {
target = mkOption {
type = types.str;
};
};
httpConnectionOptions.options = {
target = mkOption {
type = types.str;
};
};
lookupService = name: type: context:
if hasAttr name cfg.services.${type} then
cfg.services.${type}.${name}
else
throw "Unknown ${type} service ${name} at ${context}";
lookupIds = hostName: context:
if hasAttr hostName subConfigurations then
{
inherit (subConfigurations.${hostName}.config.config.microvm)
groupId
taskId;
}
else
throw "Unknown hostName ${hostName} at ${context}";
subConfigurations = cfg.vms // config.containers;
in
{
options.microvm = {
services = {
tcpUdp = mkOption {
type = with types; types.attrsOf (submodule tcpUdpServiceOptions);
};
icmp = mkOption {
type = with types; types.attrsOf (submodule icmpServiceOptions);
};
http = mkOption {
type = with types; types.attrsOf (submodule httpServiceOptions);
};
};
connections = {
tcpUdp = mkOption {
type = with types;
listOf (submodule tcpUdpConnectionOptions);
default = [];
};
icmp = mkOption {
type = with types;
listOf (submodule icmpConnectionOptions);
default = [];
};
http = mkOption {
type = with types;
listOf (submodule httpConnectionOptions);
default = [];
};
};
};
config.microvm.services.tcpUdp = flip mapAttrs' cfg.services.http
(n: v:
nameValuePair
(n + "@http")
{
inherit (v)
hostName
port;
protocol = [ "tcp" ];
}
);
config.microvm.connections.tcpUdp = flip map cfg.connections.http
(v:
{
target = v.target + "@http";
}
);
config.networking.notnft.rules =
with notnft.dsl; with payload; ruleset {
bridge-t = add table { family = f: f.bridge; } {
output-body = lib.foldl (acc: x: acc x) (add chain) ((flip mapAttrsToList subConfigurations
(n: v:
let
microvmConfig = v.config.config.microvm;
tcpUdpRules =
flip map microvmConfig.connections.tcpUdp (connection:
let
service = lookupService connection.target "tcpUdp" n;
ids = lookupIds service.hostName n;
in
[
(is.eq meta.oifname "mvm-${microvmConfig.hostName}")
(is.eq ip.protocol (f: with f; set (map (protocolEnumToNft f) service.protocol)))
(is.eq ip.saddr "10.80.${toString microvmConfig.groupId}.${toString microvmConfig.taskId}")
(is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}")
(is.eq th.dport service.port)
accept
]);
icmpRules =
flip map microvmConfig.connections.icmp (connection:
let
service = lookupService connection.target "icmp" n;
ids = lookupIds service.hostName n;
in
[
(is.eq meta.oifname "mvm-${microvmConfig.hostName}")
(is.eq ip.protocol (f: with f; icmp))
(is.eq ip.saddr "10.80.${toString microvmConfig.groupId}.${toString microvmConfig.taskId}")
(is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}")
accept
]);
in
tcpUdpRules ++ icmpRules
)) ++ (flip map cfg.connections.icmp (connection:
let
service = lookupService connection.target "icmp" "host";
ids = lookupIds service.hostName "host";
in
[
(is.eq meta.oifname "mvm-${service.hostName}")
(is.eq ip.protocol (f: with f; icmp))
(is.eq ip.saddr "10.80.${toString ids.groupId}.1")
(is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}")
accept
]
)) ++ (flip map cfg.connections.tcpUdp (connection:
let
service = lookupService connection.target "tcpUdp" "host";
ids = lookupIds service.hostName "host";
in
[
(is.eq meta.oifname "mvm-${service.hostName}")
(is.eq ip.protocol (f: with f; set (map (protocolEnumToNft f) service.protocol)))
(is.eq ip.saddr "10.80.${toString ids.groupId}.1")
(is.eq ip.daddr "10.80.${toString ids.groupId}.${toString ids.taskId}")
(is.eq th.dport service.port)
accept
]
)));
};
};
}

View file

@ -0,0 +1,348 @@
{ config, lib, ... }:
let
inherit (lib)
mkOption
mkEnableOption
types;
cfg = config.microvm;
intToHex = int:
{
"0" = "00";
"1" = "01";
"2" = "02";
"3" = "03";
"4" = "04";
"5" = "05";
"6" = "06";
"7" = "07";
"8" = "08";
"9" = "09";
"10" = "0a";
"11" = "0b";
"12" = "0c";
"13" = "0d";
"14" = "0e";
"15" = "0f";
"16" = "10";
"17" = "11";
"18" = "12";
"19" = "13";
"20" = "14";
"21" = "15";
"22" = "16";
"23" = "17";
"24" = "18";
"25" = "19";
"26" = "1a";
"27" = "1b";
"28" = "1c";
"29" = "1d";
"30" = "1e";
"31" = "1f";
"32" = "20";
"33" = "21";
"34" = "22";
"35" = "23";
"36" = "24";
"37" = "25";
"38" = "26";
"39" = "27";
"40" = "28";
"41" = "29";
"42" = "2a";
"43" = "2b";
"44" = "2c";
"45" = "2d";
"46" = "2e";
"47" = "2f";
"48" = "30";
"49" = "31";
"50" = "32";
"51" = "33";
"52" = "34";
"53" = "35";
"54" = "36";
"55" = "37";
"56" = "38";
"57" = "39";
"58" = "3a";
"59" = "3b";
"60" = "3c";
"61" = "3d";
"62" = "3e";
"63" = "3f";
"64" = "40";
"65" = "41";
"66" = "42";
"67" = "43";
"68" = "44";
"69" = "45";
"70" = "46";
"71" = "47";
"72" = "48";
"73" = "49";
"74" = "4a";
"75" = "4b";
"76" = "4c";
"77" = "4d";
"78" = "4e";
"79" = "4f";
"80" = "50";
"81" = "51";
"82" = "52";
"83" = "53";
"84" = "54";
"85" = "55";
"86" = "56";
"87" = "57";
"88" = "58";
"89" = "59";
"90" = "5a";
"91" = "5b";
"92" = "5c";
"93" = "5d";
"94" = "5e";
"95" = "5f";
"96" = "60";
"97" = "61";
"98" = "62";
"99" = "63";
"100" = "64";
"101" = "65";
"102" = "66";
"103" = "67";
"104" = "68";
"105" = "69";
"106" = "6a";
"107" = "6b";
"108" = "6c";
"109" = "6d";
"110" = "6e";
"111" = "6f";
"112" = "70";
"113" = "71";
"114" = "72";
"115" = "73";
"116" = "74";
"117" = "75";
"118" = "76";
"119" = "77";
"120" = "78";
"121" = "79";
"122" = "7a";
"123" = "7b";
"124" = "7c";
"125" = "7d";
"126" = "7e";
"127" = "7f";
"128" = "80";
"129" = "81";
"130" = "82";
"131" = "83";
"132" = "84";
"133" = "85";
"134" = "86";
"135" = "87";
"136" = "88";
"137" = "89";
"138" = "8a";
"139" = "8b";
"140" = "8c";
"141" = "8d";
"142" = "8e";
"143" = "8f";
"144" = "90";
"145" = "91";
"146" = "92";
"147" = "93";
"148" = "94";
"149" = "95";
"150" = "96";
"151" = "97";
"152" = "98";
"153" = "99";
"154" = "9a";
"155" = "9b";
"156" = "9c";
"157" = "9d";
"158" = "9e";
"159" = "9f";
"160" = "a0";
"161" = "a1";
"162" = "a2";
"163" = "a3";
"164" = "a4";
"165" = "a5";
"166" = "a6";
"167" = "a7";
"168" = "a8";
"169" = "a9";
"170" = "aa";
"171" = "ab";
"172" = "ac";
"173" = "ad";
"174" = "ae";
"175" = "af";
"176" = "b0";
"177" = "b1";
"178" = "b2";
"179" = "b3";
"180" = "b4";
"181" = "b5";
"182" = "b6";
"183" = "b7";
"184" = "b8";
"185" = "b9";
"186" = "ba";
"187" = "bb";
"188" = "bc";
"189" = "bd";
"190" = "be";
"191" = "bf";
"192" = "c0";
"193" = "c1";
"194" = "c2";
"195" = "c3";
"196" = "c4";
"197" = "c5";
"198" = "c6";
"199" = "c7";
"200" = "c8";
"201" = "c9";
"202" = "ca";
"203" = "cb";
"204" = "cc";
"205" = "cd";
"206" = "ce";
"207" = "cf";
"208" = "d0";
"209" = "d1";
"210" = "d2";
"211" = "d3";
"212" = "d4";
"213" = "d5";
"214" = "d6";
"215" = "d7";
"216" = "d8";
"217" = "d9";
"218" = "da";
"219" = "db";
"220" = "dc";
"221" = "dd";
"222" = "de";
"223" = "df";
"224" = "e0";
"225" = "e1";
"226" = "e2";
"227" = "e3";
"228" = "e4";
"229" = "e5";
"230" = "e6";
"231" = "e7";
"232" = "e8";
"233" = "e9";
"234" = "ea";
"235" = "eb";
"236" = "ec";
"237" = "ed";
"238" = "ee";
"239" = "ef";
"240" = "f0";
"241" = "f1";
"242" = "f2";
"243" = "f3";
"244" = "f4";
"245" = "f5";
"246" = "f6";
"247" = "f7";
"248" = "f8";
"249" = "f9";
"250" = "fa";
"251" = "fb";
"252" = "fc";
"253" = "fd";
"254" = "fe";
"255" = "ff";
}.${toString int};
groupIdOption = mkOption {
type = types.int;
default = config.microvm.groupId;
};
taskIdOption = mkOption {
type = types.int;
};
tcpUdpConnectionOptions.options = {
target = mkOption {
type = types.str;
};
};
icmpConnectionOptions.options = {
target = mkOption {
type = types.str;
};
};
in
{
options.microvm = {
enableExtras = mkEnableOption "Extras";
groupId = mkOption {
type = types.int;
};
taskId = mkOption {
type = types.int;
};
hostsHostName = mkOption {
type = types.str;
};
hostName = mkOption {
type = types.str;
};
connections = {
tcpUdp = mkOption {
type = with types;
listOf (submodule tcpUdpConnectionOptions);
default = [];
};
icmp = mkOption {
type = with types;
listOf (submodule icmpConnectionOptions);
default = [];
};
};
};
config = {
networking.hostName = "${cfg.hostName}-${cfg.hostsHostName}";
microvm.interfaces = [
{
type = "tap";
# interface name on the host
id = "mvm-${cfg.hostName}";
# Ethernet address of the MicroVM's interface, not the host's
#
# Locally administered have one of 2/6/A/E in the second nibble.
mac = "02:00:00:00:${intToHex cfg.groupId}:${intToHex cfg.taskId}";
}
];
networking.interfaces."eth0" = {
ipv4.addresses = [
{
address = "10.80.${toString cfg.groupId}.${toString cfg.taskId}";
prefixLength = 24;
}
];
};
};
}

122
nixos/modules/notnft.nix Normal file
View file

@ -0,0 +1,122 @@
{ pkgs, config, lib, notnft, ... }:
let
inherit (lib)
types
mkOption
mkDefault
mkEnableOption
flip
concatMapStringsSep
optionalAttrs
listToAttrs
optional
filter;
cfg = config.networking.notnft;
jsonFormat = (pkgs.formats.json {});
in
{
options.networking.notnft = {
enable = mkEnableOption "notnft";
preRules = mkOption {
type = types.listOf jsonFormat.type;
default = [];
};
rules = mkOption {
type = notnft.types.ruleset;
default = {};
};
postRules = mkOption {
type = types.listOf jsonFormat.type;
default = [];
};
json = mkOption {
type = jsonFormat.type;
readOnly = true;
};
jsonFile = mkOption {
type = types.path;
readOnly = true;
};
flush = mkOption {
type = types.bool;
default = true;
};
chains = {
dnsDrop = {
enable = mkEnableOption "Add dns-drop chain";
rule = mkOption {
type = notnft.type.rule;
readOnly = true;
default = with notnft.dsl; with payload;
[ jump "dns-drop" ];
};
};
};
};
config = {
networking.notnft.rules = with notnft.dsl; with payload; ruleset {
filter = add table { family = f: f.inet; }
(listToAttrs (filter (x: x != {}) [
(optionalAttrs cfg.chains.dnsDrop.enable {
name = "dns-drop";
value = add chain
[ (is.ne ip.daddr "10.64.2.1") (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ];
})
]));
};
networking.notnft.json = builtins.toJSON {
nftables = (optional cfg.flush { flush.ruleset = null; }) ++ cfg.preRules ++ cfg.rules.nftables ++ cfg.postRules;
};
networking.notnft.jsonFile = pkgs.writeText "rules.json" cfg.json;
boot.blacklistedKernelModules = [ "ip_tables" ];
environment.systemPackages = [ pkgs.nftables ];
networking.networkmanager.firewallBackend = mkDefault "nftables";
systemd.services.notnftables = {
description = "notnftables firewall";
before = [ "network-pre.target" ];
wants = [ "network-pre.target" ];
wantedBy = [ "multi-user.target" ];
reloadIfChanged = true;
serviceConfig = let
startScript = pkgs.writeShellScript "start-nft.sh" ''
${pkgs.buildPackages.nftables}/bin/nft -j -f ${cfg.jsonFile}
'';
# rulesScript = pkgs.writeTextFile {
# name = "nftables-rules";
# executable = true;
# text = ''
# #! ${pkgs.nftables}/bin/nft -f
# flush ruleset
# ${if cfg.rulesetFile != null then ''
# include "${cfg.rulesetFile}"
# '' else cfg.ruleset}
# '';
# checkPhase = lib.optionalString cfg.checkRuleset ''
# cp $out ruleset.conf
# ${cfg.preCheckRuleset}
# export NIX_REDIRECTS=/etc/protocols=${pkgs.buildPackages.iana-etc}/etc/protocols:/etc/services=${pkgs.buildPackages.iana-etc}/etc/services
# LD_PRELOAD="${pkgs.buildPackages.libredirect}/lib/libredirect.so ${pkgs.buildPackages.lklWithFirewall.lib}/lib/liblkl-hijack.so" \
# ${pkgs.buildPackages.nftables}/bin/nft --check -j < ${cfg.jsonFile}
# '';
# };
in {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = startScript;
ExecReload = startScript;
ExecStop = "${pkgs.nftables}/bin/nft flush ruleset";
};
};
};
}

View file

@ -46,7 +46,14 @@ in
./users.nix ./users.nix
./sol.nix ./sol.nix
../../common/remote_access.nix ../../common/remote_access.nix
./microvms.nix
inputs.serokell-nix.nixosModules.acme-sh inputs.serokell-nix.nixosModules.acme-sh
inputs.notnft.nixosModules.default
inputs.self.nixosModules.notnft
inputs.microvm.nixosModules.host
inputs.self.nixosModules.microvm-extras-host
config'.flake.nixosModules.hashicorp config'.flake.nixosModules.hashicorp
config'.flake.nixosModules.hashicorp-envoy config'.flake.nixosModules.hashicorp-envoy
config'.flake.nixosModules.telegraf config'.flake.nixosModules.telegraf

View file

@ -145,7 +145,7 @@ in
iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface" iifname { "nomad", "ve-monitor", "ve-klipper" } oifname { "nomad", "ve-monitor", "ve-klipper" } accept comment "Allow Nomad to do whatever it wants in its interface"
iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router" iifname { "${wlan}", "${lan}", "lo" } accept comment "Allow local network to access the router"
iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper" } jump input_out iifname { "${wan}", "${doVPN}", "nomad", "docker0", "ve-monitor", "ve-klipper", "mvm0" } jump input_out
iifname { "${doVPN}" } jump input_doVPN iifname { "${doVPN}" } jump input_doVPN
# Allow containers to reach the DNS server # Allow containers to reach the DNS server

View file

@ -0,0 +1,179 @@
# SPDX-FileCopyrightText: 2022 Richard Brežák <richard@brezak.sk>
#
# SPDX-License-Identifier: LGPL-3.0-or-later
{ notnft, inputs', lib, config, ... }:
let
inherit (lib)
mkBefore
flip
genAttrs;
in
{
networking.notnft = {
enable = true;
flush = false;
};
networking.notnft.preRules = [
{ add.table = { family = "bridge"; name = "bridge-t"; }; }
{ flush.table = { family = "bridge"; name = "bridge-t"; }; }
];
networking.notnft.rules =
let
interfaces = [ "mvm-test" "mvm0" ];
logRule = with notnft.dsl; with payload; prefix:
[
(log { prefix = "${prefix} dropped: "; flags = (f: [ f.all ]); } )
];
dropRule = with notnft.dsl; with payload;
[ drop ];
in
with notnft.dsl; with payload; ruleset {
bridge-t = add table { family = f: f.bridge; } {
input-body = add chain;
input-mvm = add chain
[ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ]
[ (is.eq meta.protocol (f: f.arp)) accept ]
[ (jump "input-body") ]
(logRule "Bridge input")
(dropRule);
input = add chain
{ type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.accept; }
[ (vmap meta.iifname (genAttrs interfaces (_: (goto "input-mvm")))) ]
[ (vmap meta.oifname (genAttrs interfaces (_: (goto "input-mvm")))) ];
output-body = add chain;
output-mvm = add chain
[ (is.eq ether.type (f: f.arp)) accept ]
[ (jump "output-body") ]
(logRule "Bridge output")
(dropRule);
output = add chain
{ type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.accept; }
[ (vmap meta.iifname (genAttrs interfaces (_: (goto "output-mvm")))) ]
[ (vmap meta.oifname (genAttrs interfaces (_: (goto "output-mvm")))) ];
forward-body = add chain;
forward-mvm = add chain
[ (jump "forward-body") ]
(logRule "Bridge forward")
(dropRule);
forward = add chain
{ type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.accept; }
[ (vmap meta.iifname (genAttrs interfaces (_: (goto "input-mvm")))) ]
[ (vmap meta.oifname (genAttrs interfaces (_: (goto "input-mvm")))) ];
# prerouting = add chain
# { type = f: f.filter; hook = f: f.prerouting; prio = -300; policy = f: f.accept; }
# ;
# postrouting = add chain
# { type = f: f.filter; hook = f: f.postrouting; prio = -300; policy = f: f.accept; }
# ;
};
};
systemd.services.notnftables = {
requires = [ "nftables.service" ];
after = [ "nftables.service" ];
};
networking.bridges.mvm0 = {
interfaces = [];
};
networking.interfaces.mvm0 = {
useDHCP = false;
ipv4.addresses = [
{
address = "10.80.1.1";
prefixLength = 24;
}
];
};
microvm.services.tcpUdp.test-ssh = {
hostName = "test";
port = 22;
protocol = [ "tcp" ];
};
microvm.services.tcpUdp.test-http = {
hostName = "test";
port = 80;
protocol = [ "tcp" ];
};
microvm.services.icmp.test = {
hostName = "test";
};
microvm.connections.tcpUdp = [
{
target = "test-ssh";
}
{
target = "test-http";
}
];
microvm.connections.icmp = [
{
target = "test";
}
];
microvm.vms = {
test.config = {
imports = [ inputs'.self.nixosModules.microvm-extras ];
microvm = {
hostName = "test";
hostsHostName = "omen";
groupId = 1;
taskId = 2;
};
microvm.hypervisor = "cloud-hypervisor";
microvm.shares = [{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "ro-store";
proto = "virtiofs";
}];
microvm.storeOnDisk = false;
networking.firewall.allowedTCPPorts = [ 80 22 ];
services.nginx = {
enable = true;
virtualHosts."example.com" = {
root = "/var/www/blog";
};
};
users.users.root.password = "";
services.getty.helpLine = ''
Log in as "root" with an empty password.
'';
services.openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
};
}

View file

@ -7,7 +7,8 @@ let
flip flip
mapAttrs mapAttrs
singleton singleton
loadSecrets; loadSecrets
mkAfter;
config' = config; config' = config;
in in
@ -23,7 +24,7 @@ in
}; };
modules = singleton modules = singleton
({ pkgs, config, ... }: ({ pkgs, lib, config, ... }:
{ {
imports = [ imports = [
./xserver.nix ./xserver.nix
@ -35,13 +36,19 @@ in
./users.nix ./users.nix
./nixpkgs.nix ./nixpkgs.nix
../../common/sound.nix ../../common/sound.nix
# ./test-vm.nix
inputs.dwarffs.nixosModules.dwarffs inputs.dwarffs.nixosModules.dwarffs
inputs.microvm.nixosModules.host
inputs.notnft.nixosModules.default
inputs.self.nixosModules.notnft
inputs.self.nixosModules.microvm-extras-host
]; ];
_module.args.nixinate = { _module.args.nixinate = {
host = secret.network.ips.omen.vpn or ""; host = secret.network.ips.omen.vpn or "";
sshUser = "main"; sshUser = "main";
buildOn = "local"; buildOn = "local";
substituteOnTarget = true; substituteOnTarget = true;
hermetic = false; hermetic = false;
@ -71,6 +78,302 @@ in
virtualisation.podman.enable = true; virtualisation.podman.enable = true;
virtualisation.podman.dockerCompat = true; virtualisation.podman.dockerCompat = true;
nixpkgs.overlays = [
(final: prev:
let
nixpkgs = final.fetchFromGitHub {
owner = "NixOS";
repo = "nixpkgs";
rev = "2ca2346b60f72fc75bcc570367e24e1d68d55b18";
sha256 = "sha256-GNUI2MRZGe8rm27DgY503aLyXRm/Dfcao6McHrHkA7s=";
};
pkgs = import nixpkgs { system = final.stdenv.system; };
virtiofsd = pkgs.virtiofsd;
in
{
virtiofsd = final.writeShellScriptBin "virtiofsd" ''
ok_args=()
while [[ $# -gt 0 ]] ; do
case "$1" in
--posix-acl)
;;
*)
ok_args+=("$1")
;;
esac
shift 1
done
exec ${lib.getExe virtiofsd} "''${ok_args[@]}"
'';
}
)
];
# networking.nftables = {
# enable = true;
# rulesetFile =
networking.notnft.rules =
let
notnft = (inputs.notnft.lib.${pkgs.stdenv.system});
logRule = with notnft.dsl; with payload; prefix:
[ (log { prefix = "${prefix} dropped: "; flags = (f: [ f.all ]); } ) ];
traceChain = with notnft.dsl; with payload;
add chain
[ (is.eq th.dport 53) (mangle meta.nftrace 1) ]
[ (is.eq th.dport 53) (mangle meta.nftrace 1) ]
[ (is.eq th.dport 22) (mangle meta.nftrace 1) ]
[ (is.eq th.sport 22) (mangle meta.nftrace 1) ]
[ (is.eq meta.oifname "mvm0") (mangle meta.nftrace 1) ]
[ (is.eq meta.iifname "mvm0") (mangle meta.nftrace 1) ];
in
# pkgs.writeText "nftables.json" (builtins.toJSON (with notnft.dsl; with payload; ruleset
with notnft.dsl; with payload; ruleset {
filter = add table { family = f: f.inet; } {
trace = traceChain;
### lo
input-lo = add chain
[ (is.ne ip.saddr (set [ "127.0.0.1" "127.0.0.53" ])) drop ]
[ (is.ne ip.daddr (set [ "127.0.0.1" "127.0.0.53" ])) drop ]
[ accept ];
output-lo = add chain
[ accept ];
###
### mvm
input-mvm = add chain
[ (is.eq ip.protocol (f: f.icmp)) accept ];
output-mvm = add chain
[ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ]
[ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr "10.80.1.2") (is.eq ip.daddr "10.80.1.1") accept ]
[ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport 22) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ]
[ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport 80) (is.eq ip.saddr "10.80.1.1") (is.eq ip.daddr "10.80.1.2") accept ];
###
### wlan0
input-wlan0 = add chain
[ drop ];
output-wlan0 = add chain
[ (is.ne ip.daddr (secret.network.ips.blowhole.ip or "")) (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ]
[ accept ];
###
### wlan0
input-eth0 = add chain
[ drop ];
output-eth0 = add chain
[ (is.ne ip.daddr (secret.network.ips.blowhole.ip or "")) (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) drop ]
[ accept ];
###
### wg0
input-wg0 = add chain
# accept syncthing sharing
[ (is.eq ip.protocol (f: f.udp)) (is.eq th.sport "22000") (is.eq th.dport "22000") accept ]
[ (is.eq ip.protocol (f: f.tcp)) (is.eq th.dport "22000") accept ]
[ (is.eq ip.protocol (f: f.icmp)) accept ];
output-wg0 = add chain
# TCP, UDP 53 to blowhole
[ (is.eq ip.protocol (f: with f; set [ udp tcp ])) (is.eq th.dport 53) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accep t]
# TCP 22, 80, 4646, 8200, 8500, 2049 to blowhole
[ (is.eq ip.protocol (f: with f; set [ tcp ])) (is.eq th.dport (set [ 22 80 4646 8200 8500 2049 ])) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accept ]
# ICMP to blowhole, toothpick
[ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.saddr (secret.network.ips.omen.vpn or "")) (is.eq ip.daddr (set [ (secret.network.ips.toothpick or "") (secret.network.ips.blowhole.ip or "") ])) accept ]
# accept syncthing sharing
[ (is.eq ip.protocol (f: f.udp)) (is.eq th.sport "22000") (is.eq th.dport "22000") accept ]
[ (is.eq ip.protocol (f: f.tcp)) (is.eq th.dport "22000") accept ]
;
###
input = add chain { type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.drop; }
# accept related, established and drop invalid
[ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ]
# # accept icmp between the same IP
# [ (is.eq ip.protocol (f: f.icmp)) (is.eq ip.daddr ip.saddr) accept ]
[ (jump "trace") ]
[ (is.eq meta.iifname "wlan0") (jump "input-wlan0") ]
[ (is.eq meta.iifname "eth0") (jump "input-eth0") ]
[ (is.eq meta.iifname "mvm0") (jump "input-mvm") ]
[ (is.eq meta.iifname "lo") (jump "input-lo") ]
[ (is.eq meta.iifname "wg0") (jump "input-wg0") ]
[ (is.eq ip.protocol (f: f.icmp)) accept ]
(logRule "Input");
output = add chain { type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.drop; }
[ (jump "trace") ]
[ (is.eq meta.oifname "wlan0") (jump "output-wlan0") ]
[ (is.eq meta.oifname "eth0") (jump "output-eth0") ]
[ (is.eq meta.oifname "lo") (jump "output-lo") ]
[ (is.eq meta.oifname "mvm0") (jump "output-mvm") ]
[ (is.eq meta.oifname "wg0") (jump "output-wg0") ]
(logRule "Output");
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.drop; }
[ (jump "trace") ]
# accept masquaraded packets incoming from wg0
[ (is.eq meta.iifname "wg0") (vmap ct.state { established = accept; related = accept; }) ]
# accept TCP, UDP 53 from 10.80.1.2 to blowhole
[ (is.eq meta.iifname "mvm0") (is.eq meta.oifname "wg0") (is.eq ip.protocol (f: with f; set [ tcp udp ])) (is.eq th.dport 53) (is.eq ip.saddr "10.80.1.2") (is.eq ip.daddr (secret.network.ips.blowhole.ip or "")) accept ]
(logRule "Forward");
prerouting = add chain { type = f: f.nat; hook = f: f.prerouting; prio = -199; policy = f: f.accept; }
;
postrouting = add chain { type = f: f.nat; hook = f: f.postrouting; prio = -199; policy = f: f.accept; }
# masquarade from 10.80.1.2 heading to wg0
[ (is.eq meta.iifname "mvm0") (is.eq meta.oifname "wg0") (is.eq ip.saddr (set [ "10.80.1.2" ])) masquerade ];
};
bridge-t = add table { family = f: f.bridge; } {
trace = traceChain;
input-body = add chain;
input = add chain { type = f: f.filter; hook = f: f.input; prio = -300; policy = f: f.drop; }
[ (jump "trace") ]
[ (vmap ct.state { established = accept; related = accept; invalid = drop; }) ]
[ (is.eq meta.protocol (f: f.arp)) accept ]
[ (jump "input-body") ]
(logRule "Bridge input");
output-body = add chain;
output = add chain { type = f: f.filter; hook = f: f.output; prio = -300; policy = f: f.drop; }
[ (jump "trace") ]
[ (is.eq ether.type (f: f.arp)) accept ]
[ (jump "output-body") ]
(logRule "Bridge output");
forward-body = add chain;
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = -300; policy = f: f.drop; }
[ (jump "trace") ]
[ (jump "forward-body") ]
(logRule "Bridge forward");
prerouting = add chain { type = f: f.filter; hook = f: f.prerouting; prio = -300; policy = f: f.accept; }
;
postrouting = add chain { type = f: f.filter; hook = f: f.postrouting; prio = -300; policy = f: f.accept; }
;
};
};
# ));
# };
systemd.network.netdevs."mvm0" = {
netdevConfig = {
Name = "mvm0";
Kind = "bridge";
};
};
systemd.network.networks."10-mvm0" = {
matchConfig.Name = "mvm0";
networkConfig.Address = "10.80.1.1/24";
linkConfig.RequiredForOnline = "yes";
};
systemd.network.networks."11-mvm-test" = {
matchConfig.Name = "mvm-test";
networkConfig.Bridge = "mvm0";
linkConfig.RequiredForOnline = "no";
};
microvm.services.tcpUdp.test-ssh = {
hostName = "test";
port = 22;
protocol = [ "tcp" ];
};
microvm.services.http.test = {
hostName = "test";
port = 80;
};
microvm.services.icmp.test = {
hostName = "test";
};
microvm.connections.http = [
{
target = "test";
}
];
microvm.connections.tcpUdp = [
{
target = "test-ssh";
}
];
microvm.connections.icmp = [
{
target = "test";
}
];
microvm.vms = {
test.config = {
imports = [ inputs.self.nixosModules.microvm-extras ];
microvm = {
hostName = "test";
hostsHostName = "omen";
groupId = 1;
taskId = 2;
};
microvm.hypervisor = "cloud-hypervisor";
microvm.shares = [{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "ro-store";
proto = "virtiofs";
}];
microvm.storeOnDisk = false;
networking.firewall.allowedTCPPorts = [ 80 22 ];
services.nginx = {
enable = true;
virtualHosts."example.com" = {
root = "/var/www/blog";
};
};
users.users.root.password = "";
services.getty.helpLine = ''
Log in as "root" with an empty password.
'';
services.openssh = {
enable = true;
settings.PermitRootLogin = "yes";
};
};
};
}); });
}; };
} }

View file

@ -1,24 +1,100 @@
{ pkgs, lib, inputs', secret, ... }: { pkgs, lib, inputs', secret, notnft, ... }:
let let
inherit (lib) inherit (lib)
concatStringsSep; concatStringsSep;
in in
{ {
systemd.network.enable = true;
networking = { networking = {
hostName = "omen"; hostName = "omen";
useDHCP = false;
# interfaces.eno1.useDHCP = true;
hostId = "10c7ffc5";
networkmanager.dns = "none";
nameservers = [ "10.64.2.1" ];
firewall.allowedTCPPorts = [22000]; hostId = "10c7ffc5";
nameservers = [ secret.network.ips.blowhole.ip ];
firewall.enable = false;
wireguard.interfaces."wg0" = wireguard.interfaces."wg0" =
secret.wireguard."omen" or { privateKey = ""; }; secret.wireguard."omen" or { privateKey = ""; };
}; };
networking.networkmanager.enable = true; networking.notnft.rules = with notnft.dsl; with payload; ruleset {
filter = add table { family = f: f.inet; } {
trace = add chain
[ (is.eq ip.protocol (f: f.icmp)) (mangle meta.nftrace 1) ];
};
};
services.networkd-dispatcher = {
enable = true;
rules.wlan-eth-switch = {
onState = [ "no-carrier" "configured" ];
script = ''
#!${pkgs.runtimeShell}
export PATH=$PATH:${pkgs.iwd}/bin
echo "entered state: '$STATE' on interface '$IFACE' with IPs '$IP_ADDRS'"
case $IFACE in
eth0)
echo $IP_ADDRS | ${lib.getExe pkgs.grepcidr} ${secret.network.networks.home.amsterdam} > /dev/null
home_net=$?
case $STATE in
no-carrier)
if [ "$(iwctl station wlan0 show | grep -i State | tr -s ' ' | cut -f 3 -d ' ')" == "disconnected" ] ; then
iwctl device wlan0 set-property Powered off
iwctl device wlan0 set-property Powered on
fi
;;
configured)
if [ "$home_net" == "0" ] ; then
iwctl station wlan0 disconnect
fi
;;
*)
;;
esac
;;
*)
;;
esac
'';
};
};
systemd.network.links."50-eth0" = {
matchConfig.MACAddress = secret.network.mac.usbc-omen;
linkConfig.Name = "eth0";
};
systemd.network.networks."50-eth0" = {
matchConfig.Name = "eth0";
networkConfig.DHCP = "ipv4";
linkConfig.RequiredForOnline = "no";
};
systemd.network.networks."50-wlan0" = {
matchConfig.Name = "wlan0";
linkConfig.RequiredForOnline = "no";
# networkConfig.DHCP = "ipv4";
# networkConfig.DNS = "${secret.network.ips.blowhole.ip}";
# dhcpV4Config.UseDNS = false;
# dhcpV6Config.UseDNS = false;
};
services.resolved.enable = false;
environment.etc."resolv.conf".text = ''
nameserver ${secret.network.ips.blowhole.ip}
'';
services.resolved.extraConfig = ''
[Resolve]
DNS=${secret.network.ips.blowhole.ip}
FallbackDNS=
'';
networking.wireless.iwd.enable = true;
hardware.bluetooth = { hardware.bluetooth = {
enable = true; enable = true;
settings = { settings = {