dotfiles/nixos/systems/blowhole/kubernetes.nix

{
  config,
  pkgs,
  lib,
  inputs',
  ...
}: let
  k3s = pkgs.runCommandNoCC pkgs.k3s_1_29.name {} ''
    cp --no-preserve=mode,ownership -r ${pkgs.k3s_1_29} $out
    rm $out/bin/{crictl,ctr}
    chmod +x $out/bin/*
  '';
  inherit
    (lib)
    singleton
    concatStringsSep
    getExe'
    mkForce
    ;

  k3sFormat = pkgs.formats.json {};

  k3sConfigDir = "/run/secrets/k3s.d";
  k3sConfig = k3sFormat.generate "k3s-config.json" {
    disable = [
      "traefik"
      "servicelb"
    ];
    cluster-cidr = "10.64.48.0/22";
    service-cidr = "10.64.48.0/23";
    flannel-iface = "eno1";
    container-runtime-endpoint = "unix:///run/containerd/containerd.sock";
    flannel-backend = "vxlan";
    # disable-network-policy = true;
    node-ip = "10.64.2.1";
    node-external-ip = "10.64.2.1";
  };

  k3s-pgpassPath = "${k3sConfigDir}/pgpass.yaml";

  cilium-cni = pkgs.callPackage (
    {
      lib,
      buildGoModule,
      fetchFromGitHub,
      installShellFiles,
    }:
      buildGoModule rec {
        pname = "cilium-cni";
        version = "1.15.1";

        src = fetchFromGitHub {
          owner = "cilium";
          repo = "cilium";
          rev = "v${version}";
          hash = "sha256-RFuSaBiobN9V7u1EEWGoMl/g9iKu/yAJana6wUSQ/+I=";
        };

        vendorHash = null;

        subPackages = ["plugins/cilium-cni"];

        ldflags = [
          "-s"
          "-w"
          # "-X github.com/cilium/cilium-cli/cli.Version=${version}"
        ];

        # Required to workaround install check error:
        # 2022/06/25 10:36:22 Unable to start gops: mkdir /homeless-shelter: permission denied
        HOME = "$TMPDIR";

        # doInstallCheck = true;
        # installCheckPhase = ''
        #   $out/bin/cilium version | grep ${version} > /dev/null
        # '';

        # nativeBuildInputs = [installShellFiles];
        # postInstall = ''
        #   installShellCompletion --cmd cilium \
        #     --bash <($out/bin/cilium completion bash) \
        #     --fish <($out/bin/cilium completion fish) \
        #     --zsh <($out/bin/cilium completion zsh)
        # '';

        meta = with lib; {
          description = "CLI to install, manage & troubleshoot Kubernetes clusters running Cilium";
          license = licenses.asl20;
          homepage = "https://www.cilium.io/";
          maintainers = with maintainers; [humancalico bryanasdev000 qjoly];
          mainProgram = "cilium";
        };
      }
  ) {};
in {
  uk3s.helmCharts.main = {
    istio-base = {
      chart = pkgs.istioctl.src + "/manifests/charts/base";
      namespace = "istio-system";
      values = {
        defaultRevision = "foo";
      };
    };
    istio-operator = {
      chart = pkgs.istioctl.src + "/manifests/charts/istio-operator";
      namespace = "istio-system";
      values = {
        enableCRDTemplates = true;
        revision = "foo";
      };
    };
    metallb = {
      chart =
        (pkgs.fetchFromGitHub {
          owner = "metallb";
          repo = "metallb";
          rev = "v0.14.3";
          hash = "sha256-H9F2g4N/RGeMViPpK9jiJG1ABI8luFmfk6d2mjQobKQ=";
        })
        + "/charts/metallb";
      namespace = "metallb-system";
      values = {
      };
    };
  };

  environment.systemPackages = [k3s];

  systemd.services."k3s-secrets" = {
    requiredBy = ["k3s.service"];
    before = ["k3s.service"];
    restartIfChanged = true;
    serviceConfig.Type = "oneshot";
    script = ''
      mkdir -p ${k3sConfigDir}
      touch ${k3s-pgpassPath}
      chmod 400 ${k3s-pgpassPath}
      chown root:root ${k3s-pgpassPath}
      cat <<EOF > ${k3s-pgpassPath}
      datastore-endpoint: postgres://k3s:$(cat /mnt/kyle/infrastructure/k3s/postgresql/pgpass)@${config.containers.k3s-psql.localAddress}:5432/k3s
      EOF
      ln -sf ${k3sConfig} ${k3sConfigDir}/main.yaml
    '';
  };

  systemd.services."k3s-pre-setup" = {
    requiredBy = ["k3s.service"];
    before = ["k3s.service"];
    restartIfChanged = true;
    serviceConfig.Type = "oneshot";
    script = let
      k3sCniDir = "/var/lib/rancher/k3s/agent/etc/cni/net.d";
    in ''
      mkdir -p /var/lib/rancher/k3s/agent/etc/cni /etc/cni/net.d
      [ -e ${k3sCniDir} ] && ! [ -L ${k3sCniDir} ] && rm -r ${k3sCniDir}
      ! [ -L ${k3sCniDir} ] && ln -sf /etc/cni/net.d /var/lib/rancher/k3s/agent/etc/cni
      exit 0
    '';
  };

  systemd.tmpfiles.rules = [
    "D /run/cilium/cni 600 root root 0 -"
  ];

  systemd.services."k3s-setup" = {
    wantedBy = ["k3s.service"];
    after = ["k3s.service"];
    restartIfChanged = true;
    serviceConfig.Type = "oneshot";
    script = let
      sanitizedRevision = lib.stringAsChars (c:
        {"." = "-";}.${c} or c)
      pkgs.istioctl.src.rev;
      istio-base = pkgs.callPackage kubernetes/templateHelmChart.nix {} {
        helmChart = pkgs.istioctl.src + "/manifests/charts/base";
        name = "istio-base";
        namespace = "istio-system";
        values = {
          defaultRevision = sanitizedRevision;
        };
      };
      istio-istiod = pkgs.callPackage kubernetes/templateHelmChart.nix {} {
        helmChart = pkgs.istioctl.src + "/manifests/charts/istio-control/istio-discovery";
        name = "istio-istiod";
        namespace = "istio-system";
        values = {
          revision = sanitizedRevision;
        };
      };
      istio-gateway = pkgs.callPackage kubernetes/templateHelmChart.nix {} {
        helmChart = pkgs.istioctl.src + "/manifests/charts/gateway";
        name = "istio-gateway";
        namespace = "istio-system";
        values = {
          revision = sanitizedRevision;
        };
      };
      istio-ingressgateway = pkgs.callPackage kubernetes/templateHelmChart.nix {} {
        helmChart = pkgs.istioctl.src + "/manifests/charts/gateways/istio-ingress";
        name = "istio-ingress";
        namespace = "istio-ingress";
        values = {
          revision = sanitizedRevision;
        };
      };
      kubectl = getExe' k3s "kubectl";
    in ''
      # if ! ${kubectl} get namespace istio-system ; then
      #   ${kubectl} create namespace istio-system
      # fi
      ${kubectl} apply -f ${istio-base}
      ${kubectl} apply -f ${istio-istiod}
      ${kubectl} apply -f ${istio-gateway}
      ${kubectl}  apply -f ${istio-ingressgateway}
      ${kubectl} apply -f ${./kubernetes/test-website.yaml}
    '';
  };

  systemd.services."container@k3s-psql".environment.SYSTEMD_LOG_LEVEL = "debug";
  systemd.services."container@k3s-psql".script = lib.mkBefore ''
    if [ -z $OVERWRITTEN ] ; then
      _tmp=$(mktemp)
      cp $0 $_tmp
      sed -i -e 's/--notify-ready=yes/--notify-ready=no/' \
          -e 's/--kill-signal=SIGRTMIN+3/--kill-signal=SIGHUP/' \
          $_tmp
      export OVERWRITTEN="$_tmp"
      chmod +x $_tmp
      exec -a $0 $_tmp
    fi
    rm "$OVERWRITTEN"
    unset OVERWRITTEN
  '';

  containers.k3s-psql = {
    ephemeral = true;
    autoStart = true;
    privateNetwork = true;

    localAddress = "10.64.99.8";
    hostAddress = "10.64.99.7";

    bindMounts = {
      "/var/lib/postgresql" = {
        hostPath = "/mnt/kyle/infrastructure/k3s/postgresql";
        isReadOnly = false;
      };
    };

    path =
      (inputs'.nixng.nglib.makeSystem {
        inherit (pkgs.stdenv) system;
        name = "k3s-psql";
        inherit (inputs') nixpkgs;
        config = {config, ...}: {
          dumb-init = {
            enable = true;
            type.services = {};
          };

          init.services.sdnotify = {
            enabled = true;
            shutdownOnExit = false;
            script = pkgs.writeShellScript "notify-ready" ''
              NOTIFY_SOCKET=/run/host/notify ${getExe' pkgs.sdnotify-wrapper "sdnotify-wrapper"} sh -c 'echo ; sleep infinity'
            '';
          };

          init.services.postgresql-secrets = {
            enabled = true;
            shutdownOnExit = false;
            script = pkgs.writeShellScript "" ''
              mkdir -p /secrets
              cat <<EOF > /secrets/init.sql
              alter user "k3s" with password '$(cat /var/lib/postgresql/pgpass)';
              alter database "k3s" owner to "k3s";
              EOF
              exec sleep infinity
            '';
          };

          init.services.postgresql = {
            dependencies = [
              "postgresql-secrets"
            ];
          };

          services.postgresql = {
            enable = true;
            package = pkgs.postgresql_16;
            enableTCPIP = true;
            authentication = "host  all all all          md5";
            initialScript = "/secrets/init.sql";
            ensureDatabases = {
              "k3s" = {
                ENCODING = "UTF8";
                TEMPLATE = "template0";
              };
            };
            ensureExtensions = {};
            ensureUsers = [
              {
                name = "k3s";
                ensurePermissions."DATABASE \"k3s\"" = "ALL PRIVILEGES";
              }
            ];
          };
        };
      })
      .config
      .system
      .build
      .toplevel;
  };

  systemd.services.containerd.path = with pkgs; [
    iptables-nftables-compat
  ];
  systemd.services.k3s.path = with pkgs; [
    iptables-nftables-compat
  ];
  virtualisation.containerd = {
    enable = true;
    settings = {
      plugins."io.containerd.grpc.v1.cri".containerd.snapshotter = mkForce "overlayfs";
      plugins."io.containerd.grpc.v1.cri".cni.bin_dir = mkForce ((pkgs.symlinkJoin {
          name = "cni-containerd-plugins";
          paths = with pkgs; [
            cni-plugins
            cni-plugin-flannel
          ];
          postBuild = "echo links added";
        })
        + "/bin");
    };
  };

  services = {
    k3s = {
      package = k3s;
      enable = true;
      role = "server";
      serverAddr = "https://10.64.2.1:6443";
      extraFlags = concatStringsSep " " [
        "--config /run/secrets/k3s"
        "--debug"
      ];
    };
  };
}