dotfiles/nixos/systems/blowhole/monitoring.nix

# SPDX-FileCopyrightText: 2023 Richard Brežák <richard@brezak.sk>
#
# SPDX-License-Identifier: LGPL-3.0-or-later
{ pkgs, roots, lib, inputs', config, secret, config', ... }:
let
  inherit (lib)
    singleton
    nixosTests
    concatStringsSep;
in
{
  uterranix.config = { tflib, ... }:
    let
      inherit (tflib)
        tf;
    in
      {
        output."envoy_grafana".value = tf "vault_consul_secret_backend_role.envoy-grafana";
        output."envoy_blowhole".value = tf "vault_consul_secret_backend_role.envoy-blowhole";

        data."influxdb-v2_organization"."redalder" = {
          name = "redalder";
        };

        resource."influxdb-v2_bucket"."metrics_bucket" = {
          name        = "metrics";
          description = "Metrics bucket";
          org_id      = "\${data.influxdb-v2_organization.redalder.id}";
          retention_rules = {
            every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m
          };
        };

        resource."influxdb-v2_bucket"."metrics_preprocessed_bucket" = {
          name        = "metrics-preprocessed";
          description = "Preprocessed bucket";
          org_id      = "\${data.influxdb-v2_organization.redalder.id}";
          retention_rules = {
            every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m
          };
        };

        resource."influxdb-v2_bucket"."logs_bucket" = {
          org_id      = "\${data.influxdb-v2_organization.redalder.id}";
          name        = "logs";
          description = "Logs bucket";
          retention_rules = {
            every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m
          };
        };

        resource."influxdb-v2_authorization"."telegraf_authorization" = {
          org_id      = "\${data.influxdb-v2_organization.redalder.id}";
          description = "Token for telegraf ingestion";
          status      = "active";
          permissions = [
            {
              action = "write";
              resource = {
                id     = "\${influxdb-v2_bucket.logs_bucket.id}";
                org_id = "\${data.influxdb-v2_organization.redalder.id}";
                type   = "buckets";
              };
            }
            {
              action = "write";
              resource = {
                id     = "\${influxdb-v2_bucket.metrics_bucket.id}";
                org_id = "\${data.influxdb-v2_organization.redalder.id}";
                type   = "buckets";
              };
            }
          ];
        };

        resource."influxdb-v2_authorization"."grafana_authorization" = {
          org_id      = "\${data.influxdb-v2_organization.redalder.id}";
          description = "Token for Grafana";
          status      = "active";
          permissions = [
            {
              action = "read";
              resource = {
                id     = "\${influxdb-v2_bucket.logs_bucket.id}";
                org_id = "\${data.influxdb-v2_organization.redalder.id}";
                type   = "buckets";
              };
            }
            {
              action = "read";
              resource = {
                id     = "\${influxdb-v2_bucket.metrics_preprocessed_bucket.id}";
                org_id = "\${data.influxdb-v2_organization.redalder.id}";
                type   = "buckets";
              };
            }
            {
              action = "read";
              resource = {
                id     = "\${influxdb-v2_bucket.metrics_bucket.id}";
                org_id = "\${data.influxdb-v2_organization.redalder.id}";
                type   = "buckets";
              };
            }
          ];
        };

        resource."vault_mount"."kv" = {
          path = "kv";
          type = "kv";
          options = { version = 2; };
          description = "KV Version 2 secret engine mount";
        };

        resource."vault_kv_secret_v2"."telegraf_secret" = {
          mount = "\${vault_mount.kv.path}";
          name = "homelab-1/blowhole/monitor/telegraf";
          options = { version = 2; };
          data_json = builtins.toJSON {
            influxdb_token = "\${influxdb-v2_authorization.telegraf_authorization.token}";
          };
        };

        resource."vault_kv_secret_v2"."grafana_secret" = {
          mount = "\${vault_mount.kv.path}";
          name = "homelab-1/blowhole/monitor/grafana";
          options = { version = 2; };
          data_json = builtins.toJSON {
            influxdb_token = "\${influxdb-v2_authorization.grafana_authorization.token}";
          };
        };
      };

  nixpkgs.overlays = singleton (_: _:
    {
      telegraf =
        pkgs.buildGoModule rec {
          pname = "telegraf";
          version = "1.25.3";

          excludedPackages = "test";
          doCheck = false;

          subPackages = singleton "cmd/telegraf";

          src = pkgs.fetchFromGitHub {
            owner = "influxdata";
            repo = "telegraf";
            rev = "v${version}";
            sha256 = "sha256-FUZDS4As9qP2Dn0NSBM/e8udDLMk5OZol4CQSI39T4s=";
          };

          vendorHash = "sha256-uWoWvS9ZZzhpE+PiJv0fqblMLOAGIrhCdi0ugvF/lQI=";
          proxyVendor = true;

          ldflags = [
            "-w" "-s" "-X main.version=${version}"
          ];

          passthru.tests = { inherit (nixosTests) telegraf; };

          meta = with lib; {
            description = "The plugin-driven server agent for collecting & reporting metrics";
            license = licenses.mit;
            homepage = "https://www.influxdata.com/time-series-platform/telegraf/";
            maintainers = with maintainers; [ mic92 roblabla timstott ];
          };
        };
    });

  services.hashicorp.vault-agent =
    {
      settings.template = [
        {
          source = pkgs.writeText "envoy-grafana.token.vtmpl" ''
            {{ with secret "consul/creds/envoy-grafana" }}{{ .Data.token }}{{ end }}
          '';
          destination = "/run/secrets/monitor/envoy-grafana.token";
          command =
            let
              serviceList =
                [ "hashicorp-envoy-grafana" "hashicorp-envoy-influx" "hashicorp-envoy-telegraf" ];
            in
              pkgs.writeShellScript "envoy-grafana-reload.sh" ''
                sudo systemd-run -P --machine monitor /run/current-system/sw/bin/bash -l -c \
                  'systemctl try-reload-or-restart ${concatStringsSep " " serviceList}' || true
              '';
        }
        {
          source = pkgs.writeText "envoy-blowhole.token.vtmpl" ''
            {{ with secret "consul/creds/envoy-blowhole" }}{{ .Data.token }}{{ end }}
          '';
          destination = "/run/secrets/envoy-blowhole.token";
          command = pkgs.writeShellScript "envoy-blowhole-reload.sh" ''
            sudo systemctl try-reload-or-restart hashicorp-envoy-telegraf || true
          '';
        }
        {
          source = pkgs.writeText "telegraf.env.vtmpl" ''
            INFLUXDB_TOKEN={{ with secret "kv/data/homelab-1/blowhole/monitor/telegraf" }}{{ .Data.data.influxdb_token }}{{ end }}
          '';
          destination = "/run/secrets/monitor/telegraf.env";
          command = pkgs.writeShellScript "monitor-telegraf-reload.sh" ''
            sudo systemd-run -P --machine monitor /run/current-system/sw/bin/bash -l -c \
              'systemctl try-reload-or-restart telegraf' || true
          '';
        }
        {
          source = pkgs.writeText "grafana-influx.token.vtmpl" ''
            {{ with secret "kv/data/homelab-1/blowhole/monitor/grafana" }}
            {{ .Data.data.influxdb_token }}
            {{ end }}
          '';
          destination = "/run/secrets/monitor/grafana-influx.token";
          perms = "0644";
          command = pkgs.writeShellScript "monitor-telegraf-reload.sh" ''
            sudo systemd-run -P --machine monitor /run/current-system/sw/bin/bash -l -c \
              'systemctl try-reload-or-restart grafana' || true
          '';
        }
        {
          source = pkgs.writeText "itp.env.vtmpl" ''
            {{ with secret "kv/data/homelab-1/blowhole/monitor/itp" }}
            INFLUX_HOST={{ .Data.data.host }}
            INFLUX_TOKEN={{ .Data.data.token }}
            {{ end }}
          '';
          destination = "/run/secrets/monitor/itp.env";
        }
      ];
    };


  systemd.services."hashicorp-envoy-telegraf" = {
    requires = [ "vault-unsealed.service" ];
    after = [ "vault-unsealed.service" ];
  };

  ## There is no way to say, hey, listen on localhost. The listeners option is missing the `address` field
  ## and the `name` field so it's impossible to configure....
  services.hashicorp-envoy.telegraf = {
    type = "ingress";
    address = "${secret.network.ips.blowhole.ip or ""}:19000";
    service = {
      kind = "ingress-gateway";
      name = "telegraf-blowhole";

      listeners = singleton {
        port = 8086;
        protocol = "tcp";
        services = singleton {
          name = "telegraf";
        };
      };
    };

    environment = {
      "CONSUL_HTTP_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8500";
      "CONSUL_GRPC_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8502";
      "CONSUL_HTTP_TOKEN_FILE" = "/run/secrets/envoy-blowhole.token";
    };

    adminBind = "127.0.0.1:19100";
    hotRestart = false;
    consulPackage = inputs'.nixpkgs-hashicorp.legacyPackages.${pkgs.stdenv.system}.consul;
    extraConsulArgs = [ "-ignore-envoy-compatibility" ];
  };

  systemd.services."telegraf-magic" = {
    requires = [ "vault-unsealed.service" ];
    after = [ "vault-unsealed.service" ];
  };

  services.telegraf-magic = {
    enable = true;
    settings = {
      inputs.cpu = {
        percpu = true;
        totalcpu = true;
        tags.host = "blowhole";
        tags.bucket = "metrics";
      };

      inputs.mem = {
        tags.host = "blowhole";
        tags.bucket = "metrics";
      };

      inputs.nomad = {
        url = "http://${secret.network.ips.blowhole.ip or ""}:4646";
        tags.host = "blowhole";
        tags.bucket = "metrics";
      };

      # aggregators.minmax = {
      #   period = "30s";
      #   drop_original = true;
      #   namepass = [ "nomad" ];
      # };

      inputs.zfs = {
        tags.host = "blowhole";
        tags.bucket = "metrics";
      };

      # inputs.tail = [
      #   {
      #     files = ["/var/lib/nomad/alloc/*/alloc/logs/*.stdout.*"];
      #     data_format = "value";
      #     data_type = "string";

      #     name_override = "nomad_alloc_log";
      #     tags.bucket = "logs";
      #   }
      #   {
      #     files = ["/var/lib/nomad/alloc/*/alloc/logs/*.stderr.*"];
      #     data_format = "value";
      #     data_type = "string";

      #     name_override = "nomad_alloc_log";
      #     tags.bucket = "logs";
      #   }
      # ];

      inputs.docker_log = {
        tags.bucket = "logs";
        source_tag = true;
      };

      processors.parser = [
        {
          parse_fields = [ "message" ];
          merge = "override";
          data_format = "grok";

          grok_patterns = ["%{COMBINED_LOG_FORMAT}"];

          tagpass = {
            "grok_type" = [ "nginx" "apache" ];
          };
          namepass = [ "docker_log" ];
        }
        {
          parse_fields = [ "message" ];
          merge = "override";
          data_format = "json_v2";

          json_v2 = [
            # the TOML generator won't create the structure required by telegraf without this
            {}
            {
              object = [
                {
                  path = "@this";
                  timestamp_key = "time";
                  timestamp_format = "unix";
                  tags = [
                    "level"
                    "server_name"
                    "namespace"
                    "level"
                    "request"
                  ];
                  disable_prepend_keys = true;
                }
              ];
            }
          ];

          tagpass = {
            "grok_type" = [ "synapse" ];
          };
          namepass = [ "docker_log" ];
        }
      ];

      outputs.influxdb_v2 = [
        {
          urls = singleton "http://${secret.network.ips.blowhole.ip or ""}:8086";
          bucket = "metrics";
          tagpass.bucket = singleton "metrics";
        }
        {
          urls = singleton "http://${secret.network.ips.blowhole.ip or ""}:8086";
          bucket = "logs";
          tagpass.bucket = singleton "logs";
        }
      ];
    };

    systemd = {
      serviceConfig.SupplementaryGroups = [ "docker" ];
    };
  };

  fileSystems."/var/lib/grafana" = {
    device = "blowhole-zpool/persist/grafana";
    fsType = "zfs";
  };

  fileSystems."/var/lib/grafana-postgres" = {
    device = "blowhole-zpool/persist/grafana-postgres";
    fsType = "zfs";
  };

  fileSystems."/var/lib/grafana-influxdb2" = {
    device = "blowhole-zpool/persist/grafana-influxdb2";
    fsType = "zfs";
  };

  systemd.services."container@monitor" = {
    requires = [ "vault-unsealed.service" ];
    after = [ "vault-unsealed.service" ];

    serviceConfig.LimitNOFILE = "infinity";
  };

  # TODO: split interface name and container name, i.e. rewrite the container module....... again
  containers.monitor = {
    ephemeral = true;
    autoStart = true;
    privateNetwork = true;

    localAddress = "10.64.99.2";
    hostAddress = "10.64.99.1";

    extraFlags = [
      "--capability=CAP_IPC_LOCK"
    ];

    bindMounts = {
      "/run/secrets" = {
        hostPath = "/run/secrets/monitor";
        isReadOnly = true;
      };
      "/var/lib/grafana" = {
        hostPath = "/var/lib/grafana";
        isReadOnly = false;
      };
      "/var/lib/postgresql" = {
        hostPath = "/var/lib/grafana-postgres";
        isReadOnly = false;
      };
      "/var/lib/influxdb2" = {
        hostPath = "/var/lib/grafana-influxdb2";
        isReadOnly = false;
      };
    };
    config = {
      nixpkgs.overlays = config.nixpkgs.overlays;

      imports = with config'.flake.nixosModules; [
        hashicorp
        hashicorp-envoy
        telegraf
        grafana
        influx-provisioning
      ];

      services.hashicorp-envoy.grafana = {
        service = {
          name = "grafana";
          id = "grafana";
          address = "10.64.99.2";
          port = 3000;

          connect.sidecar_service = {};
        };

        environment = {
          "CONSUL_HTTP_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8500";
          "CONSUL_GRPC_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8502";
          "CONSUL_HTTP_TOKEN_FILE" = "/run/secrets/envoy-grafana.token";
        };

        address = "10.64.99.2:19000";
        adminBind = "127.0.0.1:19100";
        hotRestart = false;
        consulPackage = inputs'.nixpkgs-hashicorp.legacyPackages.${pkgs.stdenv.system}.consul;
        extraConsulArgs = [ "-ignore-envoy-compatibility" ];
      };

      services.postgresql = {
        enable = true;
        ensureDatabases = singleton "grafana";
        ensureUsers = singleton {
          name = "grafana";
          ensurePermissions."DATABASE grafana" = "ALL PRIVILEGES";
        };
      };

      systemd.services.grafana = {
        serviceConfig = {
          Restart = "always";
          RestartSec = "10s";
        };
      };

      services.grafana-magic = {
        enable = true;
        settings = {
          security = {
            content_security_policy = true;
            disable_gravatar = true;
            data_source_proxy_whitelist = concatStringsSep " " [
              "127.0.0.1:8086"
            ];
          };
          server = {
            domain = "grafana.in.redalder.org";
          };
          system = {
            http_addr = "127.0.0.1";
          };
          database = {
            type = "postgres";
            host = "/var/run/postgresql";
            name = "grafana";
            user = "grafana";
          };

          paths.provisioning = {
            datasources.datasources = [
              {
                name = "InfluxDB";
                type = "influxdb";
                access = "proxy";
                orgId = 1;
                uid = "influxdb";
                url = "http://127.0.0.1:8086";
                jsonData = {
                  version = "Flux";
                  organization = "redalder";
                  defaultBucket = "bucket";
                };
                secureJsonData = {
                  token = "$__file{/run/secrets/grafana-influx.token}";
                };
              }
            ];
          };
        };
      };

      services.hashicorp-envoy.influx = {
        service = {
          name = "influx";
          id = "influx";
          address = "10.64.99.2";
          port = 8086;

          connect.sidecar_service = {};
        };

        environment = {
          "CONSUL_HTTP_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8500";
          "CONSUL_GRPC_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8502";
          "CONSUL_HTTP_TOKEN_FILE" = "/run/secrets/envoy-grafana.token";
        };

        address = "10.64.99.2:19001";
        adminBind = "127.0.0.1:19101";
        hotRestart = false;
        consulPackage = inputs'.nixpkgs-hashicorp.legacyPackages.${pkgs.stdenv.system}.consul;
        extraConsulArgs = [ "-ignore-envoy-compatibility" ];
      };

      systemd.services."influxdb2-provision".serviceConfig.EnvironmentFile = [
        "/run/secrets/itp.env"
      ];

      services.influxdb2 = {
        enable = true;
        provision = {
          stateFile = "/var/lib/influxdb2/itp.state";
          organization = "redalder";

          # tasks.test = {
          #   every = "30s";
          #   fluxFile = ./influx-tasks/system-memory.flux;
          # };
        };
        settings = {
          http-bind-address = "127.0.0.1:8086";
          hardening-enabled = true;
          reporting-disabled = true;
        };
      };

      services.hashicorp-envoy.telegraf = {
        service = {
          name = "telegraf";
          id = "telegraf";
          address = "10.64.99.2";
          port = 8087;

          connect.sidecar_service = {};
        };

        environment = {
          "CONSUL_HTTP_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8500";
          "CONSUL_GRPC_ADDR" = "http://${secret.network.ips.blowhole.ip or ""}:8502";
          "CONSUL_HTTP_TOKEN_FILE" = "/run/secrets/envoy-grafana.token";
        };

        address = "10.64.99.2:19002";
        adminBind = "127.0.0.1:19102";
        hotRestart = false;
        extraConsulArgs = [ "-ignore-envoy-compatibility" ];
      };

      services.telegraf-magic = {
        enable = true;
        settings = {
          inputs.influxdb_v2_listener = {
            service_address = "127.0.0.1:8087";
            bucket_tag = "bucket";
            parser_type = "upstream";
          };

          inputs.systemd_units = {
            unittype = "service";
            tags = {
              host = "blowhole#monitoring";
              bucket = "metrics";
            };
          };

          outputs.influxdb_v2 = singleton {
            urls = [ "http://127.0.0.1:8086" ];
            token = "\${INFLUXDB_TOKEN}";
            organization = "redalder";
            bucket_tag = "bucket";
          };
        };

        systemd.serviceConfig = {
          EnvironmentFile = "/run/secrets/telegraf.env";
        };
      };
    };
  };
}