diff --git a/flake.nix b/flake.nix index a8fdff7..4b190ba 100644 --- a/flake.nix +++ b/flake.nix @@ -192,13 +192,13 @@ p.random p.null (hpkgs.terraform.plugins.mkProvider { - owner = "Janrupf"; + owner = "MagicRB"; repo = "terraform-provider-influxdb-v2"; - rev = "ce868cc190d41e459d40152d11220542a6af4eee"; + rev = "4f10e465f9526b47d1ef97a8f2e109aa85a7d647"; version = "0.4.6"; - hash = "sha256-NZgKkIvw2H+vP4qawDVyURcl56ze+3K2cqIolm2GM8E="; + hash = "sha256-/IQoA1CwYIafHbHKSZq7pZKFxefgd09fm0lnBW3r11Q="; vendorHash = "sha256-g7Njs7psHFFSWk44CiV+blLrzpnB+L9HgMTx3lLMA8Q="; - provider-source-address = "registry.terraform.io/Janrupf/influxdb-v2"; + provider-source-address = "registry.terraform.io/MagicRB/influxdb-v2"; }) ] ); diff --git a/nixos/systems/blowhole/monitoring.nix b/nixos/systems/blowhole/monitoring.nix index 8da0693..47eb55f 100644 --- a/nixos/systems/blowhole/monitoring.nix +++ b/nixos/systems/blowhole/monitoring.nix @@ -19,68 +19,100 @@ in output."envoy_grafana".value = tf "vault_consul_secret_backend_role.envoy-grafana"; output."envoy_blowhole".value = tf "vault_consul_secret_backend_role.envoy-blowhole"; - # data."influxdb-v2_organization"."redalder" = { - # name = "redalder"; - # }; + data."influxdb-v2_organization"."redalder" = { + name = "redalder"; + }; - # resource."influxdb-v2_bucket"."metrics_bucket" = { - # name = "metrics"; - # description = "Metrics bucket"; - # org_id = "\${data.influxdb-v2_organization.redalder.id}"; - # retention_rules = { - # every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m - # }; - # }; + resource."influxdb-v2_bucket"."metrics_bucket" = { + name = "metrics"; + description = "Metrics bucket"; + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + retention_rules = { + every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m + }; + }; - # resource."influxdb-v2_bucket"."logs_bucket" = { - # org_id = "\${data.influxdb-v2_organization.redalder.id}"; - # name = "logs"; - # description = "Logs bucket"; - # retention_rules = { - # every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m - # }; - # }; + resource."influxdb-v2_bucket"."logs_bucket" = { + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + name = "logs"; + description = "Logs bucket"; + retention_rules = { + every_seconds = 30 * 24 * 60 * 60; # days * h/d * m/h * s/m + }; + }; - # resource."influxdb-v2_authorization"."telegraf_authorization" = { - # org_id = "\${data.influxdb-v2_organization.redalder.id}"; - # description = "Token for telegraf ingestion"; - # status = "active"; - # permissions = [ - # { - # action = "write"; - # resource = { - # id = "\${influxdb-v2_bucket.logs_bucket.id}"; - # org_id = "\${data.influxdb-v2_organization.redalder.id}"; - # type = "buckets"; - # }; - # } - # { - # action = "write"; - # resource = { - # id = "\${influxdb-v2_bucket.metrics_bucket.id}"; - # org_id = "\${data.influxdb-v2_organization.redalder.id}"; - # type = "buckets"; - # }; - # } - # ]; - # }; + resource."influxdb-v2_authorization"."telegraf_authorization" = { + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + description = "Token for telegraf ingestion"; + status = "active"; + permissions = [ + { + action = "write"; + resource = { + id = "\${influxdb-v2_bucket.logs_bucket.id}"; + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + type = "buckets"; + }; + } + { + action = "write"; + resource = { + id = "\${influxdb-v2_bucket.metrics_bucket.id}"; + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + type = "buckets"; + }; + } + ]; + }; - # resource."vault_mount"."kvv2" = { - # path = "kvv2"; - # type = "kv"; - # options = { version = 2; }; - # description = "KV Version 2 secret engine mount"; - # }; + resource."influxdb-v2_authorization"."grafana_authorization" = { + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + description = "Token for Grefana"; + status = "active"; + permissions = [ + { + action = "read"; + resource = { + id = "\${influxdb-v2_bucket.logs_bucket.id}"; + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + type = "buckets"; + }; + } + { + action = "read"; + resource = { + id = "\${influxdb-v2_bucket.metrics_bucket.id}"; + org_id = "\${data.influxdb-v2_organization.redalder.id}"; + type = "buckets"; + }; + } + ]; + }; + resource."vault_mount"."kv" = { + path = "kv"; + type = "kv"; + options = { version = 2; }; + description = "KV Version 2 secret engine mount"; + }; - # resource."vault_kv_secret_v2"."telegraf_secret" = { - # mount = "\${vault_mount.kvv2.path}"; - # name = "homelab-1/blowhole/monitor/telegraf"; - # options = { version = 2; }; - # data_json = builtins.toJSON { - # influxdb_token = "\${influxdb-v2_authorization.telegraf_authorization.token}"; - # }; - # }; + resource."vault_kv_secret_v2"."telegraf_secret" = { + mount = "\${vault_mount.kv.path}"; + name = "homelab-1/blowhole/monitor/telegraf"; + options = { version = 2; }; + data_json = builtins.toJSON { + influxdb_token = "\${influxdb-v2_authorization.telegraf_authorization.token}"; + }; + }; + + resource."vault_kv_secret_v2"."grafana_secret" = { + mount = "\${vault_mount.kv.path}"; + name = "homelab-1/blowhole/monitor/grafana"; + options = { version = 2; }; + data_json = builtins.toJSON { + influxdb_token = "\${influxdb-v2_authorization.grafana_authorization.token}"; + }; + }; }; nixpkgs.overlays = singleton (_: _: @@ -147,7 +179,7 @@ in ''; destination = "/run/secrets/envoy-blowhole.token"; command = pkgs.writeShellScript "envoy-blowhole-reload.sh" - '' + '' sudo systemctl try-reload-or-restart hashicorp-envoy-telegraf ''; } @@ -156,6 +188,25 @@ in INFLUXDB_TOKEN={{ with secret "kv/data/homelab-1/blowhole/monitor/telegraf" }}{{ .Data.data.influxdb_token }}{{ end }} ''; destination = "/run/secrets/monitor/telegraf.env"; + command = pkgs.writeShellScript "monitor-telegraf-reload.sh" + '' + sudo systemd-run -P --machine monitor /run/current-system/sw/bin/bash -l -c \ + 'systemctl try-reload-or-restart telegraf' + ''; + } + { + source = pkgs.writeText "grafana-influx.token.vtmpl" '' + {{ with secret "kv/data/homelab-1/blowhole/monitor/grafana" }} + {{ .Data.data.influxdb_token }} + {{ end }} + ''; + destination = "/run/secrets/monitor/grafana-influx.token"; + perms = "0644"; + command = pkgs.writeShellScript "monitor-telegraf-reload.sh" + '' + sudo systemd-run -P --machine monitor /run/current-system/sw/bin/bash -l -c \ + 'systemctl try-reload-or-restart grafana' + ''; } ]; }; @@ -188,7 +239,7 @@ in }; adminBind = "127.0.0.1:19100"; - hotRestart = true; + hotRestart = false; }; services.telegraf-magic = { @@ -198,23 +249,23 @@ in percpu = true; totalcpu = true; tags.host = "blowhole"; - tags.bucket = "telegraf"; + tags.bucket = "metrics"; }; inputs.mem = { tags.host = "blowhole"; - tags.bucket = "telegraf"; + tags.bucket = "metrics"; }; inputs.nomad = { url = "http://${secret.network.ips.blowhole.ip}:4646"; tags.host = "blowhole"; - tags.bucket = "telegraf"; + tags.bucket = "metrics"; }; inputs.zfs = { tags.host = "blowhole"; - tags.bucket = "telegraf"; + tags.bucket = "metrics"; }; # inputs.tail = [ @@ -239,20 +290,18 @@ in outputs.influxdb_v2 = [ { urls = [ "http://${secret.network.ips.blowhole.ip}:8086" ]; - bucket = "telegraf"; - # tagdrop = [ "bucket" ]; - # tagpass = { - # bucket = "telegraf"; - # }; + bucket = "metrics"; + tagpass = { + bucket = [ "metrics" ]; + }; + } + { + urls = [ "http://${secret.network.ips.blowhole.ip}:8086" ]; + bucket = "logs"; + tagpass = { + bucket = [ "logs" ]; + }; } - # { - # urls = [ "http://${secret.network.ips.blowhole.ip}:8086" ]; - # bucket = "logs"; - # tagdrop = [ "bucket" ]; - # tagpass = { - # bucket = "logs"; - # }; - # } ]; }; }; @@ -336,7 +385,7 @@ in address = "10.64.99.2:19000"; adminBind = "127.0.0.1:19100"; - hotRestart = true; + hotRestart = false; }; services.postgresql = { @@ -382,7 +431,29 @@ in name = "grafana"; user = "grafana"; }; + + paths.provisioning = { + datasources.datasources = [ + { + name = "InfluxDB"; + type = "influxdb"; + access = "proxy"; + orgId = 1; + uid = "influxdb"; + url = "http://127.0.0.1:8086"; + jsonData = { + version = "Flux"; + organization = "redalder"; + defaultBucket = "bucket"; + }; + secureJsonData = { + token = "$__file{/run/secrets/grafana-influx.token}"; + }; + } + ]; + }; }; + enable = true; }; @@ -404,7 +475,7 @@ in address = "10.64.99.2:19001"; adminBind = "127.0.0.1:19101"; - hotRestart = true; + hotRestart = false; }; services.influxdb2 = { @@ -434,7 +505,7 @@ in address = "10.64.99.2:19002"; adminBind = "127.0.0.1:19102"; - hotRestart = true; + hotRestart = false; }; services.telegraf-magic = { @@ -450,29 +521,16 @@ in unittype = "service"; tags = { host = "blowhole#monitoring"; + bucket = "metrics"; }; }; outputs.influxdb_v2 = [ - # { - # urls = [ "http://127.0.0.1:8086" ]; - # token = "\${INFLUXDB_TOKEN}"; - # organization = "redalder"; - # bucket = "logs"; - # tagdrop = [ "bucket" ]; - # tagpass = { - # bucket = "logs"; - # }; - # } { urls = [ "http://127.0.0.1:8086" ]; token = "\${INFLUXDB_TOKEN}"; organization = "redalder"; - bucket = "telegraf"; - tagdrop = [ "bucket" ]; - # tagpass = { - # bucket = "telegraf"; - # }; + bucket_tag = "bucket"; } ]; }; diff --git a/nixos/systems/blowhole/nomad.nix b/nixos/systems/blowhole/nomad.nix index 78378aa..bcd861e 100644 --- a/nixos/systems/blowhole/nomad.nix +++ b/nixos/systems/blowhole/nomad.nix @@ -82,6 +82,11 @@ in enabled = true; }; + telemetry = { + publish_allocation_metrics = true; + publish_node_metrics = true; + }; + client = { cni_path = "${pkgs.cni-plugins}/bin"; diff --git a/nixos/systems/blowhole/vault-agent.nix b/nixos/systems/blowhole/vault-agent.nix index f3445d4..9899479 100644 --- a/nixos/systems/blowhole/vault-agent.nix +++ b/nixos/systems/blowhole/vault-agent.nix @@ -7,8 +7,8 @@ in let config = pkgs.writeText "hashicorp-vault-agent-tmpfiles.d" '' d /run/secrets 0750 root root 0 - x /run/secrets/monitor 0750 root root - - d /run/secrets/monitor 0750 root root 0 + x /run/secrets/monitor 0755 root root - + d /run/secrets/monitor 0755 root root 0 ''; in { diff --git a/terranix/blowhole.nix b/terranix/blowhole.nix index 5218c1c..3424640 100644 --- a/terranix/blowhole.nix +++ b/terranix/blowhole.nix @@ -78,6 +78,10 @@ in path "${vaultKvMount}/data/homelab-1/blowhole/monitor/telegraf" { capabilities = ["read"] } + + path "${vaultKvMount}/data/homelab-1/blowhole/monitor/grafana" { + capabilities = ["read"] + } ''; }; diff --git a/terranix/default.nix b/terranix/default.nix index 2826817..48377e6 100644 --- a/terranix/default.nix +++ b/terranix/default.nix @@ -34,9 +34,9 @@ in address = "http://10.64.2.1:4646"; }; - # provider."influxdb-v2" = { - # url = "http://influx.in.redalder.org"; - # }; + provider."influxdb-v2" = { + url = "http://influx.in.redalder.org"; + }; imports = [ ./modules/push_approles.nix @@ -55,7 +55,7 @@ in terraform.required_providers = { influxdb-v2 = { - source = "Janrupf/influxdb-v2"; + source = "MagicRB/influxdb-v2"; }; }; @@ -78,7 +78,7 @@ in path = "kv"; type = "kv"; options.version = "2"; - description = "KV Version 2 secret envine mount"; + description = "KV Version 2 secret engine mount"; }; resource."vault_kv_secret_backend_v2"."config" = {