diff --git a/ops/glesys/main.tf b/ops/glesys/main.tf index 2e84f7ca5..cefd5d697 100644 --- a/ops/glesys/main.tf +++ b/ops/glesys/main.tf @@ -95,7 +95,6 @@ locals { # Hostnames of all public services on whitby whitby_services = [ "deploys", - "status", ] # Hostnames of all public services on nevsky @@ -106,6 +105,7 @@ locals { "code", "cs", "grep", + "status", ] # Hostnames of all public services on bugry diff --git a/ops/machines/nevsky/default.nix b/ops/machines/nevsky/default.nix index c1c42d426..2ea75aba3 100644 --- a/ops/machines/nevsky/default.nix +++ b/ops/machines/nevsky/default.nix @@ -15,6 +15,7 @@ in (mod "josh.nix") (mod "known-hosts.nix") (mod "livegrep.nix") + (mod "monitoring.nix") (mod "monorepo-gerrit.nix") (mod "owothia.nix") (mod "panettone.nix") @@ -33,6 +34,7 @@ in (mod "www/grep.tvl.fyi.nix") (mod "www/self-cache.tvl.fyi.nix") (mod "www/self-redirect.nix") + (mod "www/status.tvl.su.nix") (depot.third_party.agenix.src + "/modules/age.nix") ]; diff --git a/ops/modules/monitoring.nix b/ops/modules/monitoring.nix new file mode 100644 index 000000000..654f8d87e --- /dev/null +++ b/ops/modules/monitoring.nix @@ -0,0 +1,106 @@ +# Runs the TVL Monitoring setup (currently Grafana + Prometheus). +{ depot, pkgs, config, lib, ... }: + +{ + # Required for prometheus to be able to scrape stats + services.nginx.statusPage = true; + + # Configure Prometheus & Grafana. Exporter configuration for + # Prometheus is inside the respective service modules. + services.prometheus = { + enable = true; + retentionTime = "90d"; + + exporters = { + node = { + enable = true; + + enabledCollectors = [ + "logind" + "processes" + "systemd" + ]; + }; + + nginx = { + enable = true; + sslVerify = false; + constLabels = [ "host=whitby" ]; + }; + }; + + scrapeConfigs = [{ + job_name = "node"; + scrape_interval = "5s"; + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; + }]; + } + { + job_name = "nginx"; + scrape_interval = "5s"; + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ]; + }]; + }]; + }; + + services.grafana = { + enable = true; + + settings = { + server = { + http_port = 4723; # "graf" on phone keyboard + domain = "status.tvl.su"; + root_url = "https://status.tvl.su"; + }; + + analytics.reporting_enabled = false; + + "auth.generic_oauth" = { + enabled = true; + client_id = "grafana"; + scopes = "openid profile email"; + name = "TVL"; + email_attribute_path = "mail"; + login_attribute_path = "sub"; + name_attribute_path = "displayName"; + auth_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/auth"; + token_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/token"; + api_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/userinfo"; + + # Give lukegb, aspen, tazjin "Admin" rights. + role_attribute_path = "((sub == 'lukegb' || sub == 'aspen' || sub == 'tazjin') && 'Admin') || 'Editor'"; + + # Allow creating new Grafana accounts from OAuth accounts. + allow_sign_up = true; + }; + + "auth.anonymous" = { + enabled = true; + org_name = "The Virus Lounge"; + org_role = "Viewer"; + }; + + "auth.basic".enabled = false; + + auth = { + oauth_auto_login = true; + disable_login_form = true; + }; + }; + + provision = { + enable = true; + datasources.settings.datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + }]; + }; + }; + + # Contains GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. + systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secretsDir + "/grafana"; +} +