diff --git a/ops/glesys/dns-tvl-fyi.tf b/ops/glesys/dns-tvl-fyi.tf index 2edef3a45..26b4a5e42 100644 --- a/ops/glesys/dns-tvl-fyi.tf +++ b/ops/glesys/dns-tvl-fyi.tf @@ -104,6 +104,40 @@ resource "glesys_dnsdomain_record" "tvl_fyi_net_CNAME" { host = "net" } +# Binary cache round-robin setup (experimental; only on .fyi) + +resource "glesys_dnsdomain_record" "cache_tvl_fyi_A" { + domain = glesys_dnsdomain.tvl_fyi.id + host = "cache" + type = "A" + data = each.key + for_each = toset([var.whitby_ipv4, var.nevsky_ipv4]) +} + +resource "glesys_dnsdomain_record" "cache_tvl_fyi_AAAA" { + domain = glesys_dnsdomain.tvl_fyi.id + host = "cache" + type = "AAAA" + data = each.key + for_each = toset([var.whitby_ipv6, var.nevsky_ipv6]) +} + +# Builderball cache records + +resource "glesys_dnsdomain_record" "tvl_fyi_cache_whitby_CNAME" { + domain = glesys_dnsdomain.tvl_fyi.id + type = "CNAME" + data = "whitby.tvl.fyi." + host = "whitby.cache" +} + +resource "glesys_dnsdomain_record" "tvl_fyi_cache_nevsky_CNAME" { + domain = glesys_dnsdomain.tvl_fyi.id + type = "CNAME" + data = "nevsky.tvl.fyi." + host = "nevsky.cache" +} + # Google Domains mail forwarding configuration (no sending) resource "glesys_dnsdomain_record" "tvl_fyi_MX_5" { domain = glesys_dnsdomain.tvl_fyi.id diff --git a/ops/glesys/dns-tvl-su.tf b/ops/glesys/dns-tvl-su.tf index f2286cf1c..97c7d60ac 100644 --- a/ops/glesys/dns-tvl-su.tf +++ b/ops/glesys/dns-tvl-su.tf @@ -67,6 +67,13 @@ resource "glesys_dnsdomain_record" "tvl_su_sanduny_AAAA" { data = var.sanduny_ipv6 } +resource "glesys_dnsdomain_record" "cache_tvl_su_whitby_CNAME" { + domain = glesys_dnsdomain.tvl_su.id + host = "cache" + type = "CNAME" + data = "whitby.tvl.su." +} + # Explicit records for all services running on whitby resource "glesys_dnsdomain_record" "tvl_su_whitby_services" { domain = glesys_dnsdomain.tvl_su.id diff --git a/ops/glesys/main.tf b/ops/glesys/main.tf index eef612027..d92740d08 100644 --- a/ops/glesys/main.tf +++ b/ops/glesys/main.tf @@ -98,7 +98,6 @@ locals { "atward", "auth", "b", - "cache", "cl", "code", "cs", diff --git a/ops/machines/nevsky/default.nix b/ops/machines/nevsky/default.nix index 811c3d5a1..8453415da 100644 --- a/ops/machines/nevsky/default.nix +++ b/ops/machines/nevsky/default.nix @@ -6,8 +6,13 @@ let in { imports = [ + (mod "builderball.nix") + (mod "harmonia.nix") (mod "known-hosts.nix") (mod "tvl-users.nix") + (mod "www/cache.tvl.fyi.nix") + (mod "www/self-cache.tvl.fyi.nix") + (mod "www/self-redirect.nix") (depot.third_party.agenix.src + "/modules/age.nix") ]; @@ -85,9 +90,25 @@ in }; }; - age.secrets = { - wg-privkey.file = depot.ops.secrets."wg-nevsky.age"; - }; + age.secrets = + let + secretFile = name: depot.ops.secrets."${name}.age"; + in + { + wg-privkey.file = depot.ops.secrets."wg-nevsky.age"; + + nix-cache-priv = { + file = secretFile "nix-cache-priv"; + mode = "0440"; + group = "harmonia"; + }; + + # Not actually a secret + nix-cache-pub = { + file = secretFile "nix-cache-pub"; + mode = "0444"; + }; + }; networking = { hostName = "nevsky"; @@ -175,12 +196,22 @@ in useRoutingFeatures = "both"; }; - security.sudo.extraRules = [ - { - groups = [ "wheel" ]; - commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; - } - ]; + # Run a Harmonia binary cache. + # + # TODO(tazjin): switch to upstream module after fix for Nix 2.3 + services.depot.harmonia = { + enable = true; + signKeyPaths = [ (config.age.secretsDir + "/nix-cache-priv") ]; + settings.bind = "127.0.0.1:6443"; + settings.priority = 50; + }; + + services.depot.builderball.enable = true; + + security.sudo.extraRules = [{ + groups = [ "wheel" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; zramSwap.enable = true; diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix index 7bc02a3fe..8c0b7d184 100644 --- a/ops/machines/whitby/default.nix +++ b/ops/machines/whitby/default.nix @@ -10,6 +10,7 @@ in { imports = [ (mod "atward.nix") + (mod "builderball.nix") (mod "cgit.nix") (mod "cheddar.nix") (mod "clbot.nix") @@ -33,11 +34,13 @@ in (mod "www/atward.tvl.fyi.nix") (mod "www/auth.tvl.fyi.nix") (mod "www/b.tvl.fyi.nix") + (mod "www/cache.tvl.fyi.nix") (mod "www/cache.tvl.su.nix") (mod "www/cl.tvl.fyi.nix") (mod "www/code.tvl.fyi.nix") (mod "www/cs.tvl.fyi.nix") (mod "www/deploys.tvl.fyi.nix") + (mod "www/self-cache.tvl.fyi.nix") (mod "www/self-redirect.nix") (mod "www/signup.tvl.fyi.nix") (mod "www/static.tvl.fyi.nix") @@ -386,6 +389,9 @@ in # Run a livegrep code search instance livegrep.enable = true; + # Run Nix cache proxy + builderball.enable = true; + # Run the Panettone issue tracker panettone = { enable = true; diff --git a/ops/modules/builderball.nix b/ops/modules/builderball.nix new file mode 100644 index 000000000..d56c69513 --- /dev/null +++ b/ops/modules/builderball.nix @@ -0,0 +1,51 @@ +# Configuration for builderball, the Nix cache proxy for substituting between +# builders. +# +# This is in experimental state, not yet supporting any dynamic private builders. +{ depot, config, lib, ... }: + +let + cfg = config.services.depot.builderball; + description = "Nix cache proxy for distribution between builders"; + hostname = config.networing.hostName; +in +{ + options.services.depot.builderball = { + enable = lib.mkEnableOption description; + + caches = lib.mkOption { + type = with lib.types; listOf string; + description = "Public addresses of caches to use"; + + default = [ + "whitby.cache.tvl.fyi" + "nevsky.cache.tvl.fyi" + ]; + }; + + port = lib.mkOption { + type = lib.types.int; + description = "port on which to listen locally"; + default = 26862; # bounc + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.builderball = + let + caches = lib.concatStringsSep " " (map (c: "-cache https://${c}") cfg.caches); + in + { + inherit description; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + serviceConfig = { + ExecStart = "${depot.ops.builderball}/bin/builderball ${caches} -port ${toString cfg.port} -debug"; + DynamicUser = true; + Restart = "always"; + }; + }; + }; +} diff --git a/ops/modules/www/cache.tvl.fyi.nix b/ops/modules/www/cache.tvl.fyi.nix new file mode 100644 index 000000000..b0c90267a --- /dev/null +++ b/ops/modules/www/cache.tvl.fyi.nix @@ -0,0 +1,50 @@ +# Publicly serve builderball cache. This is an experimental setup, and separate +# from the "normal" harmonia cache on cache.tvl.su. +{ config, ... }: + +let + # This attrset forms a linked list of hosts, which delegate ACME fallbacks to + # each other. These *must* form a circle, otherwise we may end up walking only + # part of the ring. + acmeFallback = host: ({ + whitby = "nevsky.cache.tvl.fyi"; + nevsky = "whitby.cache.tvl.fyi"; # GOTO 1 + })."${host}"; +in +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."cache.tvl.fyi" = { + serverName = "cache.tvl.fyi"; + enableACME = true; + forceSSL = true; + + # This enables fetching TLS certificates for the same domain on different + # hosts. This config is kind of messy; it would be nice to generate a + # correct ring from the depot fixpoint, but this may be impossible due to + # infinite recursion. Please read the comment on `acmeFallback` above. + acmeFallbackHost = acmeFallback config.networking.hostName; + + extraConfig = '' + location = /cache-key.pub { + alias /run/agenix/nix-cache-pub; + } + + location = / { + proxy_pass http://${config.services.depot.harmonia.settings.bind}; + } + + location / { + proxy_pass http://localhost:${toString config.services.depot.builderball.port}; + } + ''; + }; + + # participating hosts should use their local cache, otherwise they might end + # up querying themselves from afar for data they don't have. + networking.extraHosts = "127.0.0.1 cache.tvl.fyi"; + }; +} diff --git a/ops/modules/www/cache.tvl.su.nix b/ops/modules/www/cache.tvl.su.nix index 27d1c06dd..69f0f1e93 100644 --- a/ops/modules/www/cache.tvl.su.nix +++ b/ops/modules/www/cache.tvl.su.nix @@ -8,7 +8,6 @@ config = { services.nginx.virtualHosts."cache.tvl.su" = { serverName = "cache.tvl.su"; - serverAliases = [ "cache.tvl.fyi" ]; enableACME = true; forceSSL = true; diff --git a/ops/modules/www/self-cache.tvl.fyi.nix b/ops/modules/www/self-cache.tvl.fyi.nix new file mode 100644 index 000000000..e0f87651d --- /dev/null +++ b/ops/modules/www/self-cache.tvl.fyi.nix @@ -0,0 +1,26 @@ +# per-host addresses for publicly reachable caches, for use with builderball +# TODO(tazjin): merge with the public cache module; but needs ACME fixes +{ config, lib, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = lib.mkIf config.services.depot.harmonia.enable { + services.nginx.virtualHosts."${config.networking.hostName}.cache.tvl.fyi" = { + enableACME = true; + forceSSL = true; + + extraConfig = '' + location = /cache-key.pub { + alias /run/agenix/nix-cache-pub; + } + + location / { + proxy_pass http://${config.services.depot.harmonia.settings.bind}; + } + ''; + }; + }; +}