feat(ops/modules): configure builderball cache setup

Configures an experimental setup for a builderball-based public cache.

This cache only includes the two build machines (whitby & nevsky), for the time
period where both of them exist simultaneously.

The idea is this:

All participating hosts run a harmonia binary cache locally (whitby already
does). They then run builderball instances pointing at each other's harmonia
caches (through dedicated public hostnames).

When a request comes in, the first matching cache address is returned and Nix
will substitute from there.

Change-Id: Ia7d5357fd5e04f77b460205544fa24e82b100230
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12975
Autosubmit: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>

ops/glesys/dns-tvl-fyi.tf

@@ -104,6 +104,40 @@ resource "glesys_dnsdomain_record" "tvl_fyi_net_CNAME" {
   host   = "net"
 }
 
+# Binary cache round-robin setup (experimental; only on .fyi)
+
+resource "glesys_dnsdomain_record" "cache_tvl_fyi_A" {
+  domain   = glesys_dnsdomain.tvl_fyi.id
+  host     = "cache"
+  type     = "A"
+  data     = each.key
+  for_each = toset([var.whitby_ipv4, var.nevsky_ipv4])
+}
+
+resource "glesys_dnsdomain_record" "cache_tvl_fyi_AAAA" {
+  domain   = glesys_dnsdomain.tvl_fyi.id
+  host     = "cache"
+  type     = "AAAA"
+  data     = each.key
+  for_each = toset([var.whitby_ipv6, var.nevsky_ipv6])
+}
+
+# Builderball cache records
+
+resource "glesys_dnsdomain_record" "tvl_fyi_cache_whitby_CNAME" {
+  domain = glesys_dnsdomain.tvl_fyi.id
+  type   = "CNAME"
+  data   = "whitby.tvl.fyi."
+  host   = "whitby.cache"
+}
+
+resource "glesys_dnsdomain_record" "tvl_fyi_cache_nevsky_CNAME" {
+  domain = glesys_dnsdomain.tvl_fyi.id
+  type   = "CNAME"
+  data   = "nevsky.tvl.fyi."
+  host   = "nevsky.cache"
+}
+
 # Google Domains mail forwarding configuration (no sending)
 resource "glesys_dnsdomain_record" "tvl_fyi_MX_5" {
   domain = glesys_dnsdomain.tvl_fyi.id

ops/glesys/dns-tvl-su.tf

@@ -67,6 +67,13 @@ resource "glesys_dnsdomain_record" "tvl_su_sanduny_AAAA" {
   data   = var.sanduny_ipv6
 }
 
+resource "glesys_dnsdomain_record" "cache_tvl_su_whitby_CNAME" {
+  domain = glesys_dnsdomain.tvl_su.id
+  host   = "cache"
+  type   = "CNAME"
+  data   = "whitby.tvl.su."
+}
+
 # Explicit records for all services running on whitby
 resource "glesys_dnsdomain_record" "tvl_su_whitby_services" {
   domain   = glesys_dnsdomain.tvl_su.id

ops/glesys/main.tf

@@ -98,7 +98,6 @@ locals {
     "atward",
     "auth",
     "b",
-    "cache",
     "cl",
     "code",
     "cs",

ops/machines/nevsky/default.nix

@@ -6,8 +6,13 @@ let
 in
 {
   imports = [
+    (mod "builderball.nix")
+    (mod "harmonia.nix")
     (mod "known-hosts.nix")
     (mod "tvl-users.nix")
+    (mod "www/cache.tvl.fyi.nix")
+    (mod "www/self-cache.tvl.fyi.nix")
+    (mod "www/self-redirect.nix")
     (depot.third_party.agenix.src + "/modules/age.nix")
   ];
 
@@ -85,9 +90,25 @@ in
     };
   };
 
-  age.secrets = {
-    wg-privkey.file = depot.ops.secrets."wg-nevsky.age";
-  };
+  age.secrets =
+    let
+      secretFile = name: depot.ops.secrets."${name}.age";
+    in
+    {
+      wg-privkey.file = depot.ops.secrets."wg-nevsky.age";
+
+      nix-cache-priv = {
+        file = secretFile "nix-cache-priv";
+        mode = "0440";
+        group = "harmonia";
+      };
+
+      # Not actually a secret
+      nix-cache-pub = {
+        file = secretFile "nix-cache-pub";
+        mode = "0444";
+      };
+    };
 
   networking = {
     hostName = "nevsky";
@@ -175,12 +196,22 @@ in
     useRoutingFeatures = "both";
   };
 
-  security.sudo.extraRules = [
-    {
-      groups = [ "wheel" ];
-      commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
-    }
-  ];
+  # Run a Harmonia binary cache.
+  #
+  # TODO(tazjin): switch to upstream module after fix for Nix 2.3
+  services.depot.harmonia = {
+    enable = true;
+    signKeyPaths = [ (config.age.secretsDir + "/nix-cache-priv") ];
+    settings.bind = "127.0.0.1:6443";
+    settings.priority = 50;
+  };
+
+  services.depot.builderball.enable = true;
+
+  security.sudo.extraRules = [{
+    groups = [ "wheel" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
 
   zramSwap.enable = true;
 

ops/machines/whitby/default.nix

@@ -10,6 +10,7 @@ in
 {
   imports = [
     (mod "atward.nix")
+    (mod "builderball.nix")
     (mod "cgit.nix")
     (mod "cheddar.nix")
     (mod "clbot.nix")
@@ -33,11 +34,13 @@ in
     (mod "www/atward.tvl.fyi.nix")
     (mod "www/auth.tvl.fyi.nix")
     (mod "www/b.tvl.fyi.nix")
+    (mod "www/cache.tvl.fyi.nix")
     (mod "www/cache.tvl.su.nix")
     (mod "www/cl.tvl.fyi.nix")
     (mod "www/code.tvl.fyi.nix")
     (mod "www/cs.tvl.fyi.nix")
     (mod "www/deploys.tvl.fyi.nix")
+    (mod "www/self-cache.tvl.fyi.nix")
     (mod "www/self-redirect.nix")
     (mod "www/signup.tvl.fyi.nix")
     (mod "www/static.tvl.fyi.nix")
@@ -386,6 +389,9 @@ in
     # Run a livegrep code search instance
     livegrep.enable = true;
 
+    # Run Nix cache proxy
+    builderball.enable = true;
+
     # Run the Panettone issue tracker
     panettone = {
       enable = true;

ops/modules/builderball.nix

@@ -0,0 +1,51 @@
+# Configuration for builderball, the Nix cache proxy for substituting between
+# builders.
+#
+# This is in experimental state, not yet supporting any dynamic private builders.
+{ depot, config, lib, ... }:
+
+let
+  cfg = config.services.depot.builderball;
+  description = "Nix cache proxy for distribution between builders";
+  hostname = config.networing.hostName;
+in
+{
+  options.services.depot.builderball = {
+    enable = lib.mkEnableOption description;
+
+    caches = lib.mkOption {
+      type = with lib.types; listOf string;
+      description = "Public addresses of caches to use";
+
+      default = [
+        "whitby.cache.tvl.fyi"
+        "nevsky.cache.tvl.fyi"
+      ];
+    };
+
+    port = lib.mkOption {
+      type = lib.types.int;
+      description = "port on which to listen locally";
+      default = 26862; # bounc
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.builderball =
+      let
+        caches = lib.concatStringsSep " " (map (c: "-cache https://${c}") cfg.caches);
+      in
+      {
+        inherit description;
+        wantedBy = [ "multi-user.target" ];
+        wants = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+
+        serviceConfig = {
+          ExecStart = "${depot.ops.builderball}/bin/builderball ${caches} -port ${toString cfg.port} -debug";
+          DynamicUser = true;
+          Restart = "always";
+        };
+      };
+  };
+}

ops/modules/www/cache.tvl.fyi.nix

@@ -0,0 +1,50 @@
+# Publicly serve builderball cache. This is an experimental setup, and separate
+# from the "normal" harmonia cache on cache.tvl.su.
+{ config, ... }:
+
+let
+  # This attrset forms a linked list of hosts, which delegate ACME fallbacks to
+  # each other. These *must* form a circle, otherwise we may end up walking only
+  # part of the ring.
+  acmeFallback = host: ({
+    whitby = "nevsky.cache.tvl.fyi";
+    nevsky = "whitby.cache.tvl.fyi"; # GOTO 1
+  })."${host}";
+in
+{
+  imports = [
+    ./base.nix
+  ];
+
+  config = {
+    services.nginx.virtualHosts."cache.tvl.fyi" = {
+      serverName = "cache.tvl.fyi";
+      enableACME = true;
+      forceSSL = true;
+
+      # This enables fetching TLS certificates for the same domain on different
+      # hosts. This config is kind of messy; it would be nice to generate a
+      # correct ring from the depot fixpoint, but this may be impossible due to
+      # infinite recursion. Please read the comment on `acmeFallback` above.
+      acmeFallbackHost = acmeFallback config.networking.hostName;
+
+      extraConfig = ''
+        location = /cache-key.pub {
+            alias /run/agenix/nix-cache-pub;
+        }
+
+        location = / {
+            proxy_pass http://${config.services.depot.harmonia.settings.bind};
+        }
+
+        location / {
+            proxy_pass http://localhost:${toString config.services.depot.builderball.port};
+        }
+      '';
+    };
+
+    # participating hosts should use their local cache, otherwise they might end
+    # up querying themselves from afar for data they don't have.
+    networking.extraHosts = "127.0.0.1 cache.tvl.fyi";
+  };
+}

ops/modules/www/cache.tvl.su.nix

@@ -8,7 +8,6 @@
   config = {
     services.nginx.virtualHosts."cache.tvl.su" = {
       serverName = "cache.tvl.su";
-      serverAliases = [ "cache.tvl.fyi" ];
       enableACME = true;
       forceSSL = true;
 

ops/modules/www/self-cache.tvl.fyi.nix

@@ -0,0 +1,26 @@
+# per-host addresses for publicly reachable caches, for use with builderball
+# TODO(tazjin): merge with the public cache module; but needs ACME fixes
+{ config, lib, ... }:
+
+{
+  imports = [
+    ./base.nix
+  ];
+
+  config = lib.mkIf config.services.depot.harmonia.enable {
+    services.nginx.virtualHosts."${config.networking.hostName}.cache.tvl.fyi" = {
+      enableACME = true;
+      forceSSL = true;
+
+      extraConfig = ''
+        location = /cache-key.pub {
+          alias /run/agenix/nix-cache-pub;
+        }
+
+        location / {
+          proxy_pass http://${config.services.depot.harmonia.settings.bind};
+        }
+      '';
+    };
+  };
+}