feat(ops/modules): configure builderball cache setup

Configures an experimental setup for a builderball-based public cache.

This cache only includes the two build machines (whitby & nevsky), for the time
period where both of them exist simultaneously.

The idea is this:

All participating hosts run a harmonia binary cache locally (whitby already
does). They then run builderball instances pointing at each other's harmonia
caches (through dedicated public hostnames).

When a request comes in, the first matching cache address is returned and Nix
will substitute from there.

Change-Id: Ia7d5357fd5e04f77b460205544fa24e82b100230
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12975
Autosubmit: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>

ops/modules/www/cache.tvl.fyi.nix

@@ -0,0 +1,50 @@
+# Publicly serve builderball cache. This is an experimental setup, and separate
+# from the "normal" harmonia cache on cache.tvl.su.
+{ config, ... }:
+
+let
+  # This attrset forms a linked list of hosts, which delegate ACME fallbacks to
+  # each other. These *must* form a circle, otherwise we may end up walking only
+  # part of the ring.
+  acmeFallback = host: ({
+    whitby = "nevsky.cache.tvl.fyi";
+    nevsky = "whitby.cache.tvl.fyi"; # GOTO 1
+  })."${host}";
+in
+{
+  imports = [
+    ./base.nix
+  ];
+
+  config = {
+    services.nginx.virtualHosts."cache.tvl.fyi" = {
+      serverName = "cache.tvl.fyi";
+      enableACME = true;
+      forceSSL = true;
+
+      # This enables fetching TLS certificates for the same domain on different
+      # hosts. This config is kind of messy; it would be nice to generate a
+      # correct ring from the depot fixpoint, but this may be impossible due to
+      # infinite recursion. Please read the comment on `acmeFallback` above.
+      acmeFallbackHost = acmeFallback config.networking.hostName;
+
+      extraConfig = ''
+        location = /cache-key.pub {
+            alias /run/agenix/nix-cache-pub;
+        }
+
+        location = / {
+            proxy_pass http://${config.services.depot.harmonia.settings.bind};
+        }
+
+        location / {
+            proxy_pass http://localhost:${toString config.services.depot.builderball.port};
+        }
+      '';
+    };
+
+    # participating hosts should use their local cache, otherwise they might end
+    # up querying themselves from afar for data they don't have.
+    networking.extraHosts = "127.0.0.1 cache.tvl.fyi";
+  };
+}

ops/modules/www/cache.tvl.su.nix

@@ -8,7 +8,6 @@
   config = {
     services.nginx.virtualHosts."cache.tvl.su" = {
       serverName = "cache.tvl.su";
-      serverAliases = [ "cache.tvl.fyi" ];
       enableACME = true;
       forceSSL = true;
 

ops/modules/www/self-cache.tvl.fyi.nix

@@ -0,0 +1,26 @@
+# per-host addresses for publicly reachable caches, for use with builderball
+# TODO(tazjin): merge with the public cache module; but needs ACME fixes
+{ config, lib, ... }:
+
+{
+  imports = [
+    ./base.nix
+  ];
+
+  config = lib.mkIf config.services.depot.harmonia.enable {
+    services.nginx.virtualHosts."${config.networking.hostName}.cache.tvl.fyi" = {
+      enableACME = true;
+      forceSSL = true;
+
+      extraConfig = ''
+        location = /cache-key.pub {
+          alias /run/agenix/nix-cache-pub;
+        }
+
+        location / {
+          proxy_pass http://${config.services.depot.harmonia.settings.bind};
+        }
+      '';
+    };
+  };
+}