2d989ef6cb
Adds a new tagging system to Buildkite agents, where agents are tagged with large/small slots. All agents have small slots, only some agents have large slots. The small slots are purely informative - nothing targets them, whereas large slots will be used for filtering agents. This allows us to target large slots in some builds and minimise the concurrent execution of extremely large builds, while keeping a large number of small slots around for all the light targets. This will need some tuning over time (also because tagging is a manual process). Change-Id: I15aa657773ed874d84d98e55238fb31c75d4efa7 Reviewed-on: https://cl.tvl.fyi/c/depot/+/13120 Tested-by: BuildkiteCI Reviewed-by: sterni <sternenseemann@systemli.org>
95 lines
2.7 KiB
Nix
95 lines
2.7 KiB
Nix
# Configuration for the TVL buildkite agents.
|
|
{ config, depot, pkgs, lib, ... }:
|
|
|
|
let
|
|
cfg = config.services.depot.buildkite;
|
|
agents = lib.range 1 cfg.agentCount;
|
|
description = "Buildkite agents for TVL";
|
|
hostname = config.networking.hostName;
|
|
|
|
besadiiWithConfig = name: pkgs.writeShellScript "besadii-${hostname}" ''
|
|
export BESADII_CONFIG=/run/agenix/buildkite-besadii-config
|
|
exec -a ${name} ${depot.ops.besadii}/bin/besadii "$@"
|
|
'';
|
|
|
|
# All Buildkite hooks are actually besadii, but it's being invoked
|
|
# with different names.
|
|
buildkiteHooks = pkgs.runCommand "buildkite-hooks" { } ''
|
|
mkdir -p $out/bin
|
|
ln -s ${besadiiWithConfig "post-command"} $out/bin/post-command
|
|
'';
|
|
|
|
credentialHelper = pkgs.writeShellScriptBin "git-credential-gerrit-creds" ''
|
|
echo 'username=buildkite'
|
|
echo "password=$(jq -r '.gerritPassword' /run/agenix/buildkite-besadii-config)"
|
|
'';
|
|
in
|
|
{
|
|
options.services.depot.buildkite = {
|
|
enable = lib.mkEnableOption description;
|
|
|
|
agentCount = lib.mkOption {
|
|
type = lib.types.int;
|
|
description = "Number of Buildkite agents to launch";
|
|
};
|
|
|
|
largeSlots = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = cfg.agentCount;
|
|
description = "Number of agents with 'large=true'";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# Run the Buildkite agents using the default upstream module.
|
|
services.buildkite-agents = builtins.listToAttrs (map
|
|
(n: rec {
|
|
name = "${hostname}-${toString n}";
|
|
value = {
|
|
inherit name;
|
|
enable = true;
|
|
tokenPath = config.age.secretsDir + "/buildkite-agent-token";
|
|
privateSshKeyPath = config.age.secretsDir + "/buildkite-private-key";
|
|
hooks.post-command = "${buildkiteHooks}/bin/post-command";
|
|
hooks.environment = ''
|
|
export PATH=$PATH:/run/wrappers/bin
|
|
'';
|
|
|
|
tags.hostname = hostname;
|
|
|
|
# all agents support small jobs
|
|
tags.small = "true";
|
|
|
|
runtimePackages = with pkgs; [
|
|
bash
|
|
coreutils
|
|
credentialHelper
|
|
curl
|
|
git
|
|
gnutar
|
|
gzip
|
|
jq
|
|
nix
|
|
];
|
|
} // (lib.optionalAttrs (n <= cfg.largeSlots) {
|
|
tags.large = "true";
|
|
});
|
|
})
|
|
agents);
|
|
|
|
# Set up a group for all Buildkite agent users
|
|
users = {
|
|
groups.buildkite-agents = { };
|
|
users = builtins.listToAttrs (map
|
|
(n: rec {
|
|
name = "buildkite-agent-${hostname}-${toString n}";
|
|
value = {
|
|
isSystemUser = true;
|
|
group = lib.mkForce "buildkite-agents";
|
|
extraGroups = [ name "docker" ];
|
|
};
|
|
})
|
|
agents);
|
|
};
|
|
};
|
|
}
|