maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ops/pipelines): Dynamically generate CI pipeline from targets

Browse source 
Create the pipeline by outputting a file that contains nix-build
invocations for each target's *derivation path*.

Each invocation has a generated Nix expression passed to it with `-E`
which fetches the correct target from the tree while correctly
handling targets with strange characters (such as in Go-packages).

This makes it possible to run target-level granular pipelines. We're
getting somewhere!

Change-Id: Ia6946e389dafd1d4926130bb8891446d6e17133b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1855
Tested-by: BuildkiteCI
Reviewed-by: glittershark <grfn@gws.fyi>
Reviewed-by: lukegb <lukegb@tvl.fyi>
This commit is contained in:
Vincent Ambo 2020-08-27 01:05:45 +01:00 • committed by tazjin
parent 21690c644b
commit 61d2d2d503
6 changed files with 78 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

79
ops/pipelines/depot.nix
View file

@ -4,22 +4,75 @@
# It outputs a "YAML" (actually JSON) file which is evaluated and
# submitted to Buildkite at the start of each build. This means we can
# dynamically configure the pipeline execution here.
{ depot, pkgs, ... }:
{ depot, lib, pkgs, ... }:
let
inherit (builtins) toJSON;
inherit (builtins) concatStringsSep foldl' map toJSON;
inherit (lib) singleton;
inherit (pkgs) writeText;
# Create an expression that builds the target at the specified
# location.
mkBuildExpr =
let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
in foldl' descend "import ./. {}";
# Create a pipeline label from the targets tree location.
mkLabel = concatStringsSep "/";
# Create a pipeline step from a single target.
#
# If the build fails, Buildkite metadata is updated to mark the
# pipeline as failed. Buildkite has a concept of a failed pipeline
# regardless, but this data is not accessible.
mkStep = target: {
command = ''
nix-build -E '${mkBuildExpr target.__readTree}' || (buildkite-agent meta-data set "failure" "1"; exit 1)
'';
label = ":nix: ${mkLabel target.__readTree}";
};
# Protobuf check step which validates that changes to .proto files
# between revisions don't cause backwards-incompatible or otherwise
# flawed changes.
protoCheck = {
command = "${depot.nix.bufCheck}/bin/ci-buf-check";
label = ":water_buffalo:";
};
# This defines the build pipeline, using the pipeline format
# documented on https://buildkite.com/docs/pipelines/defining-steps
pipeline.steps = [
{
command = "nix-build -A ci.targets --show-trace";
label = ":duck:";
}
{
command = "${depot.nix.bufCheck}/bin/ci-buf-check";
label = ":water_buffalo:";
}
];
in writeText "depot.yaml" (toJSON pipeline)
#
# Pipeline steps need to stay in order.
pipeline.steps =
# Zero the failure status
[
{
command = "buildkite-agent meta-data set 'failure' '0'";
label = ":buildkite:";
}
{ wait = null; }
]
# Create build steps for each CI target
++ (map mkStep depot.ci.targets)
++ [
# Simultaneously run protobuf checks
protoCheck
# Wait for all previous checks to complete
({
wait = null;
continue_on_failure = true;
})
# Wait for all steps to complete, then exit with success or
# failure depending on whether any failure status was written.
# This step must be :duck:! (yes, really!)
({
command = "exit $(buildkite-agent meta-data get 'failure')";
label = ":duck:";
})
];
in (writeText "depot.yaml" (toJSON pipeline))