From 61d2d2d50379e8e445255ec7863f1610ce984b26 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 27 Aug 2020 01:05:45 +0100 Subject: [PATCH] feat(ops/pipelines): Dynamically generate CI pipeline from targets Create the pipeline by outputting a file that contains nix-build invocations for each target's *derivation path*. Each invocation has a generated Nix expression passed to it with `-E` which fetches the correct target from the tree while correctly handling targets with strange characters (such as in Go-packages). This makes it possible to run target-level granular pipelines. We're getting somewhere! Change-Id: Ia6946e389dafd1d4926130bb8891446d6e17133b Reviewed-on: https://cl.tvl.fyi/c/depot/+/1855 Tested-by: BuildkiteCI Reviewed-by: glittershark Reviewed-by: lukegb --- default.nix | 11 ++- ops/nixos/default.nix | 2 +- ops/pipelines/depot.nix | 79 ++++++++++++++++---- users/glittershark/system/home/default.nix | 2 +- users/glittershark/system/system/default.nix | 2 +- users/tazjin/nixos/default.nix | 2 +- 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/default.nix b/default.nix index b0b5399c4..151d8987e 100644 --- a/default.nix +++ b/default.nix @@ -73,9 +73,14 @@ in fix(self: { # List of all buildable targets, for CI purposes. # - # Note: This *must* be a nested attribute, otherwise we will get - # infinite recursion and everything blows up. - ci.targets = gather self; + # Note: To prevent infinite recursion, this *must* be a nested + # attribute set (which does not have a __readTree attribute). + ci.targets = gather (self // { + # remove the pipelines themselves from the set over which to + # generate pipelines because that also leads to infinite + # recursion. + ops = self.ops // { pipelines = null; }; + }); } # Add local packages as structured by readTree diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix index 407e667f4..917a56b76 100644 --- a/ops/nixos/default.nix +++ b/ops/nixos/default.nix @@ -47,5 +47,5 @@ rec { # # TODO(tazjin): Refactor the whole systems setup, it's a bit # inconsistent at the moment. - whitbySystem = (nixosFor whitby).system // { __readTree = true; }; + whitbySystem = (nixosFor whitby).system; } diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix index 2e99bf8d1..5d1f2babe 100644 --- a/ops/pipelines/depot.nix +++ b/ops/pipelines/depot.nix @@ -4,22 +4,75 @@ # It outputs a "YAML" (actually JSON) file which is evaluated and # submitted to Buildkite at the start of each build. This means we can # dynamically configure the pipeline execution here. -{ depot, pkgs, ... }: +{ depot, lib, pkgs, ... }: let - inherit (builtins) toJSON; + inherit (builtins) concatStringsSep foldl' map toJSON; + inherit (lib) singleton; inherit (pkgs) writeText; + # Create an expression that builds the target at the specified + # location. + mkBuildExpr = + let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; + in foldl' descend "import ./. {}"; + + # Create a pipeline label from the targets tree location. + mkLabel = concatStringsSep "/"; + + # Create a pipeline step from a single target. + # + # If the build fails, Buildkite metadata is updated to mark the + # pipeline as failed. Buildkite has a concept of a failed pipeline + # regardless, but this data is not accessible. + mkStep = target: { + command = '' + nix-build -E '${mkBuildExpr target.__readTree}' || (buildkite-agent meta-data set "failure" "1"; exit 1) + ''; + label = ":nix: ${mkLabel target.__readTree}"; + }; + + # Protobuf check step which validates that changes to .proto files + # between revisions don't cause backwards-incompatible or otherwise + # flawed changes. + protoCheck = { + command = "${depot.nix.bufCheck}/bin/ci-buf-check"; + label = ":water_buffalo:"; + }; + # This defines the build pipeline, using the pipeline format # documented on https://buildkite.com/docs/pipelines/defining-steps - pipeline.steps = [ - { - command = "nix-build -A ci.targets --show-trace"; - label = ":duck:"; - } - { - command = "${depot.nix.bufCheck}/bin/ci-buf-check"; - label = ":water_buffalo:"; - } - ]; -in writeText "depot.yaml" (toJSON pipeline) + # + # Pipeline steps need to stay in order. + pipeline.steps = + # Zero the failure status + [ + { + command = "buildkite-agent meta-data set 'failure' '0'"; + label = ":buildkite:"; + } + { wait = null; } + ] + + # Create build steps for each CI target + ++ (map mkStep depot.ci.targets) + + ++ [ + # Simultaneously run protobuf checks + protoCheck + + # Wait for all previous checks to complete + ({ + wait = null; + continue_on_failure = true; + }) + + # Wait for all steps to complete, then exit with success or + # failure depending on whether any failure status was written. + # This step must be :duck:! (yes, really!) + ({ + command = "exit $(buildkite-agent meta-data get 'failure')"; + label = ":duck:"; + }) + ]; +in (writeText "depot.yaml" (toJSON pipeline)) diff --git a/users/glittershark/system/home/default.nix b/users/glittershark/system/home/default.nix index d896ba340..a10e3f8df 100644 --- a/users/glittershark/system/home/default.nix +++ b/users/glittershark/system/home/default.nix @@ -20,7 +20,7 @@ rec { lib.depot = depot; }; - }) // { __readTree = true; }; + }); chupacabra = home ./machines/chupacabra.nix; } diff --git a/users/glittershark/system/system/default.nix b/users/glittershark/system/system/default.nix index f6710eff7..e19468105 100644 --- a/users/glittershark/system/system/default.nix +++ b/users/glittershark/system/system/default.nix @@ -5,7 +5,7 @@ rec { chupacabraSystem = (pkgs.nixos { configuration = chupacabra; - }).system // { __readTree = true; }; + }).system; rebuilder = let diff --git a/users/tazjin/nixos/default.nix b/users/tazjin/nixos/default.nix index 11f2eef13..9747f5c00 100644 --- a/users/tazjin/nixos/default.nix +++ b/users/tazjin/nixos/default.nix @@ -8,7 +8,7 @@ let configuration = lib.fix(config: foldl' lib.recursiveUpdate {} (map (c: c config) configs) ); - }).system // { __readTree = true; }; + }).system; caseFor = hostname: '' ${hostname})