maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

style: format entire depot with nixpkgs-fmt

Browse source 
This CL can be used to compare the style of nixpkgs-fmt against other
formatters (nixpkgs, alejandra).

Change-Id: I87c6abff6bcb546b02ead15ad0405f81e01b6d9e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4397
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>
Reviewed-by: lukegb <lukegb@tvl.fyi>
Reviewed-by: wpcarro <wpcarro@gmail.com>
Reviewed-by: Profpatsch <mail@profpatsch.de>
Reviewed-by: kanepyork <rikingcoding@gmail.com>
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: cynthia <cynthia@tvl.fyi>
Reviewed-by: edef <edef@edef.eu>
Reviewed-by: eta <tvl@eta.st>
Reviewed-by: grfn <grfn@gws.fyi>
This commit is contained in:
Vincent Ambo 2022-01-30 19:06:58 +03:00 • committed by tazjin
parent 2d10d60fac
commit aa122cbae7
310 changed files with 7278 additions and 5490 deletions
Download patch file Download diff file Expand all files Collapse all files

288
nix/buildkite/default.nix
View file

@ -29,7 +29,8 @@ let
unsafeDiscardStringContext;
inherit (pkgs) lib runCommandNoCC writeText;
in rec {
in
rec {
# Creates a Nix expression that yields the target at the specified
# location in the repository.
#
@ -42,14 +43,15 @@ in rec {
descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
targetExpr = foldl' descend "import ./. {}" target.__readTree;
subtargetExpr = descend targetExpr target.__subtarget;
in if target ? __subtarget then subtargetExpr else targetExpr;
in
if target ? __subtarget then subtargetExpr else targetExpr;
# Create a pipeline label from the target's tree location.
mkLabel = target:
let label = concatStringsSep "/" target.__readTree;
in if target ? __subtarget
then "${label}:${target.__subtarget}"
else label;
then "${label}:${target.__subtarget}"
else label;
# Determine whether to skip a target if it has not diverged from the
# HEAD branch.
@ -74,33 +76,36 @@ in rec {
# Create a pipeline step from a single target.
mkStep = headBranch: parentTargetMap: target:
let
label = mkLabel target;
drvPath = unsafeDiscardStringContext target.drvPath;
shouldSkip' = shouldSkip parentTargetMap;
in {
label = ":nix: " + label;
key = hashString "sha1" label;
skip = shouldSkip' label drvPath;
command = mkBuildCommand target drvPath;
env.READTREE_TARGET = label;
let
label = mkLabel target;
drvPath = unsafeDiscardStringContext target.drvPath;
shouldSkip' = shouldSkip parentTargetMap;
in
{
label = ":nix: " + label;
key = hashString "sha1" label;
skip = shouldSkip' label drvPath;
command = mkBuildCommand target drvPath;
env.READTREE_TARGET = label;
# Add a dependency on the initial static pipeline step which
# always runs. This allows build steps uploaded in batches to
# start running before all batches have been uploaded.
depends_on = ":init:";
};
# Add a dependency on the initial static pipeline step which
# always runs. This allows build steps uploaded in batches to
# start running before all batches have been uploaded.
depends_on = ":init:";
};
# Helper function to inelegantly divide a list into chunks of at
# most n elements.
#
# This works by assigning each element a chunk ID based on its
# index, and then grouping all elements by their chunk ID.
chunksOf = n: list: let
chunkId = idx: toString (idx / n + 1);
assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
unchunk = mapAttrs (_: elements: map (e: e.value) elements);
in unchunk (lib.groupBy (e: e.chunk) assigned);
chunksOf = n: list:
let
chunkId = idx: toString (idx / n + 1);
assigned = lib.imap1 (idx: value: { inherit value; chunk = chunkId idx; }) list;
unchunk = mapAttrs (_: elements: map (e: e.value) elements);
in
unchunk (lib.groupBy (e: e.chunk) assigned);
# Define a build pipeline chunk as a JSON file, using the pipeline
# format documented on
@ -120,104 +125,112 @@ in rec {
attrValues (mapAttrs (makePipelineChunk name) (chunksOf 192 steps));
# Create a pipeline structure for the given targets.
mkPipeline = {
# HEAD branch of the repository on which release steps, GC
# anchoring and other "mainline only" steps should run.
headBranch,
mkPipeline =
{
# HEAD branch of the repository on which release steps, GC
# anchoring and other "mainline only" steps should run.
headBranch
, # List of derivations as read by readTree (in most cases just the
# output of readTree.gather) that should be built in Buildkite.
#
# These are scheduled as the first build steps and run as fast as
# possible, in order, without any concurrency restrictions.
drvTargets
, # Derivation map of a parent commit. Only targets which no longer
# correspond to the content of this map will be built. Passing an
# empty map will always build all targets.
parentTargetMap ? { }
, # A list of plain Buildkite step structures to run alongside the
# build for all drvTargets, but before proceeding with any
# post-build actions such as status reporting.
#
# Can be used for things like code formatting checks.
additionalSteps ? [ ]
, # A list of plain Buildkite step structures to run after all
# previous steps succeeded.
#
# Can be used for status reporting steps and the like.
postBuildSteps ? [ ]
}:
let
# Convert a target into all of its build and post-build steps,
# treated separately as they need to be in different chunks.
targetToSteps = target:
let
step = mkStep headBranch parentTargetMap target;
# List of derivations as read by readTree (in most cases just the
# output of readTree.gather) that should be built in Buildkite.
#
# These are scheduled as the first build steps and run as fast as
# possible, in order, without any concurrency restrictions.
drvTargets,
# Split build/post-build steps
splitExtraSteps = partition ({ postStep, ... }: postStep)
(attrValues (mapAttrs
(name: value: {
inherit name value;
postStep = (value ? prompt) || (value.postBuild or false);
})
(target.meta.ci.extraSteps or { })));
# Derivation map of a parent commit. Only targets which no longer
# correspond to the content of this map will be built. Passing an
# empty map will always build all targets.
parentTargetMap ? {},
mkExtraStep' = { name, value, ... }: mkExtraStep step name value;
extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt
extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt
in
{
buildSteps = [ step ] ++ extraBuildSteps;
postSteps = extraPostSteps;
};
# A list of plain Buildkite step structures to run alongside the
# build for all drvTargets, but before proceeding with any
# post-build actions such as status reporting.
#
# Can be used for things like code formatting checks.
additionalSteps ? [],
# Combine all target steps into separate build and post-build step lists.
steps = foldl'
(acc: t: {
buildSteps = acc.buildSteps ++ t.buildSteps;
postSteps = acc.postSteps ++ t.postSteps;
})
{ buildSteps = [ ]; postSteps = [ ]; }
(map targetToSteps drvTargets);
# A list of plain Buildkite step structures to run after all
# previous steps succeeded.
#
# Can be used for status reporting steps and the like.
postBuildSteps ? []
}: let
# Convert a target into all of its build and post-build steps,
# treated separately as they need to be in different chunks.
targetToSteps = target: let
step = mkStep headBranch parentTargetMap target;
buildSteps =
# Add build steps for each derivation target and their extra
# steps.
steps.buildSteps
# Split build/post-build steps
splitExtraSteps = partition ({ postStep, ... }: postStep)
(attrValues (mapAttrs (name: value: {
inherit name value;
postStep = (value ? prompt) || (value.postBuild or false);
}) (target.meta.ci.extraSteps or {})));
# Add additional steps (if set).
++ additionalSteps;
mkExtraStep' = { name, value, ... }: mkExtraStep step name value;
extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt
extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt
in {
buildSteps = [ step ] ++ extraBuildSteps;
postSteps = extraPostSteps;
};
postSteps =
# Add post-build steps for each derivation target.
steps.postSteps
# Combine all target steps into separate build and post-build step lists.
steps = foldl' (acc: t: {
buildSteps = acc.buildSteps ++ t.buildSteps;
postSteps = acc.postSteps ++ t.postSteps;
}) { buildSteps = []; postSteps = []; } (map targetToSteps drvTargets);
# Add any globally defined post-build steps.
++ postBuildSteps;
buildSteps =
# Add build steps for each derivation target and their extra
# steps.
steps.buildSteps
# Add additional steps (if set).
++ additionalSteps;
postSteps =
# Add post-build steps for each derivation target.
steps.postSteps
# Add any globally defined post-build steps.
++ postBuildSteps;
buildChunks = pipelineChunks "build" buildSteps;
postBuildChunks = pipelineChunks "post" postSteps;
chunks = buildChunks ++ postBuildChunks;
in runCommandNoCC "buildkite-pipeline" {} ''
mkdir $out
echo "Generated ${toString (length chunks)} pipeline chunks"
${
lib.concatMapStringsSep "\n"
(chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
}
'';
buildChunks = pipelineChunks "build" buildSteps;
postBuildChunks = pipelineChunks "post" postSteps;
chunks = buildChunks ++ postBuildChunks;
in
runCommandNoCC "buildkite-pipeline" { } ''
mkdir $out
echo "Generated ${toString (length chunks)} pipeline chunks"
${
lib.concatMapStringsSep "\n"
(chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
}
'';
# Create a drvmap structure for the given targets, containing the
# mapping of all target paths to their derivations. The mapping can
# be persisted for future use.
mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: {
name = mkLabel target;
value = {
drvPath = unsafeDiscardStringContext target.drvPath;
mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map
(target: {
name = mkLabel target;
value = {
drvPath = unsafeDiscardStringContext target.drvPath;
# Include the attrPath in the output to reconstruct the drv
# without parsing the human-readable label.
attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
target.__subtarget
];
};
}) drvTargets)));
# Include the attrPath in the output to reconstruct the drv
# without parsing the human-readable label.
attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
target.__subtarget
];
};
})
drvTargets)));
# Implementation of extra step logic.
#
@ -278,34 +291,37 @@ in rec {
# Create the Buildkite configuration for an extra step, optionally
# wrapping it in a gate group.
mkExtraStep = parent: key: {
command,
label ? key,
prompt ? false,
needsOutput ? false,
branches ? null,
alwaysRun ? false,
postBuild ? false
}@cfg: let
parentLabel = parent.env.READTREE_TARGET;
mkExtraStep = parent: key: { command
, label ? key
, prompt ? false
, needsOutput ? false
, branches ? null
, alwaysRun ? false
, postBuild ? false
}@cfg:
let
parentLabel = parent.env.READTREE_TARGET;
step = {
label = ":gear: ${label} (from ${parentLabel})";
skip = if alwaysRun then false else parent.skip or false;
depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key;
branches = if branches != null then lib.concatStringsSep " " branches else null;
step = {
label = ":gear: ${label} (from ${parentLabel})";
skip = if alwaysRun then false else parent.skip or false;
depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key;
branches = if branches != null then lib.concatStringsSep " " branches else null;
command = pkgs.writeShellScript "${key}-script" ''
set -ueo pipefail
${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"}
${lib.optionalString needsOutput parent.command}
echo '+++ Running extra step command'
exec ${command}
'';
};
in if (isString prompt)
then mkGatedStep {
inherit step label parent prompt;
}
command = pkgs.writeShellScript "${key}-script" ''
set -ueo pipefail
${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"}
${lib.optionalString needsOutput parent.command}
echo '+++ Running extra step command'
exec ${command}
'';
};
in
if (isString prompt)
then
mkGatedStep
{
inherit step label parent prompt;
}
else step;
}