From c05bf02a856121cdf40f77a21cdb26667d449615 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 7 Jul 2024 19:12:28 +0100 Subject: [PATCH] chore(3p/gerrit): create buildBazelPackageNG and migrate gerrit to it This bumps Gerrit to 3.10.0, and also introduces a new mechanism for building it that should hopefully have some more stable hashes than the previous bodgery. In this world, we only cache what we explicitly want to. There are some hooks implemented for `rules_java` and `rules_nodejs` (before version 6) that force use of local binaries; this means we can drop the use of the FHSUserEnv and use the java and nodejs binaries provided by nixpkgs instead. detzip is deleted; it hasn't been used in yonks. We also add https://gerrit-review.googlesource.com/c/gerrit/+/431977, which bumps the SSHd version so that we can have U2F-based SSH keys. Change-Id: Ie12a9a33bbb1e4bd96aa252580aca3b8bc4a1205 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11963 Reviewed-by: lukegb Autosubmit: lukegb Tested-by: BuildkiteCI --- nix/buildBazelPackageNG/.skip-subtree | 0 .../bazelRulesJavaHook/default.nix | 8 + .../bazelRulesJavaHook/local_java/BUILD.bazel | 3 + .../bazelRulesJavaHook/local_java/WORKSPACE | 1 + .../bazelRulesJavaHook/setup-hook.sh | 17 ++ .../bazelRulesNodeJS5Hook/default.nix | 53 ++++++ .../bazelRulesNodeJS5Hook/local_node/BUILD | 20 ++ .../local_node/WORKSPACE | 1 + .../bazelRulesNodeJS5Hook/local_node/bin/node | 3 + .../bazelRulesNodeJS5Hook/local_node/bin/npm | 3 + .../bazelRulesNodeJS5Hook/local_yarn/BUILD | 0 .../local_yarn/WORKSPACE | 1 + .../bazelRulesNodeJS5Hook/local_yarn/bin/yarn | 2 + .../bazelRulesNodeJS5Hook/setup-hook.sh | 63 +++++++ .../buildBazelPackageNG.nix | 105 +++++++++++ nix/buildBazelPackageNG/default.nix | 6 + .../gerrit/0001-Syntax-highlight-nix.patch | 16 +- .../0002-Syntax-highlight-rules.pl.patch | 12 +- .../0003-Add-titles-to-CLs-over-HTTP.patch | 35 ++-- third_party/gerrit/bazelrc | 11 ++ third_party/gerrit/default.nix | 171 ++++++------------ third_party/gerrit/detzip.go | 97 ---------- .../gerrit/gerrit-cl-431977-bump-sshd.patch | 40 ++++ third_party/gerrit/workspace_overrides.bzl | 5 + third_party/gerrit_plugins/builder.nix | 25 +-- .../gerrit_plugins/code-owners/default.nix | 6 +- third_party/gerrit_plugins/oauth/default.nix | 11 +- 27 files changed, 455 insertions(+), 260 deletions(-) create mode 100644 nix/buildBazelPackageNG/.skip-subtree create mode 100644 nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix create mode 100644 nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel create mode 100644 nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE create mode 100644 nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn create mode 100644 nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh create mode 100644 nix/buildBazelPackageNG/buildBazelPackageNG.nix create mode 100644 nix/buildBazelPackageNG/default.nix create mode 100644 third_party/gerrit/bazelrc delete mode 100644 third_party/gerrit/detzip.go create mode 100644 third_party/gerrit/gerrit-cl-431977-bump-sshd.patch create mode 100644 third_party/gerrit/workspace_overrides.bzl diff --git a/nix/buildBazelPackageNG/.skip-subtree b/nix/buildBazelPackageNG/.skip-subtree new file mode 100644 index 000000000..e69de29bb diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix new file mode 100644 index 000000000..eb8332e44 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/default.nix @@ -0,0 +1,8 @@ +{ makeSetupHook }: + +makeSetupHook { + name = "rules_java_bazel_hook"; + substitutions = { + local_java = ./local_java; + }; +} ./setup-hook.sh diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel new file mode 100644 index 000000000..8bea4954c --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/BUILD.bazel @@ -0,0 +1,3 @@ +alias(name = "jdk", actual = "@local_jdk//:jdk") +alias(name = "toolchain", actual = "@local_jdk//:toolchain") +alias(name = "bootstrap_runtime_toolchain", actual = "@local_jdk//:bootstrap_runtime_toolchain") diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE new file mode 100644 index 000000000..5b3107898 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/local_java/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "local_java") diff --git a/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh new file mode 100644 index 000000000..f7f7e3afe --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesJavaHook/setup-hook.sh @@ -0,0 +1,17 @@ +prePatchHooks+=(_setupLocalJavaRepo) + +javaVersions=(11 17 21) +javaPlatforms=( + "linux" "linux_aarch64" "linux_ppc64le" "linux_s390x" + "macos" "macos_aarch64" + "win" "win_arm64") + +_setupLocalJavaRepo() { + for javaVersion in ${javaVersions[@]}; do + for javaPlatform in ${javaPlatforms[@]}; do + bazelFlagsArray+=( + "--override_repository=remotejdk${javaVersion}_${javaPlatform}=@local_java@" + ) + done + done +} diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix new file mode 100644 index 000000000..c99cc39e9 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/default.nix @@ -0,0 +1,53 @@ +{ stdenvNoCC +, lib +, makeSetupHook +, fetchFromGitHub +, coreutils +, gnugrep +, nodejs +, yarn +, git +, cacert +}: +let + rulesNodeJS = stdenvNoCC.mkDerivation rec { + pname = "bazelbuild-rules_nodejs"; + version = "5.8.5"; + + src = fetchFromGitHub { + owner = "bazelbuild"; + repo = "rules_nodejs"; + rev = version; + hash = "sha256-6UbYRrOnS93+pK4VI016gQZv2jLCzkJn6wJ4vZNCNjY="; + }; + + dontBuild = true; + + postPatch = '' + shopt -s globstar + for i in **/*.bzl **/*.sh **/*.cjs; do + substituteInPlace "$i" \ + --replace-quiet '#!/usr/bin/env bash' '#!${stdenvNoCC.shell}' \ + --replace-quiet '#!/bin/bash' '#!${stdenvNoCC.shell}' + done + sed -i '/^#!/a export PATH=${lib.makeBinPath [ coreutils gnugrep ]}:$PATH' internal/node/launcher.sh + ''; + + installPhase = '' + cp -R . $out + ''; + }; +in makeSetupHook { + name = "bazelbuild-rules_nodejs-5-hook"; + propagatedBuildInputs = [ + nodejs + yarn + git + cacert + ]; + substitutions = { + inherit nodejs yarn cacert rulesNodeJS; + local_node = ./local_node; + local_yarn = ./local_yarn; + }; +} ./setup-hook.sh diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD new file mode 100644 index 000000000..d764d23ff --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/BUILD @@ -0,0 +1,20 @@ +load("@build_bazel_rules_nodejs//nodejs:toolchain.bzl", _node_toolchain = "node_toolchain") + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "bin/node", + "bin/npm", +]) + +_node_toolchain( + name = "node_toolchain", + target_tool_path = "__NODEJS__/bin/node", + npm_path = "__NODEJS__/bin/npm", +) + +toolchain( + name = "nodejs", + toolchain = ":node_toolchain", + toolchain_type = "@build_bazel_rules_nodejs//nodejs:toolchain_type", +) diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE new file mode 100644 index 000000000..5bc1698b6 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "nodejs") diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node new file mode 100644 index 000000000..ef1f010f0 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/node @@ -0,0 +1,3 @@ +#!/bin/sh + +exec "__NODEJS__/bin/node" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm new file mode 100644 index 000000000..63a985dbd --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_node/bin/npm @@ -0,0 +1,3 @@ +#!/bin/sh + +exec "__NODEJS__/bin/npm" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/BUILD new file mode 100644 index 000000000..e69de29bb diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE new file mode 100644 index 000000000..2a1b7d465 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "yarn") diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn new file mode 100644 index 000000000..2009572e4 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/local_yarn/bin/yarn @@ -0,0 +1,2 @@ +#!/bin/sh +exec "__YARN__/bin/yarn" "$@" diff --git a/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh new file mode 100644 index 000000000..5e3cf1eb9 --- /dev/null +++ b/nix/buildBazelPackageNG/bazelRulesNodeJS5Hook/setup-hook.sh @@ -0,0 +1,63 @@ +prePatchHooks+=(_setupLocalNodeRepos) +preBuildHooks+=(_setupYarnCache) + +case "$bazelPhase" in + cache) + postInstallHooks+=(_copyYarnCache) + ;; + build) + preBuildHooks+=(_linkYarnCache) + ;; + *) + echo "Unexpected bazelPhase '$bazelPhase' (want cache or build)" >&2 + exit 1 + ;; +esac + + +_setupLocalNodeRepos() { + cp -R @local_node@ $HOME/local_node + chmod -R +w $HOME/local_node + substituteInPlace $HOME/local_node/bin/node \ + --replace-fail '__NODEJS__' '@nodejs@' + substituteInPlace $HOME/local_node/bin/npm \ + --replace-fail '__NODEJS__' '@nodejs@' + substituteInPlace $HOME/local_node/BUILD \ + --replace-fail '__NODEJS__' '@nodejs@' + chmod -R +x $HOME/local_node/bin/* + + cp -R @local_yarn@ $HOME/local_yarn + chmod -R +w $HOME/local_yarn + substituteInPlace $HOME/local_yarn/bin/yarn \ + --replace-fail '__YARN__' '@yarn@' + chmod -R +x $HOME/local_yarn/bin/* + + bazelFlagsArray+=( + "--override_repository=build_bazel_rules_nodejs=@rulesNodeJS@" + + "--override_repository=nodejs_linux_amd64=$HOME/local_node" + "--override_repository=nodejs_linux_arm64=$HOME/local_node" + "--override_repository=nodejs_linux_s390x=$HOME/local_node" + "--override_repository=nodejs_linux_ppc64le=$HOME/local_node" + "--override_repository=nodejs_darwin_amd64=$HOME/local_node" + "--override_repository=nodejs_darwin_arm64=$HOME/local_node" + "--override_repository=nodejs_windows_amd64=$HOME/local_node" + "--override_repository=nodejs_windows_arm64=$HOME/local_node" + "--override_repository=nodejs=$HOME/local_node" + + "--override_repository=yarn=$HOME/local_yarn" + ) +} + +_setupYarnCache() { + @yarn@/bin/yarn config set cafile "@cacert@/etc/ssl/certs/ca-bundle.crt" + @yarn@/bin/yarn config set yarn-offline-mirror "$HOME/yarn-offline-mirror" +} + +_copyYarnCache() { + cp -R "$HOME/yarn-offline-mirror" "$out/yarn-offline-mirror" +} + +_linkYarnCache() { + ln -sf "$cache/yarn-offline-mirror" "$HOME/yarn-offline-mirror" +} diff --git a/nix/buildBazelPackageNG/buildBazelPackageNG.nix b/nix/buildBazelPackageNG/buildBazelPackageNG.nix new file mode 100644 index 000000000..5195d3e89 --- /dev/null +++ b/nix/buildBazelPackageNG/buildBazelPackageNG.nix @@ -0,0 +1,105 @@ +{ stdenv +, lib +, pkgs +, coreutils +}: + +{ name ? "${baseAttrs.pname}-${baseAttrs.version}" +, bazelTargets +, bazel ? pkgs.bazel +, depsHash +, extraCacheInstall ? "" +, extraBuildSetup ? "" +, extraBuildInstall ? "" +, ... +}@baseAttrs: + +let + cleanAttrs = lib.flip removeAttrs [ + "bazelTargets" "depsHash" "extraCacheInstall" "extraBuildSetup" "extraBuildInstall" + ]; + attrs = cleanAttrs baseAttrs; + + base = stdenv.mkDerivation (attrs // { + nativeBuildInputs = (attrs.nativeBuildInputs or []) ++ [ + bazel + ]; + + preUnpack = '' + if [[ ! -d $HOME ]]; then + export HOME=$NIX_BUILD_TOP/home + mkdir -p $HOME + fi + ''; + + bazelTargetNames = builtins.attrNames bazelTargets; + }); + + cache = base.overrideAttrs (base: { + name = "${name}-deps"; + + bazelPhase = "cache"; + + buildPhase = '' + runHook preBuild + + bazel sync --repository_cache=repository-cache $bazelFlags "''${bazelFlagsArray[@]}" + bazel build --repository_cache=repository-cache --nobuild $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + echo "${bazel.version}" > $out/bazel_version + cp -R repository-cache $out/repository-cache + ${extraCacheInstall} + + runHook postInstall + ''; + + outputHashMode = "recursive"; + outputHash = depsHash; + }); + + build = base.overrideAttrs (base: { + bazelPhase = "build"; + + inherit cache; + + nativeBuildInputs = (base.nativeBuildInputs or []) ++ [ + coreutils + ]; + + buildPhase = '' + runHook preBuild + + ${extraBuildSetup} + bazel build --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" $bazelTargetNames + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (target: outPath: lib.optionalString (outPath != null) '' + TARGET_OUTPUTS="$(bazel cquery --repository_cache=$cache/repository-cache $bazelFlags "''${bazelFlagsArray[@]}" --output=files "${target}")" + if [[ "$(echo "$TARGET_OUTPUTS" | wc -l)" -gt 1 ]]; then + echo "Installing ${target}'s outputs ($TARGET_OUTPUTS) into ${outPath} as a directory" + mkdir -p "${outPath}" + cp $TARGET_OUTPUTS "${outPath}" + else + echo "Installing ${target}'s output ($TARGET_OUTPUTS) to ${outPath}" + mkdir -p "${dirOf outPath}" + cp "$TARGET_OUTPUTS" "${outPath}" + fi + '') bazelTargets)} + ${extraBuildInstall} + + runHook postInstall + ''; + }); +in build diff --git a/nix/buildBazelPackageNG/default.nix b/nix/buildBazelPackageNG/default.nix new file mode 100644 index 000000000..c1584e66a --- /dev/null +++ b/nix/buildBazelPackageNG/default.nix @@ -0,0 +1,6 @@ +{ pkgs, ... }: + +(pkgs.callPackage ./buildBazelPackageNG.nix { }) // { + bazelRulesJavaHook = pkgs.callPackage ./bazelRulesJavaHook { }; + bazelRulesNodeJS5Hook = pkgs.callPackage ./bazelRulesNodeJS5Hook { }; +} diff --git a/third_party/gerrit/0001-Syntax-highlight-nix.patch b/third_party/gerrit/0001-Syntax-highlight-nix.patch index bdc3fd3b5..d17dc27db 100644 --- a/third_party/gerrit/0001-Syntax-highlight-nix.patch +++ b/third_party/gerrit/0001-Syntax-highlight-nix.patch @@ -1,4 +1,4 @@ -From 084e4f92fb58f7cd85303ba602fb3c40133c8fcc Mon Sep 17 00:00:00 2001 +From 216843cff4a8e41ad9887118751a412c1a22ce72 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Thu, 2 Jul 2020 23:02:32 +0100 Subject: [PATCH 1/3] Syntax highlight nix @@ -9,23 +9,23 @@ Subject: [PATCH 1/3] Syntax highlight nix 2 files changed, 2 insertions(+) diff --git a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -index a9f88bdd81..385249f280 100644 +index 50742903de..d1e89920cc 100644 --- a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts +++ b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -@@ -93,6 +93,7 @@ const LANGUAGE_MAP = new Map([ +@@ -98,6 +98,7 @@ const LANGUAGE_MAP = new Map([ ['text/x-vhdl', 'vhdl'], ['text/x-yaml', 'yaml'], ['text/vbscript', 'vbscript'], + ['text/x-nix', 'nix'], ]); - const CLASS_PREFIX = 'gr-diff gr-syntax gr-syntax-'; + const CLASS_PREFIX = 'gr-syntax gr-syntax-'; diff --git a/resources/com/google/gerrit/server/mime/mime-types.properties b/resources/com/google/gerrit/server/mime/mime-types.properties -index 2f9561ba2e..739818ec05 100644 +index 642ef474a5..97f1ff835b 100644 --- a/resources/com/google/gerrit/server/mime/mime-types.properties +++ b/resources/com/google/gerrit/server/mime/mime-types.properties -@@ -149,6 +149,7 @@ mscin = text/x-mscgen - msgenny = text/x-msgenny +@@ -154,6 +154,7 @@ msgenny = text/x-msgenny + mts = application/typescript nb = text/x-mathematica nginx.conf = text/x-nginx-conf +nix = text/x-nix @@ -33,5 +33,5 @@ index 2f9561ba2e..739818ec05 100644 nsi = text/x-nsis nt = text/n-triples -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch b/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch index 4b91e2c35..d0da61dd0 100644 --- a/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch +++ b/third_party/gerrit/0002-Syntax-highlight-rules.pl.patch @@ -1,4 +1,4 @@ -From aedf8ac8fa5113843bcd83ff85e2d9f3bffdb16c Mon Sep 17 00:00:00 2001 +From 63f1ff6ea749ae2af29a53463bca81bc3f4bf25b Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Thu, 2 Jul 2020 23:02:43 +0100 Subject: [PATCH 2/3] Syntax highlight rules.pl @@ -9,10 +9,10 @@ Subject: [PATCH 2/3] Syntax highlight rules.pl 2 files changed, 2 insertions(+) diff --git a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -index 385249f280..7cb3068494 100644 +index d1e89920cc..5d62af1c64 100644 --- a/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts +++ b/polygerrit-ui/app/embed/diff/gr-syntax-layer/gr-syntax-layer-worker.ts -@@ -68,6 +68,7 @@ const LANGUAGE_MAP = new Map([ +@@ -72,6 +72,7 @@ const LANGUAGE_MAP = new Map([ ['text/x-perl', 'perl'], ['text/x-pgsql', 'pgsql'], // postgresql ['text/x-php', 'php'], @@ -21,10 +21,10 @@ index 385249f280..7cb3068494 100644 ['text/x-protobuf', 'protobuf'], ['text/x-puppet', 'puppet'], diff --git a/resources/com/google/gerrit/server/mime/mime-types.properties b/resources/com/google/gerrit/server/mime/mime-types.properties -index 739818ec05..58eb727bf9 100644 +index 97f1ff835b..85d630340f 100644 --- a/resources/com/google/gerrit/server/mime/mime-types.properties +++ b/resources/com/google/gerrit/server/mime/mime-types.properties -@@ -200,6 +200,7 @@ rq = application/sparql-query +@@ -208,6 +208,7 @@ rq = application/sparql-query rs = text/x-rustsrc rss = application/xml rst = text/x-rst @@ -33,5 +33,5 @@ index 739818ec05..58eb727bf9 100644 s = text/x-gas sas = text/x-sas -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch b/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch index c4edee3a4..a5881e5a6 100644 --- a/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch +++ b/third_party/gerrit/0003-Add-titles-to-CLs-over-HTTP.patch @@ -1,19 +1,19 @@ -From f49c50ca9a84ca374b7bd91c171bbea0457f2c7a Mon Sep 17 00:00:00 2001 +From ca2df6d7f53441d443d42908e30bf60fbfc15392 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Thu, 2 Jul 2020 23:03:02 +0100 Subject: [PATCH 3/3] Add titles to CLs over HTTP --- - .../gerrit/httpd/raw/IndexHtmlUtil.java | 13 +++- + .../gerrit/httpd/raw/IndexHtmlUtil.java | 12 +++- .../google/gerrit/httpd/raw/IndexServlet.java | 8 ++- .../google/gerrit/httpd/raw/StaticModule.java | 5 +- .../gerrit/httpd/raw/TitleComputer.java | 67 +++++++++++++++++++ .../gerrit/httpd/raw/PolyGerritIndexHtml.soy | 4 +- - 5 files changed, 89 insertions(+), 8 deletions(-) + 5 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 java/com/google/gerrit/httpd/raw/TitleComputer.java diff --git a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java -index 72bfe40c3b..439bd73b44 100644 +index a92dd18f04..f87c46d321 100644 --- a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +++ b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java @@ -41,6 +41,7 @@ import java.util.Collections; @@ -24,7 +24,7 @@ index 72bfe40c3b..439bd73b44 100644 import java.util.Set; import java.util.function.Function; -@@ -62,13 +63,14 @@ public class IndexHtmlUtil { +@@ -63,13 +64,14 @@ public class IndexHtmlUtil { String faviconPath, Map urlParameterMap, Function urlInScriptTagOrdainer, @@ -36,22 +36,21 @@ index 72bfe40c3b..439bd73b44 100644 data.putAll( staticTemplateData( canonicalURL, cdnPath, faviconPath, urlParameterMap, urlInScriptTagOrdainer)) -- .putAll(dynamicTemplateData(gerritApi, requestedURL)); -+ .putAll(dynamicTemplateData(gerritApi, requestedURL, titleComputer)); +- .putAll(dynamicTemplateData(gerritApi, requestedURL, canonicalURL)); ++ .putAll(dynamicTemplateData(gerritApi, requestedURL, canonicalURL, titleComputer)); Set enabledExperiments = new HashSet<>(); enabledExperiments.addAll(experimentFeatures.getEnabledExperimentFeatures()); // Add all experiments enabled through url -@@ -81,7 +83,8 @@ public class IndexHtmlUtil { +@@ -82,7 +84,7 @@ public class IndexHtmlUtil { /** Returns dynamic parameters of {@code index.html}. */ public static ImmutableMap dynamicTemplateData( -- GerritApi gerritApi, String requestedURL) throws RestApiException, URISyntaxException { -+ GerritApi gerritApi, String requestedURL, TitleComputer titleComputer) -+ throws RestApiException, URISyntaxException { +- GerritApi gerritApi, String requestedURL, String canonicalURL) ++ GerritApi gerritApi, String requestedURL, String canonicalURL, TitleComputer titleComputer) + throws RestApiException, URISyntaxException { ImmutableMap.Builder data = ImmutableMap.builder(); Map initialData = new HashMap<>(); - Server serverApi = gerritApi.config().server(); -@@ -129,6 +132,10 @@ public class IndexHtmlUtil { +@@ -141,6 +143,10 @@ public class IndexHtmlUtil { } data.put("gerritInitialData", initialData); @@ -102,17 +101,17 @@ index fcb821e5ae..e1464b992b 100644 } catch (URISyntaxException | RestApiException e) { throw new IOException(e); diff --git a/java/com/google/gerrit/httpd/raw/StaticModule.java b/java/com/google/gerrit/httpd/raw/StaticModule.java -index 15dcf42e0e..9f56bf33ce 100644 +index b00294f73e..f1c1aae12c 100644 --- a/java/com/google/gerrit/httpd/raw/StaticModule.java +++ b/java/com/google/gerrit/httpd/raw/StaticModule.java -@@ -241,10 +241,11 @@ public class StaticModule extends ServletModule { +@@ -224,10 +224,11 @@ public class StaticModule extends ServletModule { @CanonicalWebUrl @Nullable String canonicalUrl, @GerritServerConfig Config cfg, GerritApi gerritApi, - ExperimentFeatures experimentFeatures) { + ExperimentFeatures experimentFeatures, + TitleComputer titleComputer) { - String cdnPath = options.devCdn().orElse(cfg.getString("gerrit", null, "cdnPath")); + String cdnPath = options.devCdn().orElseGet(() -> cfg.getString("gerrit", null, "cdnPath")); String faviconPath = cfg.getString("gerrit", null, "faviconPath"); - return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures); + return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures, titleComputer); @@ -193,7 +192,7 @@ index 0000000000..8fd2053ad0 + } +} diff --git a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy -index dbfef44dfe..347ee75aab 100644 +index 5ff1822cd9..81c3cdf0e1 100644 --- a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy +++ b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy @@ -33,10 +33,12 @@ @@ -211,5 +210,5 @@ index dbfef44dfe..347ee75aab 100644 {\n} -- -2.37.3 +2.45.1 diff --git a/third_party/gerrit/bazelrc b/third_party/gerrit/bazelrc new file mode 100644 index 000000000..f77da77b4 --- /dev/null +++ b/third_party/gerrit/bazelrc @@ -0,0 +1,11 @@ +# Not using common --repository_cache because Gerrit's bazelrc overrides this... +build --repository_cache=repository-cache +build --action_env=SSL_CERT_FILE +build --action_env=GERRIT_CACHE_HOME +build --tool_java_runtime_version=local_jdk --java_runtime_version=local_jdk +build --workspace_status_command="cat .version" + +# Disable errorprone +build --javacopt="-XepDisableAllChecks" + +sync --repository_cache=repository-cache diff --git a/third_party/gerrit/default.nix b/third_party/gerrit/default.nix index a13794626..94d68a1dc 100644 --- a/third_party/gerrit/default.nix +++ b/third_party/gerrit/default.nix @@ -1,139 +1,88 @@ { depot, pkgs, ... }: let - bazelRunScript = pkgs.writeShellScriptBin "bazel-run" '' - yarn config set cache-folder "$bazelOut/external/yarn_cache" - export HOME="$bazelOut/external/home" - mkdir -p "$bazelOut/external/home" - exec /bin/bazel "$@" - ''; - bazelTop = pkgs.buildFHSUserEnv { - name = "bazel"; - targetPkgs = pkgs: [ - (pkgs.bazel_5.override { enableNixHacks = true; }) - pkgs.jdk17_headless - pkgs.zlib - pkgs.python3 - pkgs.curl - pkgs.nodejs - pkgs.yarn - pkgs.git - bazelRunScript - ]; - runScript = "/bin/bazel-run"; - }; - bazel = bazelTop // { override = x: bazelTop; }; - version = "3.9.1"; + inherit (depot.nix) buildBazelPackageNG; + inherit (buildBazelPackageNG) bazelRulesJavaHook bazelRulesNodeJS5Hook; in -pkgs.lib.makeOverridable pkgs.buildBazelPackage { +pkgs.lib.makeOverridable depot.nix.buildBazelPackageNG rec { pname = "gerrit"; - inherit version; + version = "3.10.0"; - src = pkgs.fetchgit { + bazel = pkgs.bazel_7; + + src = (pkgs.fetchgit { url = "https://gerrit.googlesource.com/gerrit"; - rev = "620a819cbf3c64fff7a66798822775ad42c91d8e"; - branchName = "v${version}"; - sha256 = "sha256:1mdxbgnx3mpxand4wq96ic38bb4yh45q271h40jrk7dk23sgmz02"; + rev = "v${version}"; fetchSubmodules = true; - }; + deepClone = true; + hash = "sha256-FpKuzityHuHNYBIOL8YUjCLlkuVBfxjvHECb26NsZNE="; + }).overrideAttrs (_: { + env.NIX_PREFETCH_GIT_CHECKOUT_HOOK = '' + pushd "$dir" >/dev/null + ${pkgs.python3}/bin/python tools/workspace_status_release.py | sort > .version + popd >/dev/null + + # delete all the .git; we can't do this using fetchgit if deepClone is on, + # but our mischief has already been achieved by the python command above :) + find "$dir" -name .git -print0 | xargs -0 rm -rf + ''; + }); + depsHash = "sha256-Z6pvnL8bv09K4wKaatUCsnXAtSno+BJO6rTKhDsHYv4="; patches = [ ./0001-Syntax-highlight-nix.patch ./0002-Syntax-highlight-rules.pl.patch ./0003-Add-titles-to-CLs-over-HTTP.patch + + ./gerrit-cl-431977-bump-sshd.patch ]; - bazelTargets = [ "release" "api-skip-javadoc" ]; - inherit bazel; + nativeBuildInputs = with pkgs; [ + bazelRulesJavaHook + bazelRulesNodeJS5Hook - bazelFlags = [ - "--repository_cache=" - "--disk_cache=" + curl + jdk + python3 + unzip ]; - removeRulesCC = false; - fetchConfigured = true; + prePatch = '' + rm .bazelversion - fetchAttrs = { - sha256 = "sha256:119mqli75c9fy05ddrlh2brjxb354yvv1ijjkk1y1yqcaqppwwb8"; - preBuild = '' - rm .bazelversion - ''; + ln -sf ${./bazelrc} user.bazelrc - installPhase = '' - runHook preInstall + ln -sf ${./workspace_overrides.bzl} workspace_overrides.bzl + substituteInPlace WORKSPACE \ + --replace-fail 'load("@io_bazel_rules_webtesting//web:repositories.bzl"' 'load("//:workspace_overrides.bzl"' \ + --replace-fail 'load("@io_bazel_rules_webtesting//web/versioned:browsers-0.3.3.bzl"' 'load("//:workspace_overrides.bzl"' - # Remove all built in external workspaces, Bazel will recreate them when building - rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker} - rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker} - rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker} - rm -rf $bazelOut/external/{local_*,\@local_*.marker} + patchShebangs Documentation/replace_macros.py + ''; - # Clear markers - find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \; + postPatch = '' + sed -Ei 's,^(STABLE_BUILD_GERRIT_LABEL.*)$,\1-dirty-nix,' .version + ''; - # Remove all vcs files - rm -rf $(find $bazelOut/external -type d -name .git) - rm -rf $(find $bazelOut/external -type d -name .svn) - rm -rf $(find $bazelOut/external -type d -name .hg) + preBuild = '' + export GERRIT_CACHE_HOME=$HOME/gerrit-cache + ''; - # Removing top-level symlinks along with their markers. - # This is needed because they sometimes point to temporary paths (?). - # For example, in Tensorflow-gpu build: - #sha256:06bmzbcb9717s4b016kcbn8nr9pgaz04i8bnzg7ybkbdwpl8vxvv"; platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms - find $bazelOut/external -maxdepth 1 -type l | while read symlink; do - name="$(basename "$symlink")" - rm -rf "$symlink" "$bazelOut/external/@$name.marker" - done + extraCacheInstall = '' + cp -R $GERRIT_CACHE_HOME $out/gerrit-cache + ''; - # Patching symlinks to remove build directory reference - find $bazelOut/external -type l | while read symlink; do - new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")" - rm "$symlink" - ln -sf "$new_target" "$symlink" - done + extraBuildSetup = '' + ln -sf $cache/gerrit-cache $GERRIT_CACHE_HOME + ''; + extraBuildInstall = '' + mkdir -p "$out"/share/api/ + unzip bazel-bin/api-skip-javadoc.zip -d "$out"/share/api + ''; - echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version - - # Gerrit fixups: - # Normalize permissions on .yarn-{tarball,metadata} files - test -d $bazelOut/external/yarn_cache && find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} + - - mkdir $bazelOut/_bits/ - find . -name node_modules -prune -print | while read d; do - echo "$d" "$(dirname $d)" - mkdir -p $bazelOut/_bits/$(dirname $d) - cp -R "$d" "$bazelOut/_bits/$(dirname $d)/node_modules" - done - - (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/ _bits/) - - runHook postInstall - ''; - }; - - buildAttrs = { - preConfigure = '' - rm .bazelversion - - [ "$(ls -A $bazelOut/_bits)" ] && cp -R $bazelOut/_bits/* ./ || true - ''; - postPatch = '' - # Disable all errorprone checks, since we might be using a different version. - sed -i \ - -e '/-Xep:/d' \ - -e '/-XepExcludedPaths:/a "-XepDisableAllChecks",' \ - tools/BUILD - ''; - installPhase = '' - mkdir -p "$out"/webapps/ "$out"/share/api/ - cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war - unzip bazel-bin/api-skip-javadoc.zip -d "$out"/share/api - ''; - - nativeBuildInputs = with pkgs; [ - unzip - ]; + bazelTargets = { + "//:release" = "$out/webapps/gerrit-${version}.war"; + "//:api-skip-javadoc" = null; }; passthru = { @@ -154,5 +103,5 @@ pkgs.lib.makeOverridable pkgs.buildBazelPackage { ]; }; - meta.ci.targets = [ "deps" ]; + meta.ci.targets = [ "cache" ]; } diff --git a/third_party/gerrit/detzip.go b/third_party/gerrit/detzip.go deleted file mode 100644 index 511c18ecf..000000000 --- a/third_party/gerrit/detzip.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "archive/zip" - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" - "sort" - "strings" -) - -var ( - exclude = flag.String("exclude", "", "comma-separated list of filenames to exclude (in any directory)") -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s [zip file] [directory]:\n", os.Args[0]) - flag.PrintDefaults() - } -} - -func listToMap(ss []string) map[string]bool { - m := make(map[string]bool) - for _, s := range ss { - m[s] = true - } - return m -} - -func main() { - flag.Parse() - if flag.NArg() != 2 { - flag.Usage() - os.Exit(1) - } - - outPath := flag.Arg(0) - dirPath := flag.Arg(1) - - excludeFiles := listToMap(strings.Split(*exclude, ",")) - - // Aggregate all files first. - var files []string - filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if excludeFiles[info.Name()] { - return nil - } - files = append(files, path) - return nil - }) - - // Create zip - outW, err := os.Create(outPath) - if err != nil { - log.Fatalf("Create(%q): %v", outPath, err) - } - - zipW := zip.NewWriter(outW) - - // Output files in alphabetical order - sort.Strings(files) - for _, f := range files { - fw, err := zipW.CreateHeader(&zip.FileHeader{ - Name: f, - Method: zip.Store, - }) - if err != nil { - log.Fatalf("creating %q in zip: %v", f, err) - } - - ff, err := os.Open(f) - if err != nil { - log.Fatalf("opening %q: %v", f, err) - } - if _, err := io.Copy(fw, ff); err != nil { - log.Fatalf("copying %q to zip: %v", f, err) - } - ff.Close() - } - - if err := zipW.Close(); err != nil { - log.Fatalf("writing ZIP central directory: %v", err) - } - if err := outW.Close(); err != nil { - log.Fatalf("closing ZIP file: %v", err) - } -} diff --git a/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch b/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch new file mode 100644 index 000000000..3a08df21b --- /dev/null +++ b/third_party/gerrit/gerrit-cl-431977-bump-sshd.patch @@ -0,0 +1,40 @@ +Bump SSHD version to 2.13.1 + +Release-Notes: Update SSHD version to 2.13.1 +Change-Id: Ib7bc185bfd9e7eda0cc04230da8bd87ee1bb2358 + +diff --git a/tools/nongoogle.bzl b/tools/nongoogle.bzl +index 91caf31..d9b90d8 100644 +--- a/tools/nongoogle.bzl ++++ b/tools/nongoogle.bzl +@@ -137,18 +137,18 @@ + sha1 = "cb2f351bf4463751201f43bb99865235d5ba07ca", + ) + +- SSHD_VERS = "2.12.0" ++ SSHD_VERS = "2.13.1" + + maven_jar( + name = "sshd-osgi", + artifact = "org.apache.sshd:sshd-osgi:" + SSHD_VERS, +- sha1 = "32b8de1cbb722ba75bdf9898e0c41d42af00ce57", ++ sha1 = "50958cc44076749e790d7332021cff546707624c", + ) + + maven_jar( + name = "sshd-sftp", + artifact = "org.apache.sshd:sshd-sftp:" + SSHD_VERS, +- sha1 = "0f96f00a07b186ea62838a6a4122e8f4cad44df6", ++ sha1 = "e1b6da4ef604718e32cad59ef32618610da7a170", + ) + + maven_jar( +@@ -166,7 +166,7 @@ + maven_jar( + name = "sshd-mina", + artifact = "org.apache.sshd:sshd-mina:" + SSHD_VERS, +- sha1 = "8b202f7d4c0d7b714fd0c93a1352af52aa031149", ++ sha1 = "ff4a9fac41a111d806f6a058d23278b0819da7ce", + ) + + maven_jar( diff --git a/third_party/gerrit/workspace_overrides.bzl b/third_party/gerrit/workspace_overrides.bzl new file mode 100644 index 000000000..839368606 --- /dev/null +++ b/third_party/gerrit/workspace_overrides.bzl @@ -0,0 +1,5 @@ +def web_test_repositories(): + pass + +def browser_repositories(*args, **kwargs): + pass diff --git a/third_party/gerrit_plugins/builder.nix b/third_party/gerrit_plugins/builder.nix index 50a4e78ae..84515a9f6 100644 --- a/third_party/gerrit_plugins/builder.nix +++ b/third_party/gerrit_plugins/builder.nix @@ -2,30 +2,33 @@ { buildGerritBazelPlugin = { name + , version , src - , depsOutputHash + , depsHash ? null , overlayPluginCmd ? '' cp -R "${src}" "$out/plugins/${name}" + echo "STABLE_BUILD_${lib.toUpper name}_LABEL v${version}-nix${if patches != [] then "-dirty" else ""}" >> $out/.version '' + , postOverlayPlugin ? "" , postPatch ? "" , patches ? [ ] - }: ((depot.third_party.gerrit.override { + }: ((depot.third_party.gerrit.override (old: { name = "${name}.jar"; src = pkgs.runCommandLocal "${name}-src" { } '' cp -R "${depot.third_party.gerrit.src}" "$out" - chmod +w "$out/plugins" + chmod -R +w "$out" ${overlayPluginCmd} + ${postOverlayPlugin} ''; + depsHash = (if depsHash != null then depsHash else old.depsHash); - bazelTargets = [ "//plugins/${name}" ]; - }).overrideAttrs (super: { - deps = super.deps.overrideAttrs (superDeps: { - outputHash = depsOutputHash; - }); - installPhase = '' - cp "bazel-bin/plugins/${name}/${name}.jar" "$out" - ''; + bazelTargets = { + "//plugins/${name}" = "$out"; + }; + + extraBuildInstall = ""; + })).overrideAttrs (super: { postPatch = '' ${super.postPatch or ""} pushd "plugins/${name}" diff --git a/third_party/gerrit_plugins/code-owners/default.nix b/third_party/gerrit_plugins/code-owners/default.nix index 0dc3ef83a..c849bfb52 100644 --- a/third_party/gerrit_plugins/code-owners/default.nix +++ b/third_party/gerrit_plugins/code-owners/default.nix @@ -5,11 +5,11 @@ let in buildGerritBazelPlugin rec { name = "code-owners"; - depsOutputHash = "sha256:0jv62cc1kpgsmwk119i9njmqn6w6k8frlbgcw87y8nfbpprmcf01"; + version = "7de40d8"; src = pkgs.fetchgit { url = "https://gerrit.googlesource.com/plugins/code-owners"; - rev = "e654ae5bda2085bce9a99942bec440e004a114f3"; - sha256 = "sha256:14d3x3iqskgw16pvyaa0swh252agj84p9pzlf24l8lgx9d7y4biz"; + rev = "7de40d8b30e55eb64316b6fc3d0d00da9caddade"; + hash = "sha256-0sLwUcG9RN1o9vZGW8ErwL7UgJapgYoo8XMGsWLO25Q="; }; patches = [ ./using-usernames.patch diff --git a/third_party/gerrit_plugins/oauth/default.nix b/third_party/gerrit_plugins/oauth/default.nix index e7626fa88..a949596b0 100644 --- a/third_party/gerrit_plugins/oauth/default.nix +++ b/third_party/gerrit_plugins/oauth/default.nix @@ -5,15 +5,14 @@ let in buildGerritBazelPlugin rec { name = "oauth"; - depsOutputHash = "sha256:01z7rn8hnms3cp7mq27yk063lpy4pmqwpfrcc3cfl0r43k889zz3"; + version = "982316"; src = pkgs.fetchgit { url = "https://gerrit.googlesource.com/plugins/oauth"; - rev = "b27cf3ea820eec2ddd22d217fc839261692ccdb0"; - sha256 = "1m654ibgzprrhcl0wpzqrmq8drpgx6rzlw0ha16l1fi2zv5idkk2"; + rev = "98231604d60788bb43490f1a301d792817ac8008"; + hash = "sha256-AuVO1Yys8BYqGHZI/adszCUg0JM2v4Td4fe26LdOPLM="; }; - overlayPluginCmd = '' - chmod +w "$out" "$out/plugins/external_plugin_deps.bzl" - cp -R "${src}" "$out/plugins/${name}" + depsHash = "sha256-qEeKQPyt3uIS+BStrputBUnE3ft5c4oeazgsWd+lbN8="; + postOverlayPlugin = '' cp "${src}/external_plugin_deps.bzl" "$out/plugins/external_plugin_deps.bzl" ''; }