refactor: Move nixpkgs attribute to third_party.nixpkgs

Please read b/108 to make sense of this.

This gets rid of the explicit list of exposed packages from nixpkgs,
and instead makes the entire package set available at
`third_party.nixpkgs`.

To accommodate this, a LOT of things have to be very slightly shuffled
around. Some of this was done in already submitted CLs, but this
change is unfortunately still quite noisy.

Pay extra attention to:

* overlay-like functionality that was partially moved to actual
  overlays (partially as in, the minimum required to get a green
  build)

* modified uses of the package set path, esp. in NixOS systems

Special notes:

* xanthous has been disabled in CI because of issues with the Haskell
  overlay
* //third_party/nix has been disabled because of other unclear
  dependency issues

Both of these will be tackled in a followup CL.

Change-Id: I2f9c60a4d275fdb5209264be0addfd7e06c53118
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2910
Reviewed-by: glittershark <grfn@gws.fyi>
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

third_party/default.nix

@@ -1,73 +1,23 @@
-# This file controls the import of external dependencies (i.e.
-# third-party code) into my package tree.
+# This file defines the root of all external dependency imports (i.e.
+# third-party code) in the TVL package tree.
 #
-# This includes *all packages needed from nixpkgs*.
-{ ... }:
+# There are two categories of third-party programs:
+#
+# 1) Programs in nixpkgs, the NixOS package set. For these, you might
+#    want to look at //third_party/nixpkgs (for the package set
+#    imports) and //third_party/overlays (for modifications in these
+#    imported package sets).
+#
+# 2) Third-party software packaged in this repository. This is all
+#    other folders below //third_party, other than the ones mentioned
+#    above.
 
-let
-  # Tracking nixos-unstable as of 2021-03-25.
-  nixpkgsCommit = "60dd94fb7e01a8288f6638eee71d7cb354c49327";
-  nixpkgsSrc = fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs/archive/${nixpkgsCommit}.tar.gz";
-    sha256 = "0skdwk9bdld295kzrymirs8xrzycqmhsclaz8s18jhcz75hb8sk3";
-  };
-  nixpkgs = import nixpkgsSrc {
-    config.allowUnfree = true;
-    config.allowBroken = true;
+{ pkgs, ... }:
 
-    # Lutris depends on p7zip, which is considered insecure.
-    config.permittedInsecurePackages = [
-      "p7zip-16.02"
-    ];
-  };
-
-  # Tracking nixos-20.09 as of 2021-03-25.
-  stableCommit = "223d0d733a66b46504ea6b4c15f88b7cc4db58fb";
-  stableNixpkgsSrc = fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs/archive/${stableCommit}.tar.gz";
-    sha256 = "073327ris0frqa3kpid3nsjr9w8yx2z83xpsc24w898mrs9r7d5v";
-  };
-  stableNixpkgs = import stableNixpkgsSrc {};
-
-  exposed = import ./nixpkgs-exposed/exposed { inherit nixpkgs stableNixpkgs; };
-
-in exposed.lib.fix(self: exposed // {
-  callPackage = nixpkgs.lib.callPackageWith self;
-
-  # Provide the source code of nixpkgs, but do not provide an imported
-  # version of it.
-  inherit nixpkgsCommit nixpkgsSrc stableNixpkgsSrc;
-
-  # Expose upstream attributes so they can be overridden in readTree nodes
-  originals = {
-    inherit (nixpkgs) gtest openldap go grpc notmuch rr;
-    inherit (stableNixpkgs) git tdlib;
-    ffmpeg = nixpkgs.ffmpeg-full;
-    telega = stableNixpkgs.emacsPackages.telega;
-
-  };
-
-  # Use LLVM 11
-  llvmPackages = nixpkgs.llvmPackages_11;
-  clangStdenv = nixpkgs.llvmPackages_11.stdenv;
-  stdenv = nixpkgs.llvmPackages_11.stdenv;
-
-  clang-tools = (nixpkgs.clang-tools.override {
-    llvmPackages = nixpkgs.llvmPackages_11;
-  });
-
-  # Provide Emacs 27
-  #
-  # The assert exists because the name of the attribute is unversioned
-  # (which is different from previous versions).
-  emacs27 = assert ((exposed.lib.versions.major nixpkgs.emacs.version) == "27");
-    nixpkgs.emacs.overrideAttrs(old: {
-      configureFlags = old.configureFlags ++ [ "--with-cairo" ];
-    });
-
-  emacs27-nox = assert ((exposed.lib.versions.major nixpkgs.emacs.version) == "27");
-    nixpkgs.emacs-nox;
-
-  # Make NixOS available
-  nixos = import "${nixpkgsSrc}/nixos";
-})
+{
+  # Expose a partially applied NixOS, expecting an attribute set with
+  # a `configuration` key. Exposing it like this makes it possible to
+  # modify some of the base configuration used by NixOS. passed to
+  # this.
+  nixos = import "${pkgs.path}/nixos";
+}

third_party/ffmpeg/default.nix

@@ -1,12 +0,0 @@
-{ pkgs, ... }:
-
-pkgs.originals.ffmpeg.overrideAttrs(old: {
-  buildInputs = old.buildInputs ++ [
-    pkgs.cudatoolkit.out
-  ];
-
-  configureFlags = old.configureFlags ++ [
-    "--enable-libnpp"
-    "--enable-nonfree"
-  ];
-})

third_party/git/default.nix

@@ -1,15 +1,13 @@
 # Use the upstream git derivation (there's a lot of stuff happening in
 # there!) and just override the source:
-{ depot, ... }:
+{ pkgs, ... }:
 
-with depot.third_party;
-
-(originals.git.overrideAttrs(_: {
+(pkgs.git.overrideAttrs(_: {
   version = "2.29.2";
   src = ./.;
   doInstallCheck = false;
   preConfigure = ''
-    ${autoconf}/bin/autoreconf -i
+    ${pkgs.autoconf}/bin/autoreconf -i
   '';
 })).override {
   sendEmailSupport = true;

third_party/grpc/default.nix

@@ -1,7 +1,7 @@
-{ pkgs, ... }:
+{ depot, pkgs, ... }:
 
-(pkgs.originals.grpc.override {
-  protobuf = pkgs.protobuf;
+(pkgs.grpc.override {
+  protobuf = depot.third_party.protobuf;
   stdenv = pkgs.llvmPackages.libcxxStdenv;
 }).overrideAttrs(orig: rec {
   version = "1.30.0";

third_party/gtest/default.nix

@@ -1,6 +1,6 @@
 { pkgs, ... }:
 
-(pkgs.originals.gtest.override {
+(pkgs.gtest.override {
   stdenv = pkgs.llvmPackages.libcxxStdenv;
 }).overrideAttrs(_: {
   src = pkgs.fetchFromGitHub {

third_party/nix/default.nix

@@ -1,5 +1,6 @@
 args@{
-  pkgs ? (import ../.. {}).third_party
+  depot ? (import ../.. {})
+, pkgs ? depot.third_party.nixpkgs
 , lib
 , buildType ? "release"
 , depotPath ? ../..
@@ -43,10 +44,11 @@ let
   protoSrcs = pkgs.runCommand "nix-proto-srcs" {} ''
     export PROTO_SRCS=${./src/proto}
     mkdir -p $out/libproto
-    ${pkgs.protobuf}/bin/protoc -I=$PROTO_SRCS \
+    ${depot.third_party.protobuf}/bin/protoc -I=$PROTO_SRCS \
       --cpp_out=$out/libproto \
-      --plugin=protoc-gen-grpc=${pkgs.grpc}/bin/grpc_cpp_plugin --grpc_out=$out/libproto \
-      $PROTO_SRCS/*.proto
+      --plugin=protoc-gen-grpc=${depot.third_party.grpc}/bin/grpc_cpp_plugin \
+        --grpc_out=$out/libproto \
+        $PROTO_SRCS/*.proto
   '';
 
   # Derivation for busybox that just has the `busybox` binary in bin/, not all
@@ -73,7 +75,6 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
  # TODO(tazjin): Some of these might only be required for native inputs
   buildInputs = with pkgs; [
-    abseil_cpp
     aws-s3-cpp
     brotli
     bzip2
@@ -81,16 +82,18 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
     curl
     editline
     flex
-    glog
-    grpc
     libseccomp
     libsodium
     systemd.dev
     openssl
-    protobuf
     sqlite
     xz
-  ];
+  ] ++ (with depot.third_party; [
+    abseil_cpp
+    glog
+    grpc
+    protobuf
+  ]);
 
   doCheck = false;
   doInstallCheck = true;
@@ -99,8 +102,8 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
   dontStrip = true;
 
   installCheckInputs = with pkgs; [
+    depot.third_party.gtest
     fd
-    gtest
     rapidcheck
   ];
 
@@ -185,6 +188,9 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
   # TODO(tazjin): integration test setup?
   # TODO(tazjin): docs generation?
 
+  # TODO(tazjin): Sort out after CL/2910 lands
+  meta.ci = false;
+
   passthru = {
     build-shell = self.overrideAttrs (up: rec {
       run_clang_tidy = pkgs.writeShellScriptBin "run-clang-tidy" ''

third_party/nix/test-vm.nix

@@ -1,10 +1,9 @@
 { depot, pkgs, ... }:
 
 let
-
   configuration = { ... }: {
     imports = [
-      "${pkgs.nixpkgsSrc}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix"
+      "${pkgs.path}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix"
     ];
 
     nix.package = depot.third_party.nix;
@@ -16,6 +15,5 @@ let
     ];
   };
 
-  system = pkgs.nixos { inherit configuration; };
-
+  system = depot.third_party.nixos { inherit configuration; };
 in system.vm

third_party/nixpkgs-exposed/.skip-subtree

@@ -1 +0,0 @@
-This subtree is responsible for setting up the read tree arguments in the first place.

third_party/nixpkgs-exposed/OWNERS

@@ -1,8 +0,0 @@
-# The nixpkgs whitelist needs to be changed every time we reference
-# a new attribute, so every commiter should be able to change it,
-# otherwise we create undue blockers on superowners.
-inherited: true
-owners:
-  - Profpatsch
-  - sterni
-  - glittershark

third_party/nixpkgs-exposed/exposed/default.nix

@@ -1,210 +0,0 @@
-# This file has to be in yet another subdir
-# because of how readTree interprets .skip-subtree
-# see https://b.tvl.fyi/issues/89
-{ nixpkgs, stableNixpkgs }:
-{
-  # Inherit the packages from nixos-unstable that should be available inside
-  # of the repo. They become available under `pkgs.third_party.<name>`
-  inherit (nixpkgs)
-    age
-    atk
-    autoconf
-    autoreconfHook
-    avrdude
-    avrlibc
-    bashInteractive
-    bat
-    bc
-    bind
-    buildBazelPackage
-    buildFHSUserEnv
-    buildGoModule
-    buildGoPackage
-    buildPackages
-    buildRustCrate
-    buildkite-agent
-    busybox
-    bzip2
-    c-ares
-    cacert
-    cachix
-    cairo
-    cargo
-    cgit
-    clang_11
-    cmake
-    coreutils
-    cudatoolkit
-    darwin
-    dbus-glib
-    dbus_libs
-    dfu-programmer
-    dfu-util
-    diffutils
-    docker-compose
-    dockerTools
-    emacs27
-    emacs27-nox
-    emacsPackages
-    emacsPackagesGen
-    execline
-    fd
-    fdtools
-    fetchFromGitHub
-    fetchFromGitLab
-    fetchgit
-    fetchurl
-    fetchzip
-    ffmpeg-full
-    findutils
-    fira
-    fira-code
-    fira-mono
-    flamegraph
-    fontconfig
-    freetype
-    gdk-pixbuf
-    gettext
-    glibc
-    glib
-    gtk3
-    gmock
-    gnused
-    gnutar
-    go
-    google-cloud-sdk
-    graphviz
-    gzip
-    haskell
-    iana-etc
-    imagemagickBig
-    installShellFiles
-    jdk
-    jdk11
-    jdk11_headless
-    jetbrains-mono
-    jq
-    kontemplate
-    lib
-    libappindicator-gtk3
-    libredirect
-    libffi
-    linuxPackages
-    linuxPackages_5_11
-    lr
-    luajit
-    lutris
-    makeFontsConf
-    makeWrapper
-    mandoc
-    mdbook
-    meson
-    mime-types
-    minify
-    mkShell
-    moreutils
-    nano
-    ncurses
-    nginx
-    ninja
-    nix
-    ocamlPackages
-    openssh
-    openssl
-    overrideCC
-    pandoc
-    pango
-    parallel
-    path
-    pkgconfig
-    pkgsCross
-    plantuml
-    postgresql
-    pounce
-    pulseaudio
-    python3
-    python3Packages
-    quassel
-    remarshal
-    rink
-    ripgrep
-    rsync
-    runCommand
-    runCommandLocal
-    runCommandNoCC
-    rustPlatform
-    rustc
-    s6-portable-utils
-    sbcl
-    shellcheck
-    sqlite
-    stdenvNoCC
-    stern
-    substituteAll
-    symlinkJoin
-    systemd
-    tdlib
-    teensy-loader-cli
-    terraform_0_12
-    texlive
-    thttpd
-    tree
-    tree-sitter
-    unzip
-    which
-    writers
-    writeShellScript
-    writeShellScriptBin
-    writeText
-    xorg
-    xz
-    zlib
-    zstd;
-
-  # Inherit packages from the stable channel for things that are
-  # broken on unstable
-  inherit (stableNixpkgs)
-    awscli # TODO(grfn): Move back to unstable once it is fixed
-    ;
-
-  # Required by //third_party/nix
-  inherit (nixpkgs)
-    aws-sdk-cpp
-    bison
-    boehmgc
-    boost # urgh
-    brotli
-    busybox-sandbox-shell
-    curl
-    docbook5
-    docbook_xsl_ns
-    editline
-    flex
-    libseccomp
-    libsodium
-    libxml2
-    libxslt
-    mercurial
-    perl
-    perlPackages
-    ;
-
-  haskellPackages = (nixpkgs.haskellPackages.override {
-    overrides = (import ../../overlays/haskell
-    { /* empty readTree arg */ }
-    { pkgs = nixpkgs; });
-  });
-
-  gradle_6 = (nixpkgs.gradleGen.override {
-    java = nixpkgs.jdk11;
-    jdk = nixpkgs.jdk11;
-  }).gradleGen rec {
-    name = "gradle-6.5.1";
-    nativeVersion = "0.22-milestone-3";
-
-    src = builtins.fetchurl {
-      url = "https://services.gradle.org/distributions/${name}-bin.zip";
-      sha256 = "0jmmipjh4fbsn92zpifa5cqg5ws2a4ha0s4jzqhrg4zs542x79sh";
-    };
-  };
-}

third_party/nixpkgs/default.nix

@@ -0,0 +1,56 @@
+# This file imports the pinned nixpkgs sets and applies relevant
+# modifications, such as our overlays.
+#
+# Note that the attribute exposed by this (third_party.nixpkgs) is
+# "special" in that the fixpoint used as readTree's config parameter
+# in //default.nix passes this attribute as the `pkgs` argument to all
+# readTree derivations.
+
+{ depot, ... }:
+
+let
+  # This provides the sources of nixpkgs. We track both
+  # nixos-unstable, and the current stable channel of the latest NixOS
+  # release.
+
+  # Tracking nixos-unstable as of 2021-03-25.
+  unstableHashes = {
+    commit = "60dd94fb7e01a8288f6638eee71d7cb354c49327";
+    sha256 = "0skdwk9bdld295kzrymirs8xrzycqmhsclaz8s18jhcz75hb8sk3";
+  };
+
+  # Tracking nixos-20.09 as of 2021-03-25.
+  stableHashes = {
+    commit = "223d0d733a66b46504ea6b4c15f88b7cc4db58fb";
+    sha256 = "073327ris0frqa3kpid3nsjr9w8yx2z83xpsc24w898mrs9r7d5v";
+  };
+
+  nixpkgsSrc = fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${unstableHashes.commit}.tar.gz";
+    sha256 = unstableHashes.sha256;
+  };
+  stableNixpkgsSrc = fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${stableHashes.commit}.tar.gz";
+    sha256 = stableHashes.sha256;
+  };
+
+  # Stable package set is imported, but not exposed, to overlay
+  # required packages into the unstable set.
+  stableNixpkgs = import stableNixpkgsSrc {};
+
+  # Overlay for packages that should come from the stable channel
+  # instead (e.g. because something is broken in unstable).
+  stableOverlay = self: super: {
+    inherit (stableNixpkgs)
+      awscli # TODO(grfn): Move back to unstable once it is fixed
+      ;
+  };
+in import nixpkgsSrc {
+  config.allowUnfree = true;
+  config.allowBroken = true;
+  overlays = [
+    stableOverlay
+    depot.third_party.overlays.tvl
+    depot.third_party.overlays.haskell
+  ];
+}

third_party/notmuch/default.nix

@@ -1,6 +1,7 @@
+# Notmuch, but with support for https://dotti.me/
 { pkgs, ... }:
 
-pkgs.originals.notmuch.overrideAttrs(old: {
+pkgs.notmuch.overrideAttrs(old: {
   doCheck = false;
   patches = [ ./dottime.patch ] ++ (if old ? patches then old.patches else []);
 })

third_party/openldap/default.nix

@@ -6,7 +6,7 @@
 # derivation to include this module.
 { pkgs, ... }:
 
-pkgs.originals.openldap.overrideAttrs(old: {
+pkgs.openldap.overrideAttrs(old: {
   buildInputs = old.buildInputs ++ [ pkgs.libsodium ];
 
   postBuild = ''

third_party/overlays/haskell/default.nix

@@ -1,34 +1,41 @@
-# Defines overrides for Haskell packages, for example to avoid
-# breakage currently present in nixpkgs or to modify package versions.
+# Defines an overlay for overriding Haskell packages, for example to
+# avoid breakage currently present in nixpkgs or to modify package
+# versions.
 
 { ... }: # This file needs nothing from readTree
-{ pkgs }: # ... but is called with a separate package set in the overlay
 
-self: super: with pkgs.haskell.lib; rec {
-  generic-arbitrary = appendPatch
-    super.generic-arbitrary
-    [ ./patches/generic-arbitrary-export-garbitrary.patch ];
+self: super: # overlay parameters for the nixpkgs overlay
 
-  random = dontCheck (self.callHackageDirect {
-    pkg = "random";
-    ver = "1.2.0";
-    sha256 = "06s3mmqbsfwv09j2s45qnd66nrxfp9280gnl9ng8yh128pfr7bjh";
-  } {});
+let
+  overrides = hsSelf: hsSuper: with super.haskell.lib; rec {
+    generic-arbitrary = appendPatch hsSuper.generic-arbitrary
+      [ ./patches/generic-arbitrary-export-garbitrary.patch ];
 
-  # random <1.2
-  test-framework = doJailbreak super.test-framework;
-  hashable = doJailbreak super.hashable;
-  test-framework-quickcheck2 = doJailbreak super.test-framework-quickcheck2;
+    # random = dontCheck (hsSuper.callHackageDirect {
+    #   pkg = "random";
+    #   ver = "1.2.0";
+    #   sha256 = "06s3mmqbsfwv09j2s45qnd66nrxfp9280gnl9ng8yh128pfr7bjh";
+    # } {});
 
-  # can be removed if we have the following PR or equivalent
-  # https://github.com/NixOS/nixpkgs/pull/116931
-  hedgehog-classes = overrideCabal super.hedgehog-classes (attrs: {
-    # remove version bound on semirings which is inside a
-    # conditional, so doJailbreak doesn't work
-    prePatch = ''
-      sed -i 's|semirings.*0.6|semirings|g' hedgehog-classes.cabal
-    '';
-  });
+    # random <1.2
+    test-framework = doJailbreak hsSuper.test-framework;
+    hashable = doJailbreak hsSuper.hashable;
+    test-framework-quickcheck2 = doJailbreak hsSuper.test-framework-quickcheck2;
 
-  hgeometry-combinatorial = dontCheck super.hgeometry-combinatorial;
+    # can be removed if we have the following PR or equivalent
+    # https://github.com/NixOS/nixpkgs/pull/116931
+    hedgehog-classes = overrideCabal hsSuper.hedgehog-classes (attrs: {
+      # remove version bound on semirings which is inside a
+      # conditional, so doJailbreak doesn't work
+      prePatch = ''
+        sed -i 's|semirings.*0.6|semirings|g' hedgehog-classes.cabal
+      '';
+    });
+
+    hgeometry-combinatorial = dontCheck hsSuper.hgeometry-combinatorial;
+  };
+in {
+  haskellPackages = super.haskellPackages.override {
+    inherit overrides;
+  };
 }

third_party/overlays/tvl.nix

@@ -0,0 +1,27 @@
+# This overlay is used to make TVL-specific modifications in the
+# nixpkgs tree, where required.
+{ ... }:
+
+self: super: {
+  # Required for apereo-cas
+  # TODO(lukegb): Document why?
+  gradle_6 = (super.gradleGen.override {
+    java = self.jdk11;
+    jdk = self.jdk11;
+  }).gradleGen rec {
+    name = "gradle-6.5.1";
+    nativeVersion = "0.22-milestone-3";
+
+    src = builtins.fetchurl {
+      url = "https://services.gradle.org/distributions/${name}-bin.zip";
+      sha256 = "0jmmipjh4fbsn92zpifa5cqg5ws2a4ha0s4jzqhrg4zs542x79sh";
+    };
+  };
+
+  # Use LLVM 11
+  llvmPackages = self.llvmPackages_11;
+  clangStdenv = self.llvmPackages_11.stdenv;
+  clang-tools = (super.clang-tools.override {
+    llvmPackages = self.llvmPackages_11;
+  });
+}

third_party/protobuf/default.nix

@@ -1,7 +1,7 @@
 # Pin protobuf to version 3.12, with LLVM.
 { depot, pkgs, ... }:
 
-pkgs.callPackage "${depot.third_party.nixpkgsSrc}/pkgs/development/libraries/protobuf/generic-v3.nix" {
+pkgs.callPackage "${pkgs.path}/pkgs/development/libraries/protobuf/generic-v3.nix" {
   version = "3.12.2";
   sha256 = "1lp368aa206vpic9fmax4k6llnmf28plfvkkm4vqhgphmjqykvl2";
   stdenv = pkgs.llvmPackages.libcxxStdenv;

third_party/rr/default.nix

@@ -1,16 +0,0 @@
-{ pkgs, ... }:
-
-pkgs.originals.rr.overrideAttrs(_: {
-  src = pkgs.fetchFromGitHub {
-    owner = "mozilla";
-    repo = "rr";
-    rev = "8fc7d2a09a739fee1883d262501e88613165c1dd";
-    sha256 = "0avq5lv082z2sasggfn2awnfrh08cr8f0i9iw1dnrcxa6pc3bi9k";
-    fetchSubmodules = false;
-  };
-
-  # Workaround as documented on https://github.com/mozilla/rr/wiki/Zen
-  postInstall = ''
-    cp $src/scripts/zen_workaround.py $out/bin/rr_zen_workaround
-  '';
-})