feat(nix/buildLisp): add ecl

Adds ECL as a second supported implementation, specifically a statically
linked ECL. This is interesting because we can create statically linked
binaries, but has a few drawbacks which doesn't make it generally
useful:

* Loading things is very slow: The statically linked ECL only has byte
  compilation available, so when we do load things or use the REPL it is
  significantly worse than with e. g. SBCL.

* We can't load shared objects via the FFI since ECL's dffi is not
  available when linked statically. This means that as it stands, we
  can't build a statically linked //web/panettone for example.

Since ECL is quite slow anyways, I think these drawbacks are worth it
since the biggest reason for using ECL would be to get a statically
linked binary. If we change our minds, it shouldn't be too hard to
provide ecl-static and ecl-dynamic as separate implementations.

ECL is LGPL and some libraries it uses as part of its runtime are as
well. I've outlined in the ecl-static overlay why this should be of no
concern in the context of depot even though we are statically linking.

Currently everything is building except projects that are using cffi to
load shared libaries which have gotten an appropriate
`badImplementations` entry. To get the rest building the following
changes were made:

* Anywhere a dependency on UIOP is expressed as `bundled "uiop"` we now
  use `bundled "asdf"` for all implementations except SBCL. From my
  testing, SBCL seems to be the only implementation to support using
  `(require 'uiop)` to only load the UIOP package. Where both a
  dependency on ASDF and UIOP exists, we just delete the UIOP one.
  `(require 'asdf)` always causes UIOP to be available.

* Where appropriate only conditionally compile SBCL-specific code and
  if any build the corresponding files for ECL.

* //lisp/klatre: Use the standard condition parse-error for all
  implementations except SBCL in try-parse-integer.

* //3p/lisp/ironclad: disable SBCL assembly optimization hack for all
  other platforms as it may interfere with compilation.

* //3p/lisp/trivial-mimes: prevent call to asdf function by substituting
  it out of the source since it always errors out in ECL and we hardcode
  the correct path elsewhere anyways.

As it stands ECL still suffers from a very weird problem which happens
when compiling postmodern and moptilities:
https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651

Change-Id: I0285924f92ac154126b4c42145073c3fb33702ed
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3297
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
Reviewed-by: eta <tvl@eta.st>

third_party/lisp/bordeaux-threads.nix

@@ -2,18 +2,25 @@
 # in Common Lisp simple.
 { depot, ... }:
 
-let src = builtins.fetchGit {
-  url = "https://github.com/sionescu/bordeaux-threads.git";
-  rev = "499b6d3f0ce635417d6096acf0a671d8bf3f6e5f";
-};
+let
+  src = builtins.fetchGit {
+    url = "https://github.com/sionescu/bordeaux-threads.git";
+    rev = "499b6d3f0ce635417d6096acf0a671d8bf3f6e5f";
+  };
+  getSrc = f: "${src}/src/${f}";
 in depot.nix.buildLisp.library {
   name = "bordeaux-threads";
   deps = [ depot.third_party.lisp.alexandria ];
 
-  srcs = map (f: src + ("/src/" + f)) [
+  srcs = map getSrc [
     "pkgdcl.lisp"
     "bordeaux-threads.lisp"
-    "impl-sbcl.lisp"
+  ] ++ [
+    {
+      sbcl = getSrc "impl-sbcl.lisp";
+      ecl = getSrc "impl-ecl.lisp";
+    }
+  ] ++ map getSrc [
     "default-implementations.lisp"
   ];
 }

third_party/lisp/cffi.nix

@@ -13,11 +13,14 @@ in buildLisp.library {
     babel
     trivial-features
     (buildLisp.bundled "asdf")
-    (buildLisp.bundled "uiop")
   ];
 
-  srcs = map (f: src + ("/src/" + f)) [
-    "cffi-sbcl.lisp"
+  srcs = [
+    {
+      ecl = src + "/src/cffi-ecl.lisp";
+      sbcl = src + "/src/cffi-sbcl.lisp";
+    }
+  ] ++ map (f: src + ("/src/" + f)) [
     "package.lisp"
     "utils.lisp"
     "libraries.lisp"

third_party/lisp/cl-fad.nix

@@ -15,7 +15,9 @@ in buildLisp.library {
   deps = with depot.third_party.lisp; [
     alexandria
     bordeaux-threads
-    (buildLisp.bundled "sb-posix")
+    {
+      sbcl = buildLisp.bundled "sb-posix";
+    }
   ];
 
   srcs = map (f: src + ("/" + f)) [

third_party/lisp/cl-plus-ssl.nix

@@ -37,4 +37,8 @@ in buildLisp.library {
     "context.lisp"
     "verify-hostname.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/cl-smtp.nix

@@ -25,4 +25,8 @@ in depot.nix.buildLisp.library {
     "cl-smtp.lisp"
     "mime-types.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/cl-unicode.nix

@@ -29,7 +29,10 @@ let
     deps = with depot.third_party.lisp; [
       cl-unicode-base
       flexi-streams
-      (bundled "uiop")
+      {
+        ecl = bundled "asdf";
+        default = bundled "uiop";
+      }
     ];
 
     srcs = (map (f: src + ("/build/" + f)) [

third_party/lisp/closer-mop.nix

@@ -15,6 +15,9 @@ in depot.nix.buildLisp.library {
   srcs = [
     "${src}/closer-mop-packages.lisp"
     "${src}/closer-mop-shared.lisp"
-    "${src}/closer-sbcl.lisp"
+    {
+      sbcl = "${src}/closer-sbcl.lisp";
+      ecl = "${src}/closer-ecl.lisp";
+    }
   ];
 }

third_party/lisp/drakma.nix

@@ -32,4 +32,8 @@ in depot.nix.buildLisp.library {
     "encoding.lisp"
     "request.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/easy-routes.nix

@@ -23,4 +23,7 @@ in depot.nix.buildLisp.library {
     "routes-map-printer.lisp"
   ];
 
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/hunchentoot.nix

@@ -58,4 +58,8 @@ in depot.nix.buildLisp.library {
     "acceptor.lisp"
     "easy-handlers.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/ironclad.nix

@@ -10,33 +10,40 @@ let
     sha256 = "0k4bib9mbrzalbl9ivkw4a7g4c7bbad1l5jw4pzkifqszy2swkr5";
   };
 
+  getSrc = f: "${src}/src/${f}";
+
 in depot.nix.buildLisp.library {
   name = "ironclad";
 
   deps = with depot.third_party.lisp; [
     (bundled "asdf")
-    (bundled "sb-rotate-byte")
-    (bundled "sb-posix")
+    { sbcl = bundled "sb-rotate-byte"; }
+    { sbcl = bundled "sb-posix"; }
     alexandria
     bordeaux-threads
     nibbles
   ];
 
   srcs = [
-    "${src}/ironclad.asd"
-    # TODO(grfn): Figure out how to get this compiling with the assembly
-    # optimization eventually - see https://cl.tvl.fyi/c/depot/+/1333
-    (runCommand "package.lisp" {} ''
-      substitute ${src}/src/package.lisp $out \
-        --replace \#-ecl-bytecmp "" \
-        --replace '(pushnew :ironclad-assembly *features*)' ""
-    '')
-  ] ++ (map (f: src + ("/src/" + f)) [
+    {
+      # TODO(grfn): Figure out how to get this compiling with the assembly
+      # optimization eventually - see https://cl.tvl.fyi/c/depot/+/1333
+      sbcl = runCommand "package.lisp" {} ''
+        substitute ${src}/src/package.lisp $out \
+          --replace \#-ecl-bytecmp "" \
+          --replace '(pushnew :ironclad-assembly *features*)' ""
+      '';
+      default = getSrc "package.lisp";
+    }
+  ] ++ map getSrc [
     "macro-utils.lisp"
+  ] ++ [
+    { sbcl = getSrc "opt/sbcl/fndb.lisp"; }
+    { sbcl = getSrc "opt/sbcl/cpu-features.lisp"; }
+    { sbcl = getSrc "opt/sbcl/x86oid-vm.lisp"; }
 
-    "opt/sbcl/fndb.lisp"
-    "opt/sbcl/cpu-features.lisp"
-    "opt/sbcl/x86oid-vm.lisp"
+    { ecl = getSrc "opt/ecl/c-functions.lisp"; }
+  ] ++ map getSrc [
 
     "common.lisp"
     "conditions.lisp"
@@ -142,5 +149,5 @@ in depot.nix.buildLisp.library {
     "public-key/elgamal.lisp"
     "public-key/pkcs1.lisp"
     "public-key/rsa.lisp"
-  ]);
+  ];
 }

third_party/lisp/lisp-binary.nix

@@ -28,4 +28,8 @@ in depot.nix.buildLisp.library {
     "binary-2.lisp"
     "types.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/md5.nix

@@ -11,6 +11,11 @@ let src = pkgs.fetchFromGitHub {
 };
 in buildLisp.library {
   name = "md5";
-  deps = [ (buildLisp.bundled "sb-rotate-byte") ];
+  deps = [
+    {
+      sbcl = buildLisp.bundled "sb-rotate-byte";
+      default = depot.third_party.lisp.flexi-streams;
+    }
+  ];
   srcs = [ (src + "/md5.lisp") ];
 }

third_party/lisp/moptilities.nix

@@ -11,4 +11,8 @@ in depot.nix.buildLisp.library {
   name = "moptilities";
   deps = [ depot.third_party.lisp.closer-mop ];
   srcs = [ "${src}/dev/moptilities.lisp" ];
+
+  brokenOn = [
+    "ecl" # TODO(sterni): https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651
+  ];
 }

third_party/lisp/nibbles.nix

@@ -24,9 +24,10 @@ in depot.nix.buildLisp.library {
     "types.lisp"
     "vectors.lisp"
     "streams.lisp"
-    "sbcl-opt/fndb.lisp"
-    "sbcl-opt/nib-tran.lisp"
-    "sbcl-opt/x86-vm.lisp"
-    "sbcl-opt/x86-64-vm.lisp"
+  ] ++ [
+    { sbcl = "${src}/sbcl-opt/fndb.lisp"; }
+    { sbcl = "${src}/sbcl-opt/nib-tran.lisp"; }
+    { sbcl = "${src}/sbcl-opt/x86-vm.lisp"; }
+    { sbcl = "${src}/sbcl-opt/x86-64-vm.lisp"; }
   ];
 }

third_party/lisp/postmodern.nix

@@ -83,6 +83,10 @@ let
       "table.lisp"
       "deftable.lisp"
     ]);
+
+    brokenOn = [
+      "ecl" # TODO(sterni): https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651
+    ];
   };
 
 in postmodern // {

third_party/lisp/restas.nix

@@ -35,4 +35,7 @@ in depot.nix.buildLisp.library {
     "policy.lisp"
   ];
 
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/trivial-features.nix

@@ -7,6 +7,9 @@ let src = builtins.fetchGit {
 in depot.nix.buildLisp.library {
   name = "trivial-features";
   srcs = [
-    (src + "/src/tf-sbcl.lisp")
+    {
+      sbcl = src + "/src/tf-sbcl.lisp";
+      ecl = src + "/src/tf-ecl.lisp";
+    }
   ];
 }

third_party/lisp/trivial-ldap.nix

@@ -19,4 +19,8 @@ in depot.nix.buildLisp.library {
     "package.lisp"
     "trivial-ldap.lisp"
   ];
+
+  brokenOn = [
+    "ecl" # dynamic cffi
+  ];
 }

third_party/lisp/trivial-mimes.nix

@@ -10,14 +10,20 @@ let
 
   mime-types = pkgs.runCommand "mime-types.lisp" {} ''
     substitute ${src}/mime-types.lisp $out \
-      --replace /etc/mime.types ${src}/mime.types
+      --replace /etc/mime.types ${src}/mime.types \
+      --replace "(asdf:system-source-directory :trivial-mimes)" '"/bogus-dir"'
+      # We want to prevent an ASDF lookup at build time since this will
+      # generally fail — we are not using ASDF after all.
   '';
 
 in depot.nix.buildLisp.library {
   name = "trivial-mimes";
 
   deps = [
-    (depot.nix.buildLisp.bundled "uiop")
+    {
+      sbcl = depot.nix.buildLisp.bundled "uiop";
+      default = depot.nix.buildLisp.bundled "asdf";
+    }
   ];
 
   srcs = [ mime-types ];

third_party/lisp/uax-15.nix

@@ -19,7 +19,6 @@ in depot.nix.buildLisp.library {
   deps = with depot.third_party.lisp; [
     split-sequence
     cl-ppcre
-    (bundled "uiop")
     (bundled "asdf")
   ];
 

third_party/lisp/usocket.nix

@@ -33,6 +33,12 @@ in buildLisp.library {
     "package.lisp"
     "usocket.lisp"
     "condition.lisp"
-    "backend/sbcl.lisp"
+  ] ++ [
+    { sbcl = "${src}/backend/sbcl.lisp"; }
+
+    # ECL actually has two files, it supports the SBCL backend,
+    # but usocket also has some ECL specific code
+    { ecl = "${src}/backend/sbcl.lisp"; }
+    { ecl = "${src}/backend/ecl.lisp"; }
   ]);
 }

third_party/nixpkgs/default.nix

@@ -66,5 +66,6 @@ in import nixpkgsSrc {
     depot.third_party.overlays.tvl
     depot.third_party.overlays.haskell
     depot.third_party.overlays.emacs
+    depot.third_party.overlays.ecl-static
   ];
 }

third_party/overlays/ecl-static.nix

@@ -0,0 +1,41 @@
+{ ... }:
+
+self: super:
+
+{
+  # Statically linked ECL with statically linked dependencies.
+  # Works quite well, but solving this properly in a nixpkgs
+  # context will require figuring out cross compilation (for
+  # pkgsStatic), so we're gonna use this override for now.
+  #
+  # Note that ecl-static does mean that we have things
+  # statically linked against GMP and ECL which are LGPL.
+  # I believe this should be alright: The way ppl are gonna
+  # interact with the distributed binaries (i. e. the binary
+  # cache) is Nix in the depot monorepo, so the separability
+  # requirement should be satisfied: Source code or overriding
+  # would be available as ways to swap out the used GMP in the
+  # program.
+  # See https://www.gnu.org/licenses/gpl-faq.en.html#LGPLStaticVsDynamic
+  ecl-static = (super.pkgsMusl.ecl.override {
+    inherit (self.pkgsStatic) gmp libffi boehmgc;
+  }).overrideAttrs (drv: {
+    # Patches that make .fasc files concatenable again
+    patches = drv.patches ++ [
+      (self.fetchpatch {
+        name = "make-bytecode-fasl-concatenatable-1.patch";
+        url = "https://gitlab.com/embeddable-common-lisp/ecl/-/commit/fbb75a0fc524e3280d89d8abf3be2ee9924955c8.patch";
+        sha256 = "0k6cx1bh835rl0j0wbbi5nj0aa2rwbyfyz5q2jw643iqc62l16kv";
+      })
+      (self.fetchpatch {
+        name = "make-bytecode-fasl-concatenatable-2.patch";
+        url = "https://gitlab.com/embeddable-common-lisp/ecl/-/commit/a8b1c0da43f89800d09c23a27832d0b4c9dcc1e8.patch";
+        sha256 = "18hl79lss0dxglpa34hszqb6ajvs8rs4b4g1qlrqrvgh1gs667n0";
+      })
+    ];
+    configureFlags = drv.configureFlags ++ [
+      "--disable-shared"
+      "--with-dffi=no" # will fail at runtime anyways if statically linked
+    ];
+  });
+}