From 9c159d65caa8bd951f0c576e1fc150c67042d50b Mon Sep 17 00:00:00 2001
From: sterni <sternenseemann@systemli.org>
Date: Sun, 16 Feb 2025 00:52:29 +0100
Subject: [PATCH] fix(tvix/cli): don't apply overlays for expectedPath in eval
 tests

The TVL overlay can lead to diverging store paths, e.g. starting with
CL/13119, firefox depends on some FODs that in turn depend on Nix. Thus
the TVL overlay (with Nix 2.3) will come up with a different drvPath
than just using (import pkgs.path {}).

To fix this, just automatically use the same arguments to nixpkgs as
Tvix for expectedPath.

Change-Id: I77d39df9092442a49c8ee94b012dec046ff6a07e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/13147
Autosubmit: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
---
 tvix/cli/default.nix | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/tvix/cli/default.nix b/tvix/cli/default.nix
index 175e088c2..a2b841026 100644
--- a/tvix/cli/default.nix
+++ b/tvix/cli/default.nix
@@ -52,16 +52,19 @@ let
   mkNixpkgsEvalTest =
     { attrPath ? null # An attribute that must already be accessible from `pkgs`. Should evaluate to a store path.
     , expr ? null # A Nix expression that should evaluate to a store path.
-    , expectedPath # The expected store path that should match one of the above.
-    }:
+    , expectedPath ? null # The expected store path that should match one of the above.
+    }@args:
       assert lib.assertMsg (attrPath != null || expr != null) "Either 'attrPath' or 'expr' must be set.";
+      assert lib.assertMsg (expr != null -> expectedPath != null) "For 'expr', 'expectedPath' must be given explicitly.";
       let
-        name = "tvix-eval-test-${builtins.replaceStrings [".drv"] ["-drv"] (if expr != null then "custom-expr" else attrPath)}";
+        expectedPath' = args.expectedPath or (lib.getAttrFromPath attrPath (import pkgs.path { }));
+        renderedAttrPath = lib.concatStringsSep "." attrPath;
+        name = "tvix-eval-test-${builtins.replaceStrings [".drv"] ["-drv"] (if expr != null then "custom-expr" else renderedAttrPath)}";
       in
       (pkgs.runCommand name { } ''
         export SSL_CERT_FILE=/dev/null
-        TVIX_OUTPUT=$(${tvix-cli}/bin/tvix --no-warnings -E '${if expr != null then expr else "(import ${pkgs.path} {}).${attrPath}"}')
-        EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath}\" :: string"}'
+        TVIX_OUTPUT=$(${tvix-cli}/bin/tvix --no-warnings -E '${if expr != null then expr else "(import ${pkgs.path} {}).${renderedAttrPath}"}')
+        EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath'}\" :: string"}'
 
         echo "Tvix output: ''${TVIX_OUTPUT}"
         if [ "$TVIX_OUTPUT" != "$EXPECTED" ]; then
@@ -84,13 +87,13 @@ let
   };
 
   evalTests = {
-    eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalTest { attrPath = "stdenv.drvPath"; expectedPath = pkgs.stdenv.drvPath; });
-    eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalTest { attrPath = "stdenv.outPath"; expectedPath = pkgs.stdenv.outPath; });
-    eval-nixpkgs-hello-outpath = (mkNixpkgsEvalTest { attrPath = "hello.outPath"; expectedPath = pkgs.hello.outPath; });
-    eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalTest { attrPath = "firefox.outPath"; expectedPath = pkgs.firefox.outPath; });
-    eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalTest { attrPath = "firefox.drvPath"; expectedPath = pkgs.firefox.drvPath; });
-    eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalTest { attrPath = "pkgsCross.aarch64-multiplatform.stdenv.outPath"; expectedPath = pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath; });
-    eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalTest { attrPath = "pkgsCross.aarch64-multiplatform.hello.outPath"; expectedPath = pkgs.pkgsCross.aarch64-multiplatform.hello.outPath; });
+    eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalTest { attrPath = [ "stdenv" "drvPath" ]; });
+    eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalTest { attrPath = [ "stdenv" "outPath" ]; });
+    eval-nixpkgs-hello-outpath = (mkNixpkgsEvalTest { attrPath = [ "hello" "outPath" ]; });
+    eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalTest { attrPath = [ "firefox" "outPath" ]; });
+    eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalTest { attrPath = [ "firefox" "drvPath" ]; });
+    eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalTest { attrPath = [ "pkgsCross" "aarch64-multiplatform" "stdenv" "outPath" ]; });
+    eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalTest { attrPath = [ "pkgsCross" "aarch64-multiplatform" "hello" "outPath" ]; });
     # Our CI runner currently uses Nix version lower than 2.12, which means it uses the old JSON library.
     # The NixOS docs generate a JSON file with all the NixOS options, and so output is different between Tvix (and Nix 2.12+) and our CI runner's Nix version,
     # so we disable the NixOS docs generation for now. TODO(kranzes): Re-enable NixOS docs once the CI runner is using a newer Nix version.