snix/tvix/verify-lang-tests/default.nix

# SPDX-License-Identifier: LGPL-2.1-only
# SPDX-FileCopyrightText: © 2022 The TVL Contributors
# SPDX-FileCopyrightText: © 2004-2022 The Nix Contributors
#
# Execute language tests found in tvix_tests and nix_tests
# using the C++ Nix implementation. Based on NixOS/nix:tests/lang.sh.
{ depot, pkgs, lib, ... }:

let
  testRoot = ../eval/src/tests;

  inherit (pkgs.nixVersions) nix_2_3;
  # The latest Nix version we've verified to work for our testing suite.
  nix_latest_verified = pkgs.nixVersions.nix_2_23;

  parseTest = dir: baseName:
    let
      tokens = builtins.match "(eval|parse)-(okay|fail).+\\.nix" baseName;
    in
    if tokens == null
    then null
    else {
      type = builtins.elemAt tokens 0;
      expectedSuccess = (builtins.elemAt tokens 1) == "okay";
      fileName = "${dir}/${baseName}";
    };

  allLangTests =
    lib.concatMap
      (
        dir:
        lib.pipe
          (builtins.readDir (testRoot + "/${dir}"))
          [
            builtins.attrNames
            (builtins.map (parseTest dir))
            (builtins.filter (t: t != null))
          ]
      ) [ "nix_tests" "nix_tests/notyetpassing" "tvix_tests" "tvix_tests/notyetpassing" ];

  skippedLangTests = {
    # TODO(sterni): set up NIX_PATH in sandbox
    "eval-okay-search-path.nix" = true;
    # Floating point precision differs between tvix and Nix
    "eval-okay-fromjson.nix" = true;
    # C++ Nix can't TCO
    "eval-okay-tail-call-1.nix" = true;
    # Ordering change after 2.3
    "eval-okay-xml.nix" = [ nix_2_3 ];
    # Missing builtins in Nix 2.3
    "eval-okay-ceil.nix" = [ nix_2_3 ];
    "eval-okay-floor-ceil.nix" = [ nix_2_3 ];
    "eval-okay-floor.nix" = [ nix_2_3 ];
    "eval-okay-groupBy.nix" = [ nix_2_3 ];
    "eval-okay-zipAttrsWith.nix" = [ nix_2_3 ];
    "eval-okay-builtins-group-by-propagate-catchable.nix" = [ nix_2_3 ];
    # Comparable lists are not in Nix 2.3
    "eval-okay-sort.nix" = [ nix_2_3 ];
    "eval-okay-compare-lists.nix" = [ nix_2_3 ];
    "eval-okay-value-pointer-compare.nix" = [ nix_2_3 ];
    "eval-okay-builtins-genericClosure-pointer-equality.nix" = [ nix_2_3 ];
    "eval-okay-list-comparison.nix" = [ nix_2_3 ];
    # getAttrPos gains support for functionArgs-returned sets after 2.3
    "eval-okay-getattrpos-functionargs.nix" = [ nix_2_3 ];
    # groupBy appeared (long) after 2.3
    "eval-okay-builtins-groupby-thunk.nix" = [ nix_2_3 ];
    # import is no longer considered a curried primop in Nix > 2.3
    "eval-okay-import-display.nix" = [ nix_2_3 ];
    # Cycle detection and formatting changed sometime after Nix 2.3
    "eval-okay-cycle-display-cpp-nix-2.13.nix" = [ nix_2_3 ];
    # builtins.replaceStrings becomes lazier in Nix 2.16
    "eval-okay-replacestrings.nix" = [ nix_2_3 ];
    # builtins.readFileType is added in Nix 2.15
    "eval-okay-readFileType.nix" = [ nix_2_3 ];
    # builtins.fromTOML gains support for timestamps in Nix 2.16
    "eval-okay-fromTOML-timestamps.nix" = [ nix_2_3 ];
    # identifier formatting changed in Nix 2.17 due to cppnix commit
    # b72bc4a972fe568744d98b89d63adcd504cb586c
    "eval-okay-identifier-formatting.nix" = [ nix_2_3 ];

    # Differing strictness in the function argument for some builtins in Nix 2.18
    # https://github.com/NixOS/nix/issues/9779
    "eval-okay-builtins-map-propagate-catchable.nix" = [ nix_latest_verified ];
    "eval-okay-builtins-gen-list-propagate-catchable.nix" = [ nix_latest_verified ];
    "eval-okay-builtins-replace-strings-propagate-catchable.nix" =
      [ nix_latest_verified ];
    "eval-okay-builtins-map-function-strictness.nix" = [ nix_latest_verified ];
    "eval-okay-builtins-genList-function-strictness.nix" = [ nix_latest_verified ];

    # TODO(sterni): support diffing working directory and home relative paths
    # like C++ Nix test suite (using string replacement).
    "eval-okay-path-antiquotation.nix" = true;

    # The output of dirOf (and maybe other filesystem builtins) have changed in Nix 2.23
    # when they switched to using std::filesystem, https://github.com/NixOS/nix/pull/10658.
    "eval-okay-dirof.nix" = [ nix_latest_verified ];
  };

  runCppNixLangTests = cpp-nix:
    let
      testCommand = { fileName, type, expectedSuccess, ... }:
        let
          testBase = lib.removeSuffix ".nix" fileName;
          expFile =
            let
              possibleFiles =
                builtins.filter
                  (path: builtins.pathExists (testRoot + "/${path}"))
                  (builtins.map
                    (ext: "${testBase}.${ext}")
                    [ "exp" "exp.xml" ]);
            in
            if possibleFiles == [ ] then null else builtins.head possibleFiles;
          outFile = "${testBase}.out";

          # Skip if skippedLangTests prescribes it (possibly just for the current nix)
          # or if we are missing an exp file for an eval-okay test.
          skip =
            let
              doSkip = skippedLangTests.${builtins.baseNameOf fileName} or false;
            in
            if type == "eval" && expectedSuccess && (expFile == null) then true
            else if builtins.isBool doSkip then doSkip
            else builtins.any (drv: cpp-nix == drv) doSkip;

          flagsFile = "${testBase}.flags";

          instantiateFlags =
            lib.escapeShellArgs
              (
                [ "--${type}" fileName ]
                ++ lib.optionals (type == "eval") [ "--strict" ]
                ++ lib.optionals (expFile != null && lib.hasSuffix "xml" expFile)
                  [
                    "--no-location"
                    "--xml"
                  ]
              )
            + lib.optionalString (builtins.pathExists (testRoot + "/${flagsFile}"))
              " $(cat '${flagsFile}')";
        in

        if skip
        then "echo \"SKIP ${type} ${fileName}\"\n"
        else ''
          thisTestPassed=true

          echo "RUN  ${type} ${fileName} ${
            lib.optionalString (!expectedSuccess) "(expecting failure)"
          }"

          if ! expect ${if expectedSuccess then "0" else "1"} \
                 nix-instantiate ${instantiateFlags} \
                 ${if expectedSuccess then "1" else "2"}> \
                 ${if expFile != null then outFile else "/dev/null"};
          then
            echo -n "FAIL"
            thisTestPassed=false
          fi
        '' + lib.optionalString (expFile != null) ''
          if ! diff --color=always -u '${outFile}' '${expFile}'; then
            thisTestPassed=false
          fi
        '' + ''
          if $thisTestPassed; then
            echo -n "PASS"
          else
            echo -n "FAIL"
            passed=false
          fi

          echo " ${type} ${fileName}"

          unset thisTestPassed
        '';
    in

    pkgs.stdenv.mkDerivation {
      name = "cpp-${cpp-nix.name}-run-lang-tests";

      nativeBuildInputs = [ cpp-nix ];

      # Obtain tests via the unpackPhase
      src = testRoot;
      dontConfigure = true;

      # Environment expected by the test suite
      TEST_VAR = "foo";

      buildPhase = ''
        # Make nix-instantiate happy in the sandbox
        export NIX_STORE_DIR="$(realpath "$(mktemp -d store.XXXXXXXXXX)")"
        export NIX_STATE_DIR="$(realpath "$(mktemp -d state.XXXXXXXXXX)")"

        # Helper function to check expected exit code
        expect() {
          local expected res
          expected="$1"
          shift
          set +e
          "$@"
          res="$?"
          set -e
          [[ $res -eq $expected ]]
        }

        # Track test results so far
        passed=true

        source "$testCommandsPath"
      '';

      # Actually runs into the argv limit
      passAsFile = [ "testCommands" ];
      testCommands = lib.concatMapStrings testCommand allLangTests;

      installPhase = ''
        if $passed; then
          touch $out
        else
          echo "Some test(s) failed!"
          exit 1
        fi
      '';
    };

in
depot.nix.readTree.drvTargets {
  "nix-${lib.versions.majorMinor nix_2_3.version}" = runCppNixLangTests nix_2_3;
  "nix-${lib.versions.majorMinor nix_latest_verified.version}" = lib.warnIf (lib.versionOlder nix_latest_verified.version pkgs.nixVersions.latest.version)
    "The latest verified Nix version is out of date, consider updating the value of `nix_latest_verified` and verifying that the tests still pass."
    runCppNixLangTests
    nix_latest_verified;
}