feat(tvix/eval): move away from test_generator to rstest

`test-generator` has not been updated in the past 2 years.
`rstest` has not been updated in the past 5 months.

This is an improvement in the maintenance state… I guess?
We get also new features, it changes the name of the tests with numbers too.

Change-Id: I5376104c7704f525dba7524da78daa09867cc669
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10623
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>

tvix/Cargo.lock

@@ -954,6 +954,12 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
+
 [[package]]
 name = "futures-util"
 version = "0.3.30"
@@ -2216,6 +2222,12 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
 
+[[package]]
+name = "relative-path"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc"
+
 [[package]]
 name = "reqwest"
 version = "0.11.22"
@@ -2295,6 +2307,35 @@ dependencies = [
  "text-size",
 ]
 
+[[package]]
+name = "rstest"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "rstest_macros",
+ "rustc_version",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro2 1.0.75",
+ "quote 1.0.35",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn 2.0.48",
+ "unicode-ident",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -3382,12 +3423,12 @@ dependencies = [
  "regex",
  "rnix",
  "rowan",
+ "rstest",
  "serde",
  "serde_json",
  "smol_str",
  "tabwriter",
  "tempfile",
- "test-generator",
  "test-strategy",
  "toml",
  "tvix-eval-builtin-macros",

tvix/Cargo.nix

@@ -2791,6 +2791,20 @@ rec {
         };
         resolvedDefaultFeatures = [ "alloc" "std" ];
       };
+      "futures-timer" = rec {
+        crateName = "futures-timer";
+        version = "3.0.2";
+        edition = "2018";
+        sha256 = "0b5v7lk9838ix6jdcrainsyrh7xrf24pwm61dp13907qkn806jz6";
+        authors = [
+          "Alex Crichton <alex@alexcrichton.com>"
+        ];
+        features = {
+          "gloo-timers" = [ "dep:gloo-timers" ];
+          "send_wrapper" = [ "dep:send_wrapper" ];
+          "wasm-bindgen" = [ "gloo-timers" "send_wrapper" ];
+        };
+      };
       "futures-util" = rec {
         crateName = "futures-util";
         version = "0.3.30";
@@ -6586,6 +6600,19 @@ rec {
         };
         resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ];
       };
+      "relative-path" = rec {
+        crateName = "relative-path";
+        version = "1.9.2";
+        edition = "2021";
+        sha256 = "1g0gc604zwm73gvpcicn8si25j9j5agqz50r0x1bkmgx6f7mi678";
+        authors = [
+          "John-John Tedro <udoprog@tedro.se>"
+        ];
+        features = {
+          "serde" = [ "dep:serde" ];
+        };
+        resolvedDefaultFeatures = [ "default" ];
+      };
       "reqwest" = rec {
         crateName = "reqwest";
         version = "0.11.22";
@@ -6987,6 +7014,98 @@ rec {
           "serde1" = [ "serde" "text-size/serde" ];
         };
       };
+      "rstest" = rec {
+        crateName = "rstest";
+        version = "0.18.2";
+        edition = "2021";
+        sha256 = "1681ncnlzhc8894idm3pqf40nndn4k4kcp0kpv29n68a7hpspvlp";
+        authors = [
+          "Michele d'Amico <michele.damico@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "futures";
+            packageId = "futures";
+            optional = true;
+          }
+          {
+            name = "futures-timer";
+            packageId = "futures-timer";
+            optional = true;
+          }
+          {
+            name = "rstest_macros";
+            packageId = "rstest_macros";
+            usesDefaultFeatures = false;
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "rustc_version";
+            packageId = "rustc_version";
+          }
+        ];
+        features = {
+          "async-timeout" = [ "dep:futures" "dep:futures-timer" "rstest_macros/async-timeout" ];
+          "default" = [ "async-timeout" ];
+        };
+        resolvedDefaultFeatures = [ "async-timeout" "default" ];
+      };
+      "rstest_macros" = rec {
+        crateName = "rstest_macros";
+        version = "0.18.2";
+        edition = "2021";
+        sha256 = "01g6rg60snmscipc9xiili7nsn0v25sv64713gp99y2jg0jgha6l";
+        procMacro = true;
+        authors = [
+          "Michele d'Amico <michele.damico@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "cfg-if";
+            packageId = "cfg-if";
+          }
+          {
+            name = "glob";
+            packageId = "glob";
+          }
+          {
+            name = "proc-macro2";
+            packageId = "proc-macro2 1.0.75";
+          }
+          {
+            name = "quote";
+            packageId = "quote 1.0.35";
+          }
+          {
+            name = "regex";
+            packageId = "regex";
+          }
+          {
+            name = "relative-path";
+            packageId = "relative-path";
+          }
+          {
+            name = "syn";
+            packageId = "syn 2.0.48";
+            features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ];
+          }
+          {
+            name = "unicode-ident";
+            packageId = "unicode-ident";
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "rustc_version";
+            packageId = "rustc_version";
+          }
+        ];
+        features = {
+          "default" = [ "async-timeout" ];
+        };
+        resolvedDefaultFeatures = [ "async-timeout" ];
+      };
       "rustc-demangle" = rec {
         crateName = "rustc-demangle";
         version = "0.1.23";
@@ -10689,12 +10808,12 @@ rec {
             packageId = "pretty_assertions";
           }
           {
-            name = "tempfile";
+            name = "rstest";
-            packageId = "tempfile";
+            packageId = "rstest";
           }
           {
-            name = "test-generator";
+            name = "tempfile";
-            packageId = "test-generator";
+            packageId = "tempfile";
           }
         ];
         features = {

tvix/eval/Cargo.toml

@@ -35,14 +35,9 @@ xml-rs = "0.8.4"
 criterion = "0.5"
 itertools = "0.12.0"
 pretty_assertions = "1.2.1"
+rstest = "0.18.2"
 tempfile = "3.3.0"
 
-[dev-dependencies.test-generator]
-# This fork of test-generator adds support for cargo workspaces, see
-# also https://github.com/frehberg/test-generator/pull/14
-git = "https://github.com/JamesGuthrie/test-generator.git"
-rev = "82e799979980962aec1aa324ec6e0e4cad781f41"
-
 [features]
 default = ["impure", "arbitrary", "nix_tests"]
 

tvix/eval/src/tests/mod.rs

@@ -1,7 +1,8 @@
 use crate::value::Value;
 use builtin_macros::builtins;
 use pretty_assertions::assert_eq;
-use test_generator::test_resources;
+use rstest::rstest;
+use std::path::PathBuf;
 
 /// Module for one-off tests which do not follow the rest of the
 /// test layout.
@@ -38,14 +39,17 @@ mod mock_builtins {
     }
 }
 
-fn eval_test(code_path: &str, expect_success: bool) {
+fn eval_test(code_path: PathBuf, expect_success: bool) {
-    let base = code_path
+    eprintln!("path: {}", code_path.display());
-        .strip_suffix("nix")
+    assert_eq!(
-        .expect("test files always end in .nix");
+        code_path.extension().unwrap(),
-    let exp_path = format!("{}exp", base);
+        "nix",
-    let exp_xml_path = std::path::PathBuf::from(format!("{}exp.xml", base));
+        "test files always end in .nix"
+    );
+    let exp_path = code_path.with_extension("exp");
+    let exp_xml_path = code_path.with_extension("exp.xml");
 
-    let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
+    let code = std::fs::read_to_string(&code_path).expect("should be able to read test code");
 
     if exp_xml_path.exists() {
         // We can't test them at the moment because we don't have XML output yet.
@@ -57,14 +61,15 @@ fn eval_test(code_path: &str, expect_success: bool) {
     eval.strict = true;
     eval.builtins.extend(mock_builtins::builtins());
 
-    let result = eval.evaluate(code, Some(code_path.into()));
+    let result = eval.evaluate(code, Some(code_path.clone()));
     let failed = match result.value {
         Some(Value::Catchable(_)) => true,
         _ => !result.errors.is_empty(),
     };
     if expect_success && failed {
         panic!(
-            "{code_path}: evaluation of eval-okay test should succeed, but failed with {:?}",
+            "{}: evaluation of eval-okay test should succeed, but failed with {:?}",
+            code_path.display(),
             result.errors,
         );
     }
@@ -81,20 +86,26 @@ fn eval_test(code_path: &str, expect_success: bool) {
             assert_eq!(
                 result_str,
                 exp.trim(),
-                "{code_path}: result value representation (left) must match expectation (right)"
+                "{}: result value representation (left) must match expectation (right)",
+                code_path.display()
             );
         } else {
             assert_ne!(
                 result_str,
                 exp.trim(),
-                "{code_path}: test passed unexpectedly!  consider moving it out of notyetpassing"
+                "{}: test passed unexpectedly!  consider moving it out of notyetpassing",
+                code_path.display()
             );
         }
     } else if expect_success {
-        panic!("{code_path}: should be able to read test expectation");
+        panic!(
+            "{}: should be able to read test expectation",
+            code_path.display()
+        );
     } else {
         panic!(
-            "{code_path}: test should have failed, but succeeded with output {}",
+            "{}: test should have failed, but succeeded with output {}",
+            code_path.display(),
             result_str
         );
     }
@@ -102,8 +113,8 @@ fn eval_test(code_path: &str, expect_success: bool) {
 
 // identity-* tests contain Nix code snippets which should evaluate to
 // themselves exactly (i.e. literals).
-#[test_resources("src/tests/tvix_tests/identity-*.nix")]
+#[rstest]
-fn identity(code_path: &str) {
+fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) {
     let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
 
     let eval = crate::Evaluation {
@@ -133,15 +144,15 @@ fn identity(code_path: &str) {
 //
 // These evaluations are always supposed to succeed, i.e. all snippets
 // are guaranteed to be valid Nix code.
-#[test_resources("src/tests/tvix_tests/eval-okay-*.nix")]
+#[rstest]
-fn eval_okay(code_path: &str) {
+fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) {
     eval_test(code_path, true)
 }
 
 // eval-okay-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
-#[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
+#[rstest]
-fn nix_eval_okay(code_path: &str) {
+fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) {
     eval_test(code_path, true)
 }
 
@@ -163,27 +174,31 @@ fn nix_eval_okay(code_path: &str) {
 //
 //   https://github.com/frehberg/test-generator/pull/10
 //   https://github.com/frehberg/test-generator/pull/8
-#[test_resources("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")]
+#[rstest]
-fn nix_eval_okay_currently_failing(code_path: &str) {
+fn nix_eval_okay_currently_failing(
+    #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
+) {
     eval_test(code_path, false)
 }
 
-#[test_resources("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")]
+#[rstest]
-fn eval_okay_currently_failing(code_path: &str) {
+fn eval_okay_currently_failing(
+    #[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
+) {
     eval_test(code_path, false)
 }
 
 // eval-fail-* tests contain a snippet of Nix code, which is
 // expected to fail evaluation.  The exact type of failure
 // (assertion, parse error, etc) is not currently checked.
-#[test_resources("src/tests/tvix_tests/eval-fail-*.nix")]
+#[rstest]
-fn eval_fail(code_path: &str) {
+fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) {
     eval_test(code_path, false)
 }
 
 // eval-fail-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
-#[test_resources("src/tests/nix_tests/eval-fail-*.nix")]
+#[rstest]
-fn nix_eval_fail(code_path: &str) {
+fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) {
     eval_test(code_path, false)
 }