maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/eval): enable the XFAIL tests

Browse source 
This commit adds support for running the "expected failure" tests in
both the nix and tvix test suites.

I have disabled the eval-fail-blackhole.nix test because it gets
stuck running forever.

Signed-off-by: Adam Joseph <adam@westernsemico.com>
Change-Id: Iba75ce6c8f2becab3c834fcfdd9f4fdc5a4bdb9f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6990
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: grfn <grfn@gws.fyi>
This commit is contained in:
Adam Joseph 2022-10-12 20:07:18 -07:00
parent 706186eb5d
commit 01bc04b1d2
2 changed files with 45 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

59
tvix/eval/src/tests/mod.rs
View file

@ -4,24 +4,40 @@ use pretty_assertions::assert_eq;
use test_generator::test_resources;
fn eval_okay_test(code_path: &str) {
fn eval_test(code_path: &str, expect_success: bool) {
let base = code_path
.strip_suffix("nix")
.expect("test files always end in .nix");
let exp_path = format!("{}exp", base);
let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
let exp = std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
let result = interpret(&code, Some(code_path.into()), Options::test_options())
.expect("evaluation of eval-okay test should succeed");
let result_str = format!("{}", result);
assert_eq!(
result_str,
exp.trim(),
"result value representation (left) must match expectation (right)"
);
match interpret(&code, Some(code_path.into()), Options::test_options()) {
Ok(result) => {
if !expect_success {
panic!(
"test should have failed, but succeeded with output {}",
result
);
}
let result_str = format!("{}", result);
let exp =
std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
assert_eq!(
result_str,
exp.trim(),
"result value representation (left) must match expectation (right)"
);
}
Err(e) => {
if expect_success {
panic!(
"evaluation of eval-okay test should succeed, but failed with {:?}",
e
);
}
}
}
}
// identity-* tests contain Nix code snippets which should evaluate to
@ -48,12 +64,27 @@ fn identity(code_path: &str) {
// are guaranteed to be valid Nix code.
#[test_resources("src/tests/tvix_tests/eval-okay-*.nix")]
fn eval_okay(code_path: &str) {
eval_okay_test(code_path)
eval_test(code_path, true)
}
// eval-okay-* tests from the original Nix test suite.
// eval-fail-* tests from the original Nix test suite.
#[cfg(feature = "nix_tests")]
#[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
fn nix_eval_okay(code_path: &str) {
eval_okay_test(code_path)
eval_test(code_path, true)
}
// eval-fail-* tests contain a snippet of Nix code, which is
// expected to fail evaluation. The exact type of failure
// (assertion, parse error, etc) is not currently checked.
#[test_resources("src/tests/tvix_tests/eval-fail-*.nix")]
fn eval_fail(code_path: &str) {
eval_test(code_path, false)
}
// eval-fail-* tests from the original Nix test suite.
#[cfg(feature = "nix_tests")]
#[test_resources("src/tests/nix_tests/eval-fail-*.nix")]
fn nix_eval_fail(code_path: &str) {
eval_test(code_path, false)
}