maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/src/nar): drop NARCalculationService

Browse source 
There's only one way to calculate NAR files, by walking through them.

Things like caching such replies should be done closer to where we use
these, composing NARCalculationService doesn't actually give us much.

Instead, expose two functions, `nar::calculate_size_and_sha256` and
`nar::writer_nar`, the latter writing NAR to a writer, the former using
write_nar to only keeping the NAR size and digest.

Change-Id: Ie5d2cfea35470fdbb5cbf9da1136b0cdf0250266
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8723
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2023-06-08 23:00:37 +03:00 • committed by clbot
parent 27ff98000b
commit 8d05c0ceaa
11 changed files with 258 additions and 346 deletions
Download patch file Download diff file Expand all files Collapse all files

164
tvix/store/src/tests/nar_renderer.rs
View file

@ -1,28 +1,29 @@
use crate::directoryservice::DirectoryService;
use crate::nar::NARRenderer;
use crate::nar::calculate_size_and_sha256;
use crate::nar::writer_nar;
use crate::proto::DirectoryNode;
use crate::proto::FileNode;
use crate::proto::SymlinkNode;
use crate::tests::fixtures::*;
use crate::tests::utils::*;
use sha2::{Digest, Sha256};
use std::io;
#[test]
fn single_symlink() {
let renderer = NARRenderer::new(gen_blob_service(), gen_directory_service());
// don't put anything in the stores, as we don't actually do any requests.
let mut buf: Vec<u8> = vec![];
renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::Symlink(SymlinkNode {
name: "doesntmatter".to_string(),
target: "/nix/store/somewhereelse".to_string(),
}),
)
.expect("must succeed");
writer_nar(
&mut buf,
&crate::proto::node::Node::Symlink(SymlinkNode {
name: "doesntmatter".to_string(),
target: "/nix/store/somewhereelse".to_string(),
}),
// don't put anything in the stores, as we don't actually do any requests.
&gen_blob_service(),
gen_directory_service(),
)
.expect("must succeed");
assert_eq!(buf, NAR_CONTENTS_SYMLINK.to_vec());
}
@ -30,20 +31,21 @@ fn single_symlink() {
/// Make sure the NARRenderer fails if a referred blob doesn't exist.
#[test]
fn single_file_missing_blob() {
let renderer = NARRenderer::new(gen_blob_service(), gen_directory_service());
let mut buf: Vec<u8> = vec![];
let e = renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
)
.expect_err("must fail");
let e = writer_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
// the blobservice is empty intentionally, to provoke the error.
&gen_blob_service(),
gen_directory_service(),
)
.expect_err("must fail");
match e {
crate::nar::RenderError::NARWriterError(e) => {
@ -68,22 +70,22 @@ fn single_file_wrong_blob_size() {
.unwrap();
assert_eq!(HELLOWORLD_BLOB_DIGEST.clone(), writer.close().unwrap());
let renderer = NARRenderer::new(blob_service, gen_directory_service());
// Test with a root FileNode of a too big size
{
let mut buf: Vec<u8> = vec![];
let e = renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: 42, // <- note the wrong size here!
executable: false,
}),
)
.expect_err("must fail");
let e = writer_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: 42, // <- note the wrong size here!
executable: false,
}),
&blob_service,
gen_directory_service(),
)
.expect_err("must fail");
match e {
crate::nar::RenderError::NARWriterError(e) => {
@ -96,17 +98,19 @@ fn single_file_wrong_blob_size() {
// Test with a root FileNode of a too small size
{
let mut buf: Vec<u8> = vec![];
let e = renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: 2, // <- note the wrong size here!
executable: false,
}),
)
.expect_err("must fail");
let e = writer_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: 2, // <- note the wrong size here!
executable: false,
}),
&blob_service,
gen_directory_service(),
)
.expect_err("must fail");
match e {
crate::nar::RenderError::NARWriterError(e) => {
@ -130,20 +134,20 @@ fn single_file() {
.unwrap();
assert_eq!(HELLOWORLD_BLOB_DIGEST.clone(), writer.close().unwrap());
let renderer = NARRenderer::new(blob_service, gen_directory_service());
let mut buf: Vec<u8> = vec![];
renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
)
.expect("must succeed");
writer_nar(
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
&blob_service,
gen_directory_service(),
)
.expect("must succeed");
assert_eq!(buf, NAR_CONTENTS_HELLOWORLD.to_vec());
}
@ -168,19 +172,35 @@ fn test_complicated() {
.put(DIRECTORY_COMPLICATED.clone())
.unwrap();
let renderer = NARRenderer::new(blob_service, directory_service);
let mut buf: Vec<u8> = vec![];
renderer
.write_nar(
&mut buf,
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".to_string(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
size: DIRECTORY_COMPLICATED.size(),
}),
)
.expect("must succeed");
writer_nar(
&mut buf,
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".to_string(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
size: DIRECTORY_COMPLICATED.size(),
}),
&blob_service,
directory_service.clone(),
)
.expect("must succeed");
assert_eq!(buf, NAR_CONTENTS_COMPLICATED.to_vec());
// ensure calculate_nar does return the correct sha256 digest and sum.
let (nar_size, nar_digest) = calculate_size_and_sha256(
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".to_string(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
size: DIRECTORY_COMPLICATED.size(),
}),
&blob_service,
directory_service,
)
.expect("must succeed");
assert_eq!(NAR_CONTENTS_COMPLICATED.len() as u64, nar_size);
let d = Sha256::digest(NAR_CONTENTS_COMPLICATED.clone());
assert_eq!(d.as_slice(), nar_digest);
}