maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/store/proto): use Bytes instead of Vec<u8>

Browse source 
Makes use of https://github.com/tokio-rs/prost/pull/341, which makes our
bytes field cheaper to clone.

It's a bit annoying to configure due to
https://github.com/hyperium/tonic/issues/908, but the workaround does
get the job done.

Change-Id: I25714600b041bb5432d3adf5859b151e72b12778
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8975
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2023-07-19 18:52:50 +03:00 • committed by clbot
parent 7971d7d9ff
commit 432222f098
34 changed files with 216 additions and 164 deletions
Download patch file Download diff file Expand all files Collapse all files

53
tvix/store/src/tests/fixtures.rs
View file

@ -8,13 +8,16 @@ pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!";
pub const EMPTY_BLOB_CONTENTS: &[u8] = b"";
lazy_static! {
pub static ref DUMMY_DIGEST: Vec<u8> = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
pub static ref DUMMY_DATA_1: Vec<u8> = vec![0x01, 0x02, 0x03];
pub static ref DUMMY_DATA_2: Vec<u8> = vec![0x04, 0x05];
pub static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
u.into()
};
pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into();
pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into();
pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest =
blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into();
@ -22,19 +25,19 @@ lazy_static! {
blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into();
// 2 bytes
pub static ref BLOB_A: Vec<u8> = vec![0x00, 0x01];
pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into();
pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into();
// 1MB
pub static ref BLOB_B: Vec<u8> = (0..255).collect::<Vec<u8>>().repeat(4 * 1024);
pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into();
pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into();
// Directories
pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory {
directories: vec![],
files: vec![FileNode {
name: b".keep".to_vec(),
digest: EMPTY_BLOB_DIGEST.to_vec(),
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
@ -42,26 +45,26 @@ lazy_static! {
};
pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory {
directories: vec![DirectoryNode {
name: b"keep".to_vec(),
digest: DIRECTORY_WITH_KEEP.digest().to_vec(),
name: b"keep".to_vec().into(),
digest: DIRECTORY_WITH_KEEP.digest().into(),
size: DIRECTORY_WITH_KEEP.size(),
}],
files: vec![FileNode {
name: b".keep".to_vec(),
digest: EMPTY_BLOB_DIGEST.to_vec(),
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![SymlinkNode {
name: b"aa".to_vec(),
target: b"/nix/store/somewhereelse".to_vec(),
name: b"aa".to_vec().into(),
target: b"/nix/store/somewhereelse".to_vec().into(),
}],
};
pub static ref DIRECTORY_A: Directory = Directory::default();
pub static ref DIRECTORY_B: Directory = Directory {
directories: vec![DirectoryNode {
name: b"a".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}],
..Default::default()
@ -69,13 +72,13 @@ lazy_static! {
pub static ref DIRECTORY_C: Directory = Directory {
directories: vec![
DirectoryNode {
name: b"a".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
},
DirectoryNode {
name: b"a'".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a'".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}
],
@ -83,10 +86,10 @@ lazy_static! {
};
// output hash
pub static ref DUMMY_OUTPUT_HASH: Vec<u8> = vec![
pub static ref DUMMY_OUTPUT_HASH: bytes::Bytes = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00
];
].into();
/// The NAR representation of a symlink pointing to `/nix/store/somewhereelse`
pub static ref NAR_CONTENTS_SYMLINK: Vec<u8> = vec![

12
tvix/store/src/tests/import.rs
View file

@ -54,7 +54,7 @@ fn single_file() {
assert_eq!(
crate::proto::node::Node::File(proto::FileNode {
name: "root".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -92,8 +92,14 @@ fn complicated() {
// ensure root_node matched expectations
assert_eq!(
crate::proto::node::Node::Directory(proto::DirectoryNode {
name: tmpdir.path().file_name().unwrap().as_bytes().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
name: tmpdir
.path()
.file_name()
.unwrap()
.as_bytes()
.to_owned()
.into(),
digest: DIRECTORY_COMPLICATED.digest().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
root_node,

16
tvix/store/src/tests/nar_renderer.rs
View file

@ -36,7 +36,7 @@ fn single_file_missing_blob() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -77,7 +77,7 @@ fn single_file_wrong_blob_size() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 42, // <- note the wrong size here!
executable: false,
}),
@ -102,7 +102,7 @@ fn single_file_wrong_blob_size() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 2, // <- note the wrong size here!
executable: false,
}),
@ -127,7 +127,7 @@ fn single_file() {
// insert blob into the store
let mut writer = blob_service.open_write();
io::copy(
&mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.to_vec()),
&mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.clone()),
&mut writer,
)
.unwrap();
@ -139,7 +139,7 @@ fn single_file() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -160,7 +160,7 @@ fn test_complicated() {
// insert blob into the store
let mut writer = blob_service.open_write();
io::copy(
&mut io::Cursor::new(EMPTY_BLOB_CONTENTS.to_vec()),
&mut io::Cursor::new(EMPTY_BLOB_CONTENTS.clone()),
&mut writer,
)
.unwrap();
@ -177,7 +177,7 @@ fn test_complicated() {
&mut buf,
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().clone().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
blob_service.clone(),
@ -191,7 +191,7 @@ fn test_complicated() {
let (nar_size, nar_digest) = calculate_size_and_sha256(
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().clone().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
blob_service,