maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/castore): use Directory struct separate from proto one

Browse source 
This uses our own data type to deal with Directories in the castore model.

It makes some undesired states unrepresentable, removing the need for conversions and checking in various places:

 - In the protobuf, blake3 digests could have a wrong length, as proto doesn't know fixed-size fields. We now use `B3Digest`, which makes cloning cheaper, and removes the need to do size-checking everywhere.
 - In the protobuf, we had three different lists for `files`, `symlinks` and `directories`. This was mostly a protobuf size optimization, but made interacting with them a bit awkward. This has now been replaced with a list of enums, and convenience iterators to get various nodes, and add new ones.

Change-Id: I7b92691bb06d77ff3f58a5ccea94a22c16f84f04
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12057
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
Yureka 2024-07-29 14:34:50 +02:00 • committed by yuka
parent 5d3f3158d6
commit 3ca0b53840
53 changed files with 1429 additions and 1377 deletions
Download patch file Download diff file Expand all files Collapse all files

29
tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs
View file

@ -74,24 +74,19 @@ where
&self,
request: Request<castorepb::Node>,
) -> Result<Response<proto::CalculateNarResponse>> {
match request.into_inner().node {
None => Err(Status::invalid_argument("no root node sent")),
Some(root_node) => {
if let Err(e) = root_node.validate() {
warn!(err = %e, "invalid root node");
Err(Status::invalid_argument("invalid root node"))?
}
let root_node = (&request.into_inner()).try_into().map_err(|e| {
warn!(err = %e, "invalid root node");
Status::invalid_argument("invalid root node")
})?;
match self.nar_calculation_service.calculate_nar(&root_node).await {
Ok((nar_size, nar_sha256)) => Ok(Response::new(proto::CalculateNarResponse {
nar_size,
nar_sha256: nar_sha256.to_vec().into(),
})),
Err(e) => {
warn!(err = %e, "error during NAR calculation");
Err(e.into())
}
}
match self.nar_calculation_service.calculate_nar(&root_node).await {
Ok((nar_size, nar_sha256)) => Ok(Response::new(proto::CalculateNarResponse {
nar_size,
nar_sha256: nar_sha256.to_vec().into(),
})),
Err(e) => {
warn!(err = %e, "error during NAR calculation");
Err(e.into())
}
}
}

16
tvix/store/src/proto/mod.rs
View file

@ -9,7 +9,8 @@ use nix_compat::{
store_path::{self, StorePathRef},
};
use thiserror::Error;
use tvix_castore::proto::{self as castorepb, NamedNode, ValidateNodeError};
use tvix_castore::directoryservice::NamedNode;
use tvix_castore::ValidateNodeError;
mod grpc_pathinfoservice_wrapper;
@ -87,7 +88,7 @@ impl PathInfo {
/// validate performs some checks on the PathInfo struct,
/// Returning either a [store_path::StorePath] of the root node, or a
/// [ValidatePathInfoError].
pub fn validate(&self) -> Result<store_path::StorePathRef<'_>, ValidatePathInfoError> {
pub fn validate(&self) -> Result<store_path::StorePath, ValidatePathInfoError> {
// ensure the references have the right number of bytes.
for (i, reference) in self.references.iter().enumerate() {
if reference.len() != store_path::DIGEST_SIZE {
@ -158,14 +159,15 @@ impl PathInfo {
// Ensure there is a (root) node present, and it properly parses to a [store_path::StorePath].
let root_nix_path = match &self.node {
None | Some(castorepb::Node { node: None }) => {
Err(ValidatePathInfoError::NoNodePresent)?
}
Some(castorepb::Node { node: Some(node) }) => {
node.validate()
None => Err(ValidatePathInfoError::NoNodePresent)?,
Some(node) => {
// TODO save result somewhere
let node: tvix_castore::directoryservice::Node = node
.try_into()
.map_err(ValidatePathInfoError::InvalidRootNode)?;
// parse the name of the node itself and return
parse_node_name_root(node.get_name(), ValidatePathInfoError::InvalidNodeName)?
.to_owned()
}
};

30
tvix/store/src/proto/tests/pathinfo.rs
View file

@ -3,17 +3,18 @@ use crate::tests::fixtures::*;
use bytes::Bytes;
use data_encoding::BASE64;
use nix_compat::nixbase32;
use nix_compat::store_path::{self, StorePathRef};
use nix_compat::store_path::{self, StorePath, StorePathRef};
use rstest::rstest;
use tvix_castore::proto as castorepb;
use tvix_castore::ValidateNodeError;
#[rstest]
#[case::no_node(None, Err(ValidatePathInfoError::NoNodePresent))]
#[case::no_node_2(Some(castorepb::Node { node: None}), Err(ValidatePathInfoError::NoNodePresent))]
#[case::no_node_2(Some(castorepb::Node { node: None}), Err(ValidatePathInfoError::InvalidRootNode(ValidateNodeError::NoNodeSet)))]
fn validate_pathinfo(
#[case] node: Option<castorepb::Node>,
#[case] exp_result: Result<StorePathRef, ValidatePathInfoError>,
#[case] exp_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
@ -22,9 +23,6 @@ fn validate_pathinfo(
};
assert_eq!(exp_result, p.validate());
let err = p.validate().expect_err("validation should fail");
assert!(matches!(err, ValidatePathInfoError::NoNodePresent));
}
#[rstest]
@ -32,12 +30,12 @@ fn validate_pathinfo(
name: DUMMY_PATH.into(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
}, Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap()))]
}, Ok(StorePath::from_bytes(DUMMY_PATH.as_bytes()).unwrap()))]
#[case::invalid_digest_length(castorepb::DirectoryNode {
name: DUMMY_PATH.into(),
digest: Bytes::new(),
size: 0,
}, Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0))))]
}, Err(ValidatePathInfoError::InvalidRootNode(tvix_castore::ValidateNodeError::InvalidDigestLen(0))))]
#[case::invalid_node_name_no_storepath(castorepb::DirectoryNode {
name: "invalid".into(),
digest: DUMMY_DIGEST.clone().into(),
@ -48,7 +46,7 @@ fn validate_pathinfo(
)))]
fn validate_directory(
#[case] directory_node: castorepb::DirectoryNode,
#[case] exp_result: Result<StorePathRef, ValidatePathInfoError>,
#[case] exp_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
@ -68,7 +66,7 @@ fn validate_directory(
size: 0,
executable: false,
},
Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap())
Ok(StorePath::from_bytes(DUMMY_PATH.as_bytes()).unwrap())
)]
#[case::invalid_digest_len(
castorepb::FileNode {
@ -76,7 +74,7 @@ fn validate_directory(
digest: Bytes::new(),
..Default::default()
},
Err(ValidatePathInfoError::InvalidRootNode(castorepb::ValidateNodeError::InvalidDigestLen(0)))
Err(ValidatePathInfoError::InvalidRootNode(tvix_castore::ValidateNodeError::InvalidDigestLen(0)))
)]
#[case::invalid_node_name(
castorepb::FileNode {
@ -91,7 +89,7 @@ fn validate_directory(
)]
fn validate_file(
#[case] file_node: castorepb::FileNode,
#[case] exp_result: Result<StorePathRef, ValidatePathInfoError>,
#[case] exp_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
@ -109,7 +107,7 @@ fn validate_file(
name: DUMMY_PATH.into(),
target: "foo".into(),
},
Ok(StorePathRef::from_bytes(DUMMY_PATH.as_bytes()).unwrap())
Ok(StorePath::from_bytes(DUMMY_PATH.as_bytes()).unwrap())
)]
#[case::invalid_node_name(
castorepb::SymlinkNode {
@ -123,7 +121,7 @@ fn validate_file(
)]
fn validate_symlink(
#[case] symlink_node: castorepb::SymlinkNode,
#[case] exp_result: Result<StorePathRef, ValidatePathInfoError>,
#[case] exp_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
@ -233,7 +231,7 @@ fn validate_symlink_empty_target_invalid() {
target: "".into(),
});
node.validate().expect_err("must fail validation");
tvix_castore::directoryservice::Node::try_from(&node).expect_err("must fail validation");
}
/// Create a node with a symlink target including null bytes, and ensure it
@ -245,7 +243,7 @@ fn validate_symlink_target_null_byte_invalid() {
target: "foo\0".into(),
});
node.validate().expect_err("must fail validation");
tvix_castore::directoryservice::Node::try_from(&node).expect_err("must fail validation");
}
/// Create a PathInfo with a correct deriver field and ensure it succeeds.