maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/store/nar): pass in &proto::node::Node

Browse source 
Passing in a &proto::node::Node into all this allows us consumers to
keep ownership of the proto::node::Node.

Change-Id: I44882a86c46826b06a8a8a0b24c18adfc7052662
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8316
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2023-03-16 23:54:29 +01:00 • committed by clbot
parent 985f842e32
commit b025ebb2a1
5 changed files with 29 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tvix/store/src/nar/mod.rs
View file

@ -31,6 +31,6 @@ pub enum RenderError {
pub trait NARCalculationService { pub trait NARCalculationService {
fn calculate_nar( fn calculate_nar(
&self, &self,
root_node: proto::node::Node, root_node: &proto::node::Node,
) -> Result<proto::CalculateNarResponse, RenderError>; ) -> Result<proto::CalculateNarResponse, RenderError>;
} }

2
tvix/store/src/nar/non_caching_calculation_service.rs
View file

@ -35,7 +35,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARCalcula
{ {
fn calculate_nar( fn calculate_nar(
&self, &self,
root_node: proto::node::Node, root_node: &proto::node::Node,
) -> Result<proto::CalculateNarResponse, RenderError> { ) -> Result<proto::CalculateNarResponse, RenderError> {
let h = Sha256::new(); let h = Sha256::new();
let mut cw = CountWrite::from(h); let mut cw = CountWrite::from(h);

34
tvix/store/src/nar/renderer.rs
View file

@ -36,7 +36,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
pub fn write_nar<W: std::io::Write>( pub fn write_nar<W: std::io::Write>(
&self, &self,
w: &mut W, w: &mut W,
proto_root_node: proto::node::Node, proto_root_node: &proto::node::Node,
) -> Result<(), RenderError> { ) -> Result<(), RenderError> {
// Initialize NAR writer // Initialize NAR writer
let nar_root_node = nar::writer::open(w).map_err(RenderError::NARWriterError)?; let nar_root_node = nar::writer::open(w).map_err(RenderError::NARWriterError)?;
@ -49,7 +49,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
fn walk_node( fn walk_node(
&self, &self,
nar_node: nar::writer::Node, nar_node: nar::writer::Node,
proto_node: proto::node::Node, proto_node: &proto::node::Node,
) -> Result<(), RenderError> { ) -> Result<(), RenderError> {
match proto_node { match proto_node {
proto::node::Node::Symlink(proto_symlink_node) => { proto::node::Node::Symlink(proto_symlink_node) => {
@ -59,7 +59,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
} }
proto::node::Node::File(proto_file_node) => { proto::node::Node::File(proto_file_node) => {
// get the digest we're referring to // get the digest we're referring to
let digest = proto_file_node.digest; let digest = &proto_file_node.digest;
// query blob_service for blob_meta // query blob_service for blob_meta
let resp = self let resp = self
.blob_service .blob_service
@ -73,15 +73,18 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
match resp { match resp {
// if it's None, that's an error! // if it's None, that's an error!
None => { None => {
return Err(RenderError::BlobNotFound(digest, proto_file_node.name)); return Err(RenderError::BlobNotFound(
digest.to_vec(),
proto_file_node.name.to_owned(),
));
} }
Some(blob_meta) => { Some(blob_meta) => {
// make sure the blob_meta size matches what we expect from proto_file_node // make sure the blob_meta size matches what we expect from proto_file_node
let blob_meta_size = blob_meta.chunks.iter().fold(0, |acc, e| acc + e.size); let blob_meta_size = blob_meta.chunks.iter().fold(0, |acc, e| acc + e.size);
if blob_meta_size != proto_file_node.size { if blob_meta_size != proto_file_node.size {
return Err(RenderError::UnexpectedBlobMeta( return Err(RenderError::UnexpectedBlobMeta(
digest, digest.to_vec(),
proto_file_node.name, proto_file_node.name.to_owned(),
proto_file_node.size, proto_file_node.size,
blob_meta_size, blob_meta_size,
)); ));
@ -103,11 +106,16 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
} }
proto::node::Node::Directory(proto_directory_node) => { proto::node::Node::Directory(proto_directory_node) => {
// get the digest we're referring to // get the digest we're referring to
let digest: [u8; 32] = proto_directory_node.digest.try_into().map_err(|_e| { let digest: [u8; 32] =
RenderError::StoreError(crate::Error::StorageError( proto_directory_node
"invalid digest len in directory node".to_string(), .digest
)) .to_owned()
})?; .try_into()
.map_err(|_e| {
RenderError::StoreError(crate::Error::StorageError(
"invalid digest len in directory node".to_string(),
))
})?;
// look it up with the directory service // look it up with the directory service
let resp = self let resp = self
@ -120,7 +128,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
None => { None => {
return Err(RenderError::DirectoryNotFound( return Err(RenderError::DirectoryNotFound(
digest.to_vec(), digest.to_vec(),
proto_directory_node.name, proto_directory_node.name.to_owned(),
)) ))
} }
Some(proto_directory) => { Some(proto_directory) => {
@ -134,7 +142,7 @@ impl<BS: BlobService, CS: ChunkService + Clone, DS: DirectoryService> NARRendere
let child_node = nar_node_directory let child_node = nar_node_directory
.entry(proto_node.get_name()) .entry(proto_node.get_name())
.map_err(RenderError::NARWriterError)?; .map_err(RenderError::NARWriterError)?;
self.walk_node(child_node, proto_node)?; self.walk_node(child_node, &proto_node)?;
} }
// close the directory // close the directory

2
tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs
View file

@ -64,7 +64,7 @@ impl<
) -> Result<Response<proto::CalculateNarResponse>> { ) -> Result<Response<proto::CalculateNarResponse>> {
match request.into_inner().node { match request.into_inner().node {
None => Err(Status::invalid_argument("no root node sent")), None => Err(Status::invalid_argument("no root node sent")),
Some(root_node) => match self.nar_calculation_service.calculate_nar(root_node) { Some(root_node) => match self.nar_calculation_service.calculate_nar(&root_node) {
Ok(resp) => Ok(Response::new(resp)), Ok(resp) => Ok(Response::new(resp)),
Err(e) => Err(e.into()), Err(e) => Err(e.into()),
}, },

10
tvix/store/src/tests/nar_renderer.rs
View file

@ -23,7 +23,7 @@ fn single_symlink() {
renderer renderer
.write_nar( .write_nar(
&mut buf, &mut buf,
crate::proto::node::Node::Symlink(SymlinkNode { &crate::proto::node::Node::Symlink(SymlinkNode {
name: "doesntmatter".to_string(), name: "doesntmatter".to_string(),
target: "/nix/store/somewhereelse".to_string(), target: "/nix/store/somewhereelse".to_string(),
}), }),
@ -47,7 +47,7 @@ fn single_file_missing_blob() {
let e = renderer let e = renderer
.write_nar( .write_nar(
&mut buf, &mut buf,
crate::proto::node::Node::File(FileNode { &crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(), name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(), digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32, size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -94,7 +94,7 @@ fn single_file_wrong_blob_size() {
let e = renderer let e = renderer
.write_nar( .write_nar(
&mut buf, &mut buf,
crate::proto::node::Node::File(FileNode { &crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(), name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(), digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: 42, // <- note the wrong size here! size: 42, // <- note the wrong size here!
@ -151,7 +151,7 @@ fn single_file() {
renderer renderer
.write_nar( .write_nar(
&mut buf, &mut buf,
crate::proto::node::Node::File(FileNode { &crate::proto::node::Node::File(FileNode {
name: "doesntmatter".to_string(), name: "doesntmatter".to_string(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(), digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32, size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -196,7 +196,7 @@ fn test_complicated() {
renderer renderer
.write_nar( .write_nar(
&mut buf, &mut buf,
crate::proto::node::Node::Directory(DirectoryNode { &crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".to_string(), name: "doesntmatter".to_string(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(), digest: DIRECTORY_COMPLICATED.digest().to_vec(),
size: DIRECTORY_COMPLICATED.size(), size: DIRECTORY_COMPLICATED.size(),