feat(tvix/store/protos): add Deriver field to PathInfo

This uses the newly introduced StorePath message type to add a Deriver
field to the PathInfo message.

Support for validation is added to both the golang and rust
implementation. This includes extending unit tests.

Change-Id: Ifc3eb3263fa25b9eec260db354cd74234c40af7e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9647
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI

tvix/cli/src/tvix_store_io.rs

@@ -346,6 +346,7 @@ async fn import_path_with_pathinfo(
             // TODO: narinfo for talosctl.src contains `CA: fixed:r:sha256:1x13j5hy75221bf6kz7cpgld9vgic6bqx07w5xjs4pxnksj6lxb6`
             // do we need this anywhere?
         }),
+        deriver: None,
     };
 
     // put into [PathInfoService], and return the [PathInfo] that we get

tvix/nix-compat/src/store_path/mod.rs

@@ -116,6 +116,14 @@ impl StorePath {
         }
     }
 
+    /// Construct a [StorePath] from a name and digest.
+    pub fn from_name_and_digest(name: String, digest: &[u8]) -> Result<StorePath, Error> {
+        Ok(Self {
+            name: validate_name(name.as_bytes())?,
+            digest: digest.try_into().map_err(|_| Error::InvalidLength())?,
+        })
+    }
+
     /// Decompose a string into a [StorePath] and a [PathBuf] containing the
     /// rest of the path, or an error.
     #[cfg(target_family = "unix")]
@@ -179,6 +187,14 @@ pub(crate) fn validate_name(s: &[u8]) -> Result<String, Error> {
     Ok(String::from_utf8(s.to_vec()).unwrap())
 }
 
+/// Ensures the StorePath fulfils the requirements for store paths.
+/// Useful when populating the struct manually instead of parsing.
+pub fn validate(s: &StorePath) -> Result<(), Error> {
+    validate_name(s.name.as_bytes())?;
+
+    Ok(())
+}
+
 impl fmt::Display for StorePath {
     /// The string representation of a store path starts with a digest (20
     /// bytes), [crate::nixbase32]-encoded, followed by a `-`,

tvix/store/protos/pathinfo.go

@@ -60,42 +60,55 @@ func (p *PathInfo) Validate() (*storepath.StorePath, error) {
 	// for all three node types, ensure the name properly parses to a store path,
 	// and in case it refers to a digest, ensure it has the right length.
 
+	var storePath *storepath.StorePath
+	var err error
+
 	if node := rootNode.GetDirectory(); node != nil {
 		if len(node.Digest) != 32 {
 			return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest))
 		}
 
-		storePath, err := storepath.FromString(string(node.GetName()))
+		storePath, err = storepath.FromString(string(node.GetName()))
 
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
 		}
 
-		return storePath, nil
-
 	} else if node := rootNode.GetFile(); node != nil {
 		if len(node.Digest) != 32 {
 			return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest))
 		}
 
-		storePath, err := storepath.FromString(string(node.GetName()))
+		storePath, err = storepath.FromString(string(node.GetName()))
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
 		}
 
-		return storePath, nil
-
 	} else if node := rootNode.GetSymlink(); node != nil {
-		storePath, err := storepath.FromString(string(node.GetName()))
+		storePath, err = storepath.FromString(string(node.GetName()))
 
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
 		}
 
-		return storePath, nil
-
 	} else {
 		// this would only happen if we introduced a new type
 		panic("unreachable")
 	}
+
+	// If the Deriver field is populated, ensure it parses to a StorePath.
+	// We can't check for it to *not* end with .drv, as the .drv files produced by
+	// recursive Nix end with multiple .drv suffixes, and only one is popped when
+	// converting to this field.
+	if p.Deriver != nil {
+		storePath := storepath.StorePath{
+			Name:   string(p.Deriver.GetName()),
+			Digest: p.Deriver.GetDigest(),
+		}
+		if err := storePath.Validate(); err != nil {
+			return nil, fmt.Errorf("invalid deriver field: %w", err)
+		}
+	}
+
+	return storePath, nil
 }

tvix/store/protos/pathinfo.pb.go

@@ -39,6 +39,9 @@ type PathInfo struct {
 	References [][]byte `protobuf:"bytes,2,rep,name=references,proto3" json:"references,omitempty"`
 	// see below.
 	Narinfo *NARInfo `protobuf:"bytes,3,opt,name=narinfo,proto3" json:"narinfo,omitempty"`
+	// The StorePath of the .drv file producing this output.
+	// The .drv suffix is omitted in its `name` field.
+	Deriver *StorePath `protobuf:"bytes,4,opt,name=deriver,proto3" json:"deriver,omitempty"`
 }
 
 func (x *PathInfo) Reset() {
@@ -94,6 +97,13 @@ func (x *PathInfo) GetNarinfo() *NARInfo {
 	return nil
 }
 
+func (x *PathInfo) GetDeriver() *StorePath {
+	if x != nil {
+		return x.Deriver
+	}
+	return nil
+}
+
 // Represents a path in the Nix store (a direct child of STORE_DIR).
 // It is commonly formatted by a nixbase32-encoding the digest, and
 // concatenating the name, separated by a `-`.
@@ -306,7 +316,7 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{
 	0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
 	0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f,
 	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66,
+	0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbb, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66,
 	0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
 	0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
 	0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a,
@@ -314,28 +324,32 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{
 	0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07,
 	0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
 	0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41,
-	0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x37,
+	0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x32,
-	0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e,
+	0x0a, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
+	0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
-	0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
+	0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76,
-	0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49,
+	0x65, 0x72, 0x22, 0x37, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12,
-	0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18,
+	0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
-	0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d,
+	0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20,
-	0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01,
+	0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07,
-	0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a,
+	0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73,
-	0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28,
+	0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69,
-	0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
+	0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36,
-	0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35,
-	0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12,
+	0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18,
-	0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d,
+	0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
-	0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65,
+	0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69,
-	0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e,
+	0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75,
-	0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
+	0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
-	0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
+	0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65,
-	0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a,
+	0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09,
-	0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76,
+	0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
-	0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b,
+	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a,
-	0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74,
+	0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79,
+	0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -361,12 +375,13 @@ var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{
 var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{
 	4, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node
 	2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo
-	3, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
+	1, // 2: tvix.store.v1.PathInfo.deriver:type_name -> tvix.store.v1.StorePath
-	3, // [3:3] is the sub-list for method output_type
+	3, // 3: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
-	3, // [3:3] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for method output_type
-	3, // [3:3] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for method input_type
-	3, // [3:3] is the sub-list for extension extendee
+	4, // [4:4] is the sub-list for extension type_name
-	0, // [0:3] is the sub-list for field type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
 }
 
 func init() { file_tvix_store_protos_pathinfo_proto_init() }

tvix/store/protos/pathinfo.proto

@@ -21,6 +21,10 @@ message PathInfo {
 
     // see below.
     NARInfo narinfo = 3;
+
+    // The StorePath of the .drv file producing this output.
+    // The .drv suffix is omitted in its `name` field.
+    StorePath deriver = 4;
 }
 
 // Represents a path in the Nix store (a direct child of STORE_DIR).

tvix/store/protos/pathinfo_test.go

@@ -120,4 +120,30 @@ func TestValidate(t *testing.T) {
 		_, err := pi.Validate()
 		assert.Error(t, err, "must not validate")
 	})
+
+	t.Run("happy deriver", func(t *testing.T) {
+		pi := genPathInfoSymlink()
+
+		// add the Deriver Field.
+		pi.Deriver = &storev1pb.StorePath{
+			Digest: exampleStorePathDigest,
+			Name:   "foo",
+		}
+
+		_, err := pi.Validate()
+		assert.NoError(t, err, "must validate")
+	})
+
+	t.Run("invalid deriver", func(t *testing.T) {
+		pi := genPathInfoSymlink()
+
+		// add the Deriver Field, with a broken digest
+		pi.Deriver = &storev1pb.StorePath{
+			Digest: []byte{},
+			Name:   "foo2",
+		}
+		_, err := pi.Validate()
+		assert.Error(t, err, "must not validate")
+	})
+
 }

tvix/store/src/bin/tvix-store.rs

@@ -296,6 +296,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                                 signatures: vec![],
                                 reference_names: vec![],
                             }),
+                            deriver: None,
                         };
 
                         // put into [PathInfoService], and return the PathInfo that we get back

tvix/store/src/proto/mod.rs

@@ -48,7 +48,7 @@ pub enum ValidatePathInfoError {
     #[error("Inconsistent Number of References: {0} (references) vs {1} (narinfo)")]
     InconsistentNumberOfReferences(usize, usize),
 
-    /// A string in narinfo.reference_names does not parse to a StorePath.
+    /// A string in narinfo.reference_names does not parse to a [store_path::StorePath].
     #[error("Invalid reference_name at position {0}: {1}")]
     InvalidNarinfoReferenceName(usize, String),
 
@@ -60,6 +60,10 @@ pub enum ValidatePathInfoError {
         [u8; store_path::DIGEST_SIZE],
         [u8; store_path::DIGEST_SIZE],
     ),
+
+    /// The deriver field is invalid.
+    #[error("deriver field is invalid: {0}")]
+    InvalidDeriverField(store_path::Error),
 }
 
 /// Parses a root node name.
@@ -152,6 +156,16 @@ impl PathInfo {
             }
         };
 
+        // If the Deriver field is populated, ensure it parses to a
+        // [store_path::StorePath].
+        // We can't check for it to *not* end with .drv, as the .drv files produced by
+        // recursive Nix end with multiple .drv suffixes, and only one is popped when
+        // converting to this field.
+        if let Some(deriver) = &self.deriver {
+            store_path::StorePath::from_name_and_digest(deriver.name.clone(), &deriver.digest)
+                .map_err(ValidatePathInfoError::InvalidDeriverField)?;
+        }
+
         // return the root nix path
         Ok(root_nix_path)
     }

tvix/store/src/proto/tests/pathinfo.rs

@@ -262,3 +262,34 @@ fn validate_symlink_target_null_byte_invalid() {
 
     node.validate().expect_err("must fail validation");
 }
+
+/// Create a PathInfo with a correct deriver field and ensure it succeeds.
+#[test]
+fn validate_valid_deriver() {
+    let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone();
+
+    // add a valid deriver
+    path_info.deriver = Some(crate::proto::StorePath {
+        name: "foo".to_string(),
+        digest: DUMMY_OUTPUT_HASH.clone(),
+    });
+
+    path_info.validate().expect("must validate");
+}
+
+/// Create a PathInfo with a broken deriver field and ensure it fails.
+#[test]
+fn validate_invalid_deriver() {
+    let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone();
+
+    // add a broken deriver (invalid digest)
+    path_info.deriver = Some(crate::proto::StorePath {
+        name: "foo".to_string(),
+        digest: vec![].into(),
+    });
+
+    match path_info.validate().expect_err("must fail validation") {
+        ValidatePathInfoError::InvalidDeriverField(_) => {}
+        e => panic!("unexpected error: {:?}", e),
+    }
+}

tvix/store/src/tests/fixtures.rs

@@ -109,6 +109,7 @@ lazy_static! {
         }),
         references: vec![DUMMY_OUTPUT_HASH.clone()],
         narinfo: None,
+        deriver: None,
     };
 
     /// A PathInfo message with .narinfo populated.