feat(tvix/castore): bump [Directory,File]Node size to u64

Having more than 4GiB files is quite possible (think about the NixOS
graphical installer, and an uncompressed iso of it).

No wire format changes.

Change-Id: Ia78a07e4c554e91b93c5b9f8533266e4bd7f22b6
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9950
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

tvix/castore/protos/castore.proto

@@ -38,7 +38,7 @@ message DirectoryNode {
     // A credulous implementation won't reject an excessive size, but this is
     // harmless: you'll have some ordinals without nodes. Undersizing is
     // obvious and easy to reject: you won't have an ordinal for some nodes.
-    uint32 size = 3;
+    uint64 size = 3;
 }
 
 // A FileNode represents a regular or executable file in a Directory.
@@ -48,7 +48,7 @@ message FileNode {
     // The blake3 digest of the file contents
     bytes digest = 2;
     // The file content size
-    uint32 size = 3;
+    uint64 size = 3;
     // Whether the file is executable
     bool executable = 4;
 }

tvix/castore/src/import.rs

@@ -122,7 +122,7 @@ async fn process_entry<'a>(
         return Ok(Node::File(FileNode {
             name: entry.file_name().as_bytes().to_vec().into(),
             digest: digest.into(),
-            size: metadata.len() as u32,
+            size: metadata.len(),
             // If it's executable by the user, it'll become executable.
             // This matches nix's dump() function behaviour.
             executable: metadata.permissions().mode() & 64 != 0,

tvix/castore/src/proto/grpc_directoryservice_wrapper.rs

@@ -96,7 +96,7 @@ impl proto::directory_service_server::DirectoryService for GRPCDirectoryServiceW
         // We don't need to keep the contents around, they're stored in the DB.
         // https://github.com/rust-lang/rust-clippy/issues/5812
         #[allow(clippy::mutable_key_type)]
-        let mut seen_directories_sizes: HashMap<B3Digest, u32> = HashMap::new();
+        let mut seen_directories_sizes: HashMap<B3Digest, u64> = HashMap::new();
         let mut last_directory_dgst: Option<B3Digest> = None;
 
         // Consume directories, and insert them into the store.

tvix/castore/src/proto/mod.rs

@@ -172,23 +172,23 @@ fn insert_once<'n>(
     Ok(())
 }
 
-fn checked_sum(iter: impl IntoIterator<Item = u32>) -> Option<u32> {
-    iter.into_iter().try_fold(0u32, |acc, i| acc.checked_add(i))
+fn checked_sum(iter: impl IntoIterator<Item = u64>) -> Option<u64> {
+    iter.into_iter().try_fold(0u64, |acc, i| acc.checked_add(i))
 }
 
 impl Directory {
     /// The size of a directory is the number of all regular and symlink elements,
     /// the number of directory elements, and their size fields.
-    pub fn size(&self) -> u32 {
+    pub fn size(&self) -> u64 {
         if cfg!(debug_assertions) {
             self.size_checked()
-                .expect("Directory::size exceeds u32::MAX")
+                .expect("Directory::size exceeds u64::MAX")
         } else {
-            self.size_checked().unwrap_or(u32::MAX)
+            self.size_checked().unwrap_or(u64::MAX)
         }
     }
 
-    fn size_checked(&self) -> Option<u32> {
+    fn size_checked(&self) -> Option<u64> {
         checked_sum([
             self.files.len().try_into().ok()?,
             self.symlinks.len().try_into().ok()?,

tvix/castore/src/proto/tests/directory.rs

@@ -60,13 +60,13 @@ fn size() {
 
 #[test]
 #[cfg_attr(not(debug_assertions), ignore)]
-#[should_panic = "Directory::size exceeds u32::MAX"]
+#[should_panic = "Directory::size exceeds u64::MAX"]
 fn size_unchecked_panic() {
     let d = Directory {
         directories: vec![DirectoryNode {
             name: "foo".into(),
             digest: DUMMY_DIGEST.to_vec().into(),
-            size: u32::MAX,
+            size: u64::MAX,
         }],
         ..Default::default()
     };
@@ -81,12 +81,12 @@ fn size_unchecked_saturate() {
         directories: vec![DirectoryNode {
             name: "foo".into(),
             digest: DUMMY_DIGEST.to_vec().into(),
-            size: u32::MAX,
+            size: u64::MAX,
         }],
         ..Default::default()
     };
 
-    assert_eq!(d.size(), u32::MAX);
+    assert_eq!(d.size(), u64::MAX);
 }
 
 #[test]
@@ -98,18 +98,18 @@ fn size_checked() {
             directories: vec![DirectoryNode {
                 name: "foo".into(),
                 digest: DUMMY_DIGEST.to_vec().into(),
-                size: u32::MAX - 1,
+                size: u64::MAX - 1,
             }],
             ..Default::default()
         };
-        assert_eq!(d.size_checked(), Some(u32::MAX));
+        assert_eq!(d.size_checked(), Some(u64::MAX));
     }
     {
         let d = Directory {
             directories: vec![DirectoryNode {
                 name: "foo".into(),
                 digest: DUMMY_DIGEST.to_vec().into(),
-                size: u32::MAX,
+                size: u64::MAX,
             }],
             ..Default::default()
         };
@@ -121,12 +121,12 @@ fn size_checked() {
                 DirectoryNode {
                     name: "foo".into(),
                     digest: DUMMY_DIGEST.to_vec().into(),
-                    size: u32::MAX / 2,
+                    size: u64::MAX / 2,
                 },
                 DirectoryNode {
                     name: "foo".into(),
                     digest: DUMMY_DIGEST.to_vec().into(),
-                    size: u32::MAX / 2,
+                    size: u64::MAX / 2,
                 },
             ],
             ..Default::default()
@@ -363,7 +363,7 @@ fn validate_overflow() {
         directories: vec![DirectoryNode {
             name: "foo".into(),
             digest: DUMMY_DIGEST.to_vec().into(),
-            size: u32::MAX,
+            size: u64::MAX,
         }],
         ..Default::default()
     };

tvix/castore/src/tests/import.rs

@@ -56,7 +56,7 @@ async fn single_file() {
         proto::node::Node::File(proto::FileNode {
             name: "root".into(),
             digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
-            size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
+            size: HELLOWORLD_BLOB_CONTENTS.len() as u64,
             executable: false,
         }),
         root_node,