maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(tvix/nar-bridge): chunk blobs

Browse source 
Instead of creating one big BlobChunk containing all data, and creating
way too large proto messages, chunk blobs up to a reasonable (1MiB)
chunk size, and send them to the server like that.

Change-Id: Ia45a53956a6d7c0599cc59ac516ba37e9fb1b30e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9357
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-09-17 21:47:11 +03:00 • committed by flokli
parent 136b12ddd7
commit 0c031461c3
2 changed files with 48 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tvix/nar-bridge/pkg/reader/reader.go
View file

@ -41,7 +41,7 @@ func New(r io.Reader) *Reader {
func (r *Reader) Import(
ctx context.Context,
// callback function called with each regular file content
fileCb func(fileReader io.Reader) error,
blobCb func(fileReader io.Reader) error,
// callback function called with each finalized directory node
directoryCb func(directory *storev1pb.Directory) error,
) (*storev1pb.PathInfo, error) {
@ -219,9 +219,9 @@ func (r *Reader) Import(
// wrap reader with a reader calculating the blake3 hash
fileReader := NewHasher(narReader, blake3.New(32, nil))
err := fileCb(fileReader)
err := blobCb(fileReader)
if err != nil {
return nil, fmt.Errorf("failure from fileCb: %w", err)
return nil, fmt.Errorf("failure from blobCb: %w", err)
}
// drive the file reader to the end, in case the CB function doesn't read