maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/store): make blobstore stream chunks

Browse source 
This changes the RPC methods to return/consume a stream of chunks, instead of a
very big message containing the whole blob, to keep message sizes in manageable
sizes (less than 4MiB).

Change-Id: I2a3a50f07b059d8a2f5196860254adff98c8a352
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7651
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2022-12-27 18:32:07 +01:00 • committed by flokli
parent f879993cc4
commit 1c15154b83
3 changed files with 200 additions and 201 deletions
Download patch file Download diff file Expand all files Collapse all files

19
tvix/store/protos/rpc_blobstore.proto
View file

@ -7,8 +7,9 @@ package tvix.store.v1;
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
service BlobService {
rpc Get(GetBlobRequest) returns (GetBlobResponse);
rpc Put(PutBlobRequest) returns (PutBlobResponse);
rpc Get(GetBlobRequest) returns (stream BlobChunk);
rpc Put(stream BlobChunk) returns (PutBlobResponse);
// TODO(flokli): We can get fancy here, and add methods to retrieve
// [Bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md), and
@ -20,15 +21,13 @@ message GetBlobRequest {
bytes digest = 1;
}
message GetBlobResponse {
bytes data = 1;
}
message PutBlobRequest {
bytes data = 1;
}
message PutBlobResponse {
// The blake3 digest of the data that was sent.
bytes digest = 1;
}
// This represents a part of a chunk.
// Blobs are sent in smaller chunks to keep message sizes manageable.
message BlobChunk {
bytes data = 1;
}