feat(tvix/store): make blobstore stream chunks
This changes the RPC methods to return/consume a stream of chunks, instead of a very big message containing the whole blob, to keep message sizes in manageable sizes (less than 4MiB). Change-Id: I2a3a50f07b059d8a2f5196860254adff98c8a352 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7651 Reviewed-by: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink
flokli
parent
f879993cc4
commit
1c15154b83
3 changed files with 200 additions and 201 deletions
|
|
@ -71,100 +71,6 @@ func (x *GetBlobRequest) GetDigest() []byte {
|
|||
return nil
|
||||
}
|
||||
|
||||
type GetBlobResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (x *GetBlobResponse) Reset() {
|
||||
*x = GetBlobResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *GetBlobResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*GetBlobResponse) ProtoMessage() {}
|
||||
|
||||
func (x *GetBlobResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use GetBlobResponse.ProtoReflect.Descriptor instead.
|
||||
func (*GetBlobResponse) Descriptor() ([]byte, []int) {
|
||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *GetBlobResponse) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type PutBlobRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (x *PutBlobRequest) Reset() {
|
||||
*x = PutBlobRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *PutBlobRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*PutBlobRequest) ProtoMessage() {}
|
||||
|
||||
func (x *PutBlobRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use PutBlobRequest.ProtoReflect.Descriptor instead.
|
||||
func (*PutBlobRequest) Descriptor() ([]byte, []int) {
|
||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *PutBlobRequest) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type PutBlobResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
|
@ -177,7 +83,7 @@ type PutBlobResponse struct {
|
|||
func (x *PutBlobResponse) Reset() {
|
||||
*x = PutBlobResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -190,7 +96,7 @@ func (x *PutBlobResponse) String() string {
|
|||
func (*PutBlobResponse) ProtoMessage() {}
|
||||
|
||||
func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -203,7 +109,7 @@ func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead.
|
||||
func (*PutBlobResponse) Descriptor() ([]byte, []int) {
|
||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3}
|
||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *PutBlobResponse) GetDigest() []byte {
|
||||
|
|
@ -213,6 +119,55 @@ func (x *PutBlobResponse) GetDigest() []byte {
|
|||
return nil
|
||||
}
|
||||
|
||||
// This represents a part of a chunk.
|
||||
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
||||
type BlobChunk struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (x *BlobChunk) Reset() {
|
||||
*x = BlobChunk{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *BlobChunk) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*BlobChunk) ProtoMessage() {}
|
||||
|
||||
func (x *BlobChunk) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead.
|
||||
func (*BlobChunk) Descriptor() ([]byte, []int) {
|
||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *BlobChunk) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{
|
||||
|
|
@ -222,27 +177,24 @@ var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{
|
|||
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x28, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f,
|
||||
0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65,
|
||||
0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74,
|
||||
0x22, 0x25, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x24, 0x0a, 0x0e, 0x50, 0x75, 0x74, 0x42, 0x6c,
|
||||
0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
|
||||
0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a,
|
||||
0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
|
||||
0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x32, 0x99, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f,
|
||||
0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12,
|
||||
0x1d, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
|
||||
0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e,
|
||||
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47,
|
||||
0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44,
|
||||
0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x1d, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
|
||||
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72,
|
||||
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c,
|
||||
0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f,
|
||||
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06,
|
||||
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42,
|
||||
0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32, 0x92, 0x01, 0x0a,
|
||||
0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x03,
|
||||
0x47, 0x65, 0x74, 0x12, 0x1d, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65,
|
||||
0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
|
||||
0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41,
|
||||
0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
|
||||
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a,
|
||||
0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
|
||||
0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28,
|
||||
0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79,
|
||||
0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -257,18 +209,17 @@ func file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP() []byte {
|
|||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
|
||||
var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
|
||||
var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{
|
||||
(*GetBlobRequest)(nil), // 0: tvix.store.v1.GetBlobRequest
|
||||
(*GetBlobResponse)(nil), // 1: tvix.store.v1.GetBlobResponse
|
||||
(*PutBlobRequest)(nil), // 2: tvix.store.v1.PutBlobRequest
|
||||
(*PutBlobResponse)(nil), // 3: tvix.store.v1.PutBlobResponse
|
||||
(*PutBlobResponse)(nil), // 1: tvix.store.v1.PutBlobResponse
|
||||
(*BlobChunk)(nil), // 2: tvix.store.v1.BlobChunk
|
||||
}
|
||||
var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{
|
||||
0, // 0: tvix.store.v1.BlobService.Get:input_type -> tvix.store.v1.GetBlobRequest
|
||||
2, // 1: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.PutBlobRequest
|
||||
1, // 2: tvix.store.v1.BlobService.Get:output_type -> tvix.store.v1.GetBlobResponse
|
||||
3, // 3: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse
|
||||
2, // 1: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk
|
||||
2, // 2: tvix.store.v1.BlobService.Get:output_type -> tvix.store.v1.BlobChunk
|
||||
1, // 3: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse
|
||||
2, // [2:4] is the sub-list for method output_type
|
||||
0, // [0:2] is the sub-list for method input_type
|
||||
0, // [0:0] is the sub-list for extension type_name
|
||||
|
|
@ -295,7 +246,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
|||
}
|
||||
}
|
||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*GetBlobResponse); i {
|
||||
switch v := v.(*PutBlobResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
|
|
@ -307,19 +258,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
|||
}
|
||||
}
|
||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*PutBlobRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*PutBlobResponse); i {
|
||||
switch v := v.(*BlobChunk); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
|
|
@ -337,7 +276,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
|||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 4,
|
||||
NumMessages: 3,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue