feat(tvix/store/pathinfosvc): provide listing

This provides an additional method in the PathInfoService trait, as
well as an RPC method on the gRPC layer to list all PathInfo objects in
a PathInfoService.

Change-Id: I7378f6bbd334bd6ac4e9be92505bd099a1c2b19a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9216
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-09-03 17:09:45 +03:00 committed by clbot
parent e41b5ae3f0
commit da9d706e0a
9 changed files with 310 additions and 41 deletions

View file

@ -96,6 +96,46 @@ type GetPathInfoRequest_ByOutputHash struct {
func (*GetPathInfoRequest_ByOutputHash) isGetPathInfoRequest_ByWhat() {}
// The parameters that can be used to lookup (multiple) PathInfo objects.
// Currently no filtering is possible, all objects are returned.
type ListPathInfoRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *ListPathInfoRequest) Reset() {
*x = ListPathInfoRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ListPathInfoRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ListPathInfoRequest) ProtoMessage() {}
func (x *ListPathInfoRequest) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ListPathInfoRequest.ProtoReflect.Descriptor instead.
func (*ListPathInfoRequest) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP(), []int{1}
}
// CalculateNARResponse is the response returned by the CalculateNAR request.
//
// It contains the size of the NAR representation (in bytes), and the sha56
@ -114,7 +154,7 @@ type CalculateNARResponse struct {
func (x *CalculateNARResponse) Reset() {
*x = CalculateNARResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -127,7 +167,7 @@ func (x *CalculateNARResponse) String() string {
func (*CalculateNARResponse) ProtoMessage() {}
func (x *CalculateNARResponse) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -140,7 +180,7 @@ func (x *CalculateNARResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use CalculateNARResponse.ProtoReflect.Descriptor instead.
func (*CalculateNARResponse) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP(), []int{1}
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP(), []int{2}
}
func (x *CalculateNARResponse) GetNarSize() uint64 {
@ -170,28 +210,34 @@ var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{
0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75,
0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74,
0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f,
0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53,
0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35,
0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32,
0x35, 0x36, 0x32, 0xd7, 0x01, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65,
0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, 0x6c, 0x63, 0x75,
0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61,
0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09,
0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x32, 0x9e, 0x02, 0x0a, 0x0f, 0x50, 0x61,
0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a,
0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74,
0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78,
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e,
0x66, 0x6f, 0x12, 0x48, 0x0a, 0x0c, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e,
0x41, 0x52, 0x12, 0x13, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74,
0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x28, 0x5a, 0x26,
0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69,
0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x0c, 0x43, 0x61, 0x6c,
0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x12, 0x13, 0x2e, 0x74, 0x76, 0x69, 0x78,
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x1a, 0x23,
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43,
0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x2e, 0x74, 0x76,
0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74,
0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f,
0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -206,22 +252,25 @@ func file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP() []byte {
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescData
}
var file_tvix_store_protos_rpc_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_tvix_store_protos_rpc_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_tvix_store_protos_rpc_pathinfo_proto_goTypes = []interface{}{
(*GetPathInfoRequest)(nil), // 0: tvix.store.v1.GetPathInfoRequest
(*CalculateNARResponse)(nil), // 1: tvix.store.v1.CalculateNARResponse
(*PathInfo)(nil), // 2: tvix.store.v1.PathInfo
(*Node)(nil), // 3: tvix.store.v1.Node
(*ListPathInfoRequest)(nil), // 1: tvix.store.v1.ListPathInfoRequest
(*CalculateNARResponse)(nil), // 2: tvix.store.v1.CalculateNARResponse
(*PathInfo)(nil), // 3: tvix.store.v1.PathInfo
(*Node)(nil), // 4: tvix.store.v1.Node
}
var file_tvix_store_protos_rpc_pathinfo_proto_depIdxs = []int32{
0, // 0: tvix.store.v1.PathInfoService.Get:input_type -> tvix.store.v1.GetPathInfoRequest
2, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo
3, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.store.v1.Node
2, // 3: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo
2, // 4: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo
1, // 5: tvix.store.v1.PathInfoService.CalculateNAR:output_type -> tvix.store.v1.CalculateNARResponse
3, // [3:6] is the sub-list for method output_type
0, // [0:3] is the sub-list for method input_type
3, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo
4, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.store.v1.Node
1, // 3: tvix.store.v1.PathInfoService.List:input_type -> tvix.store.v1.ListPathInfoRequest
3, // 4: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo
3, // 5: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo
2, // 6: tvix.store.v1.PathInfoService.CalculateNAR:output_type -> tvix.store.v1.CalculateNARResponse
3, // 7: tvix.store.v1.PathInfoService.List:output_type -> tvix.store.v1.PathInfo
4, // [4:8] is the sub-list for method output_type
0, // [0:4] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
@ -247,6 +296,18 @@ func file_tvix_store_protos_rpc_pathinfo_proto_init() {
}
}
file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ListPathInfoRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CalculateNARResponse); i {
case 0:
return &v.state
@ -268,7 +329,7 @@ func file_tvix_store_protos_rpc_pathinfo_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_rpc_pathinfo_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumMessages: 3,
NumExtensions: 0,
NumServices: 1,
},

View file

@ -41,6 +41,10 @@ service PathInfoService {
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
rpc CalculateNAR(Node) returns (CalculateNARResponse);
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
rpc List(ListPathInfoRequest) returns (stream PathInfo);
}
// The parameters that can be used to lookup a (single) PathInfo object.
@ -55,6 +59,10 @@ message GetPathInfoRequest {
};
}
// The parameters that can be used to lookup (multiple) PathInfo objects.
// Currently no filtering is possible, all objects are returned.
message ListPathInfoRequest { }
// CalculateNARResponse is the response returned by the CalculateNAR request.
//
// It contains the size of the NAR representation (in bytes), and the sha56

View file

@ -25,6 +25,7 @@ const (
PathInfoService_Get_FullMethodName = "/tvix.store.v1.PathInfoService/Get"
PathInfoService_Put_FullMethodName = "/tvix.store.v1.PathInfoService/Put"
PathInfoService_CalculateNAR_FullMethodName = "/tvix.store.v1.PathInfoService/CalculateNAR"
PathInfoService_List_FullMethodName = "/tvix.store.v1.PathInfoService/List"
)
// PathInfoServiceClient is the client API for PathInfoService service.
@ -60,6 +61,9 @@ type PathInfoServiceClient interface {
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
List(ctx context.Context, in *ListPathInfoRequest, opts ...grpc.CallOption) (PathInfoService_ListClient, error)
}
type pathInfoServiceClient struct {
@ -97,6 +101,38 @@ func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *Node, opts
return out, nil
}
func (c *pathInfoServiceClient) List(ctx context.Context, in *ListPathInfoRequest, opts ...grpc.CallOption) (PathInfoService_ListClient, error) {
stream, err := c.cc.NewStream(ctx, &PathInfoService_ServiceDesc.Streams[0], PathInfoService_List_FullMethodName, opts...)
if err != nil {
return nil, err
}
x := &pathInfoServiceListClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type PathInfoService_ListClient interface {
Recv() (*PathInfo, error)
grpc.ClientStream
}
type pathInfoServiceListClient struct {
grpc.ClientStream
}
func (x *pathInfoServiceListClient) Recv() (*PathInfo, error) {
m := new(PathInfo)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// PathInfoServiceServer is the server API for PathInfoService service.
// All implementations must embed UnimplementedPathInfoServiceServer
// for forward compatibility
@ -130,6 +166,9 @@ type PathInfoServiceServer interface {
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
List(*ListPathInfoRequest, PathInfoService_ListServer) error
mustEmbedUnimplementedPathInfoServiceServer()
}
@ -146,6 +185,9 @@ func (UnimplementedPathInfoServiceServer) Put(context.Context, *PathInfo) (*Path
func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CalculateNAR not implemented")
}
func (UnimplementedPathInfoServiceServer) List(*ListPathInfoRequest, PathInfoService_ListServer) error {
return status.Errorf(codes.Unimplemented, "method List not implemented")
}
func (UnimplementedPathInfoServiceServer) mustEmbedUnimplementedPathInfoServiceServer() {}
// UnsafePathInfoServiceServer may be embedded to opt out of forward compatibility for this service.
@ -213,6 +255,27 @@ func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler)
}
func _PathInfoService_List_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(ListPathInfoRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(PathInfoServiceServer).List(m, &pathInfoServiceListServer{stream})
}
type PathInfoService_ListServer interface {
Send(*PathInfo) error
grpc.ServerStream
}
type pathInfoServiceListServer struct {
grpc.ServerStream
}
func (x *pathInfoServiceListServer) Send(m *PathInfo) error {
return x.ServerStream.SendMsg(m)
}
// PathInfoService_ServiceDesc is the grpc.ServiceDesc for PathInfoService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -233,6 +296,12 @@ var PathInfoService_ServiceDesc = grpc.ServiceDesc{
Handler: _PathInfoService_CalculateNAR_Handler,
},
},
Streams: []grpc.StreamDesc{},
Streams: []grpc.StreamDesc{
{
StreamName: "List",
Handler: _PathInfoService_List_Handler,
ServerStreams: true,
},
},
Metadata: "tvix/store/protos/rpc_pathinfo.proto",
}