feat(3p/nix): Implement AddToStore rpc client
Implement the AddToStore RPC client method, which uses an
AddToStorePathWriterSink (the dual of the AddToStorePathReaderSource on
the server side) to hook into the dumpPath machinery (which we should
refactor not to use sinks or sources, but not yet) and write dumped
paths as binary data over gRPC. With this commit and sandboxing
disabled, the following derivation builds cleanly:
    derivation {
      name = "test";
      builder = ./build.sh;
      system = "x86_64-linux";
    }
where build.sh has chmod +x and contains:
    #!/bin/sh
    echo 1 > $out
Change-Id: I94cab86f0825a3a9993262a9807130645c13bf44
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1638
Reviewed-by: kanepyork <rikingcoding@gmail.com>
Tested-by: BuildkiteCI
			
			
This commit is contained in:
		
							parent
							
								
									b15b447fcb
								
							
						
					
					
						commit
						28c7e926ea
					
				
					 4 changed files with 69 additions and 2 deletions
				
			
		
							
								
								
									
										52
									
								
								third_party/nix/src/libstore/rpc-store.cc
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										52
									
								
								third_party/nix/src/libstore/rpc-store.cc
									
										
									
									
										vendored
									
									
								
							| 
						 | 
					@ -1,6 +1,7 @@
 | 
				
			||||||
#include "rpc-store.hh"
 | 
					#include "rpc-store.hh"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#include <algorithm>
 | 
					#include <algorithm>
 | 
				
			||||||
 | 
					#include <filesystem>
 | 
				
			||||||
#include <memory>
 | 
					#include <memory>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#include <absl/status/status.h>
 | 
					#include <absl/status/status.h>
 | 
				
			||||||
| 
						 | 
					@ -20,6 +21,7 @@
 | 
				
			||||||
#include "libproto/worker.grpc.pb.h"
 | 
					#include "libproto/worker.grpc.pb.h"
 | 
				
			||||||
#include "libproto/worker.pb.h"
 | 
					#include "libproto/worker.pb.h"
 | 
				
			||||||
#include "libstore/store-api.hh"
 | 
					#include "libstore/store-api.hh"
 | 
				
			||||||
 | 
					#include "libutil/archive.hh"
 | 
				
			||||||
#include "libutil/hash.hh"
 | 
					#include "libutil/hash.hh"
 | 
				
			||||||
#include "libutil/types.hh"
 | 
					#include "libutil/types.hh"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -54,6 +56,32 @@ T FillFrom(const U& src) {
 | 
				
			||||||
  return result;
 | 
					  return result;
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class AddToStorePathWriterSink : public BufferedSink {
 | 
				
			||||||
 | 
					 public:
 | 
				
			||||||
 | 
					  explicit AddToStorePathWriterSink(
 | 
				
			||||||
 | 
					      std::unique_ptr<
 | 
				
			||||||
 | 
					          grpc_impl::ClientWriter<class nix::proto::AddToStoreRequest>>&&
 | 
				
			||||||
 | 
					          writer)
 | 
				
			||||||
 | 
					      : writer_(std::move(writer)), good_(true) {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  bool good() override { return good_; }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  void write(const unsigned char* data, size_t len) override {
 | 
				
			||||||
 | 
					    proto::AddToStoreRequest req;
 | 
				
			||||||
 | 
					    req.set_data(data, len);
 | 
				
			||||||
 | 
					    if (!writer_->Write(req)) {
 | 
				
			||||||
 | 
					      good_ = false;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  grpc::Status Finish() { return writer_->Finish(); }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					 private:
 | 
				
			||||||
 | 
					  std::unique_ptr<grpc_impl::ClientWriter<class nix::proto::AddToStoreRequest>>
 | 
				
			||||||
 | 
					      writer_;
 | 
				
			||||||
 | 
					  bool good_;
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) {
 | 
					constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) {
 | 
				
			||||||
  switch (code) {
 | 
					  switch (code) {
 | 
				
			||||||
    case grpc::StatusCode::OK:
 | 
					    case grpc::StatusCode::OK:
 | 
				
			||||||
| 
						 | 
					@ -304,7 +332,29 @@ void RpcStore::addToStore(const ValidPathInfo& info,
 | 
				
			||||||
Path RpcStore::addToStore(const std::string& name, const Path& srcPath,
 | 
					Path RpcStore::addToStore(const std::string& name, const Path& srcPath,
 | 
				
			||||||
                          bool recursive, HashType hashAlgo, PathFilter& filter,
 | 
					                          bool recursive, HashType hashAlgo, PathFilter& filter,
 | 
				
			||||||
                          RepairFlag repair) {
 | 
					                          RepairFlag repair) {
 | 
				
			||||||
  throw Unsupported(absl::StrCat("Not implemented ", __func__));
 | 
					  if (repair != 0u) {
 | 
				
			||||||
 | 
					    throw Error(
 | 
				
			||||||
 | 
					        "repairing is not supported when building through the Nix daemon");
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  ClientContext ctx;
 | 
				
			||||||
 | 
					  proto::StorePath response;
 | 
				
			||||||
 | 
					  auto writer = stub_->AddToStore(&ctx, &response);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  proto::AddToStoreRequest metadata_req;
 | 
				
			||||||
 | 
					  metadata_req.mutable_meta()->set_base_name(name);
 | 
				
			||||||
 | 
					  // TODO(grfn): what is fixed?
 | 
				
			||||||
 | 
					  metadata_req.mutable_meta()->set_fixed(!(hashAlgo == htSHA256 && recursive));
 | 
				
			||||||
 | 
					  metadata_req.mutable_meta()->set_recursive(recursive);
 | 
				
			||||||
 | 
					  metadata_req.mutable_meta()->set_hash_type(HashTypeToProto(hashAlgo));
 | 
				
			||||||
 | 
					  writer->Write(metadata_req);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  AddToStorePathWriterSink sink(std::move(writer));
 | 
				
			||||||
 | 
					  dumpPath(std::filesystem::absolute(srcPath), sink);
 | 
				
			||||||
 | 
					  sink.flush();
 | 
				
			||||||
 | 
					  SuccessOrThrow(sink.Finish());
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  return response.path();
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
Path RpcStore::addTextToStore(const std::string& name,
 | 
					Path RpcStore::addTextToStore(const std::string& name,
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
							
								
								
									
										15
									
								
								third_party/nix/src/libutil/hash.cc
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										15
									
								
								third_party/nix/src/libutil/hash.cc
									
										
									
									
										vendored
									
									
								
							| 
						 | 
					@ -35,6 +35,21 @@ std::optional<HashType> hash_type_from(nix::proto::HashType hash_type) {
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					nix::proto::HashType HashTypeToProto(HashType hash_type) {
 | 
				
			||||||
 | 
					  switch (hash_type) {
 | 
				
			||||||
 | 
					    case HashType::htMD5:
 | 
				
			||||||
 | 
					      return nix::proto::HashType::MD5;
 | 
				
			||||||
 | 
					    case HashType::htSHA1:
 | 
				
			||||||
 | 
					      return nix::proto::HashType::SHA1;
 | 
				
			||||||
 | 
					    case HashType::htSHA256:
 | 
				
			||||||
 | 
					      return nix::proto::HashType::SHA256;
 | 
				
			||||||
 | 
					    case HashType::htSHA512:
 | 
				
			||||||
 | 
					      return nix::proto::HashType::SHA512;
 | 
				
			||||||
 | 
					    default:
 | 
				
			||||||
 | 
					      return nix::proto::HashType::UNKNOWN;
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void Hash::init() {
 | 
					void Hash::init() {
 | 
				
			||||||
  if (type == htMD5) {
 | 
					  if (type == htMD5) {
 | 
				
			||||||
    hashSize = md5HashSize;
 | 
					    hashSize = md5HashSize;
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
							
								
								
									
										2
									
								
								third_party/nix/src/libutil/hash.hh
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										2
									
								
								third_party/nix/src/libutil/hash.hh
									
										
									
									
										vendored
									
									
								
							| 
						 | 
					@ -16,6 +16,8 @@ enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
 | 
				
			||||||
 | 
					
 | 
				
			||||||
std::optional<HashType> hash_type_from(nix::proto::HashType hash_type);
 | 
					std::optional<HashType> hash_type_from(nix::proto::HashType hash_type);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					nix::proto::HashType HashTypeToProto(HashType hash_type);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const int md5HashSize = 16;
 | 
					const int md5HashSize = 16;
 | 
				
			||||||
const int sha1HashSize = 20;
 | 
					const int sha1HashSize = 20;
 | 
				
			||||||
const int sha256HashSize = 32;
 | 
					const int sha256HashSize = 32;
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -146,7 +146,7 @@ class WorkerServiceImpl final : public WorkerService::Service {
 | 
				
			||||||
          proto::AddToStoreRequest metadata_request;
 | 
					          proto::AddToStoreRequest metadata_request;
 | 
				
			||||||
          auto has_metadata = reader->Read(&metadata_request);
 | 
					          auto has_metadata = reader->Read(&metadata_request);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
          if (!has_metadata || metadata_request.has_meta()) {
 | 
					          if (!has_metadata || !metadata_request.has_meta()) {
 | 
				
			||||||
            return Status(grpc::StatusCode::INVALID_ARGUMENT,
 | 
					            return Status(grpc::StatusCode::INVALID_ARGUMENT,
 | 
				
			||||||
                          "Metadata must be set before sending file content");
 | 
					                          "Metadata must be set before sending file content");
 | 
				
			||||||
          }
 | 
					          }
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue