feat(3p/nix/nix-daemon): Implement Worker::BuildDerivation handler
Implement the proto handler on the server side for Worker::BuildDerivation. This includes several additions to the proto which I had missed on the first pass, including the actual proto definition for the Derivation itself and a few sequence number reorderings which are fine because this is all provisional and not deployed yet. A couple things to note - I implemented a couple constructors for nix classes that initialize themselves based on their proto variants, which felt nice and didn't end up causing any issues. - I've made the conversions between the enum types in nix and in proto explicit via switch statements rather than using a static_cast, out of an abundance of caution that the error would get mismatched in the future and we'd convert the wrong thing to the wrong thing - this is verbose, but exceptionally future proof. Change-Id: Iecf6b88e76bc37e49efa05fd65d6cd0cb0deffed Reviewed-on: https://cl.tvl.fyi/c/depot/+/1249 Tested-by: BuildkiteCI Reviewed-by: tazjin <mail@tazj.in> Reviewed-by: Kane York <rikingcoding@gmail.com>
This commit is contained in:
		
							parent
							
								
									3f4e5050cd
								
							
						
					
					
						commit
						a79df261b4
					
				
					 6 changed files with 133 additions and 1 deletions
				
			
		
							
								
								
									
										22
									
								
								third_party/nix/src/libstore/derivations.cc
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										22
									
								
								third_party/nix/src/libstore/derivations.cc
									
										
									
									
										vendored
									
									
								
							| 
						 | 
				
			
			@ -30,6 +30,28 @@ void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
 | 
			
		|||
  hash = Hash(this->hash, hashType);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
BasicDerivation BasicDerivation::from_proto(
 | 
			
		||||
    const nix::proto::Derivation* proto_derivation, const nix::Store* store) {
 | 
			
		||||
  BasicDerivation result;
 | 
			
		||||
  result.platform = proto_derivation->platform();
 | 
			
		||||
  result.builder = proto_derivation->builder().path();
 | 
			
		||||
  store->assertStorePath(result.builder);
 | 
			
		||||
 | 
			
		||||
  result.outputs.insert(proto_derivation->outputs().begin(),
 | 
			
		||||
                        proto_derivation->outputs().end());
 | 
			
		||||
 | 
			
		||||
  result.inputSrcs.insert(proto_derivation->input_sources().paths().begin(),
 | 
			
		||||
                          proto_derivation->input_sources().paths().end());
 | 
			
		||||
 | 
			
		||||
  result.args.insert(result.args.end(), proto_derivation->args().begin(),
 | 
			
		||||
                     proto_derivation->args().end());
 | 
			
		||||
 | 
			
		||||
  result.env.insert(proto_derivation->env().begin(),
 | 
			
		||||
                    proto_derivation->env().end());
 | 
			
		||||
 | 
			
		||||
  return result;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
Path BasicDerivation::findOutput(const std::string& id) const {
 | 
			
		||||
  auto i = outputs.find(id);
 | 
			
		||||
  if (i == outputs.end()) {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										19
									
								
								third_party/nix/src/libstore/derivations.hh
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										19
									
								
								third_party/nix/src/libstore/derivations.hh
									
										
									
									
										vendored
									
									
								
							| 
						 | 
				
			
			@ -2,6 +2,7 @@
 | 
			
		|||
 | 
			
		||||
#include <map>
 | 
			
		||||
 | 
			
		||||
#include "libproto/worker.pb.h"
 | 
			
		||||
#include "libstore/store-api.hh"
 | 
			
		||||
#include "libutil/hash.hh"
 | 
			
		||||
#include "libutil/types.hh"
 | 
			
		||||
| 
						 | 
				
			
			@ -18,20 +19,31 @@ struct DerivationOutput {
 | 
			
		|||
  std::string hashAlgo; /* hash used for expected hash computation */
 | 
			
		||||
  std::string hash;     /* expected hash, may be null */
 | 
			
		||||
  DerivationOutput() {}
 | 
			
		||||
  // TODO(grfn): Make explicit
 | 
			
		||||
  DerivationOutput(Path path, std::string hashAlgo, std::string hash) {
 | 
			
		||||
    this->path = path;
 | 
			
		||||
    this->hashAlgo = hashAlgo;
 | 
			
		||||
    this->hash = hash;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  explicit DerivationOutput(
 | 
			
		||||
      const nix::proto::Derivation_DerivationOutput& proto_derivation_output)
 | 
			
		||||
      : path(proto_derivation_output.path().path()),
 | 
			
		||||
        hashAlgo(proto_derivation_output.hash_algo()),
 | 
			
		||||
        hash(proto_derivation_output.hash()) {}
 | 
			
		||||
 | 
			
		||||
  void parseHashInfo(bool& recursive, Hash& hash) const;
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
// TODO(grfn): change to absl::flat_hash_map
 | 
			
		||||
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
 | 
			
		||||
 | 
			
		||||
/* For inputs that are sub-derivations, we specify exactly which
 | 
			
		||||
   output IDs we are interested in. */
 | 
			
		||||
// TODO(grfn): change to absl::flat_hash_map
 | 
			
		||||
typedef std::map<Path, StringSet> DerivationInputs;
 | 
			
		||||
 | 
			
		||||
// TODO(grfn): change to absl::flat_hash_map
 | 
			
		||||
typedef std::map<std::string, std::string> StringPairs;
 | 
			
		||||
 | 
			
		||||
struct BasicDerivation {
 | 
			
		||||
| 
						 | 
				
			
			@ -42,6 +54,13 @@ struct BasicDerivation {
 | 
			
		|||
  Strings args;
 | 
			
		||||
  StringPairs env;
 | 
			
		||||
 | 
			
		||||
  BasicDerivation(){};
 | 
			
		||||
 | 
			
		||||
  // Convert the given proto derivation to a BasicDerivation in the given
 | 
			
		||||
  // nix::Store.
 | 
			
		||||
  static BasicDerivation from_proto(
 | 
			
		||||
      const nix::proto::Derivation* proto_derivation, const nix::Store* store);
 | 
			
		||||
 | 
			
		||||
  virtual ~BasicDerivation(){};
 | 
			
		||||
 | 
			
		||||
  /* Return the path corresponding to the output identifier `id' in
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										44
									
								
								third_party/nix/src/libstore/store-api.cc
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										44
									
								
								third_party/nix/src/libstore/store-api.cc
									
										
									
									
										vendored
									
									
								
							| 
						 | 
				
			
			@ -18,6 +18,50 @@
 | 
			
		|||
 | 
			
		||||
namespace nix {
 | 
			
		||||
 | 
			
		||||
std::optional<BuildMode> build_mode_from(nix::proto::BuildMode mode) {
 | 
			
		||||
  switch (mode) {
 | 
			
		||||
    case nix::proto::BuildMode::Normal:
 | 
			
		||||
      return BuildMode::bmNormal;
 | 
			
		||||
    case nix::proto::BuildMode::Repair:
 | 
			
		||||
      return BuildMode::bmRepair;
 | 
			
		||||
    case nix::proto::BuildMode::Check:
 | 
			
		||||
      return BuildMode::bmCheck;
 | 
			
		||||
    default:
 | 
			
		||||
      return {};
 | 
			
		||||
  }
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
nix::proto::BuildStatus BuildResult::status_to_proto() {
 | 
			
		||||
  switch (status) {
 | 
			
		||||
    case BuildResult::Status::Built:
 | 
			
		||||
      return proto::BuildStatus::Built;
 | 
			
		||||
    case BuildResult::Status::Substituted:
 | 
			
		||||
      return proto::BuildStatus::Substituted;
 | 
			
		||||
    case BuildResult::Status::AlreadyValid:
 | 
			
		||||
      return proto::BuildStatus::AlreadyValid;
 | 
			
		||||
    case BuildResult::Status::PermanentFailure:
 | 
			
		||||
      return proto::BuildStatus::PermanentFailure;
 | 
			
		||||
    case BuildResult::Status::InputRejected:
 | 
			
		||||
      return proto::BuildStatus::InputRejected;
 | 
			
		||||
    case BuildResult::Status::OutputRejected:
 | 
			
		||||
      return proto::BuildStatus::OutputRejected;
 | 
			
		||||
    case BuildResult::Status::TransientFailure:
 | 
			
		||||
      return proto::BuildStatus::TransientFailure;
 | 
			
		||||
    case BuildResult::Status::CachedFailure:
 | 
			
		||||
      return proto::BuildStatus::CachedFailure;
 | 
			
		||||
    case BuildResult::Status::TimedOut:
 | 
			
		||||
      return proto::BuildStatus::TimedOut;
 | 
			
		||||
    case BuildResult::Status::MiscFailure:
 | 
			
		||||
      return proto::BuildStatus::MiscFailure;
 | 
			
		||||
    case BuildResult::Status::DependencyFailed:
 | 
			
		||||
      return proto::BuildStatus::DependencyFailed;
 | 
			
		||||
    case BuildResult::Status::LogLimitExceeded:
 | 
			
		||||
      return proto::BuildStatus::LogLimitExceeded;
 | 
			
		||||
    case BuildResult::Status::NotDeterministic:
 | 
			
		||||
      return proto::BuildStatus::NotDeterministic;
 | 
			
		||||
  }
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
bool Store::isInStore(const Path& path) const {
 | 
			
		||||
  return isInDir(path, storeDir);
 | 
			
		||||
}
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										9
									
								
								third_party/nix/src/libstore/store-api.hh
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										9
									
								
								third_party/nix/src/libstore/store-api.hh
									
										
									
									
										vendored
									
									
								
							| 
						 | 
				
			
			@ -8,6 +8,7 @@
 | 
			
		|||
#include <unordered_map>
 | 
			
		||||
#include <unordered_set>
 | 
			
		||||
 | 
			
		||||
#include "libproto/worker.pb.h"
 | 
			
		||||
#include "libstore/crypto.hh"
 | 
			
		||||
#include "libstore/globals.hh"
 | 
			
		||||
#include "libutil/config.hh"
 | 
			
		||||
| 
						 | 
				
			
			@ -181,6 +182,10 @@ typedef std::list<ValidPathInfo> ValidPathInfos;
 | 
			
		|||
 | 
			
		||||
enum BuildMode { bmNormal, bmRepair, bmCheck };
 | 
			
		||||
 | 
			
		||||
// Convert the proto version of a `nix::proto::BuildMode` to its corresponding
 | 
			
		||||
// nix `BuildMode`
 | 
			
		||||
std::optional<BuildMode> build_mode_from(nix::proto::BuildMode mode);
 | 
			
		||||
 | 
			
		||||
struct BuildResult {
 | 
			
		||||
  /* Note: don't remove status codes, and only add new status codes
 | 
			
		||||
     at the end of the list, to prevent client/server
 | 
			
		||||
| 
						 | 
				
			
			@ -218,6 +223,10 @@ struct BuildResult {
 | 
			
		|||
  bool success() {
 | 
			
		||||
    return status == Built || status == Substituted || status == AlreadyValid;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  // Convert the status of this `BuildResult` to its corresponding
 | 
			
		||||
  // `nix::proto::BuildStatus`
 | 
			
		||||
  nix::proto::BuildStatus status_to_proto();
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
class Store : public std::enable_shared_from_this<Store>, public Config {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -6,11 +6,13 @@
 | 
			
		|||
 | 
			
		||||
#include "libproto/worker.grpc.pb.h"
 | 
			
		||||
#include "libproto/worker.pb.h"
 | 
			
		||||
#include "libstore/derivations.hh"
 | 
			
		||||
#include "libstore/store-api.hh"
 | 
			
		||||
 | 
			
		||||
namespace nix::daemon {
 | 
			
		||||
 | 
			
		||||
using ::grpc::Status;
 | 
			
		||||
using ::nix::proto::BuildStatus;
 | 
			
		||||
using ::nix::proto::PathInfo;
 | 
			
		||||
using ::nix::proto::StorePath;
 | 
			
		||||
using ::nix::proto::StorePaths;
 | 
			
		||||
| 
						 | 
				
			
			@ -204,6 +206,27 @@ class WorkerServiceImpl final : public WorkerService::Service {
 | 
			
		|||
    return Status::OK;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  Status BuildDerivation(
 | 
			
		||||
      grpc::ServerContext* context,
 | 
			
		||||
      const nix::proto::BuildDerivationRequest* request,
 | 
			
		||||
      nix::proto::BuildDerivationResponse* response) override {
 | 
			
		||||
    auto drv_path = request->drv_path().path();
 | 
			
		||||
    store_->assertStorePath(drv_path);
 | 
			
		||||
    auto drv = BasicDerivation::from_proto(&request->derivation(), store_);
 | 
			
		||||
 | 
			
		||||
    auto build_mode = nix::build_mode_from(request->build_mode());
 | 
			
		||||
    if (!build_mode) {
 | 
			
		||||
      return Status(grpc::StatusCode::INTERNAL, "Invalid build mode");
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    auto res = store_->buildDerivation(drv_path, drv, *build_mode);
 | 
			
		||||
 | 
			
		||||
    response->set_status(res.status_to_proto());
 | 
			
		||||
    response->set_error_message(res.errorMsg);
 | 
			
		||||
 | 
			
		||||
    return Status::OK;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  Status QueryMissing(grpc::ServerContext* context, const StorePaths* request,
 | 
			
		||||
                      nix::proto::QueryMissingResponse* response) override {
 | 
			
		||||
    std::set<Path> targets;
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										17
									
								
								third_party/nix/src/proto/worker.proto
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										17
									
								
								third_party/nix/src/proto/worker.proto
									
										
									
									
										vendored
									
									
								
							| 
						 | 
				
			
			@ -288,10 +288,25 @@ message VerifyStoreResponse {
 | 
			
		|||
  bool errors = 1;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
message Derivation {
 | 
			
		||||
  message DerivationOutput {
 | 
			
		||||
    StorePath path = 1;
 | 
			
		||||
    string hash_algo = 2;
 | 
			
		||||
    bytes hash = 3;
 | 
			
		||||
  }
 | 
			
		||||
  map<string, DerivationOutput> outputs = 1;
 | 
			
		||||
  StorePaths input_sources = 2;
 | 
			
		||||
  string platform = 3;
 | 
			
		||||
  StorePath builder = 4;
 | 
			
		||||
  repeated string args = 5;
 | 
			
		||||
  map<string, string> env = 6;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
message BuildDerivationRequest {
 | 
			
		||||
  // Only used for informational purposes.
 | 
			
		||||
  StorePath drv_path = 1;
 | 
			
		||||
  BuildMode build_mode = 2;
 | 
			
		||||
  Derivation derivation = 2;
 | 
			
		||||
  BuildMode build_mode = 3;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
message BuildDerivationResponse {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue