maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

style(3p/nix): Reformat project in Google C++ style

Browse source 
Reformatted with:

    fd . -e hh -e cc | xargs clang-format -i
This commit is contained in:
Vincent Ambo 2020-05-17 16:31:57 +01:00
parent 65a1aae98c
commit 0f2cf531f7
175 changed files with 32126 additions and 34689 deletions
Download patch file Download diff file Expand all files Collapse all files

449
third_party/nix/src/build-remote/build-remote.cc vendored
View file

@ -1,266 +1,275 @@
#include <algorithm>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <algorithm>
#include <set>
#include <memory>
#include <tuple>
#include <iomanip> #include <iomanip>
#include <memory>
#include <set>
#include <tuple>
#if __APPLE__ #if __APPLE__
#include <sys/time.h> #include <sys/time.h>
#endif #endif
#include "machines.hh"
#include "shared.hh"
#include "pathlocks.hh"
#include "globals.hh"
#include "serialise.hh"
#include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "local-store.hh" #include "globals.hh"
#include "legacy.hh" #include "legacy.hh"
#include "local-store.hh"
#include "machines.hh"
#include "pathlocks.hh"
#include "serialise.hh"
#include "shared.hh"
#include "store-api.hh"
using namespace nix; using namespace nix;
using std::cin; using std::cin;
static void handleAlarm(int sig) { static void handleAlarm(int sig) {}
}
std::string escapeUri(std::string uri) std::string escapeUri(std::string uri) {
{ std::replace(uri.begin(), uri.end(), '/', '_');
std::replace(uri.begin(), uri.end(), '/', '_'); return uri;
return uri;
} }
static string currentLoad; static string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) {
{ return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot),
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); true);
} }
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
for (auto & feature : requiredFeatures) for (auto& feature : requiredFeatures)
if (!settings.systemFeatures.get().count(feature)) return false; if (!settings.systemFeatures.get().count(feature)) return false;
return true; return true;
} }
static int _main(int argc, char * * argv) static int _main(int argc, char** argv) {
{ {
{ logger = makeJSONLogger(*logger);
logger = makeJSONLogger(*logger);
/* Ensure we don't get any SSH passphrase or host key popups. */ /* Ensure we don't get any SSH passphrase or host key popups. */
unsetenv("DISPLAY"); unsetenv("DISPLAY");
unsetenv("SSH_ASKPASS"); unsetenv("SSH_ASKPASS");
if (argc != 2) if (argc != 2) throw UsageError("called without required arguments");
throw UsageError("called without required arguments");
verbosity = (Verbosity) std::stoll(argv[1]); verbosity = (Verbosity)std::stoll(argv[1]);
FdSource source(STDIN_FILENO); FdSource source(STDIN_FILENO);
/* Read the parent's settings. */ /* Read the parent's settings. */
while (readInt(source)) { while (readInt(source)) {
auto name = readString(source); auto name = readString(source);
auto value = readString(source); auto value = readString(source);
settings.set(name, value); settings.set(name, value);
}
settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
initPlugins();
auto store = openStore().cast<LocalStore>();
/* It would be more appropriate to use $XDG_RUNTIME_DIR, since
that gets cleared on reboot, but it wouldn't work on macOS. */
currentLoad = store->stateDir + "/current-load";
std::shared_ptr<Store> sshStore;
AutoCloseFD bestSlotLock;
auto machines = getMachines();
debug("got %d remote builders", machines.size());
if (machines.empty()) {
std::cerr << "# decline-permanently\n";
return 0;
}
string drvPath;
string storeUri;
while (true) {
try {
auto s = readString(source);
if (s != "try") return 0;
} catch (EndOfFile&) {
return 0;
}
auto amWilling = readInt(source);
auto neededSystem = readString(source);
source >> drvPath;
auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally =
amWilling &&
(neededSystem == settings.thisSystem ||
settings.extraPlatforms.get().count(neededSystem) > 0) &&
allSupportedLocally(requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
while (true) {
bestSlotLock = -1;
AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true);
lockFile(lock.get(), ltWrite, true);
bool rightType = false;
Machine* bestMachine = nullptr;
unsigned long long bestLoad = 0;
for (auto& m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
if (m.enabled &&
std::find(m.systemTypes.begin(), m.systemTypes.end(),
neededSystem) != m.systemTypes.end() &&
m.allSupported(requiredFeatures) &&
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
unsigned long long load = 0;
for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {
free = std::move(slotLock);
}
} else {
++load;
}
}
if (!free) {
continue;
}
bool best = false;
if (!bestSlotLock) {
best = true;
} else if (load / m.speedFactor <
bestLoad / bestMachine->speedFactor) {
best = true;
} else if (load / m.speedFactor ==
bestLoad / bestMachine->speedFactor) {
if (m.speedFactor > bestMachine->speedFactor) {
best = true;
} else if (m.speedFactor == bestMachine->speedFactor) {
if (load < bestLoad) {
best = true;
}
}
}
if (best) {
bestLoad = load;
bestSlotLock = std::move(free);
bestMachine = &m;
}
}
} }
settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work if (!bestSlotLock) {
if (rightType && !canBuildLocally)
initPlugins(); std::cerr << "# postpone\n";
else
auto store = openStore().cast<LocalStore>(); std::cerr << "# decline\n";
break;
/* It would be more appropriate to use $XDG_RUNTIME_DIR, since
that gets cleared on reboot, but it wouldn't work on macOS. */
currentLoad = store->stateDir + "/current-load";
std::shared_ptr<Store> sshStore;
AutoCloseFD bestSlotLock;
auto machines = getMachines();
debug("got %d remote builders", machines.size());
if (machines.empty()) {
std::cerr << "# decline-permanently\n";
return 0;
} }
string drvPath;
string storeUri;
while (true) {
try {
auto s = readString(source);
if (s != "try") return 0;
} catch (EndOfFile &) { return 0; }
auto amWilling = readInt(source);
auto neededSystem = readString(source);
source >> drvPath;
auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
&& allSupportedLocally(requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
while (true) {
bestSlotLock = -1;
AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true);
lockFile(lock.get(), ltWrite, true);
bool rightType = false;
Machine * bestMachine = nullptr;
unsigned long long bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
if (m.enabled && std::find(m.systemTypes.begin(),
m.systemTypes.end(),
neededSystem) != m.systemTypes.end() &&
m.allSupported(requiredFeatures) &&
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
unsigned long long load = 0;
for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {
free = std::move(slotLock);
}
} else {
++load;
}
}
if (!free) {
continue;
}
bool best = false;
if (!bestSlotLock) {
best = true;
} else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) {
best = true;
} else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) {
if (m.speedFactor > bestMachine->speedFactor) {
best = true;
} else if (m.speedFactor == bestMachine->speedFactor) {
if (load < bestLoad) {
best = true;
}
}
}
if (best) {
bestLoad = load;
bestSlotLock = std::move(free);
bestMachine = &m;
}
}
}
if (!bestSlotLock) {
if (rightType && !canBuildLocally)
std::cerr << "# postpone\n";
else
std::cerr << "# decline\n";
break;
}
#if __APPLE__ #if __APPLE__
futimes(bestSlotLock.get(), NULL); futimes(bestSlotLock.get(), NULL);
#else #else
futimens(bestSlotLock.get(), NULL); futimens(bestSlotLock.get(), NULL);
#endif #endif
lock = -1; lock = -1;
try { try {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("connecting to '%s'", bestMachine->storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); Store::Params storeParams;
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
if (bestMachine->sshKey != "")
storeParams["ssh-key"] = bestMachine->sshKey;
}
Store::Params storeParams; sshStore = openStore(bestMachine->storeUri, storeParams);
if (hasPrefix(bestMachine->storeUri, "ssh://")) { sshStore->connect();
storeParams["max-connections"] ="1"; storeUri = bestMachine->storeUri;
storeParams["log-fd"] = "4";
if (bestMachine->sshKey != "")
storeParams["ssh-key"] = bestMachine->sshKey;
}
sshStore = openStore(bestMachine->storeUri, storeParams); } catch (std::exception& e) {
sshStore->connect(); auto msg = chomp(drainFD(5, false));
storeUri = bestMachine->storeUri; printError("cannot build on '%s': %s%s", bestMachine->storeUri,
e.what(), (msg.empty() ? "" : ": " + msg));
} catch (std::exception & e) { bestMachine->enabled = false;
auto msg = chomp(drainFD(5, false)); continue;
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg));
bestMachine->enabled = false;
continue;
}
goto connected;
}
} }
connected: goto connected;
close(5); }
std::cerr << "# accept\n" << storeUri << "\n";
auto inputs = readStrings<PathSet>(source);
auto outputs = readStrings<PathSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
if (!lockFile(uploadLock.get(), ltWrite, true))
printError("somebody is hogging the upload lock for '%s', continuing...");
alarm(0);
signal(SIGALRM, old);
}
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
}
uploadLock = -1;
BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
drv.inputSrcs = inputs;
auto result = sshStore->buildDerivation(drvPath, drv);
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
PathSet missing;
for (auto & path : outputs)
if (!store->isValidPath(path)) missing.insert(path);
if (!missing.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
}
return 0;
} }
connected:
close(5);
std::cerr << "# accept\n" << storeUri << "\n";
auto inputs = readStrings<PathSet>(source);
auto outputs = readStrings<PathSet>(source);
AutoCloseFD uploadLock = openLockFile(
currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown,
fmt("waiting for the upload lock to '%s'", storeUri));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
if (!lockFile(uploadLock.get(), ltWrite, true))
printError(
"somebody is hogging the upload lock for '%s', continuing...");
alarm(0);
signal(SIGALRM, old);
}
auto substitute =
settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown,
fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs,
substitute);
}
uploadLock = -1;
BasicDerivation drv(
readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
drv.inputSrcs = inputs;
auto result = sshStore->buildDerivation(drvPath, drv);
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri,
result.errorMsg);
PathSet missing;
for (auto& path : outputs)
if (!store->isValidPath(path)) missing.insert(path);
if (!missing.empty()) {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("copying outputs from '%s'", storeUri));
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs,
NoSubstitute);
}
return 0;
}
} }
static RegisterLegacyCommand s1("build-remote", _main); static RegisterLegacyCommand s1("build-remote", _main);

149
third_party/nix/src/libexpr/attr-path.cc vendored
View file

@ -2,95 +2,92 @@
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
static Strings parseAttrPath(const string& s) {
static Strings parseAttrPath(const string & s) Strings res;
{ string cur;
Strings res; string::const_iterator i = s.begin();
string cur; while (i != s.end()) {
string::const_iterator i = s.begin(); if (*i == '.') {
while (i != s.end()) { res.push_back(cur);
if (*i == '.') { cur.clear();
res.push_back(cur); } else if (*i == '"') {
cur.clear(); ++i;
} else if (*i == '"') { while (1) {
++i; if (i == s.end())
while (1) { throw Error(format("missing closing quote in selection path '%1%'") %
if (i == s.end()) s);
throw Error(format("missing closing quote in selection path '%1%'") % s); if (*i == '"') break;
if (*i == '"') break; cur.push_back(*i++);
cur.push_back(*i++); }
} } else
} else cur.push_back(*i);
cur.push_back(*i); ++i;
++i; }
} if (!cur.empty()) res.push_back(cur);
if (!cur.empty()) res.push_back(cur); return res;
return res;
} }
Value* findAlongAttrPath(EvalState& state, const string& attrPath,
Bindings& autoArgs, Value& vIn) {
Strings tokens = parseAttrPath(attrPath);
Value * findAlongAttrPath(EvalState & state, const string & attrPath, Error attrError =
Bindings & autoArgs, Value & vIn) Error(format("attribute selection path '%1%' does not match expression") %
{ attrPath);
Strings tokens = parseAttrPath(attrPath);
Error attrError = Value* v = &vIn;
Error(format("attribute selection path '%1%' does not match expression") % attrPath);
Value * v = &vIn; for (auto& attr : tokens) {
/* Is i an index (integer) or a normal attribute name? */
enum { apAttr, apIndex } apType = apAttr;
unsigned int attrIndex;
if (string2Int(attr, attrIndex)) apType = apIndex;
for (auto & attr : tokens) { /* Evaluate the expression. */
Value* vNew = state.allocValue();
state.autoCallFunction(autoArgs, *v, *vNew);
v = vNew;
state.forceValue(*v);
/* Is i an index (integer) or a normal attribute name? */ /* It should evaluate to either a set or an expression,
enum { apAttr, apIndex } apType = apAttr; according to what is specified in the attrPath. */
unsigned int attrIndex;
if (string2Int(attr, attrIndex)) apType = apIndex;
/* Evaluate the expression. */ if (apType == apAttr) {
Value * vNew = state.allocValue(); if (v->type != tAttrs)
state.autoCallFunction(autoArgs, *v, *vNew); throw TypeError(format("the expression selected by the selection path "
v = vNew; "'%1%' should be a set but is %2%") %
state.forceValue(*v); attrPath % showType(*v));
/* It should evaluate to either a set or an expression, if (attr.empty())
according to what is specified in the attrPath. */ throw Error(format("empty attribute name in selection path '%1%'") %
attrPath);
if (apType == apAttr) {
if (v->type != tAttrs)
throw TypeError(
format("the expression selected by the selection path '%1%' should be a set but is %2%")
% attrPath % showType(*v));
if (attr.empty())
throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end())
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
v = &*a->value;
}
else if (apType == apIndex) {
if (!v->isList())
throw TypeError(
format("the expression selected by the selection path '%1%' should be a list but is %2%")
% attrPath % showType(*v));
if (attrIndex >= v->listSize())
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
v = v->listElems()[attrIndex];
}
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end())
throw Error(
format("attribute '%1%' in selection path '%2%' not found") % attr %
attrPath);
v = &*a->value;
} }
return v; else if (apType == apIndex) {
if (!v->isList())
throw TypeError(format("the expression selected by the selection path "
"'%1%' should be a list but is %2%") %
attrPath % showType(*v));
if (attrIndex >= v->listSize())
throw Error(
format("list index %1% in selection path '%2%' is out of range") %
attrIndex % attrPath);
v = v->listElems()[attrIndex];
}
}
return v;
} }
} // namespace nix
}

9
third_party/nix/src/libexpr/attr-path.hh vendored
View file

@ -1,13 +1,12 @@
#pragma once #pragma once
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
namespace nix { namespace nix {
Value * findAlongAttrPath(EvalState & state, const string & attrPath, Value* findAlongAttrPath(EvalState& state, const string& attrPath,
Bindings & autoArgs, Value & vIn); Bindings& autoArgs, Value& vIn);
} }

56
third_party/nix/src/libexpr/attr-set.cc vendored
View file

@ -1,52 +1,40 @@
#include "attr-set.hh" #include "attr-set.hh"
#include <algorithm>
#include "eval-inline.hh" #include "eval-inline.hh"
#include <algorithm>
namespace nix { namespace nix {
/* Allocate a new array of attributes for an attribute set with a specific /* Allocate a new array of attributes for an attribute set with a specific
capacity. The space is implicitly reserved after the Bindings capacity. The space is implicitly reserved after the Bindings
structure. */ structure. */
Bindings * EvalState::allocBindings(size_t capacity) Bindings* EvalState::allocBindings(size_t capacity) {
{ if (capacity > std::numeric_limits<Bindings::size_t>::max())
if (capacity > std::numeric_limits<Bindings::size_t>::max()) throw Error("attribute set of size %d is too big", capacity);
throw Error("attribute set of size %d is too big", capacity); return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity))
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); Bindings((Bindings::size_t)capacity);
} }
void EvalState::mkAttrs(Value& v, size_t capacity) {
void EvalState::mkAttrs(Value & v, size_t capacity) if (capacity == 0) {
{ v = vEmptySet;
if (capacity == 0) { return;
v = vEmptySet; }
return; clearValue(v);
} v.type = tAttrs;
clearValue(v); v.attrs = allocBindings(capacity);
v.type = tAttrs; nrAttrsets++;
v.attrs = allocBindings(capacity); nrAttrsInAttrsets += capacity;
nrAttrsets++;
nrAttrsInAttrsets += capacity;
} }
/* Create a new attribute named 'name' on an existing attribute set stored /* Create a new attribute named 'name' on an existing attribute set stored
in 'vAttrs' and return the newly allocated Value which is associated with in 'vAttrs' and return the newly allocated Value which is associated with
this attribute. */ this attribute. */
Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) {
{ Value* v = allocValue();
Value * v = allocValue(); vAttrs.attrs->push_back(Attr(name, v));
vAttrs.attrs->push_back(Attr(name, v)); return v;
return v;
} }
void Bindings::sort() { std::sort(begin(), end()); }
void Bindings::sort() } // namespace nix
{
std::sort(begin(), end());
}
}

111
third_party/nix/src/libexpr/attr-set.hh vendored
View file

@ -1,95 +1,80 @@
#pragma once #pragma once
#include <algorithm>
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include <algorithm>
namespace nix { namespace nix {
class EvalState; class EvalState;
struct Value; struct Value;
/* Map one attribute name to its value. */ /* Map one attribute name to its value. */
struct Attr struct Attr {
{ Symbol name;
Symbol name; Value* value;
Value * value; Pos* pos;
Pos * pos; Attr(Symbol name, Value* value, Pos* pos = &noPos)
Attr(Symbol name, Value * value, Pos * pos = &noPos) : name(name), value(value), pos(pos){};
: name(name), value(value), pos(pos) { }; Attr() : pos(&noPos){};
Attr() : pos(&noPos) { }; bool operator<(const Attr& a) const { return name < a.name; }
bool operator < (const Attr & a) const
{
return name < a.name;
}
}; };
/* Bindings contains all the attributes of an attribute set. It is defined /* Bindings contains all the attributes of an attribute set. It is defined
by its size and its capacity, the capacity being the number of Attr by its size and its capacity, the capacity being the number of Attr
elements allocated after this structure, while the size corresponds to elements allocated after this structure, while the size corresponds to
the number of elements already inserted in this structure. */ the number of elements already inserted in this structure. */
class Bindings class Bindings {
{ public:
public: typedef uint32_t size_t;
typedef uint32_t size_t;
private: private:
size_t size_, capacity_; size_t size_, capacity_;
Attr attrs[0]; Attr attrs[0];
Bindings(size_t capacity) : size_(0), capacity_(capacity) { } Bindings(size_t capacity) : size_(0), capacity_(capacity) {}
Bindings(const Bindings & bindings) = delete; Bindings(const Bindings& bindings) = delete;
public: public:
size_t size() const { return size_; } size_t size() const { return size_; }
bool empty() const { return !size_; } bool empty() const { return !size_; }
typedef Attr * iterator; typedef Attr* iterator;
void push_back(const Attr & attr) void push_back(const Attr& attr) {
{ assert(size_ < capacity_);
assert(size_ < capacity_); attrs[size_++] = attr;
attrs[size_++] = attr; }
}
iterator find(const Symbol & name) iterator find(const Symbol& name) {
{ Attr key(name, 0);
Attr key(name, 0); iterator i = std::lower_bound(begin(), end(), key);
iterator i = std::lower_bound(begin(), end(), key); if (i != end() && i->name == name) return i;
if (i != end() && i->name == name) return i; return end();
return end(); }
}
iterator begin() { return &attrs[0]; } iterator begin() { return &attrs[0]; }
iterator end() { return &attrs[size_]; } iterator end() { return &attrs[size_]; }
Attr & operator[](size_t pos) Attr& operator[](size_t pos) { return attrs[pos]; }
{
return attrs[pos];
}
void sort(); void sort();
size_t capacity() { return capacity_; } size_t capacity() { return capacity_; }
/* Returns the attributes in lexicographically sorted order. */ /* Returns the attributes in lexicographically sorted order. */
std::vector<const Attr *> lexicographicOrder() const std::vector<const Attr*> lexicographicOrder() const {
{ std::vector<const Attr*> res;
std::vector<const Attr *> res; res.reserve(size_);
res.reserve(size_); for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]);
for (size_t n = 0; n < size_; n++) std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) {
res.emplace_back(&attrs[n]); return (const string&)a->name < (const string&)b->name;
std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { });
return (const string &) a->name < (const string &) b->name; return res;
}); }
return res;
}
friend class EvalState; friend class EvalState;
}; };
} // namespace nix
}

92
third_party/nix/src/libexpr/common-eval-args.cc vendored
View file

@ -1,59 +1,61 @@
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "shared.hh"
#include "download.hh" #include "download.hh"
#include "util.hh"
#include "eval.hh" #include "eval.hh"
#include "shared.hh"
#include "util.hh"
namespace nix { namespace nix {
MixEvalArgs::MixEvalArgs() MixEvalArgs::MixEvalArgs() {
{ mkFlag()
mkFlag() .longName("arg")
.longName("arg") .description("argument to be passed to Nix functions")
.description("argument to be passed to Nix functions") .labels({"name", "expr"})
.labels({"name", "expr"}) .handler(
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); [&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
mkFlag() mkFlag()
.longName("argstr") .longName("argstr")
.description("string-valued argument to be passed to Nix functions") .description("string-valued argument to be passed to Nix functions")
.labels({"name", "string"}) .labels({"name", "string"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); .handler(
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
mkFlag() mkFlag()
.shortName('I') .shortName('I')
.longName("include") .longName("include")
.description("add a path to the list of locations used to look up <...> file names") .description(
.label("path") "add a path to the list of locations used to look up <...> file "
.handler([&](std::string s) { searchPath.push_back(s); }); "names")
.label("path")
.handler([&](std::string s) { searchPath.push_back(s); });
} }
Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Bindings* MixEvalArgs::getAutoArgs(EvalState& state) {
{ Bindings* res = state.allocBindings(autoArgs.size());
Bindings * res = state.allocBindings(autoArgs.size()); for (auto& i : autoArgs) {
for (auto & i : autoArgs) { Value* v = state.allocValue();
Value * v = state.allocValue(); if (i.second[0] == 'E')
if (i.second[0] == 'E') state.mkThunk_(
state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); *v, state.parseExprFromString(string(i.second, 1), absPath(".")));
else else
mkString(*v, string(i.second, 1)); mkString(*v, string(i.second, 1));
res->push_back(Attr(state.symbols.create(i.first), v)); res->push_back(Attr(state.symbols.create(i.first), v));
} }
res->sort(); res->sort();
return res; return res;
} }
Path lookupFileArg(EvalState & state, string s) Path lookupFileArg(EvalState& state, string s) {
{ if (isUri(s)) {
if (isUri(s)) { CachedDownloadRequest request(s);
CachedDownloadRequest request(s); request.unpack = true;
request.unpack = true; return getDownloader()->downloadCached(state.store, request).path;
return getDownloader()->downloadCached(state.store, request).path; } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2);
Path p = s.substr(1, s.size() - 2); return state.findFile(p);
return state.findFile(p); } else
} else return absPath(s);
return absPath(s);
} }
} } // namespace nix

18
third_party/nix/src/libexpr/common-eval-args.hh vendored
View file

@ -8,19 +8,17 @@ class Store;
class EvalState; class EvalState;
class Bindings; class Bindings;
struct MixEvalArgs : virtual Args struct MixEvalArgs : virtual Args {
{ MixEvalArgs();
MixEvalArgs();
Bindings * getAutoArgs(EvalState & state); Bindings* getAutoArgs(EvalState& state);
Strings searchPath; Strings searchPath;
private: private:
std::map<std::string, std::string> autoArgs;
std::map<std::string, std::string> autoArgs;
}; };
Path lookupFileArg(EvalState & state, string s); Path lookupFileArg(EvalState& state, string s);
} } // namespace nix

117
third_party/nix/src/libexpr/eval-inline.hh vendored
View file

@ -2,94 +2,81 @@
#include "eval.hh" #include "eval.hh"
#define LocalNoInline(f) static f __attribute__((noinline)); f #define LocalNoInline(f) \
#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f static f __attribute__((noinline)); \
f
#define LocalNoInlineNoReturn(f) \
static f __attribute__((noinline, noreturn)); \
f
namespace nix { namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) {
{ throw EvalError(format(s) % pos);
throw EvalError(format(s) % pos);
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) {
{ throw TypeError(format(s) % showType(v));
throw TypeError(format(s) % showType(v));
} }
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v,
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) const Pos& pos)) {
{ throw TypeError(format(s) % showType(v) % pos);
throw TypeError(format(s) % showType(v) % pos);
} }
void EvalState::forceValue(Value& v, const Pos& pos) {
void EvalState::forceValue(Value & v, const Pos & pos) if (v.type == tThunk) {
{ Env* env = v.thunk.env;
if (v.type == tThunk) { Expr* expr = v.thunk.expr;
Env * env = v.thunk.env; try {
Expr * expr = v.thunk.expr; v.type = tBlackhole;
try { // checkInterrupt();
v.type = tBlackhole; expr->eval(*this, *env, v);
//checkInterrupt(); } catch (...) {
expr->eval(*this, *env, v); v.type = tThunk;
} catch (...) { v.thunk.env = env;
v.type = tThunk; v.thunk.expr = expr;
v.thunk.env = env; throw;
v.thunk.expr = expr;
throw;
}
} }
else if (v.type == tApp) } else if (v.type == tApp)
callFunction(*v.app.left, *v.app.right, v, noPos); callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole) else if (v.type == tBlackhole)
throwEvalError("infinite recursion encountered, at %1%", pos); throwEvalError("infinite recursion encountered, at %1%", pos);
} }
inline void EvalState::forceAttrs(Value& v) {
inline void EvalState::forceAttrs(Value & v) forceValue(v);
{ if (v.type != tAttrs)
forceValue(v); throwTypeError("value is %1% while a set was expected", v);
if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected", v);
} }
inline void EvalState::forceAttrs(Value& v, const Pos& pos) {
inline void EvalState::forceAttrs(Value & v, const Pos & pos) forceValue(v);
{ if (v.type != tAttrs)
forceValue(v); throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
} }
inline void EvalState::forceList(Value& v) {
inline void EvalState::forceList(Value & v) forceValue(v);
{ if (!v.isList()) throwTypeError("value is %1% while a list was expected", v);
forceValue(v);
if (!v.isList())
throwTypeError("value is %1% while a list was expected", v);
} }
inline void EvalState::forceList(Value& v, const Pos& pos) {
inline void EvalState::forceList(Value & v, const Pos & pos) forceValue(v);
{ if (!v.isList())
forceValue(v); throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
if (!v.isList())
throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
} }
/* Note: Various places expect the allocated memory to be zeroed. */ /* Note: Various places expect the allocated memory to be zeroed. */
inline void * allocBytes(size_t n) inline void* allocBytes(size_t n) {
{ void* p;
void * p;
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
p = GC_MALLOC(n); p = GC_MALLOC(n);
#else #else
p = calloc(n, 1); p = calloc(n, 1);
#endif #endif
if (!p) throw std::bad_alloc(); if (!p) throw std::bad_alloc();
return p; return p;
} }
} // namespace nix
}

3261
third_party/nix/src/libexpr/eval.cc vendored
View file

File diff suppressed because it is too large Load diff

478
third_party/nix/src/libexpr/eval.hh vendored
View file

@ -1,363 +1,357 @@
#pragma once #pragma once
#include "attr-set.hh"
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "hash.hh"
#include "config.hh"
#include <map> #include <map>
#include <optional> #include <optional>
#include <unordered_map> #include <unordered_map>
#include "attr-set.hh"
#include "config.hh"
#include "hash.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "value.hh"
namespace nix { namespace nix {
class Store; class Store;
class EvalState; class EvalState;
enum RepairFlag : bool; enum RepairFlag : bool;
typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args,
Value& v);
typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); struct PrimOp {
PrimOpFun fun;
size_t arity;
struct PrimOp Symbol name;
{ PrimOp(PrimOpFun fun, size_t arity, Symbol name)
PrimOpFun fun; : fun(fun), arity(arity), name(name) {}
size_t arity;
Symbol name;
PrimOp(PrimOpFun fun, size_t arity, Symbol name)
: fun(fun), arity(arity), name(name) { }
}; };
struct Env {
struct Env Env* up;
{ unsigned short size; // used by ‘valueSize’
Env * up; unsigned short prevWith : 14; // nr of levels up to next `with' environment
unsigned short size; // used by ‘valueSize’ enum { Plain = 0, HasWithExpr, HasWithAttrs } type : 2;
unsigned short prevWith:14; // nr of levels up to next `with' environment Value* values[0];
enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
Value * values[0];
}; };
Value& mkString(Value& v, const string& s, const PathSet& context = PathSet());
Value & mkString(Value & v, const string & s, const PathSet & context = PathSet()); void copyContext(const Value& v, PathSet& context);
void copyContext(const Value & v, PathSet & context);
/* Cache for calls to addToStore(); maps source paths to the store /* Cache for calls to addToStore(); maps source paths to the store
paths. */ paths. */
typedef std::map<Path, Path> SrcToStore; typedef std::map<Path, Path> SrcToStore;
std::ostream& operator<<(std::ostream& str, const Value& v);
std::ostream & operator << (std::ostream & str, const Value & v);
typedef std::pair<std::string, std::string> SearchPathElem; typedef std::pair<std::string, std::string> SearchPathElem;
typedef std::list<SearchPathElem> SearchPath; typedef std::list<SearchPathElem> SearchPath;
/* Initialise the Boehm GC, if applicable. */ /* Initialise the Boehm GC, if applicable. */
void initGC(); void initGC();
class EvalState {
public:
SymbolTable symbols;
class EvalState const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem,
{ sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn,
public: sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
SymbolTable symbols; sOutputHash, sOutputHashAlgo, sOutputHashMode;
Symbol sDerivationNix;
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, /* If set, force copying files to the Nix store even if they
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, already exist there. */
sFile, sLine, sColumn, sFunctor, sToString, RepairFlag repair;
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode;
Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they /* The allowed filesystem paths in restricted or pure evaluation
already exist there. */ mode. */
RepairFlag repair; std::optional<PathSet> allowedPaths;
/* The allowed filesystem paths in restricted or pure evaluation Value vEmptySet;
mode. */
std::optional<PathSet> allowedPaths;
Value vEmptySet; const ref<Store> store;
const ref<Store> store; private:
SrcToStore srcToStore;
private: /* A cache from path names to parse trees. */
SrcToStore srcToStore;
/* A cache from path names to parse trees. */
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache; typedef std::map<Path, Expr*, std::less<Path>,
traceable_allocator<std::pair<const Path, Expr*>>>
FileParseCache;
#else #else
typedef std::map<Path, Expr *> FileParseCache; typedef std::map<Path, Expr*> FileParseCache;
#endif #endif
FileParseCache fileParseCache; FileParseCache fileParseCache;
/* A cache from path names to values. */ /* A cache from path names to values. */
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache; typedef std::map<Path, Value, std::less<Path>,
traceable_allocator<std::pair<const Path, Value>>>
FileEvalCache;
#else #else
typedef std::map<Path, Value> FileEvalCache; typedef std::map<Path, Value> FileEvalCache;
#endif #endif
FileEvalCache fileEvalCache; FileEvalCache fileEvalCache;
SearchPath searchPath; SearchPath searchPath;
std::map<std::string, std::pair<bool, std::string>> searchPathResolved; std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
/* Cache used by checkSourcePath(). */ /* Cache used by checkSourcePath(). */
std::unordered_map<Path, Path> resolvedPaths; std::unordered_map<Path, Path> resolvedPaths;
public: public:
EvalState(const Strings& _searchPath, ref<Store> store);
~EvalState();
EvalState(const Strings & _searchPath, ref<Store> store); void addToSearchPath(const string& s);
~EvalState();
void addToSearchPath(const string & s); SearchPath getSearchPath() { return searchPath; }
SearchPath getSearchPath() { return searchPath; } Path checkSourcePath(const Path& path);
Path checkSourcePath(const Path & path); void checkURI(const std::string& uri);
void checkURI(const std::string & uri); /* When using a diverted store and 'path' is in the Nix store, map
'path' to the diverted location (e.g. /nix/store/foo is mapped
to /home/alice/my-nix/nix/store/foo). However, this is only
done if the context is not empty, since otherwise we're
probably trying to read from the actual /nix/store. This is
intended to distinguish between import-from-derivation and
sources stored in the actual /nix/store. */
Path toRealPath(const Path& path, const PathSet& context);
/* When using a diverted store and 'path' is in the Nix store, map /* Parse a Nix expression from the specified file. */
'path' to the diverted location (e.g. /nix/store/foo is mapped Expr* parseExprFromFile(const Path& path);
to /home/alice/my-nix/nix/store/foo). However, this is only Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv);
done if the context is not empty, since otherwise we're
probably trying to read from the actual /nix/store. This is
intended to distinguish between import-from-derivation and
sources stored in the actual /nix/store. */
Path toRealPath(const Path & path, const PathSet & context);
/* Parse a Nix expression from the specified file. */ /* Parse a Nix expression from the specified string. */
Expr * parseExprFromFile(const Path & path); Expr* parseExprFromString(const string& s, const Path& basePath,
Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); StaticEnv& staticEnv);
Expr* parseExprFromString(const string& s, const Path& basePath);
/* Parse a Nix expression from the specified string. */ Expr* parseStdin();
Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
Expr * parseExprFromString(const string & s, const Path & basePath);
Expr * parseStdin(); /* Evaluate an expression read from the given file to normal
form. */
void evalFile(const Path& path, Value& v);
/* Evaluate an expression read from the given file to normal void resetFileCache();
form. */
void evalFile(const Path & path, Value & v);
void resetFileCache(); /* Look up a file in the search path. */
Path findFile(const string& path);
Path findFile(SearchPath& searchPath, const string& path,
const Pos& pos = noPos);
/* Look up a file in the search path. */ /* If the specified search path element is a URI, download it. */
Path findFile(const string & path); std::pair<bool, std::string> resolveSearchPathElem(
Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); const SearchPathElem& elem);
/* If the specified search path element is a URI, download it. */ /* Evaluate an expression to normal form, storing the result in
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); value `v'. */
void eval(Expr* e, Value& v);
/* Evaluate an expression to normal form, storing the result in /* Evaluation the expression, then verify that it has the expected
value `v'. */ type. */
void eval(Expr * e, Value & v); inline bool evalBool(Env& env, Expr* e);
inline bool evalBool(Env& env, Expr* e, const Pos& pos);
inline void evalAttrs(Env& env, Expr* e, Value& v);
/* Evaluation the expression, then verify that it has the expected /* If `v' is a thunk, enter it and overwrite `v' with the result
type. */ of the evaluation of the thunk. If `v' is a delayed function
inline bool evalBool(Env & env, Expr * e); application, call the function and overwrite `v' with the
inline bool evalBool(Env & env, Expr * e, const Pos & pos); result. Otherwise, this is a no-op. */
inline void evalAttrs(Env & env, Expr * e, Value & v); inline void forceValue(Value& v, const Pos& pos = noPos);
/* If `v' is a thunk, enter it and overwrite `v' with the result /* Force a value, then recursively force list elements and
of the evaluation of the thunk. If `v' is a delayed function attributes. */
application, call the function and overwrite `v' with the void forceValueDeep(Value& v);
result. Otherwise, this is a no-op. */
inline void forceValue(Value & v, const Pos & pos = noPos);
/* Force a value, then recursively force list elements and /* Force `v', and then verify that it has the expected type. */
attributes. */ NixInt forceInt(Value& v, const Pos& pos);
void forceValueDeep(Value & v); NixFloat forceFloat(Value& v, const Pos& pos);
bool forceBool(Value& v, const Pos& pos);
inline void forceAttrs(Value& v);
inline void forceAttrs(Value& v, const Pos& pos);
inline void forceList(Value& v);
inline void forceList(Value& v, const Pos& pos);
void forceFunction(Value& v, const Pos& pos); // either lambda or primop
string forceString(Value& v, const Pos& pos = noPos);
string forceString(Value& v, PathSet& context, const Pos& pos = noPos);
string forceStringNoCtx(Value& v, const Pos& pos = noPos);
/* Force `v', and then verify that it has the expected type. */ /* Return true iff the value `v' denotes a derivation (i.e. a
NixInt forceInt(Value & v, const Pos & pos); set with attribute `type = "derivation"'). */
NixFloat forceFloat(Value & v, const Pos & pos); bool isDerivation(Value& v);
bool forceBool(Value & v, const Pos & pos);
inline void forceAttrs(Value & v);
inline void forceAttrs(Value & v, const Pos & pos);
inline void forceList(Value & v);
inline void forceList(Value & v, const Pos & pos);
void forceFunction(Value & v, const Pos & pos); // either lambda or primop
string forceString(Value & v, const Pos & pos = noPos);
string forceString(Value & v, PathSet & context, const Pos & pos = noPos);
string forceStringNoCtx(Value & v, const Pos & pos = noPos);
/* Return true iff the value `v' denotes a derivation (i.e. a std::optional<string> tryAttrsToString(const Pos& pos, Value& v,
set with attribute `type = "derivation"'). */ PathSet& context,
bool isDerivation(Value & v); bool coerceMore = false,
bool copyToStore = true);
std::optional<string> tryAttrsToString(const Pos & pos, Value & v, /* String coercion. Converts strings, paths and derivations to a
PathSet & context, bool coerceMore = false, bool copyToStore = true); string. If `coerceMore' is set, also converts nulls, integers,
booleans and lists to a string. If `copyToStore' is set,
referenced paths are copied to the Nix store as a side effect. */
string coerceToString(const Pos& pos, Value& v, PathSet& context,
bool coerceMore = false, bool copyToStore = true);
/* String coercion. Converts strings, paths and derivations to a string copyPathToStore(PathSet& context, const Path& path);
string. If `coerceMore' is set, also converts nulls, integers,
booleans and lists to a string. If `copyToStore' is set,
referenced paths are copied to the Nix store as a side effect. */
string coerceToString(const Pos & pos, Value & v, PathSet & context,
bool coerceMore = false, bool copyToStore = true);
string copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a
path. The result is guaranteed to be a canonicalised, absolute
path. Nothing is copied to the store. */
Path coerceToPath(const Pos& pos, Value& v, PathSet& context);
/* Path coercion. Converts strings, paths and derivations to a public:
path. The result is guaranteed to be a canonicalised, absolute /* The base environment, containing the builtin functions and
path. Nothing is copied to the store. */ values. */
Path coerceToPath(const Pos & pos, Value & v, PathSet & context); Env& baseEnv;
public: /* The same, but used during parsing to resolve variables. */
StaticEnv staticBaseEnv; // !!! should be private
/* The base environment, containing the builtin functions and private:
values. */ unsigned int baseEnvDispl = 0;
Env & baseEnv;
/* The same, but used during parsing to resolve variables. */ void createBaseEnv();
StaticEnv staticBaseEnv; // !!! should be private
private: Value* addConstant(const string& name, Value& v);
unsigned int baseEnvDispl = 0; Value* addPrimOp(const string& name, size_t arity, PrimOpFun primOp);
void createBaseEnv(); public:
Value& getBuiltin(const string& name);
Value * addConstant(const string & name, Value & v); private:
inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval);
Value * addPrimOp(const string & name, friend struct ExprVar;
size_t arity, PrimOpFun primOp); friend struct ExprAttrs;
friend struct ExprLet;
public: Expr* parse(const char* text, const Path& path, const Path& basePath,
StaticEnv& staticEnv);
Value & getBuiltin(const string & name); public:
/* Do a deep equality test between two values. That is, list
elements and attributes are compared recursively. */
bool eqValues(Value& v1, Value& v2);
private: bool isFunctor(Value& fun);
inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); void callFunction(Value& fun, Value& arg, Value& v, const Pos& pos);
void callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos);
friend struct ExprVar; /* Automatically call a function for which each argument has a
friend struct ExprAttrs; default value or has a binding in the `args' map. */
friend struct ExprLet; void autoCallFunction(Bindings& args, Value& fun, Value& res);
Expr * parse(const char * text, const Path & path, /* Allocation primitives. */
const Path & basePath, StaticEnv & staticEnv); Value* allocValue();
Env& allocEnv(size_t size);
public: Value* allocAttr(Value& vAttrs, const Symbol& name);
/* Do a deep equality test between two values. That is, list Bindings* allocBindings(size_t capacity);
elements and attributes are compared recursively. */
bool eqValues(Value & v1, Value & v2);
bool isFunctor(Value & fun); void mkList(Value& v, size_t length);
void mkAttrs(Value& v, size_t capacity);
void mkThunk_(Value& v, Expr* expr);
void mkPos(Value& v, Pos* pos);
void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); void concatLists(Value& v, size_t nrLists, Value** lists, const Pos& pos);
void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos);
/* Automatically call a function for which each argument has a /* Print statistics. */
default value or has a binding in the `args' map. */ void printStats();
void autoCallFunction(Bindings & args, Value & fun, Value & res);
/* Allocation primitives. */ void realiseContext(const PathSet& context);
Value * allocValue();
Env & allocEnv(size_t size);
Value * allocAttr(Value & vAttrs, const Symbol & name); private:
unsigned long nrEnvs = 0;
unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0;
unsigned long nrListElems = 0;
unsigned long nrAttrsets = 0;
unsigned long nrAttrsInAttrsets = 0;
unsigned long nrOpUpdates = 0;
unsigned long nrOpUpdateValuesCopied = 0;
unsigned long nrListConcats = 0;
unsigned long nrPrimOpCalls = 0;
unsigned long nrFunctionCalls = 0;
Bindings * allocBindings(size_t capacity); bool countCalls;
void mkList(Value & v, size_t length); typedef std::map<Symbol, size_t> PrimOpCalls;
void mkAttrs(Value & v, size_t capacity); PrimOpCalls primOpCalls;
void mkThunk_(Value & v, Expr * expr);
void mkPos(Value & v, Pos * pos);
void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); typedef std::map<ExprLambda*, size_t> FunctionCalls;
FunctionCalls functionCalls;
/* Print statistics. */ void incrFunctionCall(ExprLambda* fun);
void printStats();
void realiseContext(const PathSet & context); typedef std::map<Pos, size_t> AttrSelects;
AttrSelects attrSelects;
private: friend struct ExprOpUpdate;
friend struct ExprOpConcatLists;
unsigned long nrEnvs = 0; friend struct ExprSelect;
unsigned long nrValuesInEnvs = 0; friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args,
unsigned long nrValues = 0; Value& v);
unsigned long nrListElems = 0;
unsigned long nrAttrsets = 0;
unsigned long nrAttrsInAttrsets = 0;
unsigned long nrOpUpdates = 0;
unsigned long nrOpUpdateValuesCopied = 0;
unsigned long nrListConcats = 0;
unsigned long nrPrimOpCalls = 0;
unsigned long nrFunctionCalls = 0;
bool countCalls;
typedef std::map<Symbol, size_t> PrimOpCalls;
PrimOpCalls primOpCalls;
typedef std::map<ExprLambda *, size_t> FunctionCalls;
FunctionCalls functionCalls;
void incrFunctionCall(ExprLambda * fun);
typedef std::map<Pos, size_t> AttrSelects;
AttrSelects attrSelects;
friend struct ExprOpUpdate;
friend struct ExprOpConcatLists;
friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
}; };
/* Return a string representing the type of the value `v'. */ /* Return a string representing the type of the value `v'. */
string showType(const Value & v); string showType(const Value& v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path, /* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */ name>. */
std::pair<string, string> decodeContext(const string & s); std::pair<string, string> decodeContext(const string& s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);
struct InvalidPathError : EvalError struct InvalidPathError : EvalError {
{ Path path;
Path path; InvalidPathError(const Path& path);
InvalidPathError(const Path & path);
#ifdef EXCEPTION_NEEDS_THROW_SPEC #ifdef EXCEPTION_NEEDS_THROW_SPEC
~InvalidPathError() throw () { }; ~InvalidPathError() throw(){};
#endif #endif
}; };
struct EvalSettings : Config struct EvalSettings : Config {
{ Setting<bool> enableNativeCode{this, false,
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."}; "Whether builtin functions that allow "
"executing native code should be enabled."};
Setting<bool> restrictEval{this, false, "restrict-eval", Setting<bool> restrictEval{
"Whether to restrict file system access to paths in $NIX_PATH, " this, false, "restrict-eval",
"and network access to the URI prefixes listed in 'allowed-uris'."}; "Whether to restrict file system access to paths in $NIX_PATH, "
"and network access to the URI prefixes listed in 'allowed-uris'."};
Setting<bool> pureEval{this, false, "pure-eval", Setting<bool> pureEval{this, false, "pure-eval",
"Whether to restrict file system and network access to files specified by cryptographic hash."}; "Whether to restrict file system and network access "
"to files specified by cryptographic hash."};
Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", Setting<bool> enableImportFromDerivation{
"Whether the evaluator allows importing the result of a derivation."}; this, true, "allow-import-from-derivation",
"Whether the evaluator allows importing the result of a derivation."};
Setting<Strings> allowedUris{this, {}, "allowed-uris", Setting<Strings> allowedUris{
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; this,
{},
"allowed-uris",
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit "
"are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; "Emit log messages for each function entry "
"and exit at the 'vomit' log level (-vvvv)"};
}; };
extern EvalSettings evalSettings; extern EvalSettings evalSettings;
} } // namespace nix

16
third_party/nix/src/libexpr/function-trace.cc vendored
View file

@ -2,16 +2,16 @@
namespace nix { namespace nix {
FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) { FunctionCallTrace::FunctionCallTrace(const Pos& pos) : pos(pos) {
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
} }
FunctionCallTrace::~FunctionCallTrace() { FunctionCallTrace::~FunctionCallTrace() {
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
} }
} } // namespace nix

14
third_party/nix/src/libexpr/function-trace.hh vendored
View file

@ -1,15 +1,13 @@
#pragma once #pragma once
#include "eval.hh"
#include <chrono> #include <chrono>
#include "eval.hh"
namespace nix { namespace nix {
struct FunctionCallTrace struct FunctionCallTrace {
{ const Pos& pos;
const Pos & pos; FunctionCallTrace(const Pos& pos);
FunctionCallTrace(const Pos & pos); ~FunctionCallTrace();
~FunctionCallTrace();
}; };
} } // namespace nix

589
third_party/nix/src/libexpr/get-drvs.cc vendored
View file

@ -1,380 +1,351 @@
#include "get-drvs.hh" #include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include <cstring> #include <cstring>
#include <regex> #include <regex>
#include "derivations.hh"
#include "eval-inline.hh"
#include "util.hh"
namespace nix { namespace nix {
DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs)
: state(&state), attrs(attrs), attrPath(attrPath) {}
DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState& state, ref<Store> store,
: state(&state), attrs(attrs), attrPath(attrPath) const std::string& drvPathWithOutputs)
{ : state(&state), attrs(nullptr), attrPath("") {
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
drvPath = spec.first;
auto drv = store->derivationFromPath(drvPath);
name = storePathToName(drvPath);
if (spec.second.size() > 1)
throw Error(
"building more than one derivation output is not supported, in '%s'",
drvPathWithOutputs);
outputName = spec.second.empty() ? get(drv.env, "outputName", "out")
: *spec.second.begin();
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", drvPath,
outputName);
outPath = i->second.path;
} }
string DrvInfo::queryName() const {
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) if (name == "" && attrs) {
: state(&state), attrs(nullptr), attrPath("") auto i = attrs->find(state->sName);
{ if (i == attrs->end()) throw TypeError("derivation name missing");
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); name = state->forceStringNoCtx(*i->value);
}
drvPath = spec.first; return name;
auto drv = store->derivationFromPath(drvPath);
name = storePathToName(drvPath);
if (spec.second.size() > 1)
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
outputName =
spec.second.empty()
? get(drv.env, "outputName", "out")
: *spec.second.begin();
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
outPath = i->second.path;
} }
string DrvInfo::querySystem() const {
string DrvInfo::queryName() const if (system == "" && attrs) {
{ auto i = attrs->find(state->sSystem);
if (name == "" && attrs) { system = i == attrs->end() ? "unknown"
auto i = attrs->find(state->sName); : state->forceStringNoCtx(*i->value, *i->pos);
if (i == attrs->end()) throw TypeError("derivation name missing"); }
name = state->forceStringNoCtx(*i->value); return system;
}
return name;
} }
string DrvInfo::queryDrvPath() const {
string DrvInfo::querySystem() const if (drvPath == "" && attrs) {
{ Bindings::iterator i = attrs->find(state->sDrvPath);
if (system == "" && attrs) { PathSet context;
auto i = attrs->find(state->sSystem); drvPath = i != attrs->end()
system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); ? state->coerceToPath(*i->pos, *i->value, context)
} : "";
return system; }
return drvPath;
} }
string DrvInfo::queryOutPath() const {
if (outPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context;
outPath = i != attrs->end()
? state->coerceToPath(*i->pos, *i->value, context)
: "";
}
return outPath;
}
string DrvInfo::queryDrvPath() const DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) {
{ if (outputs.empty()) {
if (drvPath == "" && attrs) { /* Get the ‘outputs’ list. */
Bindings::iterator i = attrs->find(state->sDrvPath); Bindings::iterator i;
if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
state->forceList(*i->value, *i->pos);
/* For each output... */
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
/* Evaluate the corresponding set. */
string name =
state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
Bindings::iterator out = attrs->find(state->symbols.create(name));
if (out == attrs->end()) continue; // FIXME: throw error?
state->forceAttrs(*out->value);
/* And evaluate its ‘outPath’ attribute. */
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
if (outPath == out->value->attrs->end())
continue; // FIXME: throw error?
PathSet context; PathSet context;
drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; outputs[name] =
} state->coerceToPath(*outPath->pos, *outPath->value, context);
return drvPath; }
} else
outputs["out"] = queryOutPath();
}
if (!onlyOutputsToInstall || !attrs) return outputs;
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
const Value* outTI = queryMeta("outputsToInstall");
if (!outTI) return outputs;
const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
/* ^ this shows during `nix-env -i` right under the bad derivation */
if (!outTI->isList()) throw errMsg;
Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize();
++i) {
if ((*i)->type != tString) throw errMsg;
auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg;
result.insert(*out);
}
return result;
} }
string DrvInfo::queryOutputName() const {
string DrvInfo::queryOutPath() const if (outputName == "" && attrs) {
{ Bindings::iterator i = attrs->find(state->sOutputName);
if (outPath == "" && attrs) { outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
Bindings::iterator i = attrs->find(state->sOutPath); }
PathSet context; return outputName;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
}
return outPath;
} }
Bindings* DrvInfo::getMeta() {
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (meta) return meta;
{ if (!attrs) return 0;
if (outputs.empty()) { Bindings::iterator a = attrs->find(state->sMeta);
/* Get the ‘outputs’ list. */ if (a == attrs->end()) return 0;
Bindings::iterator i; state->forceAttrs(*a->value, *a->pos);
if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { meta = a->value->attrs;
state->forceList(*i->value, *i->pos); return meta;
/* For each output... */
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
/* Evaluate the corresponding set. */
string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
Bindings::iterator out = attrs->find(state->symbols.create(name));
if (out == attrs->end()) continue; // FIXME: throw error?
state->forceAttrs(*out->value);
/* And evaluate its ‘outPath’ attribute. */
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
PathSet context;
outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
}
} else
outputs["out"] = queryOutPath();
}
if (!onlyOutputsToInstall || !attrs)
return outputs;
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
const Value * outTI = queryMeta("outputsToInstall");
if (!outTI) return outputs;
const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
/* ^ this shows during `nix-env -i` right under the bad derivation */
if (!outTI->isList()) throw errMsg;
Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
if ((*i)->type != tString) throw errMsg;
auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg;
result.insert(*out);
}
return result;
} }
StringSet DrvInfo::queryMetaNames() {
string DrvInfo::queryOutputName() const StringSet res;
{ if (!getMeta()) return res;
if (outputName == "" && attrs) { for (auto& i : *meta) res.insert(i.name);
Bindings::iterator i = attrs->find(state->sOutputName); return res;
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
}
return outputName;
} }
bool DrvInfo::checkMeta(Value& v) {
Bindings * DrvInfo::getMeta() state->forceValue(v);
{ if (v.isList()) {
if (meta) return meta; for (unsigned int n = 0; n < v.listSize(); ++n)
if (!attrs) return 0; if (!checkMeta(*v.listElems()[n])) return false;
Bindings::iterator a = attrs->find(state->sMeta); return true;
if (a == attrs->end()) return 0; } else if (v.type == tAttrs) {
state->forceAttrs(*a->value, *a->pos); Bindings::iterator i = v.attrs->find(state->sOutPath);
meta = a->value->attrs; if (i != v.attrs->end()) return false;
return meta; for (auto& i : *v.attrs)
if (!checkMeta(*i.value)) return false;
return true;
} else
return v.type == tInt || v.type == tBool || v.type == tString ||
v.type == tFloat;
} }
Value* DrvInfo::queryMeta(const string& name) {
StringSet DrvInfo::queryMetaNames() if (!getMeta()) return 0;
{ Bindings::iterator a = meta->find(state->symbols.create(name));
StringSet res; if (a == meta->end() || !checkMeta(*a->value)) return 0;
if (!getMeta()) return res; return a->value;
for (auto & i : *meta)
res.insert(i.name);
return res;
} }
string DrvInfo::queryMetaString(const string& name) {
bool DrvInfo::checkMeta(Value & v) Value* v = queryMeta(name);
{ if (!v || v->type != tString) return "";
state->forceValue(v); return v->string.s;
if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n)
if (!checkMeta(*v.listElems()[n])) return false;
return true;
}
else if (v.type == tAttrs) {
Bindings::iterator i = v.attrs->find(state->sOutPath);
if (i != v.attrs->end()) return false;
for (auto & i : *v.attrs)
if (!checkMeta(*i.value)) return false;
return true;
}
else return v.type == tInt || v.type == tBool || v.type == tString ||
v.type == tFloat;
} }
NixInt DrvInfo::queryMetaInt(const string& name, NixInt def) {
Value * DrvInfo::queryMeta(const string & name) Value* v = queryMeta(name);
{ if (!v) return def;
if (!getMeta()) return 0; if (v->type == tInt) return v->integer;
Bindings::iterator a = meta->find(state->symbols.create(name)); if (v->type == tString) {
if (a == meta->end() || !checkMeta(*a->value)) return 0; /* Backwards compatibility with before we had support for
return a->value; integer meta fields. */
NixInt n;
if (string2Int(v->string.s, n)) return n;
}
return def;
} }
NixFloat DrvInfo::queryMetaFloat(const string& name, NixFloat def) {
string DrvInfo::queryMetaString(const string & name) Value* v = queryMeta(name);
{ if (!v) return def;
Value * v = queryMeta(name); if (v->type == tFloat) return v->fpoint;
if (!v || v->type != tString) return ""; if (v->type == tString) {
return v->string.s; /* Backwards compatibility with before we had support for
float meta fields. */
NixFloat n;
if (string2Float(v->string.s, n)) return n;
}
return def;
} }
bool DrvInfo::queryMetaBool(const string& name, bool def) {
NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) Value* v = queryMeta(name);
{ if (!v) return def;
Value * v = queryMeta(name); if (v->type == tBool) return v->boolean;
if (!v) return def; if (v->type == tString) {
if (v->type == tInt) return v->integer; /* Backwards compatibility with before we had support for
if (v->type == tString) { Boolean meta fields. */
/* Backwards compatibility with before we had support for if (strcmp(v->string.s, "true") == 0) return true;
integer meta fields. */ if (strcmp(v->string.s, "false") == 0) return false;
NixInt n; }
if (string2Int(v->string.s, n)) return n; return def;
}
return def;
} }
NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) void DrvInfo::setMeta(const string& name, Value* v) {
{ getMeta();
Value * v = queryMeta(name); Bindings* old = meta;
if (!v) return def; meta = state->allocBindings(1 + (old ? old->size() : 0));
if (v->type == tFloat) return v->fpoint; Symbol sym = state->symbols.create(name);
if (v->type == tString) { if (old)
/* Backwards compatibility with before we had support for for (auto i : *old)
float meta fields. */ if (i.name != sym) meta->push_back(i);
NixFloat n; if (v) meta->push_back(Attr(sym, v));
if (string2Float(v->string.s, n)) return n; meta->sort();
}
return def;
} }
bool DrvInfo::queryMetaBool(const string & name, bool def)
{
Value * v = queryMeta(name);
if (!v) return def;
if (v->type == tBool) return v->boolean;
if (v->type == tString) {
/* Backwards compatibility with before we had support for
Boolean meta fields. */
if (strcmp(v->string.s, "true") == 0) return true;
if (strcmp(v->string.s, "false") == 0) return false;
}
return def;
}
void DrvInfo::setMeta(const string & name, Value * v)
{
getMeta();
Bindings * old = meta;
meta = state->allocBindings(1 + (old ? old->size() : 0));
Symbol sym = state->symbols.create(name);
if (old)
for (auto i : *old)
if (i.name != sym)
meta->push_back(i);
if (v) meta->push_back(Attr(sym, v));
meta->sort();
}
/* Cache for already considered attrsets. */ /* Cache for already considered attrsets. */
typedef set<Bindings *> Done; typedef set<Bindings*> Done;
/* Evaluate value `v'. If it evaluates to a set of type `derivation', /* Evaluate value `v'. If it evaluates to a set of type `derivation',
then put information about it in `drvs' (unless it's already in `done'). then put information about it in `drvs' (unless it's already in `done').
The result boolean indicates whether it makes sense The result boolean indicates whether it makes sense
for the caller to recursively search for derivations in `v'. */ for the caller to recursively search for derivations in `v'. */
static bool getDerivation(EvalState & state, Value & v, static bool getDerivation(EvalState& state, Value& v, const string& attrPath,
const string & attrPath, DrvInfos & drvs, Done & done, DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures) bool ignoreAssertionFailures) {
{ try {
try { state.forceValue(v);
state.forceValue(v); if (!state.isDerivation(v)) return true;
if (!state.isDerivation(v)) return true;
/* Remove spurious duplicates (e.g., a set like `rec { x = /* Remove spurious duplicates (e.g., a set like `rec { x =
derivation {...}; y = x;}'. */ derivation {...}; y = x;}'. */
if (done.find(v.attrs) != done.end()) return false; if (done.find(v.attrs) != done.end()) return false;
done.insert(v.attrs); done.insert(v.attrs);
DrvInfo drv(state, attrPath, v.attrs); DrvInfo drv(state, attrPath, v.attrs);
drv.queryName(); drv.queryName();
drvs.push_back(drv); drvs.push_back(drv);
return false; return false;
} catch (AssertionError & e) { } catch (AssertionError& e) {
if (ignoreAssertionFailures) return false; if (ignoreAssertionFailures) return false;
throw; throw;
} }
} }
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) {
bool ignoreAssertionFailures) Done done;
{ DrvInfos drvs;
Done done; getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
DrvInfos drvs; if (drvs.size() != 1) return {};
getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); return std::move(drvs.front());
if (drvs.size() != 1) return {};
return std::move(drvs.front());
} }
static string addToPath(const string& s1, const string& s2) {
static string addToPath(const string & s1, const string & s2) return s1.empty() ? s2 : s1 + "." + s2;
{
return s1.empty() ? s2 : s1 + "." + s2;
} }
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
static void getDerivations(EvalState& state, Value& vIn,
const string& pathPrefix, Bindings& autoArgs,
DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures) {
Value v;
state.autoCallFunction(autoArgs, vIn, v);
static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */
const string & pathPrefix, Bindings & autoArgs, if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures))
DrvInfos & drvs, Done & done, ;
bool ignoreAssertionFailures)
{
Value v;
state.autoCallFunction(autoArgs, vIn, v);
/* Process the expression. */ else if (v.type == tAttrs) {
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; /* !!! undocumented hackery to support combining channels in
nix-env.cc. */
bool combineChannels =
v.attrs->find(state.symbols.create("_combineChannels")) !=
v.attrs->end();
else if (v.type == tAttrs) { /* Consider the attributes in sorted order to get more
deterministic behaviour in nix-env operations (e.g. when
/* !!! undocumented hackery to support combining channels in there are names clashes between derivations, the derivation
nix-env.cc. */ bound to the attribute with the "lower" name should take
bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); precedence). */
for (auto& i : v.attrs->lexicographicOrder()) {
/* Consider the attributes in sorted order to get more debug("evaluating attribute '%1%'", i->name);
deterministic behaviour in nix-env operations (e.g. when if (!std::regex_match(std::string(i->name), attrRegex)) continue;
there are names clashes between derivations, the derivation string pathPrefix2 = addToPath(pathPrefix, i->name);
bound to the attribute with the "lower" name should take if (combineChannels)
precedence). */ getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
for (auto & i : v.attrs->lexicographicOrder()) { ignoreAssertionFailures);
debug("evaluating attribute '%1%'", i->name); else if (getDerivation(state, *i->value, pathPrefix2, drvs, done,
if (!std::regex_match(std::string(i->name), attrRegex)) ignoreAssertionFailures)) {
continue; /* If the value of this attribute is itself a set,
string pathPrefix2 = addToPath(pathPrefix, i->name); should we recurse into it? => Only if it has a
if (combineChannels) `recurseForDerivations = true' attribute. */
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); if (i->value->type == tAttrs) {
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { Bindings::iterator j = i->value->attrs->find(
/* If the value of this attribute is itself a set, state.symbols.create("recurseForDerivations"));
should we recurse into it? => Only if it has a if (j != i->value->attrs->end() &&
`recurseForDerivations = true' attribute. */ state.forceBool(*j->value, *j->pos))
if (i->value->type == tAttrs) { getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); ignoreAssertionFailures);
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
}
}
} }
}
} }
}
else if (v.isList()) { else if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n) { for (unsigned int n = 0; n < v.listSize(); ++n) {
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done,
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); ignoreAssertionFailures))
} getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs,
done, ignoreAssertionFailures);
} }
}
else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); else
throw TypeError(
"expression does not evaluate to a derivation (or a set or list of "
"those)");
} }
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
void getDerivations(EvalState & state, Value & v, const string & pathPrefix, Bindings& autoArgs, DrvInfos& drvs,
Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) bool ignoreAssertionFailures) {
{ Done done;
Done done; getDerivations(state, v, pathPrefix, autoArgs, drvs, done,
getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); ignoreAssertionFailures);
} }
} // namespace nix
}

111
third_party/nix/src/libexpr/get-drvs.hh vendored
View file

@ -1,89 +1,84 @@
#pragma once #pragma once
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
namespace nix { namespace nix {
struct DrvInfo {
public:
typedef std::map<string, Path> Outputs;
struct DrvInfo private:
{ EvalState* state;
public:
typedef std::map<string, Path> Outputs;
private: mutable string name;
EvalState * state; mutable string system;
mutable string drvPath;
mutable string outPath;
mutable string outputName;
Outputs outputs;
mutable string name; bool failed = false; // set if we get an AssertionError
mutable string system;
mutable string drvPath;
mutable string outPath;
mutable string outputName;
Outputs outputs;
bool failed = false; // set if we get an AssertionError Bindings *attrs = nullptr, *meta = nullptr;
Bindings * attrs = nullptr, * meta = nullptr; Bindings* getMeta();
Bindings * getMeta(); bool checkMeta(Value& v);
bool checkMeta(Value & v); public:
string attrPath; /* path towards the derivation */
public: DrvInfo(EvalState& state) : state(&state){};
string attrPath; /* path towards the derivation */ DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs);
DrvInfo(EvalState& state, ref<Store> store,
const std::string& drvPathWithOutputs);
DrvInfo(EvalState & state) : state(&state) { }; string queryName() const;
DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); string querySystem() const;
DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs); string queryDrvPath() const;
string queryOutPath() const;
string queryOutputName() const;
/** Return the list of outputs. The "outputs to install" are determined by
* `meta.outputsToInstall`. */
Outputs queryOutputs(bool onlyOutputsToInstall = false);
string queryName() const; StringSet queryMetaNames();
string querySystem() const; Value* queryMeta(const string& name);
string queryDrvPath() const; string queryMetaString(const string& name);
string queryOutPath() const; NixInt queryMetaInt(const string& name, NixInt def);
string queryOutputName() const; NixFloat queryMetaFloat(const string& name, NixFloat def);
/** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ bool queryMetaBool(const string& name, bool def);
Outputs queryOutputs(bool onlyOutputsToInstall = false); void setMeta(const string& name, Value* v);
StringSet queryMetaNames(); /*
Value * queryMeta(const string & name); MetaInfo queryMetaInfo(EvalState & state) const;
string queryMetaString(const string & name); MetaValue queryMetaInfo(EvalState & state, const string & name) const;
NixInt queryMetaInt(const string & name, NixInt def); */
NixFloat queryMetaFloat(const string & name, NixFloat def);
bool queryMetaBool(const string & name, bool def);
void setMeta(const string & name, Value * v);
/* void setName(const string& s) { name = s; }
MetaInfo queryMetaInfo(EvalState & state) const; void setDrvPath(const string& s) { drvPath = s; }
MetaValue queryMetaInfo(EvalState & state, const string & name) const; void setOutPath(const string& s) { outPath = s; }
*/
void setName(const string & s) { name = s; } void setFailed() { failed = true; };
void setDrvPath(const string & s) { drvPath = s; } bool hasFailed() { return failed; };
void setOutPath(const string & s) { outPath = s; }
void setFailed() { failed = true; };
bool hasFailed() { return failed; };
}; };
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
#else #else
typedef list<DrvInfo> DrvInfos; typedef list<DrvInfo> DrvInfos;
#endif #endif
/* If value `v' denotes a derivation, return a DrvInfo object /* If value `v' denotes a derivation, return a DrvInfo object
describing it. Otherwise return nothing. */ describing it. Otherwise return nothing. */
std::optional<DrvInfo> getDerivation(EvalState & state, std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
Value & v, bool ignoreAssertionFailures); bool ignoreAssertionFailures);
void getDerivations(EvalState & state, Value & v, const string & pathPrefix, void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
Bindings & autoArgs, DrvInfos & drvs, Bindings& autoArgs, DrvInfos& drvs,
bool ignoreAssertionFailures); bool ignoreAssertionFailures);
} // namespace nix
}

274
third_party/nix/src/libexpr/json-to-value.cc vendored
View file

@ -1,149 +1,153 @@
#include "json-to-value.hh" #include "json-to-value.hh"
#include <cstring> #include <cstring>
namespace nix { namespace nix {
static void skipWhitespace(const char*& s) {
static void skipWhitespace(const char * & s) while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
{
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
} }
static string parseJSONString(const char*& s) {
string res;
if (*s++ != '"') throw JSONParseError("expected JSON string");
while (*s != '"') {
if (!*s) throw JSONParseError("got end-of-string in JSON string");
if (*s == '\\') {
s++;
if (*s == '"')
res += '"';
else if (*s == '\\')
res += '\\';
else if (*s == '/')
res += '/';
else if (*s == '/')
res += '/';
else if (*s == 'b')
res += '\b';
else if (*s == 'f')
res += '\f';
else if (*s == 'n')
res += '\n';
else if (*s == 'r')
res += '\r';
else if (*s == 't')
res += '\t';
else if (*s == 'u')
throw JSONParseError(
"\\u characters in JSON strings are currently not supported");
else
throw JSONParseError("invalid escaped character in JSON string");
s++;
} else
res += *s++;
}
s++;
return res;
}
static string parseJSONString(const char * & s) static void parseJSON(EvalState& state, const char*& s, Value& v) {
{ skipWhitespace(s);
string res;
if (*s++ != '"') throw JSONParseError("expected JSON string"); if (!*s) throw JSONParseError("expected JSON value");
while (*s != '"') {
if (!*s) throw JSONParseError("got end-of-string in JSON string"); if (*s == '[') {
if (*s == '\\') { s++;
s++; ValueVector values;
if (*s == '"') res += '"'; values.reserve(128);
else if (*s == '\\') res += '\\'; skipWhitespace(s);
else if (*s == '/') res += '/'; while (1) {
else if (*s == '/') res += '/'; if (values.empty() && *s == ']') break;
else if (*s == 'b') res += '\b'; Value* v2 = state.allocValue();
else if (*s == 'f') res += '\f'; parseJSON(state, s, *v2);
else if (*s == 'n') res += '\n'; values.push_back(v2);
else if (*s == 'r') res += '\r'; skipWhitespace(s);
else if (*s == 't') res += '\t'; if (*s == ']') break;
else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); if (*s != ',')
else throw JSONParseError("invalid escaped character in JSON string"); throw JSONParseError("expected ',' or ']' after JSON array element");
s++; s++;
} else
res += *s++;
} }
s++; s++;
return res; state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n) v.listElems()[n] = values[n];
}
else if (*s == '{') {
s++;
ValueMap attrs;
while (1) {
skipWhitespace(s);
if (attrs.empty() && *s == '}') break;
string name = parseJSONString(s);
skipWhitespace(s);
if (*s != ':') throw JSONParseError("expected ':' in JSON object");
s++;
Value* v2 = state.allocValue();
parseJSON(state, s, *v2);
attrs[state.symbols.create(name)] = v2;
skipWhitespace(s);
if (*s == '}') break;
if (*s != ',')
throw JSONParseError("expected ',' or '}' after JSON member");
s++;
}
state.mkAttrs(v, attrs.size());
for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second));
v.attrs->sort();
s++;
}
else if (*s == '"') {
mkString(v, parseJSONString(s));
}
else if (isdigit(*s) || *s == '-' || *s == '.') {
// Buffer into a string first, then use built-in C++ conversions
std::string tmp_number;
ValueType number_type = tInt;
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
if (*s == '.' || *s == 'e' || *s == 'E') number_type = tFloat;
tmp_number += *s++;
}
try {
if (number_type == tFloat)
mkFloat(v, stod(tmp_number));
else
mkInt(v, stol(tmp_number));
} catch (std::invalid_argument& e) {
throw JSONParseError("invalid JSON number");
} catch (std::out_of_range& e) {
throw JSONParseError("out-of-range JSON number");
}
}
else if (strncmp(s, "true", 4) == 0) {
s += 4;
mkBool(v, true);
}
else if (strncmp(s, "false", 5) == 0) {
s += 5;
mkBool(v, false);
}
else if (strncmp(s, "null", 4) == 0) {
s += 4;
mkNull(v);
}
else
throw JSONParseError("unrecognised JSON value");
} }
void parseJSON(EvalState& state, const string& s_, Value& v) {
static void parseJSON(EvalState & state, const char * & s, Value & v) const char* s = s_.c_str();
{ parseJSON(state, s, v);
skipWhitespace(s); skipWhitespace(s);
if (*s)
if (!*s) throw JSONParseError("expected JSON value"); throw JSONParseError(
format("expected end-of-string while parsing JSON value: %1%") % s);
if (*s == '[') {
s++;
ValueVector values;
values.reserve(128);
skipWhitespace(s);
while (1) {
if (values.empty() && *s == ']') break;
Value * v2 = state.allocValue();
parseJSON(state, s, *v2);
values.push_back(v2);
skipWhitespace(s);
if (*s == ']') break;
if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
s++;
}
s++;
state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n)
v.listElems()[n] = values[n];
}
else if (*s == '{') {
s++;
ValueMap attrs;
while (1) {
skipWhitespace(s);
if (attrs.empty() && *s == '}') break;
string name = parseJSONString(s);
skipWhitespace(s);
if (*s != ':') throw JSONParseError("expected ':' in JSON object");
s++;
Value * v2 = state.allocValue();
parseJSON(state, s, *v2);
attrs[state.symbols.create(name)] = v2;
skipWhitespace(s);
if (*s == '}') break;
if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
s++;
}
state.mkAttrs(v, attrs.size());
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second));
v.attrs->sort();
s++;
}
else if (*s == '"') {
mkString(v, parseJSONString(s));
}
else if (isdigit(*s) || *s == '-' || *s == '.' ) {
// Buffer into a string first, then use built-in C++ conversions
std::string tmp_number;
ValueType number_type = tInt;
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
if (*s == '.' || *s == 'e' || *s == 'E')
number_type = tFloat;
tmp_number += *s++;
}
try {
if (number_type == tFloat)
mkFloat(v, stod(tmp_number));
else
mkInt(v, stol(tmp_number));
} catch (std::invalid_argument & e) {
throw JSONParseError("invalid JSON number");
} catch (std::out_of_range & e) {
throw JSONParseError("out-of-range JSON number");
}
}
else if (strncmp(s, "true", 4) == 0) {
s += 4;
mkBool(v, true);
}
else if (strncmp(s, "false", 5) == 0) {
s += 5;
mkBool(v, false);
}
else if (strncmp(s, "null", 4) == 0) {
s += 4;
mkNull(v);
}
else throw JSONParseError("unrecognised JSON value");
} }
} // namespace nix
void parseJSON(EvalState & state, const string & s_, Value & v)
{
const char * s = s_.c_str();
parseJSON(state, s, v);
skipWhitespace(s);
if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
}
}

5
third_party/nix/src/libexpr/json-to-value.hh vendored
View file

@ -1,13 +1,12 @@
#pragma once #pragma once
#include "eval.hh"
#include <string> #include <string>
#include "eval.hh"
namespace nix { namespace nix {
MakeError(JSONParseError, EvalError) MakeError(JSONParseError, EvalError)
void parseJSON(EvalState & state, const string & s, Value & v); void parseJSON(EvalState& state, const string& s, Value& v);
} }

157
third_party/nix/src/libexpr/names.cc vendored
View file

@ -1,107 +1,98 @@
#include "names.hh" #include "names.hh"
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
DrvName::DrvName() { name = ""; }
DrvName::DrvName()
{
name = "";
}
/* Parse a derivation name. The `name' part of a derivation name is /* Parse a derivation name. The `name' part of a derivation name is
everything up to but not including the first dash *not* followed by everything up to but not including the first dash *not* followed by
a letter. The `version' part is the rest (excluding the separating a letter. The `version' part is the rest (excluding the separating
dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
'2.0.48'). */ '2.0.48'). */
DrvName::DrvName(const string & s) : hits(0) DrvName::DrvName(const string& s) : hits(0) {
{ name = fullName = s;
name = fullName = s; for (unsigned int i = 0; i < s.size(); ++i) {
for (unsigned int i = 0; i < s.size(); ++i) { /* !!! isalpha/isdigit are affected by the locale. */
/* !!! isalpha/isdigit are affected by the locale. */ if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) {
if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { name = string(s, 0, i);
name = string(s, 0, i); version = string(s, i + 1);
version = string(s, i + 1); break;
break;
}
} }
}
} }
bool DrvName::matches(DrvName& n) {
if (name != "*") {
if (!regex)
regex = std::unique_ptr<std::regex>(
new std::regex(name, std::regex::extended));
if (!std::regex_match(n.name, *regex)) return false;
}
if (version != "" && version != n.version) return false;
return true;
}
bool DrvName::matches(DrvName & n) string nextComponent(string::const_iterator& p,
{ const string::const_iterator end) {
if (name != "*") { /* Skip any dots and dashes (component separators). */
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); while (p != end && (*p == '.' || *p == '-')) ++p;
if (!std::regex_match(n.name, *regex)) return false;
} if (p == end) return "";
if (version != "" && version != n.version) return false;
/* If the first character is a digit, consume the longest sequence
of digits. Otherwise, consume the longest sequence of
non-digit, non-separator characters. */
string s;
if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++;
else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) s += *p++;
return s;
}
static bool componentsLT(const string& c1, const string& c2) {
int n1, n2;
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
if (c1Num && c2Num)
return n1 < n2;
else if (c1 == "" && c2Num)
return true; return true;
else if (c1 == "pre" && c2 != "pre")
return true;
else if (c2 == "pre")
return false;
/* Assume that `2.3a' < `2.3.1'. */
else if (c2Num)
return true;
else if (c1Num)
return false;
else
return c1 < c2;
} }
int compareVersions(const string& v1, const string& v2) {
string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin();
string nextComponent(string::const_iterator & p, while (p1 != v1.end() || p2 != v2.end()) {
const string::const_iterator end) string c1 = nextComponent(p1, v1.end());
{ string c2 = nextComponent(p2, v2.end());
/* Skip any dots and dashes (component separators). */ if (componentsLT(c1, c2))
while (p != end && (*p == '.' || *p == '-')) ++p; return -1;
else if (componentsLT(c2, c1))
return 1;
}
if (p == end) return ""; return 0;
/* If the first character is a digit, consume the longest sequence
of digits. Otherwise, consume the longest sequence of
non-digit, non-separator characters. */
string s;
if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++;
else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
s += *p++;
return s;
} }
DrvNames drvNamesFromArgs(const Strings& opArgs) {
static bool componentsLT(const string & c1, const string & c2) DrvNames result;
{ for (auto& i : opArgs) result.push_back(DrvName(i));
int n1, n2; return result;
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
if (c1Num && c2Num) return n1 < n2;
else if (c1 == "" && c2Num) return true;
else if (c1 == "pre" && c2 != "pre") return true;
else if (c2 == "pre") return false;
/* Assume that `2.3a' < `2.3.1'. */
else if (c2Num) return true;
else if (c1Num) return false;
else return c1 < c2;
} }
} // namespace nix
int compareVersions(const string & v1, const string & v2)
{
string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin();
while (p1 != v1.end() || p2 != v2.end()) {
string c1 = nextComponent(p1, v1.end());
string c2 = nextComponent(p2, v2.end());
if (componentsLT(c1, c2)) return -1;
else if (componentsLT(c2, c1)) return 1;
}
return 0;
}
DrvNames drvNamesFromArgs(const Strings & opArgs)
{
DrvNames result;
for (auto & i : opArgs)
result.push_back(DrvName(i));
return result;
}
}

34
third_party/nix/src/libexpr/names.hh vendored
View file

@ -1,32 +1,30 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include "types.hh"
#include <regex> #include <regex>
#include "types.hh"
namespace nix { namespace nix {
struct DrvName struct DrvName {
{ string fullName;
string fullName; string name;
string name; string version;
string version; unsigned int hits;
unsigned int hits;
DrvName(); DrvName();
DrvName(const string & s); DrvName(const string& s);
bool matches(DrvName & n); bool matches(DrvName& n);
private: private:
std::unique_ptr<std::regex> regex; std::unique_ptr<std::regex> regex;
}; };
typedef list<DrvName> DrvNames; typedef list<DrvName> DrvNames;
string nextComponent(string::const_iterator & p, string nextComponent(string::const_iterator& p,
const string::const_iterator end); const string::const_iterator end);
int compareVersions(const string & v1, const string & v2); int compareVersions(const string& v1, const string& v2);
DrvNames drvNamesFromArgs(const Strings & opArgs); DrvNames drvNamesFromArgs(const Strings& opArgs);
} } // namespace nix

675
third_party/nix/src/libexpr/nixexpr.cc vendored
View file

@ -1,438 +1,361 @@
#include "nixexpr.hh" #include "nixexpr.hh"
#include <cstdlib>
#include "derivations.hh" #include "derivations.hh"
#include "util.hh" #include "util.hh"
#include <cstdlib>
namespace nix { namespace nix {
/* Displaying abstract syntax trees. */ /* Displaying abstract syntax trees. */
std::ostream & operator << (std::ostream & str, const Expr & e) std::ostream& operator<<(std::ostream& str, const Expr& e) {
{ e.show(str);
e.show(str); return str;
return str;
} }
static void showString(std::ostream & str, const string & s) static void showString(std::ostream& str, const string& s) {
{ str << '"';
str << '"'; for (auto c : (string)s)
for (auto c : (string) s) if (c == '"' || c == '\\' || c == '$')
if (c == '"' || c == '\\' || c == '$') str << "\\" << c; str << "\\" << c;
else if (c == '\n') str << "\\n"; else if (c == '\n')
else if (c == '\r') str << "\\r"; str << "\\n";
else if (c == '\t') str << "\\t"; else if (c == '\r')
else str << c; str << "\\r";
str << '"'; else if (c == '\t')
} str << "\\t";
static void showId(std::ostream & str, const string & s)
{
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
str << '"' << s << '"';
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
showString(str, s);
return;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
showString(str, s);
return;
}
str << s;
}
}
std::ostream & operator << (std::ostream & str, const Symbol & sym)
{
showId(str, *sym.s);
return str;
}
void Expr::show(std::ostream & str) const
{
abort();
}
void ExprInt::show(std::ostream & str) const
{
str << n;
}
void ExprFloat::show(std::ostream & str) const
{
str << nf;
}
void ExprString::show(std::ostream & str) const
{
showString(str, s);
}
void ExprPath::show(std::ostream & str) const
{
str << s;
}
void ExprVar::show(std::ostream & str) const
{
str << name;
}
void ExprSelect::show(std::ostream & str) const
{
str << "(" << *e << ")." << showAttrPath(attrPath);
if (def) str << " or (" << *def << ")";
}
void ExprOpHasAttr::show(std::ostream & str) const
{
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
}
void ExprAttrs::show(std::ostream & str) const
{
if (recursive) str << "rec ";
str << "{ ";
for (auto & i : attrs)
if (i.second.inherited)
str << "inherit " << i.first << " " << "; ";
else
str << i.first << " = " << *i.second.e << "; ";
for (auto & i : dynamicAttrs)
str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; ";
str << "}";
}
void ExprList::show(std::ostream & str) const
{
str << "[ ";
for (auto & i : elems)
str << "(" << *i << ") ";
str << "]";
}
void ExprLambda::show(std::ostream & str) const
{
str << "(";
if (matchAttrs) {
str << "{ ";
bool first = true;
for (auto & i : formals->formals) {
if (first) first = false; else str << ", ";
str << i.name;
if (i.def) str << " ? " << *i.def;
}
if (formals->ellipsis) {
if (!first) str << ", ";
str << "...";
}
str << " }";
if (!arg.empty()) str << " @ ";
}
if (!arg.empty()) str << arg;
str << ": " << *body << ")";
}
void ExprLet::show(std::ostream & str) const
{
str << "(let ";
for (auto & i : attrs->attrs)
if (i.second.inherited) {
str << "inherit " << i.first << "; ";
}
else
str << i.first << " = " << *i.second.e << "; ";
str << "in " << *body << ")";
}
void ExprWith::show(std::ostream & str) const
{
str << "(with " << *attrs << "; " << *body << ")";
}
void ExprIf::show(std::ostream & str) const
{
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
}
void ExprAssert::show(std::ostream & str) const
{
str << "assert " << *cond << "; " << *body;
}
void ExprOpNot::show(std::ostream & str) const
{
str << "(! " << *e << ")";
}
void ExprConcatStrings::show(std::ostream & str) const
{
bool first = true;
str << "(";
for (auto & i : *es) {
if (first) first = false; else str << " + ";
str << *i;
}
str << ")";
}
void ExprPos::show(std::ostream & str) const
{
str << "__curPos";
}
std::ostream & operator << (std::ostream & str, const Pos & pos)
{
if (!pos)
str << "undefined position";
else else
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str(); str << c;
return str; str << '"';
} }
static void showId(std::ostream& str, const string& s) {
string showAttrPath(const AttrPath & attrPath) if (s.empty())
{ str << "\"\"";
std::ostringstream out; else if (s == "if") // FIXME: handle other keywords
bool first = true; str << '"' << s << '"';
for (auto & i : attrPath) { else {
if (!first) out << '.'; else first = false; char c = s[0];
if (i.symbol.set()) if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
out << i.symbol; showString(str, s);
else return;
out << "\"${" << *i.expr << "}\"";
} }
return out.str(); for (auto c : s)
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) {
showString(str, s);
return;
}
str << s;
}
} }
std::ostream& operator<<(std::ostream& str, const Symbol& sym) {
showId(str, *sym.s);
return str;
}
void Expr::show(std::ostream& str) const { abort(); }
void ExprInt::show(std::ostream& str) const { str << n; }
void ExprFloat::show(std::ostream& str) const { str << nf; }
void ExprString::show(std::ostream& str) const { showString(str, s); }
void ExprPath::show(std::ostream& str) const { str << s; }
void ExprVar::show(std::ostream& str) const { str << name; }
void ExprSelect::show(std::ostream& str) const {
str << "(" << *e << ")." << showAttrPath(attrPath);
if (def) str << " or (" << *def << ")";
}
void ExprOpHasAttr::show(std::ostream& str) const {
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
}
void ExprAttrs::show(std::ostream& str) const {
if (recursive) str << "rec ";
str << "{ ";
for (auto& i : attrs)
if (i.second.inherited)
str << "inherit " << i.first << " "
<< "; ";
else
str << i.first << " = " << *i.second.e << "; ";
for (auto& i : dynamicAttrs)
str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; ";
str << "}";
}
void ExprList::show(std::ostream& str) const {
str << "[ ";
for (auto& i : elems) str << "(" << *i << ") ";
str << "]";
}
void ExprLambda::show(std::ostream& str) const {
str << "(";
if (matchAttrs) {
str << "{ ";
bool first = true;
for (auto& i : formals->formals) {
if (first)
first = false;
else
str << ", ";
str << i.name;
if (i.def) str << " ? " << *i.def;
}
if (formals->ellipsis) {
if (!first) str << ", ";
str << "...";
}
str << " }";
if (!arg.empty()) str << " @ ";
}
if (!arg.empty()) str << arg;
str << ": " << *body << ")";
}
void ExprLet::show(std::ostream& str) const {
str << "(let ";
for (auto& i : attrs->attrs)
if (i.second.inherited) {
str << "inherit " << i.first << "; ";
} else
str << i.first << " = " << *i.second.e << "; ";
str << "in " << *body << ")";
}
void ExprWith::show(std::ostream& str) const {
str << "(with " << *attrs << "; " << *body << ")";
}
void ExprIf::show(std::ostream& str) const {
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
}
void ExprAssert::show(std::ostream& str) const {
str << "assert " << *cond << "; " << *body;
}
void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; }
void ExprConcatStrings::show(std::ostream& str) const {
bool first = true;
str << "(";
for (auto& i : *es) {
if (first)
first = false;
else
str << " + ";
str << *i;
}
str << ")";
}
void ExprPos::show(std::ostream& str) const { str << "__curPos"; }
std::ostream& operator<<(std::ostream& str, const Pos& pos) {
if (!pos)
str << "undefined position";
else
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string)pos.file %
pos.line % pos.column)
.str();
return str;
}
string showAttrPath(const AttrPath& attrPath) {
std::ostringstream out;
bool first = true;
for (auto& i : attrPath) {
if (!first)
out << '.';
else
first = false;
if (i.symbol.set())
out << i.symbol;
else
out << "\"${" << *i.expr << "}\"";
}
return out.str();
}
Pos noPos; Pos noPos;
/* Computing levels/displacements for variables. */ /* Computing levels/displacements for variables. */
void Expr::bindVars(const StaticEnv & env) void Expr::bindVars(const StaticEnv& env) { abort(); }
{
abort();
}
void ExprInt::bindVars(const StaticEnv & env) void ExprInt::bindVars(const StaticEnv& env) {}
{
}
void ExprFloat::bindVars(const StaticEnv & env) void ExprFloat::bindVars(const StaticEnv& env) {}
{
}
void ExprString::bindVars(const StaticEnv & env) void ExprString::bindVars(const StaticEnv& env) {}
{
}
void ExprPath::bindVars(const StaticEnv & env) void ExprPath::bindVars(const StaticEnv& env) {}
{
}
void ExprVar::bindVars(const StaticEnv & env) void ExprVar::bindVars(const StaticEnv& env) {
{ /* Check whether the variable appears in the environment. If so,
/* Check whether the variable appears in the environment. If so, set its level and displacement. */
set its level and displacement. */ const StaticEnv* curEnv;
const StaticEnv * curEnv; unsigned int level;
unsigned int level; int withLevel = -1;
int withLevel = -1; for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) {
if (curEnv->isWith) { if (withLevel == -1) withLevel = level;
if (withLevel == -1) withLevel = level; } else {
} else { StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); if (i != curEnv->vars.end()) {
if (i != curEnv->vars.end()) { fromWith = false;
fromWith = false; this->level = level;
this->level = level; displ = i->second;
displ = i->second; return;
return; }
}
}
} }
}
/* Otherwise, the variable must be obtained from the nearest /* Otherwise, the variable must be obtained from the nearest
enclosing `with'. If there is no `with', then we can issue an enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */ "undefined variable" error now. */
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); if (withLevel == -1)
throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name %
pos);
fromWith = true; fromWith = true;
this->level = withLevel; this->level = withLevel;
} }
void ExprSelect::bindVars(const StaticEnv & env) void ExprSelect::bindVars(const StaticEnv& env) {
{ e->bindVars(env);
e->bindVars(env); if (def) def->bindVars(env);
if (def) def->bindVars(env); for (auto& i : attrPath)
for (auto & i : attrPath) if (!i.symbol.set()) i.expr->bindVars(env);
if (!i.symbol.set())
i.expr->bindVars(env);
} }
void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprOpHasAttr::bindVars(const StaticEnv& env) {
{ e->bindVars(env);
e->bindVars(env); for (auto& i : attrPath)
for (auto & i : attrPath) if (!i.symbol.set()) i.expr->bindVars(env);
if (!i.symbol.set())
i.expr->bindVars(env);
} }
void ExprAttrs::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv& env) {
{ const StaticEnv* dynamicEnv = &env;
const StaticEnv * dynamicEnv = &env; StaticEnv newEnv(false, &env);
StaticEnv newEnv(false, &env);
if (recursive) { if (recursive) {
dynamicEnv = &newEnv; dynamicEnv = &newEnv;
unsigned int displ = 0;
for (auto & i : attrs)
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto & i : attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
}
else
for (auto & i : attrs)
i.second.e->bindVars(env);
for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(*dynamicEnv);
i.valueExpr->bindVars(*dynamicEnv);
}
}
void ExprList::bindVars(const StaticEnv & env)
{
for (auto & i : elems)
i->bindVars(env);
}
void ExprLambda::bindVars(const StaticEnv & env)
{
StaticEnv newEnv(false, &env);
unsigned int displ = 0; unsigned int displ = 0;
for (auto& i : attrs) newEnv.vars[i.first] = i.second.displ = displ++;
if (!arg.empty()) newEnv.vars[arg] = displ++; for (auto& i : attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
}
if (matchAttrs) { else
for (auto & i : formals->formals) for (auto& i : attrs) i.second.e->bindVars(env);
newEnv.vars[i.name] = displ++;
for (auto & i : formals->formals) for (auto& i : dynamicAttrs) {
if (i.def) i.def->bindVars(newEnv); i.nameExpr->bindVars(*dynamicEnv);
i.valueExpr->bindVars(*dynamicEnv);
}
}
void ExprList::bindVars(const StaticEnv& env) {
for (auto& i : elems) i->bindVars(env);
}
void ExprLambda::bindVars(const StaticEnv& env) {
StaticEnv newEnv(false, &env);
unsigned int displ = 0;
if (!arg.empty()) newEnv.vars[arg] = displ++;
if (matchAttrs) {
for (auto& i : formals->formals) newEnv.vars[i.name] = displ++;
for (auto& i : formals->formals)
if (i.def) i.def->bindVars(newEnv);
}
body->bindVars(newEnv);
}
void ExprLet::bindVars(const StaticEnv& env) {
StaticEnv newEnv(false, &env);
unsigned int displ = 0;
for (auto& i : attrs->attrs) newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs->attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
body->bindVars(newEnv);
}
void ExprWith::bindVars(const StaticEnv& env) {
/* Does this `with' have an enclosing `with'? If so, record its
level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */
const StaticEnv* curEnv;
unsigned int level;
prevWith = 0;
for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
if (curEnv->isWith) {
prevWith = level;
break;
} }
body->bindVars(newEnv); attrs->bindVars(env);
StaticEnv newEnv(true, &env);
body->bindVars(newEnv);
} }
void ExprLet::bindVars(const StaticEnv & env) void ExprIf::bindVars(const StaticEnv& env) {
{ cond->bindVars(env);
StaticEnv newEnv(false, &env); then->bindVars(env);
else_->bindVars(env);
unsigned int displ = 0;
for (auto & i : attrs->attrs)
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto & i : attrs->attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
body->bindVars(newEnv);
} }
void ExprWith::bindVars(const StaticEnv & env) void ExprAssert::bindVars(const StaticEnv& env) {
{ cond->bindVars(env);
/* Does this `with' have an enclosing `with'? If so, record its body->bindVars(env);
level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */
const StaticEnv * curEnv;
unsigned int level;
prevWith = 0;
for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
if (curEnv->isWith) {
prevWith = level;
break;
}
attrs->bindVars(env);
StaticEnv newEnv(true, &env);
body->bindVars(newEnv);
} }
void ExprIf::bindVars(const StaticEnv & env) void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); }
{
cond->bindVars(env); void ExprConcatStrings::bindVars(const StaticEnv& env) {
then->bindVars(env); for (auto& i : *es) i->bindVars(env);
else_->bindVars(env);
}
void ExprAssert::bindVars(const StaticEnv & env)
{
cond->bindVars(env);
body->bindVars(env);
}
void ExprOpNot::bindVars(const StaticEnv & env)
{
e->bindVars(env);
}
void ExprConcatStrings::bindVars(const StaticEnv & env)
{
for (auto & i : *es)
i->bindVars(env);
}
void ExprPos::bindVars(const StaticEnv & env)
{
} }
void ExprPos::bindVars(const StaticEnv& env) {}
/* Storing function names. */ /* Storing function names. */
void Expr::setName(Symbol & name) void Expr::setName(Symbol& name) {}
{
void ExprLambda::setName(Symbol& name) {
this->name = name;
body->setName(name);
} }
string ExprLambda::showNamePos() const {
void ExprLambda::setName(Symbol & name) return (format("%1% at %2%") %
{ (name.set() ? "'" + (string)name + "'" : "anonymous function") % pos)
this->name = name; .str();
body->setName(name);
} }
string ExprLambda::showNamePos() const
{
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
}
/* Symbol table. */ /* Symbol table. */
size_t SymbolTable::totalSize() const size_t SymbolTable::totalSize() const {
{ size_t n = 0;
size_t n = 0; for (auto& i : symbols) n += i.size();
for (auto & i : symbols) return n;
n += i.size();
return n;
} }
} // namespace nix
}

507
third_party/nix/src/libexpr/nixexpr.hh vendored
View file

@ -1,342 +1,309 @@
#pragma once #pragma once
#include "value.hh"
#include "symbol-table.hh"
#include <map> #include <map>
#include "symbol-table.hh"
#include "value.hh"
namespace nix { namespace nix {
MakeError(EvalError, Error) MakeError(ParseError, Error)
MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError)
MakeError(Abort, EvalError) MakeError(TypeError, EvalError)
MakeError(UndefinedVarError, Error)
MakeError(RestrictedPathError, Error)
MakeError(EvalError, Error) /* Position objects. */
MakeError(ParseError, Error)
MakeError(AssertionError, EvalError)
MakeError(ThrownError, AssertionError)
MakeError(Abort, EvalError)
MakeError(TypeError, EvalError)
MakeError(UndefinedVarError, Error)
MakeError(RestrictedPathError, Error)
struct Pos {
/* Position objects. */ Symbol file;
unsigned int line, column;
struct Pos Pos() : line(0), column(0){};
{ Pos(const Symbol& file, unsigned int line, unsigned int column)
Symbol file; : file(file), line(line), column(column){};
unsigned int line, column; operator bool() const { return line != 0; }
Pos() : line(0), column(0) { }; bool operator<(const Pos& p2) const {
Pos(const Symbol & file, unsigned int line, unsigned int column) if (!line) return p2.line;
: file(file), line(line), column(column) { }; if (!p2.line) return false;
operator bool() const int d = ((string)file).compare((string)p2.file);
{ if (d < 0) return true;
return line != 0; if (d > 0) return false;
} if (line < p2.line) return true;
bool operator < (const Pos & p2) const if (line > p2.line) return false;
{ return column < p2.column;
if (!line) return p2.line; }
if (!p2.line) return false;
int d = ((string) file).compare((string) p2.file);
if (d < 0) return true;
if (d > 0) return false;
if (line < p2.line) return true;
if (line > p2.line) return false;
return column < p2.column;
}
}; };
extern Pos noPos; extern Pos noPos;
std::ostream & operator << (std::ostream & str, const Pos & pos); std::ostream& operator<<(std::ostream& str, const Pos& pos);
struct Env; struct Env;
struct Value; struct Value;
class EvalState; class EvalState;
struct StaticEnv; struct StaticEnv;
/* An attribute path is a sequence of attribute names. */ /* An attribute path is a sequence of attribute names. */
struct AttrName struct AttrName {
{ Symbol symbol;
Symbol symbol; Expr* expr;
Expr * expr; AttrName(const Symbol& s) : symbol(s){};
AttrName(const Symbol & s) : symbol(s) {}; AttrName(Expr* e) : expr(e){};
AttrName(Expr * e) : expr(e) {};
}; };
typedef std::vector<AttrName> AttrPath; typedef std::vector<AttrName> AttrPath;
string showAttrPath(const AttrPath & attrPath); string showAttrPath(const AttrPath& attrPath);
/* Abstract syntax of Nix expressions. */ /* Abstract syntax of Nix expressions. */
struct Expr struct Expr {
{ virtual ~Expr(){};
virtual ~Expr() { }; virtual void show(std::ostream& str) const;
virtual void show(std::ostream & str) const; virtual void bindVars(const StaticEnv& env);
virtual void bindVars(const StaticEnv & env); virtual void eval(EvalState& state, Env& env, Value& v);
virtual void eval(EvalState & state, Env & env, Value & v); virtual Value* maybeThunk(EvalState& state, Env& env);
virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol& name);
virtual void setName(Symbol & name);
}; };
std::ostream & operator << (std::ostream & str, const Expr & e); std::ostream& operator<<(std::ostream& str, const Expr& e);
#define COMMON_METHODS \ #define COMMON_METHODS \
void show(std::ostream & str) const; \ void show(std::ostream& str) const; \
void eval(EvalState & state, Env & env, Value & v); \ void eval(EvalState& state, Env& env, Value& v); \
void bindVars(const StaticEnv & env); void bindVars(const StaticEnv& env);
struct ExprInt : Expr struct ExprInt : Expr {
{ NixInt n;
NixInt n; Value v;
Value v; ExprInt(NixInt n) : n(n) { mkInt(v, n); };
ExprInt(NixInt n) : n(n) { mkInt(v, n); }; COMMON_METHODS
COMMON_METHODS Value* maybeThunk(EvalState& state, Env& env);
Value * maybeThunk(EvalState & state, Env & env);
}; };
struct ExprFloat : Expr struct ExprFloat : Expr {
{ NixFloat nf;
NixFloat nf; Value v;
Value v; ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; COMMON_METHODS
COMMON_METHODS Value* maybeThunk(EvalState& state, Env& env);
Value * maybeThunk(EvalState & state, Env & env);
}; };
struct ExprString : Expr struct ExprString : Expr {
{ Symbol s;
Symbol s; Value v;
Value v; ExprString(const Symbol& s) : s(s) { mkString(v, s); };
ExprString(const Symbol & s) : s(s) { mkString(v, s); }; COMMON_METHODS
COMMON_METHODS Value* maybeThunk(EvalState& state, Env& env);
Value * maybeThunk(EvalState & state, Env & env);
}; };
/* Temporary class used during parsing of indented strings. */ /* Temporary class used during parsing of indented strings. */
struct ExprIndStr : Expr struct ExprIndStr : Expr {
{ string s;
string s; ExprIndStr(const string& s) : s(s){};
ExprIndStr(const string & s) : s(s) { };
}; };
struct ExprPath : Expr struct ExprPath : Expr {
{ string s;
string s; Value v;
Value v; ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; COMMON_METHODS
COMMON_METHODS Value* maybeThunk(EvalState& state, Env& env);
Value * maybeThunk(EvalState & state, Env & env);
}; };
struct ExprVar : Expr struct ExprVar : Expr {
{ Pos pos;
Symbol name;
/* Whether the variable comes from an environment (e.g. a rec, let
or function argument) or from a "with". */
bool fromWith;
/* In the former case, the value is obtained by going `level'
levels up from the current environment and getting the
`displ'th value in that environment. In the latter case, the
value is obtained by getting the attribute named `name' from
the set stored in the environment that is `level' levels up
from the current one.*/
unsigned int level;
unsigned int displ;
ExprVar(const Symbol& name) : name(name){};
ExprVar(const Pos& pos, const Symbol& name) : pos(pos), name(name){};
COMMON_METHODS
Value* maybeThunk(EvalState& state, Env& env);
};
struct ExprSelect : Expr {
Pos pos;
Expr *e, *def;
AttrPath attrPath;
ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def)
: pos(pos), e(e), def(def), attrPath(attrPath){};
ExprSelect(const Pos& pos, Expr* e, const Symbol& name)
: pos(pos), e(e), def(0) {
attrPath.push_back(AttrName(name));
};
COMMON_METHODS
};
struct ExprOpHasAttr : Expr {
Expr* e;
AttrPath attrPath;
ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){};
COMMON_METHODS
};
struct ExprAttrs : Expr {
bool recursive;
struct AttrDef {
bool inherited;
Expr* e;
Pos pos; Pos pos;
Symbol name; unsigned int displ; // displacement
AttrDef(Expr* e, const Pos& pos, bool inherited = false)
/* Whether the variable comes from an environment (e.g. a rec, let : inherited(inherited), e(e), pos(pos){};
or function argument) or from a "with". */ AttrDef(){};
bool fromWith; };
typedef std::map<Symbol, AttrDef> AttrDefs;
/* In the former case, the value is obtained by going `level' AttrDefs attrs;
levels up from the current environment and getting the struct DynamicAttrDef {
`displ'th value in that environment. In the latter case, the Expr *nameExpr, *valueExpr;
value is obtained by getting the attribute named `name' from
the set stored in the environment that is `level' levels up
from the current one.*/
unsigned int level;
unsigned int displ;
ExprVar(const Symbol & name) : name(name) { };
ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env);
};
struct ExprSelect : Expr
{
Pos pos; Pos pos;
Expr * e, * def; DynamicAttrDef(Expr* nameExpr, Expr* valueExpr, const Pos& pos)
AttrPath attrPath; : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos){};
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; };
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
COMMON_METHODS DynamicAttrDefs dynamicAttrs;
ExprAttrs() : recursive(false){};
COMMON_METHODS
}; };
struct ExprOpHasAttr : Expr struct ExprList : Expr {
{ std::vector<Expr*> elems;
Expr * e; ExprList(){};
AttrPath attrPath; COMMON_METHODS
ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { };
COMMON_METHODS
}; };
struct ExprAttrs : Expr struct Formal {
{ Symbol name;
bool recursive; Expr* def;
struct AttrDef { Formal(const Symbol& name, Expr* def) : name(name), def(def){};
bool inherited;
Expr * e;
Pos pos;
unsigned int displ; // displacement
AttrDef(Expr * e, const Pos & pos, bool inherited=false)
: inherited(inherited), e(e), pos(pos) { };
AttrDef() { };
};
typedef std::map<Symbol, AttrDef> AttrDefs;
AttrDefs attrs;
struct DynamicAttrDef {
Expr * nameExpr, * valueExpr;
Pos pos;
DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const Pos & pos)
: nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { };
};
typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
DynamicAttrDefs dynamicAttrs;
ExprAttrs() : recursive(false) { };
COMMON_METHODS
}; };
struct ExprList : Expr struct Formals {
{ typedef std::list<Formal> Formals_;
std::vector<Expr *> elems; Formals_ formals;
ExprList() { }; std::set<Symbol> argNames; // used during parsing
COMMON_METHODS bool ellipsis;
}; };
struct Formal struct ExprLambda : Expr {
{ Pos pos;
Symbol name; Symbol name;
Expr * def; Symbol arg;
Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; bool matchAttrs;
Formals* formals;
Expr* body;
ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs,
Formals* formals, Expr* body)
: pos(pos),
arg(arg),
matchAttrs(matchAttrs),
formals(formals),
body(body) {
if (!arg.empty() && formals &&
formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(
format("duplicate formal function argument '%1%' at %2%") % arg %
pos);
};
void setName(Symbol& name);
string showNamePos() const;
COMMON_METHODS
}; };
struct Formals struct ExprLet : Expr {
{ ExprAttrs* attrs;
typedef std::list<Formal> Formals_; Expr* body;
Formals_ formals; ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){};
std::set<Symbol> argNames; // used during parsing COMMON_METHODS
bool ellipsis;
}; };
struct ExprLambda : Expr struct ExprWith : Expr {
{ Pos pos;
Pos pos; Expr *attrs, *body;
Symbol name; size_t prevWith;
Symbol arg; ExprWith(const Pos& pos, Expr* attrs, Expr* body)
bool matchAttrs; : pos(pos), attrs(attrs), body(body){};
Formals * formals; COMMON_METHODS
Expr * body;
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(format("duplicate formal function argument '%1%' at %2%")
% arg % pos);
};
void setName(Symbol & name);
string showNamePos() const;
COMMON_METHODS
}; };
struct ExprLet : Expr struct ExprIf : Expr {
{ Expr *cond, *then, *else_;
ExprAttrs * attrs; ExprIf(Expr* cond, Expr* then, Expr* else_)
Expr * body; : cond(cond), then(then), else_(else_){};
ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { }; COMMON_METHODS
COMMON_METHODS
}; };
struct ExprWith : Expr struct ExprAssert : Expr {
{ Pos pos;
Pos pos; Expr *cond, *body;
Expr * attrs, * body; ExprAssert(const Pos& pos, Expr* cond, Expr* body)
size_t prevWith; : pos(pos), cond(cond), body(body){};
ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; COMMON_METHODS
COMMON_METHODS
}; };
struct ExprIf : Expr struct ExprOpNot : Expr {
{ Expr* e;
Expr * cond, * then, * else_; ExprOpNot(Expr* e) : e(e){};
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; COMMON_METHODS
COMMON_METHODS
}; };
struct ExprAssert : Expr #define MakeBinOp(name, s) \
{ struct name : Expr { \
Pos pos; Pos pos; \
Expr * cond, * body; Expr *e1, *e2; \
ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \
COMMON_METHODS name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \
void show(std::ostream& str) const { \
str << "(" << *e1 << " " s " " << *e2 << ")"; \
} \
void bindVars(const StaticEnv& env) { \
e1->bindVars(env); \
e2->bindVars(env); \
} \
void eval(EvalState& state, Env& env, Value& v); \
};
MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprOpConcatLists, "++")
struct ExprConcatStrings : Expr {
Pos pos;
bool forceString;
vector<Expr*>* es;
ExprConcatStrings(const Pos& pos, bool forceString, vector<Expr*>* es)
: pos(pos), forceString(forceString), es(es){};
COMMON_METHODS
}; };
struct ExprOpNot : Expr struct ExprPos : Expr {
{ Pos pos;
Expr * e; ExprPos(const Pos& pos) : pos(pos){};
ExprOpNot(Expr * e) : e(e) { }; COMMON_METHODS
COMMON_METHODS
}; };
#define MakeBinOp(name, s) \
struct name : Expr \
{ \
Pos pos; \
Expr * e1, * e2; \
name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
void show(std::ostream & str) const \
{ \
str << "(" << *e1 << " " s " " << *e2 << ")"; \
} \
void bindVars(const StaticEnv & env) \
{ \
e1->bindVars(env); e2->bindVars(env); \
} \
void eval(EvalState & state, Env & env, Value & v); \
};
MakeBinOp(ExprApp, "")
MakeBinOp(ExprOpEq, "==")
MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpAnd, "&&")
MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpImpl, "->")
MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprOpConcatLists, "++")
struct ExprConcatStrings : Expr
{
Pos pos;
bool forceString;
vector<Expr *> * es;
ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
: pos(pos), forceString(forceString), es(es) { };
COMMON_METHODS
};
struct ExprPos : Expr
{
Pos pos;
ExprPos(const Pos & pos) : pos(pos) { };
COMMON_METHODS
};
/* Static environments are used to map variable names onto (level, /* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at displacement) pairs used to obtain the value of the variable at
runtime. */ runtime. */
struct StaticEnv struct StaticEnv {
{ bool isWith;
bool isWith; const StaticEnv* up;
const StaticEnv * up; typedef std::map<Symbol, unsigned int> Vars;
typedef std::map<Symbol, unsigned int> Vars; Vars vars;
Vars vars; StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){};
StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
}; };
} // namespace nix
}

3488
third_party/nix/src/libexpr/primops.cc vendored
View file

File diff suppressed because it is too large Load diff

25
third_party/nix/src/libexpr/primops.hh vendored
View file

@ -1,26 +1,25 @@
#include "eval.hh"
#include <tuple> #include <tuple>
#include <vector> #include <vector>
#include "eval.hh"
namespace nix { namespace nix {
struct RegisterPrimOp struct RegisterPrimOp {
{ typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; static PrimOps* primOps;
static PrimOps * primOps; /* You can register a constant by passing an arity of 0. fun
/* You can register a constant by passing an arity of 0. fun will get called during EvalState initialization, so there
will get called during EvalState initialization, so there may be primops not yet added and builtins is not yet sorted. */
may be primops not yet added and builtins is not yet sorted. */ RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
}; };
/* These primops are disabled without enableNativeCode, but plugins /* These primops are disabled without enableNativeCode, but plugins
may wish to use them in limited contexts without globally enabling may wish to use them in limited contexts without globally enabling
them. */ them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */ /* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); void prim_importNative(EvalState& state, const Pos& pos, Value** args,
Value& v);
/* Execute a program and parse its output */ /* Execute a program and parse its output */
void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v);
} } // namespace nix

257
third_party/nix/src/libexpr/primops/context.cc vendored
View file

@ -1,49 +1,47 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "derivations.hh" #include "derivations.hh"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix { namespace nix {
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos,
{ Value** args, Value& v) {
PathSet context; PathSet context;
string s = state.coerceToString(pos, *args[0], context); string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet()); mkString(v, s, PathSet());
} }
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static RegisterPrimOp r1("__unsafeDiscardStringContext", 1,
prim_unsafeDiscardStringContext);
static void prim_hasContext(EvalState& state, const Pos& pos, Value** args,
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) Value& v) {
{ PathSet context;
PathSet context; state.forceString(*args[0], context, pos);
state.forceString(*args[0], context, pos); mkBool(v, !context.empty());
mkBool(v, !context.empty());
} }
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance, builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */ drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos,
{ Value** args, Value& v) {
PathSet context; PathSet context;
string s = state.coerceToString(pos, *args[0], context); string s = state.coerceToString(pos, *args[0], context);
PathSet context2; PathSet context2;
for (auto & p : context) for (auto& p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p);
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
mkString(v, s, context2); mkString(v, s, context2);
} }
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1,
prim_unsafeDiscardOutputDependency);
/* Extract the context of a string as a structured Nix value. /* Extract the context of a string as a structured Nix value.
@ -64,124 +62,131 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar
Note that for a given path any combination of the above attributes Note that for a given path any combination of the above attributes
may be present. may be present.
*/ */
static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_getContext(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
struct ContextInfo { struct ContextInfo {
bool path = false; bool path = false;
bool allOutputs = false; bool allOutputs = false;
Strings outputs; Strings outputs;
}; };
PathSet context; PathSet context;
state.forceString(*args[0], context, pos); state.forceString(*args[0], context, pos);
auto contextInfos = std::map<Path, ContextInfo>(); auto contextInfos = std::map<Path, ContextInfo>();
for (const auto & p : context) { for (const auto& p : context) {
Path drv; Path drv;
string output; string output;
const Path * path = &p; const Path* path = &p;
if (p.at(0) == '=') { if (p.at(0) == '=') {
drv = string(p, 1); drv = string(p, 1);
path = &drv; path = &drv;
} else if (p.at(0) == '!') { } else if (p.at(0) == '!') {
std::pair<string, string> ctx = decodeContext(p); std::pair<string, string> ctx = decodeContext(p);
drv = ctx.first; drv = ctx.first;
output = ctx.second; output = ctx.second;
path = &drv; path = &drv;
}
auto isPath = drv.empty();
auto isAllOutputs = (!drv.empty()) && output.empty();
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
else if (isAllOutputs)
iter->second.allOutputs = true;
else
iter->second.outputs.emplace_back(std::move(output));
}
} }
auto isPath = drv.empty();
auto isAllOutputs = (!drv.empty()) && output.empty();
state.mkAttrs(v, contextInfos.size()); auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
auto sPath = state.symbols.create("path"); contextInfos.emplace(
auto sAllOutputs = state.symbols.create("allOutputs"); *path,
for (const auto & info : contextInfos) { ContextInfo{isPath, isAllOutputs,
auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); output.empty() ? Strings{} : Strings{std::move(output)}});
state.mkAttrs(infoVal, 3); } else {
if (info.second.path) if (isPath)
mkBool(*state.allocAttr(infoVal, sPath), true); iter->second.path = true;
if (info.second.allOutputs) else if (isAllOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true); iter->second.allOutputs = true;
if (!info.second.outputs.empty()) { else
auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); iter->second.outputs.emplace_back(std::move(output));
state.mkList(outputsVal, info.second.outputs.size());
size_t i = 0;
for (const auto & output : info.second.outputs) {
mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
}
}
infoVal.attrs->sort();
} }
v.attrs->sort(); }
state.mkAttrs(v, contextInfos.size());
auto sPath = state.symbols.create("path");
auto sAllOutputs = state.symbols.create("allOutputs");
for (const auto& info : contextInfos) {
auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first));
state.mkAttrs(infoVal, 3);
if (info.second.path) mkBool(*state.allocAttr(infoVal, sPath), true);
if (info.second.allOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
if (!info.second.outputs.empty()) {
auto& outputsVal = *state.allocAttr(infoVal, state.sOutputs);
state.mkList(outputsVal, info.second.outputs.size());
size_t i = 0;
for (const auto& output : info.second.outputs) {
mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
}
}
infoVal.attrs->sort();
}
v.attrs->sort();
} }
static RegisterPrimOp r4("__getContext", 1, prim_getContext); static RegisterPrimOp r4("__getContext", 1, prim_getContext);
/* Append the given context to a given string. /* Append the given context to a given string.
See the commentary above unsafeGetContext for details of the See the commentary above unsafeGetContext for details of the
context representation. context representation.
*/ */
static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_appendContext(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
PathSet context; PathSet context;
auto orig = state.forceString(*args[0], context, pos); auto orig = state.forceString(*args[0], context, pos);
state.forceAttrs(*args[1], pos); state.forceAttrs(*args[1], pos);
auto sPath = state.symbols.create("path"); auto sPath = state.symbols.create("path");
auto sAllOutputs = state.symbols.create("allOutputs"); auto sAllOutputs = state.symbols.create("allOutputs");
for (auto & i : *args[1]->attrs) { for (auto& i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name)) if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); throw EvalError("Context key '%s' is not a store path, at %s", i.name,
if (!settings.readOnlyMode) i.pos);
state.store->ensurePath(i.name); if (!settings.readOnlyMode) state.store->ensurePath(i.name);
state.forceAttrs(*i.value, *i.pos); state.forceAttrs(*i.value, *i.pos);
auto iter = i.value->attrs->find(sPath); auto iter = i.value->attrs->find(sPath);
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) if (state.forceBool(*iter->value, *iter->pos)) context.insert(i.name);
context.insert(i.name);
}
iter = i.value->attrs->find(sAllOutputs);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
}
context.insert("=" + string(i.name));
}
}
iter = i.value->attrs->find(state.sOutputs);
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name));
}
}
} }
mkString(v, orig, context); iter = i.value->attrs->find(sAllOutputs);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError(
"Tried to add all-outputs context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
}
context.insert("=" + string(i.name));
}
}
iter = i.value->attrs->find(state.sOutputs);
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError(
"Tried to add derivation output context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name =
state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name));
}
}
}
mkString(v, orig, context);
} }
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
} } // namespace nix

436
third_party/nix/src/libexpr/primops/fetchGit.cc vendored
View file

@ -1,247 +1,253 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"
#include <sys/time.h> #include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "hash.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals; using namespace std::string_literals;
namespace nix { namespace nix {
struct GitInfo struct GitInfo {
{ Path storePath;
Path storePath; std::string rev;
std::string rev; std::string shortRev;
std::string shortRev; uint64_t revCount = 0;
uint64_t revCount = 0;
}; };
std::regex revRegex("^[0-9a-fA-F]{40}$"); std::regex revRegex("^[0-9a-fA-F]{40}$");
GitInfo exportGit(ref<Store> store, const std::string & uri, GitInfo exportGit(ref<Store> store, const std::string& uri,
std::optional<std::string> ref, std::string rev, std::optional<std::string> ref, std::string rev,
const std::string & name) const std::string& name) {
{ if (evalSettings.pureEval && rev == "")
if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
bool clean = true;
bool clean = true;
try {
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
clean = false;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
GitInfo gitInfo;
gitInfo.rev = "0000000000000000000000000000000000000000";
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
return gitInfo;
}
// clean working tree, but no ref or rev specified. Use 'HEAD'.
rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
ref = "HEAD"s;
}
if (!ref) ref = "HEAD"s;
if (rev != "" && !std::regex_match(rev, revRegex))
throw Error("invalid Git revision '%s'", rev);
deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", cacheDir });
}
Path localRefFile;
if (ref->compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + *ref;
else
localRefFile = cacheDir + "/refs/heads/" + *ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev != "") {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
try { try {
auto json = nlohmann::json::parse(readFile(storeLink)); runProgram("git", true,
{"-C", uri, "diff-index", "--quiet", "HEAD", "--"});
assert(json["name"] == name && json["rev"] == gitInfo.rev); } catch (ExecError& e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
gitInfo.storePath = json["storePath"]; clean = false;
if (store->isValidPath(gitInfo.storePath)) {
gitInfo.revCount = json["revCount"];
return gitInfo;
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
} }
// FIXME: should pipe this, or find some better way to extract a if (!clean) {
// revision. /* This is an unclean working tree. So copy all tracked
auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); files. */
Path tmpDir = createTempDir(); GitInfo gitInfo;
AutoDelete delTmpDir(tmpDir, true); gitInfo.rev = "0000000000000000000000000000000000000000";
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
runProgram("tar", true, { "x", "-C", tmpDir }, tar); auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, {"-C", uri, "ls-files", "-z"}), "\0"s);
gitInfo.storePath = store->addToStore(name, tmpDir); PathFilter filter = [&](const Path& p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); auto st = lstat(p);
nlohmann::json json; if (S_ISDIR(st.st_mode)) {
json["storePath"] = gitInfo.storePath; auto prefix = file + "/";
json["uri"] = uri; auto i = files.lower_bound(prefix);
json["name"] = name; return i != files.end() && hasPrefix(*i, prefix);
json["rev"] = gitInfo.rev;
json["revCount"] = gitInfo.revCount;
writeFile(storeLink, json.dump());
return gitInfo;
}
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
std::string url;
std::optional<std::string> ref;
std::string rev;
std::string name = "source";
PathSet context;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
} }
if (url.empty()) return files.count(file);
throw EvalError(format("'url' argument required, at %1%") % pos); };
} else gitInfo.storePath =
url = state.coerceToString(pos, *args[0], context, false, false); store->addToStore("source", uri, true, htSHA256, filter);
// FIXME: git externals probably can be used to bypass the URI return gitInfo;
// whitelist. Ah well. }
state.checkURI(url);
auto gitInfo = exportGit(state.store, url, ref, rev, name); // clean working tree, but no ref or rev specified. Use 'HEAD'.
rev = chomp(runProgram("git", true, {"-C", uri, "rev-parse", "HEAD"}));
ref = "HEAD"s;
}
state.mkAttrs(v, 8); if (!ref) ref = "HEAD"s;
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
v.attrs->sort();
if (state.allowedPaths) if (rev != "" && !std::regex_match(rev, revRegex))
state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); throw Error("invalid Git revision '%s'", rev);
deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" +
hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, {"init", "--bare", cacheDir});
}
Path localRefFile;
if (ref->compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + *ref;
else
localRefFile = cacheDir + "/refs/heads/" + *ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev != "") {
try {
runProgram("git", true, {"-C", cacheDir, "cat-file", "-e", rev});
doFetch = false;
} catch (ExecError& e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now;
}
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true,
{"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri,
fmt("%s:%s", *ref, *ref)});
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev)
.to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...",
storeLink)); // FIXME: broken
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == gitInfo.rev);
gitInfo.storePath = json["storePath"];
if (store->isValidPath(gitInfo.storePath)) {
gitInfo.revCount = json["revCount"];
return gitInfo;
}
} catch (SysError& e) {
if (e.errNo != ENOENT) throw;
}
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto tar = runProgram("git", true, {"-C", cacheDir, "archive", gitInfo.rev});
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("tar", true, {"x", "-C", tmpDir}, tar);
gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram(
"git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev}));
nlohmann::json json;
json["storePath"] = gitInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["rev"] = gitInfo.rev;
json["revCount"] = gitInfo.revCount;
writeFile(storeLink, json.dump());
return gitInfo;
}
static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args,
Value& v) {
std::string url;
std::optional<std::string> ref;
std::string rev;
std::string name = "source";
PathSet context;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto& attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s",
attr.name, *attr.pos);
}
if (url.empty())
throw EvalError(format("'url' argument required, at %1%") % pos);
} else
url = state.coerceToString(pos, *args[0], context, false, false);
// FIXME: git externals probably can be used to bypass the URI
// whitelist. Ah well.
state.checkURI(url);
auto gitInfo = exportGit(state.store, url, ref, rev, name);
state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath,
PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
gitInfo.revCount);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
} }
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
} } // namespace nix

346
third_party/nix/src/libexpr/primops/fetchMercurial.cc vendored
View file

@ -1,219 +1,229 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include <sys/time.h> #include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals; using namespace std::string_literals;
namespace nix { namespace nix {
struct HgInfo struct HgInfo {
{ Path storePath;
Path storePath; std::string branch;
std::string branch; std::string rev;
std::string rev; uint64_t revCount = 0;
uint64_t revCount = 0;
}; };
std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string & uri, HgInfo exportMercurial(ref<Store> store, const std::string& uri,
std::string rev, const std::string & name) std::string rev, const std::string& name) {
{ if (evalSettings.pureEval && rev == "")
if (evalSettings.pureEval && rev == "") throw Error(
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); "in pure evaluation mode, 'fetchMercurial' requires a Mercurial "
"revision");
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
bool clean = runProgram("hg", true,
{"status", "-R", uri, "--modified", "--added",
"--removed"}) == "";
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
if (!clean) { printTalkative("copying unclean Mercurial working tree '%s'", uri);
/* This is an unclean working tree. So copy all tracked HgInfo hgInfo;
files. */ hgInfo.rev = "0000000000000000000000000000000000000000";
hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri}));
printTalkative("copying unclean Mercurial working tree '%s'", uri); auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true,
{"status", "-R", uri, "--clean", "--modified", "--added",
"--no-status", "--print0"}),
"\0"s);
HgInfo hgInfo; PathFilter filter = [&](const Path& p) -> bool {
hgInfo.rev = "0000000000000000000000000000000000000000"; assert(hasPrefix(p, uri));
hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); std::string file(p, uri.size() + 1);
auto files = tokenizeString<std::set<std::string>>( auto st = lstat(p);
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool { if (S_ISDIR(st.st_mode)) {
assert(hasPrefix(p, uri)); auto prefix = file + "/";
std::string file(p, uri.size() + 1); auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
return hgInfo;
}
}
if (rev == "") rev = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
do so now. */
time_t now = time(0);
struct stat st;
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
{
/* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex)
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
}
} }
writeFile(stampFile, ""); return files.count(file);
};
hgInfo.storePath =
store->addToStore("source", uri, true, htSHA256, filter);
return hgInfo;
} }
}
auto tokens = tokenizeString<std::vector<std::string>>( if (rev == "") rev = "default";
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
HgInfo hgInfo; Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(),
hgInfo.rev = tokens[0]; hashString(htSHA256, uri).to_string(Base32, false));
hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir,
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); hashString(htSHA512, rev).to_string(Base32, false));
try { /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
auto json = nlohmann::json::parse(readFile(storeLink)); do so now. */
time_t now = time(0);
struct stat st;
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now) {
/* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) &&
runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev,
"--template", "1"})
.killStderr(true))
.second == "1")) {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("fetching Mercurial repository '%s'", uri));
assert(json["name"] == name && json["rev"] == hgInfo.rev); if (pathExists(cacheDir)) {
try {
hgInfo.storePath = json["storePath"]; runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
} catch (ExecError& e) {
if (store->isValidPath(hgInfo.storePath)) { string transJournal = cacheDir + "/.hg/store/journal";
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); /* hg throws "abandoned transaction" error only if this file exists */
return hgInfo; if (pathExists(transJournal)) {
runProgram("hg", true, {"recover", "-R", cacheDir});
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
} else {
throw ExecError(e.status,
fmt("'hg pull' %s", statusToString(e.status)));
}
} }
} else {
} catch (SysError & e) { createDirs(dirOf(cacheDir));
if (e.errNo != ENOENT) throw; runProgram("hg", true, {"clone", "--noupdate", "--", uri, cacheDir});
}
} }
Path tmpDir = createTempDir(); writeFile(stampFile, "");
AutoDelete delTmpDir(tmpDir, true); }
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true,
{"log", "-R", cacheDir, "-r", rev, "--template",
"{node} {rev} {branch}"}));
assert(tokens.size() == 3);
deletePath(tmpDir + "/.hg_archival.txt"); HgInfo hgInfo;
hgInfo.rev = tokens[0];
hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2];
hgInfo.storePath = store->addToStore(name, tmpDir); std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev)
.to_string(Base32, false);
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
nlohmann::json json; try {
json["storePath"] = hgInfo.storePath; auto json = nlohmann::json::parse(readFile(storeLink));
json["uri"] = uri;
json["name"] = name;
json["branch"] = hgInfo.branch;
json["rev"] = hgInfo.rev;
json["revCount"] = hgInfo.revCount;
writeFile(storeLink, json.dump()); assert(json["name"] == name && json["rev"] == hgInfo.rev);
return hgInfo; hgInfo.storePath = json["storePath"];
if (store->isValidPath(hgInfo.storePath)) {
printTalkative("using cached Mercurial store path '%s'",
hgInfo.storePath);
return hgInfo;
}
} catch (SysError& e) {
if (e.errNo != ENOENT) throw;
}
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, {"archive", "-R", cacheDir, "-r", rev, tmpDir});
deletePath(tmpDir + "/.hg_archival.txt");
hgInfo.storePath = store->addToStore(name, tmpDir);
nlohmann::json json;
json["storePath"] = hgInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["branch"] = hgInfo.branch;
json["rev"] = hgInfo.rev;
json["revCount"] = hgInfo.revCount;
writeFile(storeLink, json.dump());
return hgInfo;
} }
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
std::string url; std::string url;
std::string rev; std::string rev;
std::string name = "source"; std::string name = "source";
PathSet context; PathSet context;
state.forceValue(*args[0]); state.forceValue(*args[0]);
if (args[0]->type == tAttrs) { if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
state.forceAttrs(*args[0], pos); for (auto& attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s",
attr.name, *attr.pos);
}
for (auto & attr : *args[0]->attrs) { if (url.empty())
string n(attr.name); throw EvalError(format("'url' argument required, at %1%") % pos);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
}
if (url.empty()) } else
throw EvalError(format("'url' argument required, at %1%") % pos); url = state.coerceToString(pos, *args[0], context, false, false);
} else // FIXME: git externals probably can be used to bypass the URI
url = state.coerceToString(pos, *args[0], context, false, false); // whitelist. Ah well.
state.checkURI(url);
// FIXME: git externals probably can be used to bypass the URI auto hgInfo = exportMercurial(state.store, url, rev, name);
// whitelist. Ah well.
state.checkURI(url);
auto hgInfo = exportMercurial(state.store, url, rev, name); state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath,
PathSet({hgInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
std::string(hgInfo.rev, 0, 12));
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
v.attrs->sort();
state.mkAttrs(v, 8); if (state.allowedPaths)
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
} }
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
} } // namespace nix

146
third_party/nix/src/libexpr/primops/fromTOML.cc vendored
View file

@ -1,90 +1,90 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "cpptoml/cpptoml.h" #include "cpptoml/cpptoml.h"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix { namespace nix {
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
using namespace cpptoml; using namespace cpptoml;
auto toml = state.forceStringNoCtx(*args[0], pos); auto toml = state.forceStringNoCtx(*args[0], pos);
std::istringstream tomlStream(toml); std::istringstream tomlStream(toml);
std::function<void(Value &, std::shared_ptr<base>)> visit; std::function<void(Value&, std::shared_ptr<base>)> visit;
visit = [&](Value & v, std::shared_ptr<base> t) { visit = [&](Value& v, std::shared_ptr<base> t) {
if (auto t2 = t->as_table()) {
size_t size = 0;
for (auto& i : *t2) {
(void)i;
size++;
}
if (auto t2 = t->as_table()) { state.mkAttrs(v, size);
size_t size = 0; for (auto& i : *t2) {
for (auto & i : *t2) { (void) i; size++; } auto& v2 = *state.allocAttr(v, state.symbols.create(i.first));
state.mkAttrs(v, size); if (auto i2 = i.second->as_table_array()) {
size_t size2 = i2->get().size();
state.mkList(v2, size2);
for (size_t j = 0; j < size2; ++j)
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
} else
visit(v2, i.second);
}
for (auto & i : *t2) { v.attrs->sort();
auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
if (auto i2 = i.second->as_table_array()) {
size_t size2 = i2->get().size();
state.mkList(v2, size2);
for (size_t j = 0; j < size2; ++j)
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
}
else
visit(v2, i.second);
}
v.attrs->sort();
}
else if (auto t2 = t->as_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t i = 0; i < size; ++i)
visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
}
// Handle cases like 'a = [[{ a = true }]]', which IMHO should be
// parsed as a array containing an array containing a table,
// but instead are parsed as an array containing a table array
// containing a table.
else if (auto t2 = t->as_table_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t j = 0; j < size; ++j)
visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
}
else if (t->is_value()) {
if (auto val = t->as<int64_t>())
mkInt(v, val->get());
else if (auto val = t->as<NixFloat>())
mkFloat(v, val->get());
else if (auto val = t->as<bool>())
mkBool(v, val->get());
else if (auto val = t->as<std::string>())
mkString(v, val->get());
else
throw EvalError("unsupported value type in TOML");
}
else abort();
};
try {
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) {
throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
} }
else if (auto t2 = t->as_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t i = 0; i < size; ++i)
visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
}
// Handle cases like 'a = [[{ a = true }]]', which IMHO should be
// parsed as a array containing an array containing a table,
// but instead are parsed as an array containing a table array
// containing a table.
else if (auto t2 = t->as_table_array()) {
size_t size = t2->get().size();
state.mkList(v, size);
for (size_t j = 0; j < size; ++j)
visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
}
else if (t->is_value()) {
if (auto val = t->as<int64_t>())
mkInt(v, val->get());
else if (auto val = t->as<NixFloat>())
mkFloat(v, val->get());
else if (auto val = t->as<bool>())
mkBool(v, val->get());
else if (auto val = t->as<std::string>())
mkString(v, val->get());
else
throw EvalError("unsupported value type in TOML");
}
else
abort();
};
try {
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error& e) {
throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
}
} }
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
} } // namespace nix

87
third_party/nix/src/libexpr/symbol-table.hh vendored
View file

@ -2,7 +2,6 @@
#include <map> #include <map>
#include <unordered_set> #include <unordered_set>
#include "types.hh" #include "types.hh"
namespace nix { namespace nix {
@ -13,75 +12,49 @@ namespace nix {
they can be compared efficiently (using a pointer equality test), they can be compared efficiently (using a pointer equality test),
because the symbol table stores only one copy of each string. */ because the symbol table stores only one copy of each string. */
class Symbol class Symbol {
{ private:
private: const string* s; // pointer into SymbolTable
const string * s; // pointer into SymbolTable Symbol(const string* s) : s(s){};
Symbol(const string * s) : s(s) { }; friend class SymbolTable;
friend class SymbolTable;
public: public:
Symbol() : s(0) { }; Symbol() : s(0){};
bool operator == (const Symbol & s2) const bool operator==(const Symbol& s2) const { return s == s2.s; }
{
return s == s2.s;
}
bool operator != (const Symbol & s2) const bool operator!=(const Symbol& s2) const { return s != s2.s; }
{
return s != s2.s;
}
bool operator < (const Symbol & s2) const bool operator<(const Symbol& s2) const { return s < s2.s; }
{
return s < s2.s;
}
operator const string & () const operator const string&() const { return *s; }
{
return *s;
}
bool set() const bool set() const { return s; }
{
return s;
}
bool empty() const bool empty() const { return s->empty(); }
{
return s->empty();
}
friend std::ostream & operator << (std::ostream & str, const Symbol & sym); friend std::ostream& operator<<(std::ostream& str, const Symbol& sym);
}; };
class SymbolTable class SymbolTable {
{ private:
private: typedef std::unordered_set<string> Symbols;
typedef std::unordered_set<string> Symbols; Symbols symbols;
Symbols symbols;
public: public:
Symbol create(const string & s) Symbol create(const string& s) {
{ std::pair<Symbols::iterator, bool> res = symbols.insert(s);
std::pair<Symbols::iterator, bool> res = symbols.insert(s); return Symbol(&*res.first);
return Symbol(&*res.first); }
}
size_t size() const size_t size() const { return symbols.size(); }
{
return symbols.size();
}
size_t totalSize() const; size_t totalSize() const;
template<typename T> template <typename T>
void dump(T callback) void dump(T callback) {
{ for (auto& s : symbols) callback(s);
for (auto & s : symbols) }
callback(s);
}
}; };
} } // namespace nix

155
third_party/nix/src/libexpr/value-to-json.cc vendored
View file

@ -1,100 +1,97 @@
#include "value-to-json.hh" #include "value-to-json.hh"
#include "json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
#include "eval-inline.hh"
#include "json.hh"
#include "util.hh"
namespace nix { namespace nix {
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, JSONPlaceholder & out, PathSet & context) JSONPlaceholder& out, PathSet& context) {
{ checkInterrupt();
checkInterrupt();
if (strict) state.forceValue(v); if (strict) state.forceValue(v);
switch (v.type) { switch (v.type) {
case tInt:
out.write(v.integer);
break;
case tInt: case tBool:
out.write(v.integer); out.write(v.boolean);
break; break;
case tBool: case tString:
out.write(v.boolean); copyContext(v, context);
break; out.write(v.string.s);
break;
case tString: case tPath:
copyContext(v, context); out.write(state.copyPathToStore(context, v.path));
out.write(v.string.s); break;
break;
case tPath: case tNull:
out.write(state.copyPathToStore(context, v.path)); out.write(nullptr);
break; break;
case tNull: case tAttrs: {
out.write(nullptr); auto maybeString =
break; state.tryAttrsToString(noPos, v, context, false, false);
if (maybeString) {
case tAttrs: { out.write(*maybeString);
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); break;
if (maybeString) { }
out.write(*maybeString); auto i = v.attrs->find(state.sOutPath);
break; if (i == v.attrs->end()) {
} auto obj(out.object());
auto i = v.attrs->find(state.sOutPath); StringSet names;
if (i == v.attrs->end()) { for (auto& j : *v.attrs) names.insert(j.name);
auto obj(out.object()); for (auto& j : names) {
StringSet names; Attr& a(*v.attrs->find(state.symbols.create(j)));
for (auto & j : *v.attrs) auto placeholder(obj.placeholder(j));
names.insert(j.name); printValueAsJSON(state, strict, *a.value, placeholder, context);
for (auto & j : names) {
Attr & a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j));
printValueAsJSON(state, strict, *a.value, placeholder, context);
}
} else
printValueAsJSON(state, strict, *i->value, out, context);
break;
} }
} else
case tList1: case tList2: case tListN: { printValueAsJSON(state, strict, *i->value, out, context);
auto list(out.list()); break;
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context);
}
break;
}
case tExternal:
v.external->printValueAsJSON(state, strict, out, context);
break;
case tFloat:
out.write(v.fpoint);
break;
default:
throw TypeError(format("cannot convert %1% to JSON") % showType(v));
} }
case tList1:
case tList2:
case tListN: {
auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *v.listElems()[n], placeholder,
context);
}
break;
}
case tExternal:
v.external->printValueAsJSON(state, strict, out, context);
break;
case tFloat:
out.write(v.fpoint);
break;
default:
throw TypeError(format("cannot convert %1% to JSON") % showType(v));
}
} }
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, std::ostream & str, PathSet & context) std::ostream& str, PathSet& context) {
{ JSONPlaceholder out(str);
JSONPlaceholder out(str); printValueAsJSON(state, strict, v, out, context);
printValueAsJSON(state, strict, v, out, context);
} }
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict,
JSONPlaceholder & out, PathSet & context) const JSONPlaceholder& out,
{ PathSet& context) const {
throw TypeError(format("cannot convert %1% to JSON") % showType()); throw TypeError(format("cannot convert %1% to JSON") % showType());
} }
} // namespace nix
}

17
third_party/nix/src/libexpr/value-to-json.hh vendored
View file

@ -1,19 +1,18 @@
#pragma once #pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix { namespace nix {
class JSONPlaceholder; class JSONPlaceholder;
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, JSONPlaceholder & out, PathSet & context); JSONPlaceholder& out, PathSet& context);
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, std::ostream & str, PathSet & context); std::ostream& str, PathSet& context);
} } // namespace nix

289
third_party/nix/src/libexpr/value-to-xml.cc vendored
View file

@ -1,178 +1,173 @@
#include "value-to-xml.hh" #include "value-to-xml.hh"
#include "xml-writer.hh" #include <cstdlib>
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
#include "xml-writer.hh"
#include <cstdlib>
namespace nix { namespace nix {
static XMLAttrs singletonAttrs(const string& name, const string& value) {
static XMLAttrs singletonAttrs(const string & name, const string & value) XMLAttrs attrs;
{ attrs[name] = value;
XMLAttrs attrs; return attrs;
attrs[name] = value;
return attrs;
} }
static void printValueAsXML(EvalState& state, bool strict, bool location,
Value& v, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen);
static void printValueAsXML(EvalState & state, bool strict, bool location, static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) {
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); xmlAttrs["path"] = pos.file;
xmlAttrs["line"] = (format("%1%") % pos.line).str();
xmlAttrs["column"] = (format("%1%") % pos.column).str();
static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
{
xmlAttrs["path"] = pos.file;
xmlAttrs["line"] = (format("%1%") % pos.line).str();
xmlAttrs["column"] = (format("%1%") % pos.column).str();
} }
static void showAttrs(EvalState& state, bool strict, bool location,
Bindings& attrs, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) {
StringSet names;
static void showAttrs(EvalState & state, bool strict, bool location, for (auto& i : attrs) names.insert(i.name);
Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
{
StringSet names;
for (auto & i : attrs) for (auto& i : names) {
names.insert(i.name); Attr& a(*attrs.find(state.symbols.create(i)));
for (auto & i : names) { XMLAttrs xmlAttrs;
Attr & a(*attrs.find(state.symbols.create(i))); xmlAttrs["name"] = i;
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
XMLOpenElement _(doc, "attr", xmlAttrs);
printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen);
}
}
static void printValueAsXML(EvalState& state, bool strict, bool location,
Value& v, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) {
checkInterrupt();
if (strict) state.forceValue(v);
switch (v.type) {
case tInt:
doc.writeEmptyElement(
"int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break;
case tBool:
doc.writeEmptyElement(
"bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break;
case tString:
/* !!! show the context? */
copyContext(v, context);
doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
break;
case tPath:
doc.writeEmptyElement("path", singletonAttrs("value", v.path));
break;
case tNull:
doc.writeEmptyElement("null");
break;
case tAttrs:
if (state.isDerivation(v)) {
XMLAttrs xmlAttrs; XMLAttrs xmlAttrs;
xmlAttrs["name"] = i;
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
XMLOpenElement _(doc, "attr", xmlAttrs); Bindings::iterator a =
printValueAsXML(state, strict, location, v.attrs->find(state.symbols.create("derivation"));
*a.value, doc, context, drvsSeen);
}
}
Path drvPath;
static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath);
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) if (a != v.attrs->end()) {
{ if (strict) state.forceValue(*a->value);
checkInterrupt(); if (a->value->type == tString)
xmlAttrs["drvPath"] = drvPath = a->value->string.s;
if (strict) state.forceValue(v);
switch (v.type) {
case tInt:
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break;
case tBool:
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break;
case tString:
/* !!! show the context? */
copyContext(v, context);
doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
break;
case tPath:
doc.writeEmptyElement("path", singletonAttrs("value", v.path));
break;
case tNull:
doc.writeEmptyElement("null");
break;
case tAttrs:
if (state.isDerivation(v)) {
XMLAttrs xmlAttrs;
Bindings::iterator a = v.attrs->find(state.symbols.create("derivation"));
Path drvPath;
a = v.attrs->find(state.sDrvPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (a->value->type == tString)
xmlAttrs["drvPath"] = drvPath = a->value->string.s;
}
a = v.attrs->find(state.sOutPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (a->value->type == tString)
xmlAttrs["outPath"] = a->value->string.s;
}
XMLOpenElement _(doc, "derivation", xmlAttrs);
if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
drvsSeen.insert(drvPath);
showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
} else
doc.writeEmptyElement("repeated");
}
else {
XMLOpenElement _(doc, "attrs");
showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
}
break;
case tList1: case tList2: case tListN: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
break;
} }
case tLambda: { a = v.attrs->find(state.sOutPath);
XMLAttrs xmlAttrs; if (a != v.attrs->end()) {
if (location) posToXML(xmlAttrs, v.lambda.fun->pos); if (strict) state.forceValue(*a->value);
XMLOpenElement _(doc, "function", xmlAttrs); if (a->value->type == tString)
xmlAttrs["outPath"] = a->value->string.s;
if (v.lambda.fun->matchAttrs) {
XMLAttrs attrs;
if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
XMLOpenElement _(doc, "attrspat", attrs);
for (auto & i : v.lambda.fun->formals->formals)
doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
} else
doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
break;
} }
case tExternal: XMLOpenElement _(doc, "derivation", xmlAttrs);
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
break;
case tFloat: if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); drvsSeen.insert(drvPath);
break; showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
} else
doc.writeEmptyElement("repeated");
}
default: else {
doc.writeEmptyElement("unevaluated"); XMLOpenElement _(doc, "attrs");
showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
}
break;
case tList1:
case tList2:
case tListN: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc,
context, drvsSeen);
break;
} }
case tLambda: {
XMLAttrs xmlAttrs;
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
if (v.lambda.fun->matchAttrs) {
XMLAttrs attrs;
if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
XMLOpenElement _(doc, "attrspat", attrs);
for (auto& i : v.lambda.fun->formals->formals)
doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
} else
doc.writeEmptyElement("varpat",
singletonAttrs("name", v.lambda.fun->arg));
break;
}
case tExternal:
v.external->printValueAsXML(state, strict, location, doc, context,
drvsSeen);
break;
case tFloat:
doc.writeEmptyElement(
"float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
break;
default:
doc.writeEmptyElement("unevaluated");
}
} }
void ExternalValueBase::printValueAsXML(EvalState& state, bool strict,
void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, bool location, XMLWriter& doc,
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const PathSet& context,
{ PathSet& drvsSeen) const {
doc.writeEmptyElement("unevaluated"); doc.writeEmptyElement("unevaluated");
} }
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
void printValueAsXML(EvalState & state, bool strict, bool location, std::ostream& out, PathSet& context) {
Value & v, std::ostream & out, PathSet & context) XMLWriter doc(true, out);
{ XMLOpenElement root(doc, "expr");
XMLWriter doc(true, out); PathSet drvsSeen;
XMLOpenElement root(doc, "expr"); printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
PathSet drvsSeen;
printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
} }
} // namespace nix
}

13
third_party/nix/src/libexpr/value-to-xml.hh vendored
View file

@ -1,14 +1,13 @@
#pragma once #pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix { namespace nix {
void printValueAsXML(EvalState & state, bool strict, bool location, void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
Value & v, std::ostream & out, PathSet & context); std::ostream& out, PathSet& context);
} }

346
third_party/nix/src/libexpr/value.hh vendored
View file

@ -8,28 +8,26 @@
namespace nix { namespace nix {
typedef enum { typedef enum {
tInt = 1, tInt = 1,
tBool, tBool,
tString, tString,
tPath, tPath,
tNull, tNull,
tAttrs, tAttrs,
tList1, tList1,
tList2, tList2,
tListN, tListN,
tThunk, tThunk,
tApp, tApp,
tLambda, tLambda,
tBlackhole, tBlackhole,
tPrimOp, tPrimOp,
tPrimOpApp, tPrimOpApp,
tExternal, tExternal,
tFloat tFloat
} ValueType; } ValueType;
class Bindings; class Bindings;
struct Env; struct Env;
struct Expr; struct Expr;
@ -42,233 +40,201 @@ class EvalState;
class XMLWriter; class XMLWriter;
class JSONPlaceholder; class JSONPlaceholder;
typedef int64_t NixInt; typedef int64_t NixInt;
typedef double NixFloat; typedef double NixFloat;
/* External values must descend from ExternalValueBase, so that /* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented * type-agnostic nix functions (e.g. showType) can be implemented
*/ */
class ExternalValueBase class ExternalValueBase {
{ friend std::ostream& operator<<(std::ostream& str,
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); const ExternalValueBase& v);
protected:
/* Print out the value */
virtual std::ostream & print(std::ostream & str) const = 0;
public: protected:
/* Return a simple string describing the type */ /* Print out the value */
virtual string showType() const = 0; virtual std::ostream& print(std::ostream& str) const = 0;
/* Return a string to be used in builtins.typeOf */ public:
virtual string typeOf() const = 0; /* Return a simple string describing the type */
virtual string showType() const = 0;
/* How much space does this value take up */ /* Return a string to be used in builtins.typeOf */
virtual size_t valueSize(std::set<const void *> & seen) const = 0; virtual string typeOf() const = 0;
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an /* How much space does this value take up */
* error virtual size_t valueSize(std::set<const void*>& seen) const = 0;
*/
virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
/* Compare to another value of the same type. Defaults to uncomparable, /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* i.e. always false. * error
*/ */
virtual bool operator==(const ExternalValueBase & b) const; virtual string coerceToString(const Pos& pos, PathSet& context, bool copyMore,
bool copyToStore) const;
/* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ /* Compare to another value of the same type. Defaults to uncomparable,
virtual void printValueAsJSON(EvalState & state, bool strict, * i.e. always false.
JSONPlaceholder & out, PathSet & context) const; */
virtual bool operator==(const ExternalValueBase& b) const;
/* Print the value as XML. Defaults to unevaluated */ /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
virtual void printValueAsXML(EvalState & state, bool strict, bool location, virtual void printValueAsJSON(EvalState& state, bool strict,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; JSONPlaceholder& out, PathSet& context) const;
virtual ~ExternalValueBase() /* Print the value as XML. Defaults to unevaluated */
{ virtual void printValueAsXML(EvalState& state, bool strict, bool location,
}; XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) const;
virtual ~ExternalValueBase(){};
}; };
std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v);
struct Value {
ValueType type;
union {
NixInt integer;
bool boolean;
struct Value /* Strings in the evaluator carry a so-called `context' which
{ is a list of strings representing store paths. This is to
ValueType type; allow users to write things like
union
{
NixInt integer;
bool boolean;
/* Strings in the evaluator carry a so-called `context' which "--with-freetype2-library=" + freetype + "/lib"
is a list of strings representing store paths. This is to
allow users to write things like
"--with-freetype2-library=" + freetype + "/lib" where `freetype' is a derivation (or a source to be copied
to the store). If we just concatenated the strings without
keeping track of the referenced store paths, then if the
string is used as a derivation attribute, the derivation
will not have the correct dependencies in its inputDrvs and
inputSrcs.
where `freetype' is a derivation (or a source to be copied The semantics of the context is as follows: when a string
to the store). If we just concatenated the strings without with context C is used as a derivation attribute, then the
keeping track of the referenced store paths, then if the derivations in C will be added to the inputDrvs of the
string is used as a derivation attribute, the derivation derivation, and the other store paths in C will be added to
will not have the correct dependencies in its inputDrvs and the inputSrcs of the derivations.
inputSrcs.
The semantics of the context is as follows: when a string For canonicity, the store paths should be in sorted order. */
with context C is used as a derivation attribute, then the struct {
derivations in C will be added to the inputDrvs of the const char* s;
derivation, and the other store paths in C will be added to const char** context; // must be in sorted order
the inputSrcs of the derivations. } string;
For canonicity, the store paths should be in sorted order. */ const char* path;
struct { Bindings* attrs;
const char * s; struct {
const char * * context; // must be in sorted order size_t size;
} string; Value** elems;
} bigList;
Value* smallList[2];
struct {
Env* env;
Expr* expr;
} thunk;
struct {
Value *left, *right;
} app;
struct {
Env* env;
ExprLambda* fun;
} lambda;
PrimOp* primOp;
struct {
Value *left, *right;
} primOpApp;
ExternalValueBase* external;
NixFloat fpoint;
};
const char * path; bool isList() const {
Bindings * attrs; return type == tList1 || type == tList2 || type == tListN;
struct { }
size_t size;
Value * * elems;
} bigList;
Value * smallList[2];
struct {
Env * env;
Expr * expr;
} thunk;
struct {
Value * left, * right;
} app;
struct {
Env * env;
ExprLambda * fun;
} lambda;
PrimOp * primOp;
struct {
Value * left, * right;
} primOpApp;
ExternalValueBase * external;
NixFloat fpoint;
};
bool isList() const Value** listElems() {
{ return type == tList1 || type == tList2 ? smallList : bigList.elems;
return type == tList1 || type == tList2 || type == tListN; }
}
Value * * listElems() const Value* const* listElems() const {
{ return type == tList1 || type == tList2 ? smallList : bigList.elems;
return type == tList1 || type == tList2 ? smallList : bigList.elems; }
}
const Value * const * listElems() const size_t listSize() const {
{ return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
return type == tList1 || type == tList2 ? smallList : bigList.elems; }
}
size_t listSize() const
{
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
}
}; };
/* After overwriting an app node, be sure to clear pointers in the /* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */ Value to ensure that the target isn't kept alive unnecessarily. */
static inline void clearValue(Value & v) static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; }
{
v.app.left = v.app.right = 0; static inline void mkInt(Value& v, NixInt n) {
clearValue(v);
v.type = tInt;
v.integer = n;
} }
static inline void mkFloat(Value& v, NixFloat n) {
static inline void mkInt(Value & v, NixInt n) clearValue(v);
{ v.type = tFloat;
clearValue(v); v.fpoint = n;
v.type = tInt;
v.integer = n;
} }
static inline void mkBool(Value& v, bool b) {
static inline void mkFloat(Value & v, NixFloat n) clearValue(v);
{ v.type = tBool;
clearValue(v); v.boolean = b;
v.type = tFloat;
v.fpoint = n;
} }
static inline void mkNull(Value& v) {
static inline void mkBool(Value & v, bool b) clearValue(v);
{ v.type = tNull;
clearValue(v);
v.type = tBool;
v.boolean = b;
} }
static inline void mkApp(Value& v, Value& left, Value& right) {
static inline void mkNull(Value & v) v.type = tApp;
{ v.app.left = &left;
clearValue(v); v.app.right = &right;
v.type = tNull;
} }
static inline void mkPrimOpApp(Value& v, Value& left, Value& right) {
static inline void mkApp(Value & v, Value & left, Value & right) v.type = tPrimOpApp;
{ v.app.left = &left;
v.type = tApp; v.app.right = &right;
v.app.left = &left;
v.app.right = &right;
} }
static inline void mkStringNoCopy(Value& v, const char* s) {
static inline void mkPrimOpApp(Value & v, Value & left, Value & right) v.type = tString;
{ v.string.s = s;
v.type = tPrimOpApp; v.string.context = 0;
v.app.left = &left;
v.app.right = &right;
} }
static inline void mkString(Value& v, const Symbol& s) {
static inline void mkStringNoCopy(Value & v, const char * s) mkStringNoCopy(v, ((const string&)s).c_str());
{
v.type = tString;
v.string.s = s;
v.string.context = 0;
} }
void mkString(Value& v, const char* s);
static inline void mkString(Value & v, const Symbol & s) static inline void mkPathNoCopy(Value& v, const char* s) {
{ clearValue(v);
mkStringNoCopy(v, ((const string &) s).c_str()); v.type = tPath;
v.path = s;
} }
void mkPath(Value& v, const char* s);
void mkString(Value & v, const char * s);
static inline void mkPathNoCopy(Value & v, const char * s)
{
clearValue(v);
v.type = tPath;
v.path = s;
}
void mkPath(Value & v, const char * s);
/* Compute the size in bytes of the given value, including all values /* Compute the size in bytes of the given value, including all values
and environments reachable from it. Static expressions (Exprs) are and environments reachable from it. Static expressions (Exprs) are
not included. */ not included. */
size_t valueSize(Value & v); size_t valueSize(Value& v);
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::vector<Value *, gc_allocator<Value *> > ValueVector; typedef std::vector<Value*, gc_allocator<Value*> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap; typedef std::map<Symbol, Value*, std::less<Symbol>,
gc_allocator<std::pair<const Symbol, Value*> > >
ValueMap;
#else #else
typedef std::vector<Value *> ValueVector; typedef std::vector<Value*> ValueVector;
typedef std::map<Symbol, Value *> ValueMap; typedef std::map<Symbol, Value*> ValueMap;
#endif #endif
} // namespace nix
}

83
third_party/nix/src/libmain/common-args.cc vendored
View file

@ -3,54 +3,53 @@
namespace nix { namespace nix {
MixCommonArgs::MixCommonArgs(const string & programName) MixCommonArgs::MixCommonArgs(const string& programName)
: programName(programName) : programName(programName) {
{ mkFlag()
mkFlag() .longName("verbose")
.longName("verbose") .shortName('v')
.shortName('v') .description("increase verbosity level")
.description("increase verbosity level") .handler([]() { verbosity = (Verbosity)(verbosity + 1); });
.handler([]() { verbosity = (Verbosity) (verbosity + 1); });
mkFlag() mkFlag()
.longName("quiet") .longName("quiet")
.description("decrease verbosity level") .description("decrease verbosity level")
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); .handler([]() {
verbosity =
verbosity > lvlError ? (Verbosity)(verbosity - 1) : lvlError;
});
mkFlag() mkFlag().longName("debug").description("enable debug output").handler([]() {
.longName("debug") verbosity = lvlDebug;
.description("enable debug output") });
.handler([]() { verbosity = lvlDebug; });
mkFlag() mkFlag()
.longName("option") .longName("option")
.labels({"name", "value"}) .labels({"name", "value"})
.description("set a Nix configuration option (overriding nix.conf)") .description("set a Nix configuration option (overriding nix.conf)")
.arity(2) .arity(2)
.handler([](std::vector<std::string> ss) { .handler([](std::vector<std::string> ss) {
try { try {
globalConfig.set(ss[0], ss[1]); globalConfig.set(ss[0], ss[1]);
} catch (UsageError & e) { } catch (UsageError& e) {
warn(e.what()); warn(e.what());
} }
}); });
mkFlag() mkFlag()
.longName("max-jobs") .longName("max-jobs")
.shortName('j') .shortName('j')
.label("jobs") .label("jobs")
.description("maximum number of parallel builds") .description("maximum number of parallel builds")
.handler([=](std::string s) { .handler([=](std::string s) { settings.set("max-jobs", s); });
settings.set("max-jobs", s);
});
std::string cat = "config"; std::string cat = "config";
globalConfig.convertToArgs(*this, cat); globalConfig.convertToArgs(*this, cat);
// Backward compatibility hack: nix-env already had a --system flag. // Backward compatibility hack: nix-env already had a --system flag.
if (programName == "nix-env") longFlags.erase("system"); if (programName == "nix-env") longFlags.erase("system");
hiddenCategories.insert(cat); hiddenCategories.insert(cat);
} }
} } // namespace nix

32
third_party/nix/src/libmain/common-args.hh vendored
View file

@ -4,30 +4,24 @@
namespace nix { namespace nix {
struct MixCommonArgs : virtual Args struct MixCommonArgs : virtual Args {
{ string programName;
string programName; MixCommonArgs(const string& programName);
MixCommonArgs(const string & programName);
}; };
struct MixDryRun : virtual Args struct MixDryRun : virtual Args {
{ bool dryRun = false;
bool dryRun = false;
MixDryRun() MixDryRun() {
{ mkFlag(0, "dry-run", "show what this command would do without doing it",
mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); &dryRun);
} }
}; };
struct MixJSON : virtual Args struct MixJSON : virtual Args {
{ bool json = false;
bool json = false;
MixJSON() MixJSON() { mkFlag(0, "json", "produce JSON output", &json); }
{
mkFlag(0, "json", "produce JSON output", &json);
}
}; };
} } // namespace nix

555
third_party/nix/src/libmain/shared.cc vendored
View file

@ -1,381 +1,352 @@
#include "globals.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include <openssl/crypto.h>
#include "util.hh" #include <signal.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm> #include <algorithm>
#include <cctype> #include <cctype>
#include <cstdlib>
#include <exception> #include <exception>
#include <iostream> #include <iostream>
#include <mutex> #include <mutex>
#include "globals.hh"
#include <cstdlib> #include "store-api.hh"
#include <sys/time.h> #include "util.hh"
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#include <openssl/crypto.h>
namespace nix { namespace nix {
static bool gcWarning = true; static bool gcWarning = true;
void printGCWarning() void printGCWarning() {
{ if (!gcWarning) return;
if (!gcWarning) return; static bool haveWarned = false;
static bool haveWarned = false; warnOnce(haveWarned,
warnOnce(haveWarned, "you did not specify '--add-root'; "
"you did not specify '--add-root'; " "the result might be removed by the garbage collector");
"the result might be removed by the garbage collector");
} }
void printMissing(ref<Store> store, const PathSet& paths, Verbosity lvl) {
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl) unsigned long long downloadSize, narSize;
{ PathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize,
PathSet willBuild, willSubstitute, unknown; narSize);
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize,
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); lvl);
} }
void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize,
Verbosity lvl) {
if (!willBuild.empty()) {
printMsg(lvl, "these derivations will be built:");
Paths sorted = store->topoSortPaths(willBuild);
reverse(sorted.begin(), sorted.end());
for (auto& i : sorted) printMsg(lvl, fmt(" %s", i));
}
void printMissing(ref<Store> store, const PathSet & willBuild, if (!willSubstitute.empty()) {
const PathSet & willSubstitute, const PathSet & unknown, printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f "
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) "MiB unpacked):",
{ downloadSize / (1024.0 * 1024.0),
if (!willBuild.empty()) { narSize / (1024.0 * 1024.0)));
printMsg(lvl, "these derivations will be built:"); for (auto& i : willSubstitute) printMsg(lvl, fmt(" %s", i));
Paths sorted = store->topoSortPaths(willBuild); }
reverse(sorted.begin(), sorted.end());
for (auto & i : sorted)
printMsg(lvl, fmt(" %s", i));
}
if (!willSubstitute.empty()) { if (!unknown.empty()) {
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", printMsg(lvl, fmt("don't know how to build these paths%s:",
downloadSize / (1024.0 * 1024.0), (settings.readOnlyMode
narSize / (1024.0 * 1024.0))); ? " (may be caused by read-only store access)"
for (auto & i : willSubstitute) : "")));
printMsg(lvl, fmt(" %s", i)); for (auto& i : unknown) printMsg(lvl, fmt(" %s", i));
} }
if (!unknown.empty()) {
printMsg(lvl, fmt("don't know how to build these paths%s:",
(settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
for (auto & i : unknown)
printMsg(lvl, fmt(" %s", i));
}
} }
string getArg(const string& opt, Strings::iterator& i,
string getArg(const string & opt, const Strings::iterator& end) {
Strings::iterator & i, const Strings::iterator & end) ++i;
{ if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
++i; return *i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
return *i;
} }
#if OPENSSL_VERSION_NUMBER < 0x10101000L #if OPENSSL_VERSION_NUMBER < 0x10101000L
/* OpenSSL is not thread-safe by default - it will randomly crash /* OpenSSL is not thread-safe by default - it will randomly crash
unless the user supplies a mutex locking function. So let's do unless the user supplies a mutex locking function. So let's do
that. */ that. */
static std::vector<std::mutex> opensslLocks; static std::vector<std::mutex> opensslLocks;
static void opensslLockCallback(int mode, int type, const char * file, int line) static void opensslLockCallback(int mode, int type, const char* file,
{ int line) {
if (mode & CRYPTO_LOCK) if (mode & CRYPTO_LOCK)
opensslLocks[type].lock(); opensslLocks[type].lock();
else else
opensslLocks[type].unlock(); opensslLocks[type].unlock();
} }
#endif #endif
static void sigHandler(int signo) {}
static void sigHandler(int signo) { } void initNix() {
/* Turn on buffering for cerr. */
void initNix()
{
/* Turn on buffering for cerr. */
#if HAVE_PUBSETBUF #if HAVE_PUBSETBUF
static char buf[1024]; static char buf[1024];
std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
#endif #endif
#if OPENSSL_VERSION_NUMBER < 0x10101000L #if OPENSSL_VERSION_NUMBER < 0x10101000L
/* Initialise OpenSSL locking. */ /* Initialise OpenSSL locking. */
opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
CRYPTO_set_locking_callback(opensslLockCallback); CRYPTO_set_locking_callback(opensslLockCallback);
#endif #endif
loadConfFile(); loadConfFile();
startSignalHandlerThread(); startSignalHandlerThread();
/* Reset SIGCHLD to its default. */ /* Reset SIGCHLD to its default. */
struct sigaction act; struct sigaction act;
sigemptyset(&act.sa_mask); sigemptyset(&act.sa_mask);
act.sa_handler = SIG_DFL; act.sa_handler = SIG_DFL;
act.sa_flags = 0; act.sa_flags = 0;
if (sigaction(SIGCHLD, &act, 0)) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD");
throw SysError("resetting SIGCHLD");
/* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
act.sa_handler = sigHandler; act.sa_handler = sigHandler;
if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
#if __APPLE__ #if __APPLE__
/* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
* Instead, add a dummy sigaction handler, and signalHandlerThread * Instead, add a dummy sigaction handler, and signalHandlerThread
* can handle the rest. */ * can handle the rest. */
struct sigaction sa; struct sigaction sa;
sa.sa_handler = sigHandler; sa.sa_handler = sigHandler;
if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH");
#endif #endif
/* Register a SIGSEGV handler to detect stack overflows. */ /* Register a SIGSEGV handler to detect stack overflows. */
detectStackOverflow(); detectStackOverflow();
/* There is no privacy in the Nix system ;-) At least not for /* There is no privacy in the Nix system ;-) At least not for
now. In particular, store objects should be readable by now. In particular, store objects should be readable by
everybody. */ everybody. */
umask(0022); umask(0022);
/* Initialise the PRNG. */ /* Initialise the PRNG. */
struct timeval tv; struct timeval tv;
gettimeofday(&tv, 0); gettimeofday(&tv, 0);
srandom(tv.tv_usec); srandom(tv.tv_usec);
/* On macOS, don't use the per-session TMPDIR (as set e.g. by /* On macOS, don't use the per-session TMPDIR (as set e.g. by
sshd). This breaks build users because they don't have access sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in ‘nix-store --serve’. */ to the TMPDIR, in particular in ‘nix-store --serve’. */
#if __APPLE__ #if __APPLE__
if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/")) if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/"))
unsetenv("TMPDIR"); unsetenv("TMPDIR");
#endif #endif
} }
LegacyArgs::LegacyArgs(
const std::string& programName,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg)
: MixCommonArgs(programName), parseArg(parseArg) {
mkFlag()
.longName("no-build-output")
.shortName('Q')
.description("do not show build output")
.set(&settings.verboseBuild, false);
LegacyArgs::LegacyArgs(const std::string & programName, mkFlag()
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) .longName("keep-failed")
: MixCommonArgs(programName), parseArg(parseArg) .shortName('K')
{ .description("keep temporary directories of failed builds")
mkFlag() .set(&(bool&)settings.keepFailed, true);
.longName("no-build-output")
.shortName('Q')
.description("do not show build output")
.set(&settings.verboseBuild, false);
mkFlag() mkFlag()
.longName("keep-failed") .longName("keep-going")
.shortName('K') .shortName('k')
.description("keep temporary directories of failed builds") .description("keep going after a build fails")
.set(&(bool&) settings.keepFailed, true); .set(&(bool&)settings.keepGoing, true);
mkFlag() mkFlag()
.longName("keep-going") .longName("fallback")
.shortName('k') .description("build from source if substitution fails")
.description("keep going after a build fails") .set(&(bool&)settings.tryFallback, true);
.set(&(bool&) settings.keepGoing, true);
mkFlag() auto intSettingAlias = [&](char shortName, const std::string& longName,
.longName("fallback") const std::string& description,
.description("build from source if substitution fails") const std::string& dest) {
.set(&(bool&) settings.tryFallback, true); mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
settings.set(dest, std::to_string(n));
});
};
auto intSettingAlias = [&](char shortName, const std::string & longName, intSettingAlias(0, "cores",
const std::string & description, const std::string & dest) { "maximum number of CPU cores to use inside a build", "cores");
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { intSettingAlias(0, "max-silent-time",
settings.set(dest, std::to_string(n)); "number of seconds of silence before a build is killed",
}); "max-silent-time");
}; intSettingAlias(0, "timeout", "number of seconds before a build is killed",
"timeout");
intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); mkFlag(0, "readonly-mode", "do not write to the Nix store",
intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); &settings.readOnlyMode);
intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
mkFlag(0, "readonly-mode", "do not write to the Nix store", mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
&settings.readOnlyMode); &gcWarning, false);
mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", mkFlag()
&gcWarning, false); .longName("store")
.label("store-uri")
mkFlag() .description("URI of the Nix store to use")
.longName("store") .dest(&(std::string&)settings.storeUri);
.label("store-uri")
.description("URI of the Nix store to use")
.dest(&(std::string&) settings.storeUri);
} }
bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) {
bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end) if (MixCommonArgs::processFlag(pos, end)) return true;
{ bool res = parseArg(pos, end);
if (MixCommonArgs::processFlag(pos, end)) return true; if (res) ++pos;
bool res = parseArg(pos, end); return res;
if (res) ++pos;
return res;
} }
bool LegacyArgs::processArgs(const Strings& args, bool finish) {
bool LegacyArgs::processArgs(const Strings & args, bool finish) if (args.empty()) return true;
{ assert(args.size() == 1);
if (args.empty()) return true; Strings ss(args);
assert(args.size() == 1); auto pos = ss.begin();
Strings ss(args); if (!parseArg(pos, ss.end()))
auto pos = ss.begin(); throw UsageError(format("unexpected argument '%1%'") % args.front());
if (!parseArg(pos, ss.end())) return true;
throw UsageError(format("unexpected argument '%1%'") % args.front());
return true;
} }
void parseCmdLine(
void parseCmdLine(int argc, char * * argv, int argc, char** argv,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
{ parseArg) {
parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
} }
void parseCmdLine(
void parseCmdLine(const string & programName, const Strings & args, const string& programName, const Strings& args,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
{ parseArg) {
LegacyArgs(programName, parseArg).parseCmdline(args); LegacyArgs(programName, parseArg).parseCmdline(args);
} }
void printVersion(const string& programName) {
void printVersion(const string & programName) std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
{ if (verbosity > lvlInfo) {
std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; Strings cfg;
if (verbosity > lvlInfo) {
Strings cfg;
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
cfg.push_back("gc"); cfg.push_back("gc");
#endif #endif
#if HAVE_SODIUM #if HAVE_SODIUM
cfg.push_back("signed-caches"); cfg.push_back("signed-caches");
#endif #endif
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf"
std::cout << "Store directory: " << settings.nixStore << "\n"; << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n";
} std::cout << "State directory: " << settings.nixStateDir << "\n";
throw Exit(); }
throw Exit();
} }
void showManPage(const string& name) {
void showManPage(const string & name) restoreSignals();
{ setenv("MANPATH", settings.nixManDir.c_str(), 1);
restoreSignals(); execlp("man", "man", name.c_str(), nullptr);
setenv("MANPATH", settings.nixManDir.c_str(), 1); throw SysError(format("command 'man %1%' failed") % name.c_str());
execlp("man", "man", name.c_str(), nullptr);
throw SysError(format("command 'man %1%' failed") % name.c_str());
} }
int handleExceptions(const string& programName, std::function<void()> fun) {
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
int handleExceptions(const string & programName, std::function<void()> fun) string error = ANSI_RED "error:" ANSI_NORMAL " ";
{ try {
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
string error = ANSI_RED "error:" ANSI_NORMAL " ";
try { try {
try { fun();
fun();
} catch (...) {
/* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
}
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
printError(
format(error + "%1%\nTry '%2% --help' for more information.")
% e.what() % programName);
return 1;
} catch (BaseError & e) {
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)");
return e.status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
return 0;
}
RunPager::RunPager()
{
if (!isatty(STDOUT_FILENO)) return;
char * pager = getenv("NIX_PAGER");
if (!pager) pager = getenv("PAGER");
if (pager && ((string) pager == "" || (string) pager == "cat")) return;
Pipe toPager;
toPager.create();
pid = startProcess([&]() {
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (!getenv("LESS"))
setenv("LESS", "FRSXMK", 1);
restoreSignals();
if (pager)
execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr);
execlp("more", "more", nullptr);
throw SysError(format("executing '%1%'") % pager);
});
pid.setKillSignal(SIGINT);
if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
}
RunPager::~RunPager()
{
try {
if (pid != -1) {
std::cout.flush();
close(STDOUT_FILENO);
pid.wait();
}
} catch (...) { } catch (...) {
ignoreException(); /* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
} }
} catch (Exit& e) {
return e.status;
} catch (UsageError& e) {
printError(format(error + "%1%\nTry '%2% --help' for more information.") %
e.what() % programName);
return 1;
} catch (BaseError& e) {
printError(format(error + "%1%%2%") %
(settings.showTrace ? e.prefix() : "") % e.msg());
if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)");
return e.status;
} catch (std::bad_alloc& e) {
printError(error + "out of memory");
return 1;
} catch (std::exception& e) {
printError(error + e.what());
return 1;
}
return 0;
} }
RunPager::RunPager() {
if (!isatty(STDOUT_FILENO)) return;
char* pager = getenv("NIX_PAGER");
if (!pager) pager = getenv("PAGER");
if (pager && ((string)pager == "" || (string)pager == "cat")) return;
string showBytes(unsigned long long bytes) Pipe toPager;
{ toPager.create();
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
pid = startProcess([&]() {
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1);
restoreSignals();
if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr);
execlp("more", "more", nullptr);
throw SysError(format("executing '%1%'") % pager);
});
pid.setKillSignal(SIGINT);
if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
} }
RunPager::~RunPager() {
PrintFreed::~PrintFreed() try {
{ if (pid != -1) {
if (show) std::cout.flush();
std::cout << format("%1% store paths deleted, %2% freed\n") close(STDOUT_FILENO);
% results.paths.size() pid.wait();
% showBytes(results.bytesFreed); }
} catch (...) {
ignoreException();
}
} }
Exit::~Exit() { } string showBytes(unsigned long long bytes) {
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
} }
PrintFreed::~PrintFreed() {
if (show)
std::cout << format("%1% store paths deleted, %2% freed\n") %
results.paths.size() % showBytes(results.bytesFreed);
}
Exit::~Exit() {}
} // namespace nix

150
third_party/nix/src/libmain/shared.hh vendored
View file

@ -1,126 +1,128 @@
#pragma once #pragma once
#include "util.hh" #include <signal.h>
#include <locale>
#include "args.hh" #include "args.hh"
#include "common-args.hh" #include "common-args.hh"
#include "util.hh"
#include <signal.h>
#include <locale>
namespace nix { namespace nix {
class Exit : public std::exception class Exit : public std::exception {
{ public:
public: int status;
int status; Exit() : status(0) {}
Exit() : status(0) { } Exit(int status) : status(status) {}
Exit(int status) : status(status) { } virtual ~Exit();
virtual ~Exit();
}; };
int handleExceptions(const string & programName, std::function<void()> fun); int handleExceptions(const string& programName, std::function<void()> fun);
/* Don't forget to call initPlugins() after settings are initialized! */ /* Don't forget to call initPlugins() after settings are initialized! */
void initNix(); void initNix();
void parseCmdLine(int argc, char * * argv, void parseCmdLine(
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); int argc, char** argv,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void parseCmdLine(const string & programName, const Strings & args, void parseCmdLine(
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); const string& programName, const Strings& args,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void printVersion(const string & programName); void printVersion(const string& programName);
/* Ugh. No better place to put this. */ /* Ugh. No better place to put this. */
void printGCWarning(); void printGCWarning();
class Store; class Store;
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo); void printMissing(ref<Store> store, const PathSet& paths,
Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const PathSet & willBuild, void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet & willSubstitute, const PathSet & unknown, const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); unsigned long long downloadSize, unsigned long long narSize,
Verbosity lvl = lvlInfo);
string getArg(const string & opt, string getArg(const string& opt, Strings::iterator& i,
Strings::iterator & i, const Strings::iterator & end); const Strings::iterator& end);
template<class N> N getIntArg(const string & opt, template <class N>
Strings::iterator & i, const Strings::iterator & end, bool allowUnit) N getIntArg(const string& opt, Strings::iterator& i,
{ const Strings::iterator& end, bool allowUnit) {
++i; ++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
string s = *i; string s = *i;
N multiplier = 1; N multiplier = 1;
if (allowUnit && !s.empty()) { if (allowUnit && !s.empty()) {
char u = std::toupper(*s.rbegin()); char u = std::toupper(*s.rbegin());
if (std::isalpha(u)) { if (std::isalpha(u)) {
if (u == 'K') multiplier = 1ULL << 10; if (u == 'K')
else if (u == 'M') multiplier = 1ULL << 20; multiplier = 1ULL << 10;
else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'M')
else if (u == 'T') multiplier = 1ULL << 40; multiplier = 1ULL << 20;
else throw UsageError(format("invalid unit specifier '%1%'") % u); else if (u == 'G')
s.resize(s.size() - 1); multiplier = 1ULL << 30;
} else if (u == 'T')
multiplier = 1ULL << 40;
else
throw UsageError(format("invalid unit specifier '%1%'") % u);
s.resize(s.size() - 1);
} }
N n; }
if (!string2Int(s, n)) N n;
throw UsageError(format("'%1%' requires an integer argument") % opt); if (!string2Int(s, n))
return n * multiplier; throw UsageError(format("'%1%' requires an integer argument") % opt);
return n * multiplier;
} }
struct LegacyArgs : public MixCommonArgs {
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg;
struct LegacyArgs : public MixCommonArgs LegacyArgs(
{ const std::string& programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
LegacyArgs(const std::string & programName, bool processFlag(Strings::iterator& pos, Strings::iterator end) override;
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings& args, bool finish) override;
bool processArgs(const Strings & args, bool finish) override;
}; };
/* Show the manual page for the specified program. */ /* Show the manual page for the specified program. */
void showManPage(const string & name); void showManPage(const string& name);
/* The constructor of this class starts a pager if stdout is a /* The constructor of this class starts a pager if stdout is a
terminal and $PAGER is set. Stdout is redirected to the pager. */ terminal and $PAGER is set. Stdout is redirected to the pager. */
class RunPager class RunPager {
{ public:
public: RunPager();
RunPager(); ~RunPager();
~RunPager();
private: private:
Pid pid; Pid pid;
}; };
extern volatile ::sig_atomic_t blockInt; extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */ /* GC helpers. */
string showBytes(unsigned long long bytes); string showBytes(unsigned long long bytes);
struct GCResults; struct GCResults;
struct PrintFreed struct PrintFreed {
{ bool show;
bool show; const GCResults& results;
const GCResults & results; PrintFreed(bool show, const GCResults& results)
PrintFreed(bool show, const GCResults & results) : show(show), results(results) {}
: show(show), results(results) { } ~PrintFreed();
~PrintFreed();
}; };
/* Install a SIGSEGV handler to detect stack overflows. */ /* Install a SIGSEGV handler to detect stack overflows. */
void detectStackOverflow(); void detectStackOverflow();
} // namespace nix
}

99
third_party/nix/src/libmain/stack.cc vendored
View file

@ -1,71 +1,64 @@
#include "types.hh" #include <signal.h>
#include <unistd.h>
#include <cstring>
#include <cstddef> #include <cstddef>
#include <cstdlib> #include <cstdlib>
#include <cstring>
#include <unistd.h> #include "types.hh"
#include <signal.h>
namespace nix { namespace nix {
static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) {
static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) /* Detect stack overflows by comparing the faulting address with
{ the stack pointer. Unfortunately, getting the stack pointer is
/* Detect stack overflows by comparing the faulting address with not portable. */
the stack pointer. Unfortunately, getting the stack pointer is bool haveSP = true;
not portable. */ char* sp = 0;
bool haveSP = true;
char * sp = 0;
#if defined(__x86_64__) && defined(REG_RSP) #if defined(__x86_64__) && defined(REG_RSP)
sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP]; sp = (char*)((ucontext_t*)ctx)->uc_mcontext.gregs[REG_RSP];
#elif defined(REG_ESP) #elif defined(REG_ESP)
sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP]; sp = (char*)((ucontext_t*)ctx)->uc_mcontext.gregs[REG_ESP];
#else #else
haveSP = false; haveSP = false;
#endif #endif
if (haveSP) { if (haveSP) {
ptrdiff_t diff = (char *) info->si_addr - sp; ptrdiff_t diff = (char*)info->si_addr - sp;
if (diff < 0) diff = -diff; if (diff < 0) diff = -diff;
if (diff < 4096) { if (diff < 4096) {
char msg[] = "error: stack overflow (possible infinite recursion)\n"; char msg[] = "error: stack overflow (possible infinite recursion)\n";
[[gnu::unused]] auto res = write(2, msg, strlen(msg)); [[gnu::unused]] auto res = write(2, msg, strlen(msg));
_exit(1); // maybe abort instead? _exit(1); // maybe abort instead?
}
} }
}
/* Restore default behaviour (i.e. segfault and dump core). */ /* Restore default behaviour (i.e. segfault and dump core). */
struct sigaction act; struct sigaction act;
sigfillset(&act.sa_mask); sigfillset(&act.sa_mask);
act.sa_handler = SIG_DFL; act.sa_handler = SIG_DFL;
act.sa_flags = 0; act.sa_flags = 0;
if (sigaction(SIGSEGV, &act, 0)) abort(); if (sigaction(SIGSEGV, &act, 0)) abort();
} }
void detectStackOverflow() {
#if defined(SA_SIGINFO) && defined(SA_ONSTACK)
/* Install a SIGSEGV handler to detect stack overflows. This
requires an alternative stack, otherwise the signal cannot be
delivered when we're out of stack space. */
stack_t stack;
stack.ss_size = 4096 * 4 + MINSIGSTKSZ;
static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size);
stack.ss_sp = stackBuf->data();
if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
stack.ss_flags = 0;
if (sigaltstack(&stack, 0) == -1)
throw SysError("cannot set alternative stack");
void detectStackOverflow() struct sigaction act;
{ sigfillset(&act.sa_mask);
#if defined(SA_SIGINFO) && defined (SA_ONSTACK) act.sa_sigaction = sigsegvHandler;
/* Install a SIGSEGV handler to detect stack overflows. This act.sa_flags = SA_SIGINFO | SA_ONSTACK;
requires an alternative stack, otherwise the signal cannot be if (sigaction(SIGSEGV, &act, 0)) throw SysError("resetting SIGSEGV");
delivered when we're out of stack space. */
stack_t stack;
stack.ss_size = 4096 * 4 + MINSIGSTKSZ;
static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size);
stack.ss_sp = stackBuf->data();
if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
stack.ss_flags = 0;
if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");
struct sigaction act;
sigfillset(&act.sa_mask);
act.sa_sigaction = sigsegvHandler;
act.sa_flags = SA_SIGINFO | SA_ONSTACK;
if (sigaction(SIGSEGV, &act, 0))
throw SysError("resetting SIGSEGV");
#endif #endif
} }
} // namespace nix
}

621
third_party/nix/src/libstore/binary-cache-store.cc vendored
View file

@ -1,361 +1,364 @@
#include "archive.hh"
#include "binary-cache-store.hh" #include "binary-cache-store.hh"
#include <chrono>
#include <future>
#include "archive.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "nar-info.hh"
#include "sync.hh"
#include "remote-fs-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "nar-accessor.hh"
#include "json.hh" #include "json.hh"
#include "nar-accessor.hh"
#include <chrono> #include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include <future> #include "remote-fs-accessor.hh"
#include "sync.hh"
namespace nix { namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params) BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) {
: Store(params) if (secretKeyFile != "")
{ secretKey =
if (secretKeyFile != "") std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
StringSink sink; StringSink sink;
sink << narVersionMagic1; sink << narVersionMagic1;
narMagic = *sink.s; narMagic = *sink.s;
} }
void BinaryCacheStore::init() void BinaryCacheStore::init() {
{ std::string cacheInfoFile = "nix-cache-info";
std::string cacheInfoFile = "nix-cache-info";
auto cacheInfo = getFile(cacheInfoFile); auto cacheInfo = getFile(cacheInfoFile);
if (!cacheInfo) { if (!cacheInfo) {
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n",
} else { "text/x-nix-cache-info");
for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { } else {
size_t colon = line.find(':'); for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
if (colon == std::string::npos) continue; size_t colon = line.find(':');
auto name = line.substr(0, colon); if (colon == std::string::npos) continue;
auto value = trim(line.substr(colon + 1, std::string::npos)); auto name = line.substr(0, colon);
if (name == "StoreDir") { auto value = trim(line.substr(colon + 1, std::string::npos));
if (value != storeDir) if (name == "StoreDir") {
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") if (value != storeDir)
% getUri() % value % storeDir); throw Error(format("binary cache '%s' is for Nix stores with prefix "
} else if (name == "WantMassQuery") { "'%s', not '%s'") %
wantMassQuery_ = value == "1"; getUri() % value % storeDir);
} else if (name == "Priority") { } else if (name == "WantMassQuery") {
string2Int(value, priority); wantMassQuery_ = value == "1";
} } else if (name == "Priority") {
} string2Int(value, priority);
}
} }
}
} }
void BinaryCacheStore::getFile(const std::string & path, void BinaryCacheStore::getFile(
Callback<std::shared_ptr<std::string>> callback) noexcept const std::string& path,
{ Callback<std::shared_ptr<std::string>> callback) noexcept {
try { try {
callback(getFile(path)); callback(getFile(path));
} catch (...) { callback.rethrow(); } } catch (...) {
callback.rethrow();
}
} }
void BinaryCacheStore::getFile(const std::string & path, Sink & sink) void BinaryCacheStore::getFile(const std::string& path, Sink& sink) {
{ std::promise<std::shared_ptr<std::string>> promise;
std::promise<std::shared_ptr<std::string>> promise; getFile(path, {[&](std::future<std::shared_ptr<std::string>> result) {
getFile(path,
{[&](std::future<std::shared_ptr<std::string>> result) {
try { try {
promise.set_value(result.get()); promise.set_value(result.get());
} catch (...) { } catch (...) {
promise.set_exception(std::current_exception()); promise.set_exception(std::current_exception());
} }
}}); }});
auto data = promise.get_future().get(); auto data = promise.get_future().get();
sink((unsigned char *) data->data(), data->size()); sink((unsigned char*)data->data(), data->size());
} }
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) std::shared_ptr<std::string> BinaryCacheStore::getFile(
{ const std::string& path) {
StringSink sink; StringSink sink;
try { try {
getFile(path, sink); getFile(path, sink);
} catch (NoSuchBinaryCacheFile &) { } catch (NoSuchBinaryCacheFile&) {
return nullptr; return nullptr;
}
return sink.s;
}
Path BinaryCacheStore::narInfoFileFor(const Path& storePath) {
assertStorePath(storePath);
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) {
auto narInfoFile = narInfoFileFor(narInfo->path);
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
auto hashPart = storePathToHash(narInfo->path);
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart,
std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo& info,
const ref<std::string>& nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) {
if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
for (auto& ref : info.references) try {
if (ref != info.path) queryPathInfo(ref);
} catch (InvalidPath&) {
throw Error(format("cannot add '%s' to the binary cache because the "
"reference '%s' is not valid") %
info.path % ref);
} }
return sink.s;
}
Path BinaryCacheStore::narInfoFileFor(const Path & storePath) assert(nar->compare(0, narMagic.size(), narMagic) == 0);
{
assertStorePath(storePath);
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) auto narInfo = make_ref<NarInfo>(info);
{
auto narInfoFile = narInfoFileFor(narInfo->path);
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
auto hashPart = storePathToHash(narInfo->path); if (info.narHash && info.narHash != narInfo->narHash)
throw Error(
format("refusing to copy corrupted path '%1%' to binary cache") %
info.path);
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
std::ostringstream jsonOut;
{ {
auto state_(state.lock()); JSONObject jsonRoot(jsonOut);
state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); jsonRoot.attr("version", 1);
auto narAccessor = makeNarAccessor(nar);
if (accessor_) accessor_->addToCache(info.path, *nar, narAccessor);
{
auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true);
}
} }
if (diskCache) upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(),
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); "application/json");
}
else {
if (accessor_) accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printMsg(lvlTalkative,
format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% "
"ms) to binary cache") %
narInfo->path % narInfo->narSize %
((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) %
duration);
/* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" +
(compression == "xz" ? ".xz"
: compression == "bzip2"
? ".bz2"
: compression == "br" ? ".br" : "");
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
if (secretKey) narInfo->sign(*secretKey);
writeNarInfo(narInfo);
stats.narInfoWrite++;
} }
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool BinaryCacheStore::isValidPathUncached(const Path& storePath) {
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) // FIXME: this only checks whether a .narinfo with a matching hash
{ // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
if (!repair && isValidPath(info.path)) return; // though they shouldn't. Not easily fixed.
return fileExists(narInfoFileFor(storePath));
}
/* Verify that all references are valid. This may do some .narinfo void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) {
reads, but typically they'll already be cached. */ auto info = queryPathInfo(storePath).cast<const NarInfo>();
for (auto & ref : info.references)
uint64_t narSize = 0;
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
sink(data, len);
narSize += len;
});
auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
try {
getFile(info->url, *decompressor);
} catch (NoSuchBinaryCacheFile& e) {
throw SubstituteGone(e.what());
}
decompressor->finish();
stats.narRead++;
// stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize;
}
void BinaryCacheStore::queryPathInfoUncached(
const Path& storePath,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
auto uri = getUri();
auto act = std::make_shared<Activity>(
*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri),
Logger::Fields{storePath, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(
narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) {
try { try {
if (ref != info.path) auto data = fut.get();
queryPathInfo(ref);
} catch (InvalidPath &) { if (!data) return (*callbackPtr)(nullptr);
throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
% info.path % ref); stats.narInfoRead++;
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>(
*this, *data, narInfoFile));
(void)
act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
} }
}});
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
std::ostringstream jsonOut;
{
JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
{
auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true);
}
}
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
}
else {
if (accessor_)
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% narInfo->path % narInfo->narSize
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration);
/* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
if (secretKey) narInfo->sign(*secretKey);
writeNarInfo(narInfo);
stats.narInfoWrite++;
} }
bool BinaryCacheStore::isValidPathUncached(const Path & storePath) Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
{ bool recursive, HashType hashAlgo,
// FIXME: this only checks whether a .narinfo with a matching hash PathFilter& filter, RepairFlag repair) {
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // FIXME: some cut&paste from LocalStore::addToStore().
// though they shouldn't. Not easily fixed.
return fileExists(narInfoFileFor(storePath)); /* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
Hash h;
if (recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
} else {
auto s = readFile(srcPath);
dumpString(s, sink);
h = hashString(hashAlgo, s);
}
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
addToStore(info, sink.s, repair, CheckSigs, nullptr);
return info.path;
} }
void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) Path BinaryCacheStore::addTextToStore(const string& name, const string& s,
{ const PathSet& references,
auto info = queryPathInfo(storePath).cast<const NarInfo>(); RepairFlag repair) {
ValidPathInfo info;
info.path = computeStorePathForText(name, s, references);
info.references = references;
uint64_t narSize = 0; if (repair || !isValidPath(info.path)) {
LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
sink(data, len);
narSize += len;
});
auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
try {
getFile(info->url, *decompressor);
} catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(e.what());
}
decompressor->finish();
stats.narRead++;
//stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize;
}
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
{
auto uri = getUri();
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile,
{[=](std::future<std::shared_ptr<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
stats.narInfoRead++;
(*callbackPtr)((std::shared_ptr<ValidPathInfo>)
std::make_shared<NarInfo>(*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
}});
}
Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
// FIXME: some cut&paste from LocalStore::addToStore().
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink; StringSink sink;
Hash h; dumpString(s, sink);
if (recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
} else {
auto s = readFile(srcPath);
dumpString(s, sink);
h = hashString(hashAlgo, s);
}
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
addToStore(info, sink.s, repair, CheckSigs, nullptr); addToStore(info, sink.s, repair, CheckSigs, nullptr);
}
return info.path; return info.path;
} }
Path BinaryCacheStore::addTextToStore(const string & name, const string & s, ref<FSAccessor> BinaryCacheStore::getFSAccessor() {
const PathSet & references, RepairFlag repair) return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()),
{ localNarCache);
ValidPathInfo info; }
info.path = computeStorePathForText(name, s, references);
info.references = references;
if (repair || !isValidPath(info.path)) { void BinaryCacheStore::addSignatures(const Path& storePath,
StringSink sink; const StringSet& sigs) {
dumpString(s, sink); /* Note: this is inherently racy since there is no locking on
addToStore(info, sink.s, repair, CheckSigs, nullptr); binary caches. In particular, with S3 this unreliable, even
when addSignatures() is called sequentially on a path, because
S3 might return an outdated cached version. */
auto narInfo = make_ref<NarInfo>((NarInfo&)*queryPathInfo(storePath));
narInfo->sigs.insert(sigs.begin(), sigs.end());
auto narInfoFile = narInfoFileFor(narInfo->path);
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) {
Path drvPath;
if (isDerivation(path))
drvPath = path;
else {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (info->deriver == "") return nullptr;
drvPath = info->deriver;
} catch (InvalidPath&) {
return nullptr;
} }
}
return info.path; auto logPath = "log/" + baseNameOf(drvPath);
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
return getFile(logPath);
} }
ref<FSAccessor> BinaryCacheStore::getFSAccessor() } // namespace nix
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
}
void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
{
/* Note: this is inherently racy since there is no locking on
binary caches. In particular, with S3 this unreliable, even
when addSignatures() is called sequentially on a path, because
S3 might return an outdated cached version. */
auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath));
narInfo->sigs.insert(sigs.begin(), sigs.end());
auto narInfoFile = narInfoFileFor(narInfo->path);
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
{
Path drvPath;
if (isDerivation(path))
drvPath = path;
else {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (info->deriver == "") return nullptr;
drvPath = info->deriver;
} catch (InvalidPath &) {
return nullptr;
}
}
auto logPath = "log/" + baseNameOf(drvPath);
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
return getFile(logPath);
}
}

150
third_party/nix/src/libstore/binary-cache-store.hh vendored
View file

@ -1,115 +1,113 @@
#pragma once #pragma once
#include "crypto.hh"
#include "store-api.hh"
#include "pool.hh"
#include <atomic> #include <atomic>
#include "crypto.hh"
#include "pool.hh"
#include "store-api.hh"
namespace nix { namespace nix {
struct NarInfo; struct NarInfo;
class BinaryCacheStore : public Store class BinaryCacheStore : public Store {
{ public:
public: const Setting<std::string> compression{
this, "xz", "compression",
"NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{
this, false, "write-nar-listing",
"whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{
this, "", "secret-key",
"path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"path to a local cache of NARs"};
const Setting<bool> parallelCompression{
this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; private:
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; std::unique_ptr<SecretKey> secretKey;
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
private: protected:
BinaryCacheStore(const Params& params);
std::unique_ptr<SecretKey> secretKey; public:
virtual bool fileExists(const std::string& path) = 0;
protected: virtual void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) = 0;
BinaryCacheStore(const Params & params); /* Note: subclasses must implement at least one of the two
following getFile() methods. */
public: /* Dump the contents of the specified file to a sink. */
virtual void getFile(const std::string& path, Sink& sink);
virtual bool fileExists(const std::string & path) = 0; /* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */
virtual void getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept;
virtual void upsertFile(const std::string & path, std::shared_ptr<std::string> getFile(const std::string& path);
const std::string & data,
const std::string & mimeType) = 0;
/* Note: subclasses must implement at least one of the two protected:
following getFile() methods. */ bool wantMassQuery_ = false;
int priority = 50;
/* Dump the contents of the specified file to a sink. */ public:
virtual void getFile(const std::string & path, Sink & sink); virtual void init();
/* Fetch the specified file and call the specified callback with private:
the result. A subclass may implement this asynchronously. */ std::string narMagic;
virtual void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept;
std::shared_ptr<std::string> getFile(const std::string & path); std::string narInfoFileFor(const Path& storePath);
protected: void writeNarInfo(ref<NarInfo> narInfo);
bool wantMassQuery_ = false; public:
int priority = 50; bool isValidPathUncached(const Path& path) override;
public: void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
virtual void init(); Path queryPathFromHashPart(const string& hashPart) override {
unsupported("queryPathFromHashPart");
}
private: bool wantMassQuery() override { return wantMassQuery_; }
std::string narMagic; void addToStore(const ValidPathInfo& info, const ref<std::string>& nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
std::string narInfoFileFor(const Path & storePath); Path addToStore(const string& name, const Path& srcPath, bool recursive,
HashType hashAlgo, PathFilter& filter,
RepairFlag repair) override;
void writeNarInfo(ref<NarInfo> narInfo); Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override;
public: void narFromPath(const Path& path, Sink& sink) override;
bool isValidPathUncached(const Path & path) override; BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override {
unsupported("buildDerivation");
}
void queryPathInfoUncached(const Path & path, void ensurePath(const Path& path) override { unsupported("ensurePath"); }
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
Path queryPathFromHashPart(const string & hashPart) override ref<FSAccessor> getFSAccessor() override;
{ unsupported("queryPathFromHashPart"); }
bool wantMassQuery() override { return wantMassQuery_; } void addSignatures(const Path& storePath, const StringSet& sigs) override;
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, std::shared_ptr<std::string> getBuildLog(const Path& path) override;
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override;
void narFromPath(const Path & path, Sink & sink) override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{ unsupported("buildDerivation"); }
void ensurePath(const Path & path) override
{ unsupported("ensurePath"); }
ref<FSAccessor> getFSAccessor() override;
void addSignatures(const Path & storePath, const StringSet & sigs) override;
std::shared_ptr<std::string> getBuildLog(const Path & path) override;
int getPriority() override { return priority; }
int getPriority() override { return priority; }
}; };
MakeError(NoSuchBinaryCacheFile, Error); MakeError(NoSuchBinaryCacheFile, Error);
} } // namespace nix

7791
third_party/nix/src/libstore/build.cc vendored
View file

File diff suppressed because it is too large Load diff

6
third_party/nix/src/libstore/builtins.hh vendored
View file

@ -5,7 +5,7 @@
namespace nix { namespace nix {
// TODO: make pluggable. // TODO: make pluggable.
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData);
void builtinBuildenv(const BasicDerivation & drv); void builtinBuildenv(const BasicDerivation& drv);
} } // namespace nix

320
third_party/nix/src/libstore/builtins/buildenv.cc vendored
View file

@ -1,13 +1,12 @@
#include "builtins.hh" #include <fcntl.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <fcntl.h>
#include <algorithm> #include <algorithm>
#include "builtins.hh"
namespace nix { namespace nix {
typedef std::map<Path,int> Priorities; typedef std::map<Path, int> Priorities;
// FIXME: change into local variables. // FIXME: change into local variables.
@ -16,102 +15,104 @@ static Priorities priorities;
static unsigned long symlinks; static unsigned long symlinks;
/* For each activated package, create symlinks */ /* For each activated package, create symlinks */
static void createLinks(const Path & srcDir, const Path & dstDir, int priority) static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
{ DirEntries srcFiles;
DirEntries srcFiles;
try {
srcFiles = readDirectory(srcDir);
} catch (SysError& e) {
if (e.errNo == ENOTDIR) {
printError(
"warning: not including '%s' in the user environment because it's "
"not a directory",
srcDir);
return;
}
throw;
}
for (const auto& ent : srcFiles) {
if (ent.name[0] == '.') /* not matched by glob */
continue;
auto srcFile = srcDir + "/" + ent.name;
auto dstFile = dstDir + "/" + ent.name;
struct stat srcSt;
try { try {
srcFiles = readDirectory(srcDir); if (stat(srcFile.c_str(), &srcSt) == -1)
} catch (SysError & e) { throw SysError("getting status of '%1%'", srcFile);
if (e.errNo == ENOTDIR) { } catch (SysError& e) {
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
return; printError("warning: skipping dangling symlink '%s'", dstFile);
} continue;
throw; }
throw;
} }
for (const auto & ent : srcFiles) { /* The files below are special-cased to that they don't show up
if (ent.name[0] == '.') * in user profiles, either because they are useless, or
/* not matched by glob */ * because they would cauase pointless collisions (e.g., each
continue; * Python package brings its own
auto srcFile = srcDir + "/" + ent.name; * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
auto dstFile = dstDir + "/" + ent.name; */
if (hasSuffix(srcFile, "/propagated-build-inputs") ||
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/log"))
continue;
struct stat srcSt; else if (S_ISDIR(srcSt.st_mode)) {
try { struct stat dstSt;
if (stat(srcFile.c_str(), &srcSt) == -1) auto res = lstat(dstFile.c_str(), &dstSt);
throw SysError("getting status of '%1%'", srcFile); if (res == 0) {
} catch (SysError & e) { if (S_ISDIR(dstSt.st_mode)) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) { createLinks(srcFile, dstFile, priority);
printError("warning: skipping dangling symlink '%s'", dstFile); continue;
continue; } else if (S_ISLNK(dstSt.st_mode)) {
} auto target = canonPath(dstFile, true);
throw; if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'",
srcFile, target);
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1)
throw SysError(format("creating directory '%1%'"));
createLinks(target, dstFile, priorities[dstFile]);
createLinks(srcFile, dstFile, priority);
continue;
} }
} else if (errno != ENOENT)
/* The files below are special-cased to that they don't show up throw SysError(format("getting status of '%1%'") % dstFile);
* in user profiles, either because they are useless, or
* because they would cauase pointless collisions (e.g., each
* Python package brings its own
* `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
*/
if (hasSuffix(srcFile, "/propagated-build-inputs") ||
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") ||
hasSuffix(srcFile, "/log"))
continue;
else if (S_ISDIR(srcSt.st_mode)) {
struct stat dstSt;
auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) {
if (S_ISDIR(dstSt.st_mode)) {
createLinks(srcFile, dstFile, priority);
continue;
} else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1)
throw SysError(format("creating directory '%1%'"));
createLinks(target, dstFile, priorities[dstFile]);
createLinks(srcFile, dstFile, priority);
continue;
}
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
}
else {
struct stat dstSt;
auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) {
if (S_ISLNK(dstSt.st_mode)) {
auto prevPriority = priorities[dstFile];
if (prevPriority == priority)
throw Error(
"packages '%1%' and '%2%' have the same priority %3%; "
"use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
"to change the priority of one of the conflicting packages"
" (0 being the highest priority)",
srcFile, readLink(dstFile), priority);
if (prevPriority < priority)
continue;
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
} else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
}
createSymlink(srcFile, dstFile);
priorities[dstFile] = priority;
symlinks++;
} }
else {
struct stat dstSt;
auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) {
if (S_ISLNK(dstSt.st_mode)) {
auto prevPriority = priorities[dstFile];
if (prevPriority == priority)
throw Error(
"packages '%1%' and '%2%' have the same priority %3%; "
"use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
"to change the priority of one of the conflicting packages"
" (0 being the highest priority)",
srcFile, readLink(dstFile), priority);
if (prevPriority < priority) continue;
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
} else if (S_ISDIR(dstSt.st_mode))
throw Error(
"collision between non-directory '%1%' and directory '%2%'",
srcFile, dstFile);
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
}
createSymlink(srcFile, dstFile);
priorities[dstFile] = priority;
symlinks++;
}
} }
typedef std::set<Path> FileProp; typedef std::set<Path> FileProp;
@ -121,84 +122,87 @@ static FileProp postponed = FileProp{};
static Path out; static Path out;
static void addPkg(const Path & pkgDir, int priority) static void addPkg(const Path& pkgDir, int priority) {
{ if (done.count(pkgDir)) return;
if (done.count(pkgDir)) return; done.insert(pkgDir);
done.insert(pkgDir); createLinks(pkgDir, out, priority);
createLinks(pkgDir, out, priority);
try { try {
for (const auto & p : tokenizeString<std::vector<string>>( for (const auto& p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) readFile(pkgDir + "/nix-support/propagated-user-env-packages"),
if (!done.count(p)) " \n"))
postponed.insert(p); if (!done.count(p)) postponed.insert(p);
} catch (SysError & e) { } catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
} }
} }
struct Package { struct Package {
Path path; Path path;
bool active; bool active;
int priority; int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} Package(Path path, bool active, int priority)
: path{path}, active{active}, priority{priority} {}
}; };
typedef std::vector<Package> Packages; typedef std::vector<Package> Packages;
void builtinBuildenv(const BasicDerivation & drv) void builtinBuildenv(const BasicDerivation& drv) {
{ auto getAttr = [&](const string& name) {
auto getAttr = [&](const string & name) { auto i = drv.env.find(name);
auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second;
return i->second; };
};
out = getAttr("out"); out = getAttr("out");
createDirs(out); createDirs(out);
/* Convert the stuff we get from the environment back into a /* Convert the stuff we get from the environment back into a
* coherent data type. */ * coherent data type. */
Packages pkgs; Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations")); auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) { while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */ /* !!! We're trusting the caller to structure derivations env var correctly
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
}
}
/* Symlink to the packages that have been installed explicitly by the
* user. Process in priority order to reduce unnecessary
* symlink/unlink steps.
*/ */
std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { auto active = derivations.front();
return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); derivations.pop_front();
}); auto priority = stoi(derivations.front());
for (const auto & pkg : pkgs) derivations.pop_front();
if (pkg.active) auto outputs = stoi(derivations.front());
addPkg(pkg.path, pkg.priority); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
/* Symlink to the packages that have been "propagated" by packages auto path = derivations.front();
* installed by the user (i.e., package X declares that it wants Y derivations.pop_front();
* installed as well). We do these later because they have a lower pkgs.emplace_back(path, active != "false", priority);
* priority in case of collisions.
*/
auto priorityCounter = 1000;
while (!postponed.empty()) {
auto pkgDirs = postponed;
postponed = FileProp{};
for (const auto & pkgDir : pkgDirs)
addPkg(pkgDir, priorityCounter++);
} }
}
printError("created %d symlinks in user environment", symlinks); /* Symlink to the packages that have been installed explicitly by the
* user. Process in priority order to reduce unnecessary
* symlink/unlink steps.
*/
std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) {
return a.priority < b.priority ||
(a.priority == b.priority && a.path < b.path);
});
for (const auto& pkg : pkgs)
if (pkg.active) addPkg(pkg.path, pkg.priority);
createSymlink(getAttr("manifest"), out + "/manifest.nix"); /* Symlink to the packages that have been "propagated" by packages
* installed by the user (i.e., package X declares that it wants Y
* installed as well). We do these later because they have a lower
* priority in case of collisions.
*/
auto priorityCounter = 1000;
while (!postponed.empty()) {
auto pkgDirs = postponed;
postponed = FileProp{};
for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++);
}
printError("created %d symlinks in user environment", symlinks);
createSymlink(getAttr("manifest"), out + "/manifest.nix");
} }
} } // namespace nix

132
third_party/nix/src/libstore/builtins/fetchurl.cc vendored
View file

@ -1,78 +1,76 @@
#include "archive.hh"
#include "builtins.hh" #include "builtins.hh"
#include "compression.hh"
#include "download.hh" #include "download.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
namespace nix { namespace nix {
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) {
{ /* Make the host's netrc data available. Too bad curl requires
/* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just
this to be stored in a file. It would be nice if we could just pass a pointer to the data. */
pass a pointer to the data. */ if (netrcData != "") {
if (netrcData != "") { settings.netrcFile = "netrc";
settings.netrcFile = "netrc"; writeFile(settings.netrcFile, netrcData, 0600);
writeFile(settings.netrcFile, netrcData, 0600); }
auto getAttr = [&](const string& name) {
auto i = drv.env.find(name);
if (i == drv.env.end())
throw Error(format("attribute '%s' missing") % name);
return i->second;
};
Path storePath = getAttr("out");
auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh downloader here because we're in
a forked process. */
auto downloader = makeDownloader();
auto fetch = [&](const std::string& url) {
auto source = sinkToSource([&](Sink& sink) {
/* No need to do TLS verification, because we check the hash of
the result anyway. */
DownloadRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
downloader->download(std::move(request), *decompressor);
decompressor->finish();
});
if (unpack)
restorePath(storePath, *source);
else
writeFile(storePath, *source);
auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") {
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError(format("making '%1%' executable") % storePath);
} }
};
auto getAttr = [&](const string & name) { /* Try the hashed mirrors first. */
auto i = drv.env.find(name); if (getAttr("outputHashMode") == "flat")
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); for (auto hashedMirror : settings.hashedMirrors.get()) try {
return i->second; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
}; auto ht = parseHashType(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" +
h.to_string(Base16, false));
return;
} catch (Error& e) {
debug(e.what());
}
Path storePath = getAttr("out"); /* Otherwise try the specified URL. */
auto mainUrl = getAttr("url"); fetch(mainUrl);
bool unpack = get(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh downloader here because we're in
a forked process. */
auto downloader = makeDownloader();
auto fetch = [&](const std::string & url) {
auto source = sinkToSource([&](Sink & sink) {
/* No need to do TLS verification, because we check the hash of
the result anyway. */
DownloadRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
downloader->download(std::move(request), *decompressor);
decompressor->finish();
});
if (unpack)
restorePath(storePath, *source);
else
writeFile(storePath, *source);
auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") {
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError(format("making '%1%' executable") % storePath);
}
};
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashType(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
/* Otherwise try the specified URL. */
fetch(mainUrl);
} }
} } // namespace nix

136
third_party/nix/src/libstore/crypto.cc vendored
View file

@ -1,6 +1,6 @@
#include "crypto.hh" #include "crypto.hh"
#include "util.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh"
#if HAVE_SODIUM #if HAVE_SODIUM
#include <sodium.h> #include <sodium.h>
@ -8,119 +8,107 @@
namespace nix { namespace nix {
static std::pair<std::string, std::string> split(const string & s) static std::pair<std::string, std::string> split(const string& s) {
{ size_t colon = s.find(':');
size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""};
if (colon == std::string::npos || colon == 0) return {std::string(s, 0, colon), std::string(s, colon + 1)};
return {"", ""};
return {std::string(s, 0, colon), std::string(s, colon + 1)};
} }
Key::Key(const string & s) Key::Key(const string& s) {
{ auto ss = split(s);
auto ss = split(s);
name = ss.first; name = ss.first;
key = ss.second; key = ss.second;
if (name == "" || key == "") if (name == "" || key == "") throw Error("secret key is corrupt");
throw Error("secret key is corrupt");
key = base64Decode(key); key = base64Decode(key);
} }
SecretKey::SecretKey(const string & s) SecretKey::SecretKey(const string& s) : Key(s) {
: Key(s)
{
#if HAVE_SODIUM #if HAVE_SODIUM
if (key.size() != crypto_sign_SECRETKEYBYTES) if (key.size() != crypto_sign_SECRETKEYBYTES)
throw Error("secret key is not valid"); throw Error("secret key is not valid");
#endif #endif
} }
#if !HAVE_SODIUM #if !HAVE_SODIUM
[[noreturn]] static void noSodium() [[noreturn]] static void noSodium() {
{ throw Error(
throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); "Nix was not compiled with libsodium, required for signed binary cache "
"support");
} }
#endif #endif
std::string SecretKey::signDetached(const std::string & data) const std::string SecretKey::signDetached(const std::string& data) const {
{
#if HAVE_SODIUM #if HAVE_SODIUM
unsigned char sig[crypto_sign_BYTES]; unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen; unsigned long long sigLen;
crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), crypto_sign_detached(sig, &sigLen, (unsigned char*)data.data(), data.size(),
(unsigned char *) key.data()); (unsigned char*)key.data());
return name + ":" + base64Encode(std::string((char *) sig, sigLen)); return name + ":" + base64Encode(std::string((char*)sig, sigLen));
#else #else
noSodium(); noSodium();
#endif #endif
} }
PublicKey SecretKey::toPublicKey() const PublicKey SecretKey::toPublicKey() const {
{
#if HAVE_SODIUM #if HAVE_SODIUM
unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data());
return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); return PublicKey(name, std::string((char*)pk, crypto_sign_PUBLICKEYBYTES));
#else #else
noSodium(); noSodium();
#endif #endif
} }
PublicKey::PublicKey(const string & s) PublicKey::PublicKey(const string& s) : Key(s) {
: Key(s)
{
#if HAVE_SODIUM #if HAVE_SODIUM
if (key.size() != crypto_sign_PUBLICKEYBYTES) if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid"); throw Error("public key is not valid");
#endif #endif
} }
bool verifyDetached(const std::string & data, const std::string & sig, bool verifyDetached(const std::string& data, const std::string& sig,
const PublicKeys & publicKeys) const PublicKeys& publicKeys) {
{
#if HAVE_SODIUM #if HAVE_SODIUM
auto ss = split(sig); auto ss = split(sig);
auto key = publicKeys.find(ss.first); auto key = publicKeys.find(ss.first);
if (key == publicKeys.end()) return false; if (key == publicKeys.end()) return false;
auto sig2 = base64Decode(ss.second); auto sig2 = base64Decode(ss.second);
if (sig2.size() != crypto_sign_BYTES) if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid");
throw Error("signature is not valid");
return crypto_sign_verify_detached((unsigned char *) sig2.data(), return crypto_sign_verify_detached(
(unsigned char *) data.data(), data.size(), (unsigned char*)sig2.data(), (unsigned char*)data.data(),
(unsigned char *) key->second.key.data()) == 0; data.size(), (unsigned char*)key->second.key.data()) == 0;
#else #else
noSodium(); noSodium();
#endif #endif
} }
PublicKeys getDefaultPublicKeys() PublicKeys getDefaultPublicKeys() {
{ PublicKeys publicKeys;
PublicKeys publicKeys;
// FIXME: filter duplicates // FIXME: filter duplicates
for (auto s : settings.trustedPublicKeys.get()) { for (auto s : settings.trustedPublicKeys.get()) {
PublicKey key(s); PublicKey key(s);
publicKeys.emplace(key.name, key); publicKeys.emplace(key.name, key);
}
for (auto secretKeyFile : settings.secretKeyFiles.get()) {
try {
SecretKey secretKey(readFile(secretKeyFile));
publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
} catch (SysError& e) {
/* Ignore unreadable key files. That's normal in a
multi-user installation. */
} }
}
for (auto secretKeyFile : settings.secretKeyFiles.get()) { return publicKeys;
try {
SecretKey secretKey(readFile(secretKeyFile));
publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
} catch (SysError & e) {
/* Ignore unreadable key files. That's normal in a
multi-user installation. */
}
}
return publicKeys;
} }
} } // namespace nix

50
third_party/nix/src/libstore/crypto.hh vendored
View file

@ -1,54 +1,48 @@
#pragma once #pragma once
#include "types.hh"
#include <map> #include <map>
#include "types.hh"
namespace nix { namespace nix {
struct Key struct Key {
{ std::string name;
std::string name; std::string key;
std::string key;
/* Construct Key from a string in the format /* Construct Key from a string in the format
‘<name>:<key-in-base64>’. */ ‘<name>:<key-in-base64>’. */
Key(const std::string & s); Key(const std::string& s);
protected: protected:
Key(const std::string & name, const std::string & key) Key(const std::string& name, const std::string& key) : name(name), key(key) {}
: name(name), key(key) { }
}; };
struct PublicKey; struct PublicKey;
struct SecretKey : Key struct SecretKey : Key {
{ SecretKey(const std::string& s);
SecretKey(const std::string & s);
/* Return a detached signature of the given string. */ /* Return a detached signature of the given string. */
std::string signDetached(const std::string & s) const; std::string signDetached(const std::string& s) const;
PublicKey toPublicKey() const; PublicKey toPublicKey() const;
}; };
struct PublicKey : Key struct PublicKey : Key {
{ PublicKey(const std::string& data);
PublicKey(const std::string & data);
private: private:
PublicKey(const std::string & name, const std::string & key) PublicKey(const std::string& name, const std::string& key) : Key(name, key) {}
: Key(name, key) { } friend struct SecretKey;
friend struct SecretKey;
}; };
typedef std::map<std::string, PublicKey> PublicKeys; typedef std::map<std::string, PublicKey> PublicKeys;
/* Return true iff ‘sig’ is a correct signature over ‘data’ using one /* Return true iff ‘sig’ is a correct signature over ‘data’ using one
of the given public keys. */ of the given public keys. */
bool verifyDetached(const std::string & data, const std::string & sig, bool verifyDetached(const std::string& data, const std::string& sig,
const PublicKeys & publicKeys); const PublicKeys& publicKeys);
PublicKeys getDefaultPublicKeys(); PublicKeys getDefaultPublicKeys();
} } // namespace nix

621
third_party/nix/src/libstore/derivations.cc vendored
View file

@ -1,289 +1,294 @@
#include "derivations.hh" #include "derivations.hh"
#include "store-api.hh" #include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "istringstream_nocopy.hh"
#include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix { namespace nix {
void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
recursive = false;
string algo = hashAlgo;
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const if (string(algo, 0, 2) == "r:") {
{ recursive = true;
recursive = false; algo = string(algo, 2);
string algo = hashAlgo; }
if (string(algo, 0, 2) == "r:") { HashType hashType = parseHashType(algo);
recursive = true; if (hashType == htUnknown)
algo = string(algo, 2); throw Error(format("unknown hash algorithm '%1%'") % algo);
}
HashType hashType = parseHashType(algo); hash = Hash(this->hash, hashType);
if (hashType == htUnknown)
throw Error(format("unknown hash algorithm '%1%'") % algo);
hash = Hash(this->hash, hashType);
} }
Path BasicDerivation::findOutput(const string& id) const {
Path BasicDerivation::findOutput(const string & id) const auto i = outputs.find(id);
{ if (i == outputs.end())
auto i = outputs.find(id); throw Error(format("derivation has no output '%1%'") % id);
if (i == outputs.end()) return i->second.path;
throw Error(format("derivation has no output '%1%'") % id);
return i->second.path;
} }
bool BasicDerivation::isBuiltin() const {
bool BasicDerivation::isBuiltin() const return string(builder, 0, 8) == "builtin:";
{
return string(builder, 0, 8) == "builtin:";
} }
Path writeDerivation(ref<Store> store, const Derivation& drv,
Path writeDerivation(ref<Store> store, const string& name, RepairFlag repair) {
const Derivation & drv, const string & name, RepairFlag repair) PathSet references;
{ references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
PathSet references; for (auto& i : drv.inputDrvs) references.insert(i.first);
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); /* Note that the outputs of a derivation are *not* references
for (auto & i : drv.inputDrvs) (that can be missing (of course) and should not necessarily be
references.insert(i.first); held during a garbage collection). */
/* Note that the outputs of a derivation are *not* references string suffix = name + drvExtension;
(that can be missing (of course) and should not necessarily be string contents = drv.unparse();
held during a garbage collection). */ return settings.readOnlyMode
string suffix = name + drvExtension; ? store->computeStorePathForText(suffix, contents, references)
string contents = drv.unparse(); : store->addTextToStore(suffix, contents, references, repair);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair);
} }
/* Read string `s' from stream `str'. */ /* Read string `s' from stream `str'. */
static void expect(std::istream & str, const string & s) static void expect(std::istream& str, const string& s) {
{ char s2[s.size()];
char s2[s.size()]; str.read(s2, s.size());
str.read(s2, s.size()); if (string(s2, s.size()) != s)
if (string(s2, s.size()) != s) throw FormatError(format("expected string '%1%'") % s);
throw FormatError(format("expected string '%1%'") % s);
} }
/* Read a C-style string from stream `str'. */ /* Read a C-style string from stream `str'. */
static string parseString(std::istream & str) static string parseString(std::istream& str) {
{ string res;
string res; expect(str, "\"");
expect(str, "\""); int c;
int c; while ((c = str.get()) != '"')
while ((c = str.get()) != '"') if (c == '\\') {
if (c == '\\') { c = str.get();
c = str.get(); if (c == 'n')
if (c == 'n') res += '\n'; res += '\n';
else if (c == 'r') res += '\r'; else if (c == 'r')
else if (c == 't') res += '\t'; res += '\r';
else res += c; else if (c == 't')
} res += '\t';
else res += c; else
return res; res += c;
} else
res += c;
return res;
} }
static Path parsePath(std::istream& str) {
static Path parsePath(std::istream & str) string s = parseString(str);
{ if (s.size() == 0 || s[0] != '/')
string s = parseString(str); throw FormatError(format("bad path '%1%' in derivation") % s);
if (s.size() == 0 || s[0] != '/') return s;
throw FormatError(format("bad path '%1%' in derivation") % s);
return s;
} }
static bool endOfList(std::istream& str) {
static bool endOfList(std::istream & str) if (str.peek() == ',') {
{ str.get();
if (str.peek() == ',') {
str.get();
return false;
}
if (str.peek() == ']') {
str.get();
return true;
}
return false; return false;
}
if (str.peek() == ']') {
str.get();
return true;
}
return false;
} }
static StringSet parseStrings(std::istream& str, bool arePaths) {
static StringSet parseStrings(std::istream & str, bool arePaths) StringSet res;
{ while (!endOfList(str))
StringSet res; res.insert(arePaths ? parsePath(str) : parseString(str));
while (!endOfList(str)) return res;
res.insert(arePaths ? parsePath(str) : parseString(str));
return res;
} }
static Derivation parseDerivation(const string& s) {
Derivation drv;
istringstream_nocopy str(s);
expect(str, "Derive([");
static Derivation parseDerivation(const string & s) /* Parse the list of outputs. */
{ while (!endOfList(str)) {
Derivation drv; DerivationOutput out;
istringstream_nocopy str(s); expect(str, "(");
expect(str, "Derive(["); string id = parseString(str);
expect(str, ",");
/* Parse the list of outputs. */ out.path = parsePath(str);
while (!endOfList(str)) { expect(str, ",");
DerivationOutput out; out.hashAlgo = parseString(str);
expect(str, "("); string id = parseString(str); expect(str, ",");
expect(str, ","); out.path = parsePath(str); out.hash = parseString(str);
expect(str, ","); out.hashAlgo = parseString(str);
expect(str, ","); out.hash = parseString(str);
expect(str, ")");
drv.outputs[id] = out;
}
/* Parse the list of input derivations. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "(");
Path drvPath = parsePath(str);
expect(str, ",[");
drv.inputDrvs[drvPath] = parseStrings(str, false);
expect(str, ")");
}
expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
expect(str, ","); drv.platform = parseString(str);
expect(str, ","); drv.builder = parseString(str);
/* Parse the builder arguments. */
expect(str, ",[");
while (!endOfList(str))
drv.args.push_back(parseString(str));
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "("); string name = parseString(str);
expect(str, ","); string value = parseString(str);
expect(str, ")");
drv.env[name] = value;
}
expect(str, ")"); expect(str, ")");
return drv; drv.outputs[id] = out;
}
/* Parse the list of input derivations. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "(");
Path drvPath = parsePath(str);
expect(str, ",[");
drv.inputDrvs[drvPath] = parseStrings(str, false);
expect(str, ")");
}
expect(str, ",[");
drv.inputSrcs = parseStrings(str, true);
expect(str, ",");
drv.platform = parseString(str);
expect(str, ",");
drv.builder = parseString(str);
/* Parse the builder arguments. */
expect(str, ",[");
while (!endOfList(str)) drv.args.push_back(parseString(str));
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "(");
string name = parseString(str);
expect(str, ",");
string value = parseString(str);
expect(str, ")");
drv.env[name] = value;
}
expect(str, ")");
return drv;
} }
Derivation readDerivation(const Path& drvPath) {
Derivation readDerivation(const Path & drvPath) try {
{ return parseDerivation(readFile(drvPath));
try { } catch (FormatError& e) {
return parseDerivation(readFile(drvPath)); throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
} catch (FormatError & e) { e.msg());
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); }
}
} }
Derivation Store::derivationFromPath(const Path& drvPath) {
Derivation Store::derivationFromPath(const Path & drvPath) assertStorePath(drvPath);
{ ensurePath(drvPath);
assertStorePath(drvPath); auto accessor = getFSAccessor();
ensurePath(drvPath); try {
auto accessor = getFSAccessor(); return parseDerivation(accessor->readFile(drvPath));
try { } catch (FormatError& e) {
return parseDerivation(accessor->readFile(drvPath)); throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
} catch (FormatError & e) { e.msg());
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); }
}
} }
static void printString(string& res, const string& s) {
static void printString(string & res, const string & s) res += '"';
{ for (const char* i = s.c_str(); *i; i++)
res += '"'; if (*i == '\"' || *i == '\\') {
for (const char * i = s.c_str(); *i; i++) res += "\\";
if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } res += *i;
else if (*i == '\n') res += "\\n"; } else if (*i == '\n')
else if (*i == '\r') res += "\\r"; res += "\\n";
else if (*i == '\t') res += "\\t"; else if (*i == '\r')
else res += *i; res += "\\r";
res += '"'; else if (*i == '\t')
res += "\\t";
else
res += *i;
res += '"';
} }
template <class ForwardIterator>
template<class ForwardIterator> static void printStrings(string& res, ForwardIterator i, ForwardIterator j) {
static void printStrings(string & res, ForwardIterator i, ForwardIterator j) res += '[';
{ bool first = true;
res += '['; for (; i != j; ++i) {
bool first = true; if (first)
for ( ; i != j; ++i) { first = false;
if (first) first = false; else res += ','; else
printString(res, *i); res += ',';
} printString(res, *i);
res += ']'; }
res += ']';
} }
string Derivation::unparse() const {
string s;
s.reserve(65536);
s += "Derive([";
string Derivation::unparse() const bool first = true;
{ for (auto& i : outputs) {
string s; if (first)
s.reserve(65536); first = false;
s += "Derive(["; else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second.path);
s += ',';
printString(s, i.second.hashAlgo);
s += ',';
printString(s, i.second.hash);
s += ')';
}
bool first = true; s += "],[";
for (auto & i : outputs) { first = true;
if (first) first = false; else s += ','; for (auto& i : inputDrvs) {
s += '('; printString(s, i.first); if (first)
s += ','; printString(s, i.second.path); first = false;
s += ','; printString(s, i.second.hashAlgo); else
s += ','; printString(s, i.second.hash); s += ',';
s += ')'; s += '(';
} printString(s, i.first);
s += ',';
printStrings(s, i.second.begin(), i.second.end());
s += ')';
}
s += "],["; s += "],";
first = true; printStrings(s, inputSrcs.begin(), inputSrcs.end());
for (auto & i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printStrings(s, i.second.begin(), i.second.end());
s += ')';
}
s += "],"; s += ',';
printStrings(s, inputSrcs.begin(), inputSrcs.end()); printString(s, platform);
s += ',';
printString(s, builder);
s += ',';
printStrings(s, args.begin(), args.end());
s += ','; printString(s, platform); s += ",[";
s += ','; printString(s, builder); first = true;
s += ','; printStrings(s, args.begin(), args.end()); for (auto& i : env) {
if (first)
first = false;
else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second);
s += ')';
}
s += ",["; s += "])";
first = true;
for (auto & i : env) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, i.second);
s += ')';
}
s += "])"; return s;
return s;
} }
bool isDerivation(const string& fileName) {
bool isDerivation(const string & fileName) return hasSuffix(fileName, drvExtension);
{
return hasSuffix(fileName, drvExtension);
} }
bool BasicDerivation::isFixedOutput() const {
bool BasicDerivation::isFixedOutput() const return outputs.size() == 1 && outputs.begin()->first == "out" &&
{ outputs.begin()->second.hash != "";
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
} }
DrvHashes drvHashes; DrvHashes drvHashes;
/* Returns the hash of a derivation modulo fixed-output /* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are output (`out') for which an expected hash and hash algorithm are
@ -304,113 +309,95 @@ DrvHashes drvHashes;
paths have been replaced by the result of a recursive call to this paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of function, and that for fixed-output derivations we return a hash of
its output path. */ its output path. */
Hash hashDerivationModulo(Store & store, Derivation drv) Hash hashDerivationModulo(Store& store, Derivation drv) {
{ /* Return a fixed hash for fixed-output derivations. */
/* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) {
if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin();
DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" +
return hashString(htSHA256, "fixed:out:" i->second.hash + ":" + i->second.path);
+ i->second.hashAlgo + ":" }
+ i->second.hash + ":"
+ i->second.path); /* For other derivations, replace the inputs paths with recursive
calls to this function.*/
DerivationInputs inputs2;
for (auto& i : drv.inputDrvs) {
Hash h = drvHashes[i.first];
if (!h) {
assert(store.isValidPath(i.first));
Derivation drv2 = readDerivation(store.toRealPath(i.first));
h = hashDerivationModulo(store, drv2);
drvHashes[i.first] = h;
} }
inputs2[h.to_string(Base16, false)] = i.second;
}
drv.inputDrvs = inputs2;
/* For other derivations, replace the inputs paths with recursive return hashString(htSHA256, drv.unparse());
calls to this function.*/
DerivationInputs inputs2;
for (auto & i : drv.inputDrvs) {
Hash h = drvHashes[i.first];
if (!h) {
assert(store.isValidPath(i.first));
Derivation drv2 = readDerivation(store.toRealPath(i.first));
h = hashDerivationModulo(store, drv2);
drvHashes[i.first] = h;
}
inputs2[h.to_string(Base16, false)] = i.second;
}
drv.inputDrvs = inputs2;
return hashString(htSHA256, drv.unparse());
} }
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s) {
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) size_t n = s.find("!");
{ return n == s.npos ? DrvPathWithOutputs(s, std::set<string>())
size_t n = s.find("!"); : DrvPathWithOutputs(string(s, 0, n),
return n == s.npos tokenizeString<std::set<string> >(
? DrvPathWithOutputs(s, std::set<string>()) string(s, n + 1), ","));
: DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
} }
Path makeDrvPathWithOutputs(const Path& drvPath,
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) const std::set<string>& outputs) {
{ return outputs.empty() ? drvPath
return outputs.empty() : drvPath + "!" + concatStringsSep(",", outputs);
? drvPath
: drvPath + "!" + concatStringsSep(",", outputs);
} }
bool wantOutput(const string& output, const std::set<string>& wanted) {
bool wantOutput(const string & output, const std::set<string> & wanted) return wanted.empty() || wanted.find(output) != wanted.end();
{
return wanted.empty() || wanted.find(output) != wanted.end();
} }
PathSet BasicDerivation::outputPaths() const {
PathSet BasicDerivation::outputPaths() const PathSet paths;
{ for (auto& i : outputs) paths.insert(i.second.path);
PathSet paths; return paths;
for (auto & i : outputs)
paths.insert(i.second.path);
return paths;
} }
Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) {
drv.outputs.clear();
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash;
store.assertStorePath(o.path);
drv.outputs[name] = o;
}
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) drv.inputSrcs = readStorePaths<PathSet>(store, in);
{ in >> drv.platform >> drv.builder;
drv.outputs.clear(); drv.args = readStrings<Strings>(in);
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash;
store.assertStorePath(o.path);
drv.outputs[name] = o;
}
drv.inputSrcs = readStorePaths<PathSet>(store, in); nr = readNum<size_t>(in);
in >> drv.platform >> drv.builder; for (size_t n = 0; n < nr; n++) {
drv.args = readStrings<Strings>(in); auto key = readString(in);
auto value = readString(in);
drv.env[key] = value;
}
nr = readNum<size_t>(in); return in;
for (size_t n = 0; n < nr; n++) {
auto key = readString(in);
auto value = readString(in);
drv.env[key] = value;
}
return in;
} }
Sink& operator<<(Sink& out, const BasicDerivation& drv) {
Sink & operator << (Sink & out, const BasicDerivation & drv) out << drv.outputs.size();
{ for (auto& i : drv.outputs)
out << drv.outputs.size(); out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
for (auto & i : drv.outputs) out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; out << drv.env.size();
out << drv.inputSrcs << drv.platform << drv.builder << drv.args; for (auto& i : drv.env) out << i.first << i.second;
out << drv.env.size(); return out;
for (auto & i : drv.env)
out << i.first << i.second;
return out;
} }
std::string hashPlaceholder(const std::string& outputName) {
std::string hashPlaceholder(const std::string & outputName) // FIXME: memoize?
{ return "/" + hashString(htSHA256, "nix-output:" + outputName)
// FIXME: memoize? .to_string(Base32, false);
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
} }
} // namespace nix
}

106
third_party/nix/src/libstore/derivations.hh vendored
View file

@ -1,36 +1,28 @@
#pragma once #pragma once
#include "types.hh" #include <map>
#include "hash.hh" #include "hash.hh"
#include "store-api.hh" #include "store-api.hh"
#include "types.hh"
#include <map>
namespace nix { namespace nix {
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const string drvExtension = ".drv"; const string drvExtension = ".drv";
/* Abstract syntax of derivations. */ /* Abstract syntax of derivations. */
struct DerivationOutput struct DerivationOutput {
{ Path path;
Path path; string hashAlgo; /* hash used for expected hash computation */
string hashAlgo; /* hash used for expected hash computation */ string hash; /* expected hash, may be null */
string hash; /* expected hash, may be null */ DerivationOutput() {}
DerivationOutput() DerivationOutput(Path path, string hashAlgo, string hash) {
{ this->path = path;
} this->hashAlgo = hashAlgo;
DerivationOutput(Path path, string hashAlgo, string hash) this->hash = hash;
{ }
this->path = path; void parseHashInfo(bool& recursive, Hash& hash) const;
this->hashAlgo = hashAlgo;
this->hash = hash;
}
void parseHashInfo(bool & recursive, Hash & hash) const;
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -41,77 +33,73 @@ typedef std::map<Path, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs; typedef std::map<string, string> StringPairs;
struct BasicDerivation struct BasicDerivation {
{ DerivationOutputs outputs; /* keyed on symbolic IDs */
DerivationOutputs outputs; /* keyed on symbolic IDs */ PathSet inputSrcs; /* inputs that are sources */
PathSet inputSrcs; /* inputs that are sources */ string platform;
string platform; Path builder;
Path builder; Strings args;
Strings args; StringPairs env;
StringPairs env;
virtual ~BasicDerivation() { }; virtual ~BasicDerivation(){};
/* Return the path corresponding to the output identifier `id' in /* Return the path corresponding to the output identifier `id' in
the given derivation. */ the given derivation. */
Path findOutput(const string & id) const; Path findOutput(const string& id) const;
bool isBuiltin() const; bool isBuiltin() const;
/* Return true iff this is a fixed-output derivation. */ /* Return true iff this is a fixed-output derivation. */
bool isFixedOutput() const; bool isFixedOutput() const;
/* Return the output paths of a derivation. */
PathSet outputPaths() const;
/* Return the output paths of a derivation. */
PathSet outputPaths() const;
}; };
struct Derivation : BasicDerivation struct Derivation : BasicDerivation {
{ DerivationInputs inputDrvs; /* inputs that are sub-derivations */
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
/* Print a derivation. */ /* Print a derivation. */
std::string unparse() const; std::string unparse() const;
}; };
class Store; class Store;
/* Write a derivation to the Nix store, and return its path. */ /* Write a derivation to the Nix store, and return its path. */
Path writeDerivation(ref<Store> store, Path writeDerivation(ref<Store> store, const Derivation& drv,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair); const string& name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */ /* Read a derivation from a file. */
Derivation readDerivation(const Path & drvPath); Derivation readDerivation(const Path& drvPath);
/* Check whether a file name ends with the extension for /* Check whether a file name ends with the extension for
derivations. */ derivations. */
bool isDerivation(const string & fileName); bool isDerivation(const string& fileName);
Hash hashDerivationModulo(Store & store, Derivation drv); Hash hashDerivationModulo(Store& store, Derivation drv);
/* Memoisation of hashDerivationModulo(). */ /* Memoisation of hashDerivationModulo(). */
typedef std::map<Path, Hash> DrvHashes; typedef std::map<Path, Hash> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe extern DrvHashes drvHashes; // FIXME: global, not thread-safe
/* Split a string specifying a derivation and a set of outputs /* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path and (/nix/store/hash-foo!out1,out2,...) into the derivation path and
the outputs. */ the outputs. */
typedef std::pair<string, std::set<string> > DrvPathWithOutputs; typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); DrvPathWithOutputs parseDrvPathWithOutputs(const string& s);
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); Path makeDrvPathWithOutputs(const Path& drvPath,
const std::set<string>& outputs);
bool wantOutput(const string & output, const std::set<string> & wanted); bool wantOutput(const string& output, const std::set<string>& wanted);
struct Source; struct Source;
struct Sink; struct Sink;
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); Source& readDerivation(Source& in, Store& store, BasicDerivation& drv);
Sink & operator << (Sink & out, const BasicDerivation & drv); Sink& operator<<(Sink& out, const BasicDerivation& drv);
std::string hashPlaceholder(const std::string & outputName); std::string hashPlaceholder(const std::string& outputName);
} } // namespace nix

1716
third_party/nix/src/libstore/download.cc vendored
View file

File diff suppressed because it is too large Load diff

180
third_party/nix/src/libstore/download.hh vendored
View file

@ -1,120 +1,118 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh"
#include "globals.hh"
#include <string>
#include <future> #include <future>
#include <string>
#include "globals.hh"
#include "hash.hh"
#include "types.hh"
namespace nix { namespace nix {
struct DownloadSettings : Config struct DownloadSettings : Config {
{ Setting<bool> enableHttp2{this, true, "http2",
Setting<bool> enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."};
"Whether to enable HTTP/2 support."};
Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", Setting<std::string> userAgentSuffix{
"String appended to the user agent in HTTP requests."}; this, "", "user-agent-suffix",
"String appended to the user agent in HTTP requests."};
Setting<size_t> httpConnections{this, 25, "http-connections", Setting<size_t> httpConnections{this,
"Number of parallel HTTP connections.", 25,
{"binary-caches-parallel-connections"}}; "http-connections",
"Number of parallel HTTP connections.",
{"binary-caches-parallel-connections"}};
Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", Setting<unsigned long> connectTimeout{
"Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; this, 0, "connect-timeout",
"Timeout for connecting to servers during downloads. 0 means use curl's "
"builtin default."};
Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", Setting<unsigned long> stalledDownloadTimeout{
"Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; this, 300, "stalled-download-timeout",
"Timeout (in seconds) for receiving data from servers during download. "
"Nix cancels idle downloads after this timeout's duration."};
Setting<unsigned int> tries{this, 5, "download-attempts", Setting<unsigned int> tries{
"How often Nix will attempt to download a file before giving up."}; this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."};
}; };
extern DownloadSettings downloadSettings; extern DownloadSettings downloadSettings;
struct DownloadRequest struct DownloadRequest {
{ std::string uri;
std::string uri; std::string expectedETag;
std::string expectedETag; bool verifyTLS = true;
bool verifyTLS = true; bool head = false;
bool head = false; size_t tries = downloadSettings.tries;
size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250;
unsigned int baseRetryTimeMs = 250; ActivityId parentAct;
ActivityId parentAct; bool decompress = true;
bool decompress = true; std::shared_ptr<std::string> data;
std::shared_ptr<std::string> data; std::string mimeType;
std::string mimeType; std::function<void(char*, size_t)> dataCallback;
std::function<void(char *, size_t)> dataCallback;
DownloadRequest(const std::string & uri) DownloadRequest(const std::string& uri)
: uri(uri), parentAct(getCurActivity()) { } : uri(uri), parentAct(getCurActivity()) {}
std::string verb() std::string verb() { return data ? "upload" : "download"; }
{
return data ? "upload" : "download";
}
}; };
struct DownloadResult struct DownloadResult {
{ bool cached = false;
bool cached = false; std::string etag;
std::string etag; std::string effectiveUri;
std::string effectiveUri; std::shared_ptr<std::string> data;
std::shared_ptr<std::string> data; uint64_t bodySize = 0;
uint64_t bodySize = 0;
}; };
struct CachedDownloadRequest struct CachedDownloadRequest {
{ std::string uri;
std::string uri; bool unpack = false;
bool unpack = false; std::string name;
std::string name; Hash expectedHash;
Hash expectedHash; unsigned int ttl = settings.tarballTtl;
unsigned int ttl = settings.tarballTtl;
CachedDownloadRequest(const std::string & uri) CachedDownloadRequest(const std::string& uri) : uri(uri) {}
: uri(uri) { }
}; };
struct CachedDownloadResult struct CachedDownloadResult {
{ // Note: 'storePath' may be different from 'path' when using a
// Note: 'storePath' may be different from 'path' when using a // chroot store.
// chroot store. Path storePath;
Path storePath; Path path;
Path path; std::optional<std::string> etag;
std::optional<std::string> etag; std::string effectiveUri;
std::string effectiveUri;
}; };
class Store; class Store;
struct Downloader struct Downloader {
{ virtual ~Downloader() {}
virtual ~Downloader() { }
/* Enqueue a download request, returning a future to the result of /* Enqueue a download request, returning a future to the result of
the download. The future may throw a DownloadError the download. The future may throw a DownloadError
exception. */ exception. */
virtual void enqueueDownload(const DownloadRequest & request, virtual void enqueueDownload(const DownloadRequest& request,
Callback<DownloadResult> callback) = 0; Callback<DownloadResult> callback) = 0;
std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); std::future<DownloadResult> enqueueDownload(const DownloadRequest& request);
/* Synchronously download a file. */ /* Synchronously download a file. */
DownloadResult download(const DownloadRequest & request); DownloadResult download(const DownloadRequest& request);
/* Download a file, writing its data to a sink. The sink will be /* Download a file, writing its data to a sink. The sink will be
invoked on the thread of the caller. */ invoked on the thread of the caller. */
void download(DownloadRequest && request, Sink & sink); void download(DownloadRequest&& request, Sink& sink);
/* Check if the specified file is already in ~/.cache/nix/tarballs /* Check if the specified file is already in ~/.cache/nix/tarballs
and is more recent than ‘tarball-ttl’ seconds. Otherwise, and is more recent than ‘tarball-ttl’ seconds. Otherwise,
use the recorded ETag to verify if the server has a more use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */ recent version, and if so, download it to the Nix store. */
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); CachedDownloadResult downloadCached(ref<Store> store,
const CachedDownloadRequest& request);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
}; };
/* Return a shared Downloader object. Using this object is preferred /* Return a shared Downloader object. Using this object is preferred
@ -124,15 +122,13 @@ ref<Downloader> getDownloader();
/* Return a new Downloader object. */ /* Return a new Downloader object. */
ref<Downloader> makeDownloader(); ref<Downloader> makeDownloader();
class DownloadError : public Error class DownloadError : public Error {
{ public:
public: Downloader::Error error;
Downloader::Error error; DownloadError(Downloader::Error error, const FormatOrString& fs)
DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) {}
: Error(fs), error(error)
{ }
}; };
bool isUri(const string & s); bool isUri(const string& s);
} } // namespace nix

144
third_party/nix/src/libstore/export-import.cc vendored
View file

@ -1,106 +1,100 @@
#include "store-api.hh"
#include "archive.hh"
#include "worker-protocol.hh"
#include <algorithm> #include <algorithm>
#include "archive.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
struct HashAndWriteSink : Sink struct HashAndWriteSink : Sink {
{ Sink& writeSink;
Sink & writeSink; HashSink hashSink;
HashSink hashSink; HashAndWriteSink(Sink& writeSink)
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) : writeSink(writeSink), hashSink(htSHA256) {}
{ virtual void operator()(const unsigned char* data, size_t len) {
} writeSink(data, len);
virtual void operator () (const unsigned char * data, size_t len) hashSink(data, len);
{ }
writeSink(data, len); Hash currentHash() { return hashSink.currentHash().first; }
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
}; };
void Store::exportPaths(const Paths & paths, Sink & sink) void Store::exportPaths(const Paths& paths, Sink& sink) {
{ Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); std::reverse(sorted.begin(), sorted.end());
std::reverse(sorted.begin(), sorted.end());
std::string doneLabel("paths exported"); std::string doneLabel("paths exported");
//logger->incExpected(doneLabel, sorted.size()); // logger->incExpected(doneLabel, sorted.size());
for (auto & path : sorted) { for (auto& path : sorted) {
//Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); // Activity act(*logger, lvlInfo, format("exporting path '%s'") % path);
sink << 1; sink << 1;
exportPath(path, sink); exportPath(path, sink);
//logger->incProgress(doneLabel); // logger->incProgress(doneLabel);
} }
sink << 0; sink << 0;
} }
void Store::exportPath(const Path & path, Sink & sink) void Store::exportPath(const Path& path, Sink& sink) {
{ auto info = queryPathInfo(path);
auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink); HashAndWriteSink hashAndWriteSink(sink);
narFromPath(path, hashAndWriteSink); narFromPath(path, hashAndWriteSink);
/* Refuse to export paths that have changed. This prevents /* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines. filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */ Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash(); Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") %
% info->narHash.to_string() % hash.to_string()); path % info->narHash.to_string() % hash.to_string());
hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; hashAndWriteSink << exportMagic << path << info->references << info->deriver
<< 0;
} }
Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) Paths Store::importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
{ CheckSigsFlag checkSigs) {
Paths res; Paths res;
while (true) { while (true) {
auto n = readNum<uint64_t>(source); auto n = readNum<uint64_t>(source);
if (n == 0) break; if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); if (n != 1)
throw Error(
"input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
TeeSink tee(source); TeeSink tee(source);
parseDump(tee, tee.source); parseDump(tee, tee.source);
uint32_t magic = readInt(source); uint32_t magic = readInt(source);
if (magic != exportMagic) if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format"); throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info; ValidPathInfo info;
info.path = readStorePath(*this, source); info.path = readStorePath(*this, source);
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); // Activity act(*logger, lvlInfo, format("importing path '%s'") %
// info.path);
info.references = readStorePaths<PathSet>(*this, source); info.references = readStorePaths<PathSet>(*this, source);
info.deriver = readString(source); info.deriver = readString(source);
if (info.deriver != "") assertStorePath(info.deriver); if (info.deriver != "") assertStorePath(info.deriver);
info.narHash = hashString(htSHA256, *tee.source.data); info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.source.data->size(); info.narSize = tee.source.data->size();
// Ignore optional legacy signature. // Ignore optional legacy signature.
if (readInt(source) == 1) if (readInt(source) == 1) readString(source);
readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
res.push_back(info.path); res.push_back(info.path);
} }
return res; return res;
} }
} } // namespace nix

32
third_party/nix/src/libstore/fs-accessor.hh vendored
View file

@ -6,28 +6,26 @@ namespace nix {
/* An abstract class for accessing a filesystem-like structure, such /* An abstract class for accessing a filesystem-like structure, such
as a (possibly remote) Nix store or the contents of a NAR file. */ as a (possibly remote) Nix store or the contents of a NAR file. */
class FSAccessor class FSAccessor {
{ public:
public: enum Type { tMissing, tRegular, tSymlink, tDirectory };
enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat struct Stat {
{ Type type = tMissing;
Type type = tMissing; uint64_t fileSize = 0; // regular files only
uint64_t fileSize = 0; // regular files only bool isExecutable = false; // regular files only
bool isExecutable = false; // regular files only uint64_t narOffset = 0; // regular files only
uint64_t narOffset = 0; // regular files only };
};
virtual ~FSAccessor() { } virtual ~FSAccessor() {}
virtual Stat stat(const Path & path) = 0; virtual Stat stat(const Path& path) = 0;
virtual StringSet readDirectory(const Path & path) = 0; virtual StringSet readDirectory(const Path& path) = 0;
virtual std::string readFile(const Path & path) = 0; virtual std::string readFile(const Path& path) = 0;
virtual std::string readLink(const Path & path) = 0; virtual std::string readLink(const Path& path) = 0;
}; };
} } // namespace nix

1544
third_party/nix/src/libstore/gc.cc vendored
View file

File diff suppressed because it is too large Load diff

261
third_party/nix/src/libstore/globals.cc vendored
View file

@ -1,17 +1,14 @@
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include <dlfcn.h>
#include "archive.hh"
#include "args.hh"
#include <algorithm> #include <algorithm>
#include <map> #include <map>
#include <thread> #include <thread>
#include <dlfcn.h> #include "archive.hh"
#include "args.hh"
#include "util.hh"
namespace nix { namespace nix {
/* The default location of the daemon socket, relative to nixStateDir. /* The default location of the daemon socket, relative to nixStateDir.
The socket is in a directory to allow you to control access to the The socket is in a directory to allow you to control access to the
Nix daemon by setting the mode/ownership of the directory Nix daemon by setting the mode/ownership of the directory
@ -21,9 +18,9 @@ namespace nix {
/* chroot-like behavior from Apple's sandbox */ /* chroot-like behavior from Apple's sandbox */
#if __APPLE__ #if __APPLE__
#define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
#else #else
#define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
#endif #endif
Settings settings; Settings settings;
@ -31,157 +28,163 @@ Settings settings;
static GlobalConfig::Register r1(&settings); static GlobalConfig::Register r1(&settings);
Settings::Settings() Settings::Settings()
: nixPrefix(NIX_PREFIX) : nixPrefix(NIX_PREFIX),
, nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) nixStore(canonPath(
, nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))),
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))),
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))),
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))),
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))),
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))),
, nixManDir(canonPath(NIX_MAN_DIR)) nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))),
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) nixManDir(canonPath(NIX_MAN_DIR)),
{ nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) {
buildUsersGroup = getuid() == 0 ? "nixbld" : ""; buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
if (caFile == "") { if (caFile == "") {
for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) for (auto& fn :
if (pathExists(fn)) { {"/etc/ssl/certs/ca-certificates.crt",
caFile = fn; "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
break; if (pathExists(fn)) {
} caFile = fn;
} break;
}
}
/* Backwards compatibility. */ /* Backwards compatibility. */
auto s = getEnv("NIX_REMOTE_SYSTEMS"); auto s = getEnv("NIX_REMOTE_SYSTEMS");
if (s != "") { if (s != "") {
Strings ss; Strings ss;
for (auto & p : tokenizeString<Strings>(s, ":")) for (auto& p : tokenizeString<Strings>(s, ":")) ss.push_back("@" + p);
ss.push_back("@" + p); builders = concatStringsSep(" ", ss);
builders = concatStringsSep(" ", ss); }
}
#if defined(__linux__) && defined(SANDBOX_SHELL) #if defined(__linux__) && defined(SANDBOX_SHELL)
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
#endif #endif
allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); allowedImpureHostPrefixes =
tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
} }
void loadConfFile() void loadConfFile() {
{ globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
/* We only want to send overrides to the daemon, i.e. stuff from /* We only want to send overrides to the daemon, i.e. stuff from
~/.nix/nix.conf or the command line. */ ~/.nix/nix.conf or the command line. */
globalConfig.resetOverriden(); globalConfig.resetOverriden();
auto dirs = getConfigDirs(); auto dirs = getConfigDirs();
// Iterate over them in reverse so that the ones appearing first in the path take priority // Iterate over them in reverse so that the ones appearing first in the path
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { // take priority
globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
} globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
}
} }
unsigned int Settings::getDefaultCores() unsigned int Settings::getDefaultCores() {
{ return std::max(1U, std::thread::hardware_concurrency());
return std::max(1U, std::thread::hardware_concurrency());
} }
StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultSystemFeatures() {
{ /* For backwards compatibility, accept some "features" that are
/* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't
used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */
actually require anything special on the machines. */ StringSet features{"nixos-test", "benchmark", "big-parallel"};
StringSet features{"nixos-test", "benchmark", "big-parallel"};
#if __linux__ #if __linux__
if (access("/dev/kvm", R_OK | W_OK) == 0) if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm");
features.insert("kvm"); #endif
#endif
return features; return features;
} }
const string nixVersion = PACKAGE_VERSION; const string nixVersion = PACKAGE_VERSION;
template<> void BaseSetting<SandboxMode>::set(const std::string & str) template <>
{ void BaseSetting<SandboxMode>::set(const std::string& str) {
if (str == "true") value = smEnabled; if (str == "true")
else if (str == "relaxed") value = smRelaxed; value = smEnabled;
else if (str == "false") value = smDisabled; else if (str == "relaxed")
else throw UsageError("option '%s' has invalid value '%s'", name, str); value = smRelaxed;
else if (str == "false")
value = smDisabled;
else
throw UsageError("option '%s' has invalid value '%s'", name, str);
} }
template<> std::string BaseSetting<SandboxMode>::to_string() template <>
{ std::string BaseSetting<SandboxMode>::to_string() {
if (value == smEnabled) return "true"; if (value == smEnabled)
else if (value == smRelaxed) return "relaxed"; return "true";
else if (value == smDisabled) return "false"; else if (value == smRelaxed)
else abort(); return "relaxed";
else if (value == smDisabled)
return "false";
else
abort();
} }
template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) template <>
{ void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) {
AbstractSetting::toJSON(out); AbstractSetting::toJSON(out);
} }
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) template <>
{ void BaseSetting<SandboxMode>::convertToArg(Args& args,
args.mkFlag() const std::string& category) {
.longName(name) args.mkFlag()
.description("Enable sandboxing.") .longName(name)
.handler([=](std::vector<std::string> ss) { override(smEnabled); }) .description("Enable sandboxing.")
.category(category); .handler([=](std::vector<std::string> ss) { override(smEnabled); })
args.mkFlag() .category(category);
.longName("no-" + name) args.mkFlag()
.description("Disable sandboxing.") .longName("no-" + name)
.handler([=](std::vector<std::string> ss) { override(smDisabled); }) .description("Disable sandboxing.")
.category(category); .handler([=](std::vector<std::string> ss) { override(smDisabled); })
args.mkFlag() .category(category);
.longName("relaxed-" + name) args.mkFlag()
.description("Enable sandboxing, but allow builds to disable it.") .longName("relaxed-" + name)
.handler([=](std::vector<std::string> ss) { override(smRelaxed); }) .description("Enable sandboxing, but allow builds to disable it.")
.category(category); .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
.category(category);
} }
void MaxBuildJobsSetting::set(const std::string & str) void MaxBuildJobsSetting::set(const std::string& str) {
{ if (str == "auto")
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); value = std::max(1U, std::thread::hardware_concurrency());
else if (!string2Int(str, value)) else if (!string2Int(str, value))
throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); throw UsageError(
"configuration setting '%s' should be 'auto' or an integer", name);
} }
void initPlugins() {
void initPlugins() for (const auto& pluginFile : settings.pluginFiles.get()) {
{ Paths pluginFiles;
for (const auto & pluginFile : settings.pluginFiles.get()) { try {
Paths pluginFiles; auto ents = readDirectory(pluginFile);
try { for (const auto& ent : ents)
auto ents = readDirectory(pluginFile); pluginFiles.emplace_back(pluginFile + "/" + ent.name);
for (const auto & ent : ents) } catch (SysError& e) {
pluginFiles.emplace_back(pluginFile + "/" + ent.name); if (e.errNo != ENOTDIR) throw;
} catch (SysError & e) { pluginFiles.emplace_back(pluginFile);
if (e.errNo != ENOTDIR)
throw;
pluginFiles.emplace_back(pluginFile);
}
for (const auto & file : pluginFiles) {
/* handle is purposefully leaked as there may be state in the
DSO needed by the action of the plugin. */
void *handle =
dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle)
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
}
} }
for (const auto& file : pluginFiles) {
/* handle is purposefully leaked as there may be state in the
DSO needed by the action of the plugin. */
void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle)
throw Error("could not dynamically open plugin file '%s': %s", file,
dlerror());
}
}
/* Since plugins can add settings, try to re-apply previously /* Since plugins can add settings, try to re-apply previously
unknown settings. */ unknown settings. */
globalConfig.reapplyUnknownSettings(); globalConfig.reapplyUnknownSettings();
globalConfig.warnUnknownSettings(); globalConfig.warnUnknownSettings();
} }
} } // namespace nix

614
third_party/nix/src/libstore/globals.hh vendored
View file

@ -1,361 +1,475 @@
#pragma once #pragma once
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include <map>
#include <limits>
#include <sys/types.h> #include <sys/types.h>
#include <limits>
#include <map>
#include "config.hh"
#include "types.hh"
#include "util.hh"
namespace nix { namespace nix {
typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
struct MaxBuildJobsSetting : public BaseSetting<unsigned int> struct MaxBuildJobsSetting : public BaseSetting<unsigned int> {
{ MaxBuildJobsSetting(Config* options, unsigned int def,
MaxBuildJobsSetting(Config * options, const std::string& name, const std::string& description,
unsigned int def, const std::set<std::string>& aliases = {})
const std::string & name, : BaseSetting<unsigned int>(def, name, description, aliases) {
const std::string & description, options->addSetting(this);
const std::set<std::string> & aliases = {}) }
: BaseSetting<unsigned int>(def, name, description, aliases)
{
options->addSetting(this);
}
void set(const std::string & str) override; void set(const std::string& str) override;
}; };
class Settings : public Config { class Settings : public Config {
unsigned int getDefaultCores();
unsigned int getDefaultCores(); StringSet getDefaultSystemFeatures();
StringSet getDefaultSystemFeatures(); public:
Settings();
public: Path nixPrefix;
Settings(); /* The directory where we store sources and derived files. */
Path nixStore;
Path nixPrefix; Path nixDataDir; /* !!! fix */
/* The directory where we store sources and derived files. */ /* The directory where we log various operations. */
Path nixStore; Path nixLogDir;
Path nixDataDir; /* !!! fix */ /* The directory where state is stored. */
Path nixStateDir;
/* The directory where we log various operations. */ /* The directory where configuration files are stored. */
Path nixLogDir; Path nixConfDir;
/* The directory where state is stored. */ /* The directory where internal helper programs are stored. */
Path nixStateDir; Path nixLibexecDir;
/* The directory where configuration files are stored. */ /* The directory where the main programs are stored. */
Path nixConfDir; Path nixBinDir;
/* The directory where internal helper programs are stored. */ /* The directory where the man pages are stored. */
Path nixLibexecDir; Path nixManDir;
/* The directory where the main programs are stored. */ /* File name of the socket the daemon listens to. */
Path nixBinDir; Path nixDaemonSocketFile;
/* The directory where the man pages are stored. */ Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
Path nixManDir; "The default Nix store to use."};
/* File name of the socket the daemon listens to. */ Setting<bool> keepFailed{
Path nixDaemonSocketFile; this, false, "keep-failed",
"Whether to keep temporary directories of failed builds."};
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", Setting<bool> keepGoing{
"The default Nix store to use."}; this, false, "keep-going",
"Whether to keep building derivations when another build fails."};
Setting<bool> keepFailed{this, false, "keep-failed", Setting<bool> tryFallback{
"Whether to keep temporary directories of failed builds."}; this,
false,
"fallback",
"Whether to fall back to building when substitution fails.",
{"build-fallback"}};
Setting<bool> keepGoing{this, false, "keep-going", /* Whether to show build log output in real time. */
"Whether to keep building derivations when another build fails."}; bool verboseBuild = true;
Setting<bool> tryFallback{this, false, "fallback", Setting<size_t> logLines{
"Whether to fall back to building when substitution fails.", this, 10, "log-lines",
{"build-fallback"}}; "If verbose-build is false, the number of lines of the tail of "
"the log to show if a build fails."};
/* Whether to show build log output in real time. */ MaxBuildJobsSetting maxBuildJobs{this,
bool verboseBuild = true; 1,
"max-jobs",
"Maximum number of parallel build jobs. "
"\"auto\" means use number of cores.",
{"build-max-jobs"}};
Setting<size_t> logLines{this, 10, "log-lines", Setting<unsigned int> buildCores{
"If verbose-build is false, the number of lines of the tail of " this,
"the log to show if a build fails."}; getDefaultCores(),
"cores",
"Number of CPU cores to utilize in parallel within a build, "
"i.e. by passing this number to Make via '-j'. 0 means that the "
"number of actual CPU cores on the local host ought to be "
"auto-detected.",
{"build-cores"}};
MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", /* Read-only mode. Don't copy stuff to the store, don't change
"Maximum number of parallel build jobs. \"auto\" means use number of cores.", the database. */
{"build-max-jobs"}}; bool readOnlyMode = false;
Setting<unsigned int> buildCores{this, getDefaultCores(), "cores", Setting<std::string> thisSystem{this, SYSTEM, "system",
"Number of CPU cores to utilize in parallel within a build, " "The canonical Nix system name."};
"i.e. by passing this number to Make via '-j'. 0 means that the "
"number of actual CPU cores on the local host ought to be "
"auto-detected.", {"build-cores"}};
/* Read-only mode. Don't copy stuff to the store, don't change Setting<time_t> maxSilentTime{
the database. */ this,
bool readOnlyMode = false; 0,
"max-silent-time",
"The maximum time in seconds that a builer can go without "
"producing any output on stdout/stderr before it is killed. "
"0 means infinity.",
{"build-max-silent-time"}};
Setting<std::string> thisSystem{this, SYSTEM, "system", Setting<time_t> buildTimeout{
"The canonical Nix system name."}; this,
0,
"timeout",
"The maximum duration in seconds that a builder can run. "
"0 means infinity.",
{"build-timeout"}};
Setting<time_t> maxSilentTime{this, 0, "max-silent-time", PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote",
"The maximum time in seconds that a builer can go without " "build-hook",
"producing any output on stdout/stderr before it is killed. " "The path of the helper program that executes builds "
"0 means infinity.", "to remote machines."};
{"build-max-silent-time"}};
Setting<time_t> buildTimeout{this, 0, "timeout", Setting<std::string> builders{this, "@" + nixConfDir + "/machines",
"The maximum duration in seconds that a builder can run. " "builders",
"0 means infinity.", {"build-timeout"}}; "A semicolon-separated list of build machines, "
"in the format of nix.machines."};
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", Setting<bool> buildersUseSubstitutes{
"The path of the helper program that executes builds to remote machines."}; this, false, "builders-use-substitutes",
"Whether build machines should use their own substitutes for obtaining "
"build dependencies if possible, rather than waiting for this host to "
"upload them."};
Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", Setting<off_t> reservedSize{
"A semicolon-separated list of build machines, in the format of nix.machines."}; this, 8 * 1024 * 1024, "gc-reserved-space",
"Amount of reserved disk space for the garbage collector."};
Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
"Whether build machines should use their own substitutes for obtaining " "Whether SQLite should use fsync()."};
"build dependencies if possible, rather than waiting for this host to "
"upload them."};
Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
"Amount of reserved disk space for the garbage collector."}; "Whether SQLite should use WAL mode."};
Setting<bool> fsyncMetadata{this, true, "fsync-metadata", Setting<bool> syncBeforeRegistering{
"Whether SQLite should use fsync()."}; this, false, "sync-before-registering",
"Whether to call sync() before registering a path as valid."};
Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", Setting<bool> useSubstitutes{this,
"Whether SQLite should use WAL mode."}; true,
"substitute",
"Whether to use substitutes.",
{"build-use-substitutes"}};
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", Setting<std::string> buildUsersGroup{
"Whether to call sync() before registering a path as valid."}; this, "", "build-users-group",
"The Unix group that contains the build users."};
Setting<bool> useSubstitutes{this, true, "substitute", Setting<bool> impersonateLinux26{
"Whether to use substitutes.", this,
{"build-use-substitutes"}}; false,
"impersonate-linux-26",
"Whether to impersonate a Linux 2.6 machine on newer kernels.",
{"build-impersonate-linux-26"}};
Setting<std::string> buildUsersGroup{this, "", "build-users-group", Setting<bool> keepLog{this,
"The Unix group that contains the build users."}; true,
"keep-build-log",
"Whether to store build logs.",
{"build-keep-log"}};
Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26", Setting<bool> compressLog{this,
"Whether to impersonate a Linux 2.6 machine on newer kernels.", true,
{"build-impersonate-linux-26"}}; "compress-build-log",
"Whether to compress logs.",
{"build-compress-log"}};
Setting<bool> keepLog{this, true, "keep-build-log", Setting<unsigned long> maxLogSize{
"Whether to store build logs.", this,
{"build-keep-log"}}; 0,
"max-build-log-size",
"Maximum number of bytes a builder can write to stdout/stderr "
"before being killed (0 means no limit).",
{"build-max-log-size"}};
Setting<bool> compressLog{this, true, "compress-build-log", /* When buildRepeat > 0 and verboseBuild == true, whether to print
"Whether to compress logs.", repeated builds (i.e. builds other than the first one) to
{"build-compress-log"}}; stderr. Hack to prevent Hydra logs from being polluted. */
bool printRepeatedBuilds = true;
Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size", Setting<unsigned int> pollInterval{
"Maximum number of bytes a builder can write to stdout/stderr " this, 5, "build-poll-interval",
"before being killed (0 means no limit).", "How often (in seconds) to poll for locks."};
{"build-max-log-size"}};
/* When buildRepeat > 0 and verboseBuild == true, whether to print Setting<bool> checkRootReachability{
repeated builds (i.e. builds other than the first one) to this, false, "gc-check-reachability",
stderr. Hack to prevent Hydra logs from being polluted. */ "Whether to check if new GC roots can in fact be found by the "
bool printRepeatedBuilds = true; "garbage collector."};
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", Setting<bool> gcKeepOutputs{
"How often (in seconds) to poll for locks."}; this,
false,
"keep-outputs",
"Whether the garbage collector should keep outputs of live derivations.",
{"gc-keep-outputs"}};
Setting<bool> checkRootReachability{this, false, "gc-check-reachability", Setting<bool> gcKeepDerivations{
"Whether to check if new GC roots can in fact be found by the " this,
"garbage collector."}; true,
"keep-derivations",
"Whether the garbage collector should keep derivers of live paths.",
{"gc-keep-derivations"}};
Setting<bool> gcKeepOutputs{this, false, "keep-outputs", Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
"Whether the garbage collector should keep outputs of live derivations.", "Whether to automatically replace files with "
{"gc-keep-outputs"}}; "identical contents with hard links."};
Setting<bool> gcKeepDerivations{this, true, "keep-derivations", Setting<bool> envKeepDerivations{
"Whether the garbage collector should keep derivers of live paths.", this,
{"gc-keep-derivations"}}; false,
"keep-env-derivations",
"Whether to add derivations as a dependency of user environments "
"(to prevent them from being GCed).",
{"env-keep-derivations"}};
Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", /* Whether to lock the Nix client and worker to the same CPU. */
"Whether to automatically replace files with identical contents with hard links."}; bool lockCPU;
Setting<bool> envKeepDerivations{this, false, "keep-env-derivations", /* Whether to show a stack trace if Nix evaluation fails. */
"Whether to add derivations as a dependency of user environments " Setting<bool> showTrace{
"(to prevent them from being GCed).", this, false, "show-trace",
{"env-keep-derivations"}}; "Whether to show a stack trace on evaluation errors."};
/* Whether to lock the Nix client and worker to the same CPU. */ Setting<SandboxMode> sandboxMode {
bool lockCPU; this,
#if __linux__
smEnabled
#else
smDisabled
#endif
,
"sandbox",
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or "
"\"relaxed\".",
{
"build-use-chroot", "build-use-sandbox"
}
};
/* Whether to show a stack trace if Nix evaluation fails. */ Setting<PathSet> sandboxPaths{
Setting<bool> showTrace{this, false, "show-trace", this,
"Whether to show a stack trace on evaluation errors."}; {},
"sandbox-paths",
"The paths to make available inside the build sandbox.",
{"build-chroot-dirs", "build-sandbox-paths"}};
Setting<SandboxMode> sandboxMode{this, Setting<bool> sandboxFallback{
#if __linux__ this, true, "sandbox-fallback",
smEnabled "Whether to disable sandboxing when the kernel doesn't allow it."};
#else
smDisabled
#endif
, "sandbox",
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
{"build-use-chroot", "build-use-sandbox"}};
Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths", Setting<PathSet> extraSandboxPaths{
"The paths to make available inside the build sandbox.", this,
{"build-chroot-dirs", "build-sandbox-paths"}}; {},
"extra-sandbox-paths",
"Additional paths to make available inside the build sandbox.",
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
Setting<bool> sandboxFallback{this, true, "sandbox-fallback", Setting<size_t> buildRepeat{
"Whether to disable sandboxing when the kernel doesn't allow it."}; this,
0,
Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", "repeat",
"Additional paths to make available inside the build sandbox.", "The number of times to repeat a build in order to verify determinism.",
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; {"build-repeat"}};
Setting<size_t> buildRepeat{this, 0, "repeat",
"The number of times to repeat a build in order to verify determinism.",
{"build-repeat"}};
#if __linux__ #if __linux__
Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", Setting<std::string> sandboxShmSize{
"The size of /dev/shm in the build sandbox."}; this, "50%", "sandbox-dev-shm-size",
"The size of /dev/shm in the build sandbox."};
Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
"The build directory inside the sandbox."}; "The build directory inside the sandbox."};
#endif #endif
Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", Setting<PathSet> allowedImpureHostPrefixes{
"Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; this,
{},
"allowed-impure-host-deps",
"Which prefixes to allow derivations to ask for access to (primarily for "
"Darwin)."};
#if __APPLE__ #if __APPLE__
Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", Setting<bool> darwinLogSandboxViolations{
"Whether to log Darwin sandbox access violations to the system log."}; this, false, "darwin-log-sandbox-violations",
"Whether to log Darwin sandbox access violations to the system log."};
#endif #endif
Setting<bool> runDiffHook{this, false, "run-diff-hook", Setting<bool> runDiffHook{
"Whether to run the program specified by the diff-hook setting " this, false, "run-diff-hook",
"repeated builds produce a different result. Typically used to " "Whether to run the program specified by the diff-hook setting "
"plug in diffoscope."}; "repeated builds produce a different result. Typically used to "
"plug in diffoscope."};
PathSetting diffHook{this, true, "", "diff-hook", PathSetting diffHook{
"A program that prints out the differences between the two paths " this, true, "", "diff-hook",
"specified on its command line."}; "A program that prints out the differences between the two paths "
"specified on its command line."};
Setting<bool> enforceDeterminism{this, true, "enforce-determinism", Setting<bool> enforceDeterminism{
"Whether to fail if repeated builds produce different output."}; this, true, "enforce-determinism",
"Whether to fail if repeated builds produce different output."};
Setting<Strings> trustedPublicKeys{this, Setting<Strings> trustedPublicKeys{
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, this,
"trusted-public-keys", {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
"Trusted public keys for secure substitution.", "trusted-public-keys",
{"binary-cache-public-keys"}}; "Trusted public keys for secure substitution.",
{"binary-cache-public-keys"}};
Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", Setting<Strings> secretKeyFiles{
"Secret keys with which to sign local builds."}; this,
{},
"secret-key-files",
"Secret keys with which to sign local builds."};
Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", Setting<unsigned int> tarballTtl{
"How long downloaded files are considered up-to-date."}; this, 60 * 60, "tarball-ttl",
"How long downloaded files are considered up-to-date."};
Setting<bool> requireSigs{this, true, "require-sigs", Setting<bool> requireSigs{
"Whether to check that any non-content-addressed path added to the " this, true, "require-sigs",
"Nix store has a valid signature (that is, one signed using a key " "Whether to check that any non-content-addressed path added to the "
"listed in 'trusted-public-keys'."}; "Nix store has a valid signature (that is, one signed using a key "
"listed in 'trusted-public-keys'."};
Setting<StringSet> extraPlatforms{this, Setting<StringSet> extraPlatforms{
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, this,
"extra-platforms", std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"}
"Additional platforms that can be built on the local system. " : StringSet{},
"These may be supported natively (e.g. armv7 on some aarch64 CPUs " "extra-platforms",
"or using hacks like qemu-user."}; "Additional platforms that can be built on the local system. "
"These may be supported natively (e.g. armv7 on some aarch64 CPUs "
"or using hacks like qemu-user."};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), Setting<StringSet> systemFeatures{
"system-features", this, getDefaultSystemFeatures(), "system-features",
"Optional features that this system implements (like \"kvm\")."}; "Optional features that this system implements (like \"kvm\")."};
Setting<Strings> substituters{this, Setting<Strings> substituters{
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), this,
"substituters", nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"}
"The URIs of substituters (such as https://cache.nixos.org/).", : Strings(),
{"binary-caches"}}; "substituters",
"The URIs of substituters (such as https://cache.nixos.org/).",
{"binary-caches"}};
// FIXME: provide a way to add to option values. // FIXME: provide a way to add to option values.
Setting<Strings> extraSubstituters{this, {}, "extra-substituters", Setting<Strings> extraSubstituters{this,
"Additional URIs of substituters.", {},
{"extra-binary-caches"}}; "extra-substituters",
"Additional URIs of substituters.",
{"extra-binary-caches"}};
Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", Setting<StringSet> trustedSubstituters{
"Disabled substituters that may be enabled via the substituters option by untrusted users.", this,
{"trusted-binary-caches"}}; {},
"trusted-substituters",
"Disabled substituters that may be enabled via the substituters option "
"by untrusted users.",
{"trusted-binary-caches"}};
Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", Setting<Strings> trustedUsers{this,
"Which users or groups are trusted to ask the daemon to do unsafe things."}; {"root"},
"trusted-users",
"Which users or groups are trusted to ask the "
"daemon to do unsafe things."};
Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", Setting<unsigned int> ttlNegativeNarInfoCache{
"The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " this, 3600, "narinfo-cache-negative-ttl",
"return an invalid path result"}; "The TTL in seconds for negative lookups in the disk cache i.e binary "
"cache lookups that "
"return an invalid path result"};
Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", Setting<unsigned int> ttlPositiveNarInfoCache{
"The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
"return a valid path result."}; "The TTL in seconds for positive lookups in the disk cache i.e binary "
"cache lookups that "
"return a valid path result."};
/* ?Who we trust to use the daemon in safe ways */ /* ?Who we trust to use the daemon in safe ways */
Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", Setting<Strings> allowedUsers{
"Which users or groups are allowed to connect to the daemon."}; this,
{"*"},
"allowed-users",
"Which users or groups are allowed to connect to the daemon."};
Setting<bool> printMissing{this, true, "print-missing", Setting<bool> printMissing{
"Whether to print what paths need to be built or downloaded."}; this, true, "print-missing",
"Whether to print what paths need to be built or downloaded."};
Setting<std::string> preBuildHook{this, Setting<std::string> preBuildHook {
this,
#if __APPLE__ #if __APPLE__
nixLibexecDir + "/nix/resolve-system-dependencies", nixLibexecDir + "/nix/resolve-system-dependencies",
#else #else
"", "",
#endif #endif
"pre-build-hook", "pre-build-hook",
"A program to run just before a build to set derivation-specific build settings."}; "A program to run just before a build to set derivation-specific build "
"settings."
};
Setting<std::string> postBuildHook{this, "", "post-build-hook", Setting<std::string> postBuildHook{
"A program to run just after each successful build."}; this, "", "post-build-hook",
"A program to run just after each successful build."};
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"),
"Path to the netrc file used to obtain usernames/passwords for downloads."}; "netrc-file",
"Path to the netrc file used to obtain "
"usernames/passwords for downloads."};
/* Path to the SSL CA file used */ /* Path to the SSL CA file used */
Path caFile; Path caFile;
#if __linux__ #if __linux__
Setting<bool> filterSyscalls{this, true, "filter-syscalls", Setting<bool> filterSyscalls{
"Whether to prevent certain dangerous system calls, such as " this, true, "filter-syscalls",
"creation of setuid/setgid files or adding ACLs or extended " "Whether to prevent certain dangerous system calls, such as "
"attributes. Only disable this if you're aware of the " "creation of setuid/setgid files or adding ACLs or extended "
"security implications."}; "attributes. Only disable this if you're aware of the "
"security implications."};
Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", Setting<bool> allowNewPrivileges{
"Whether builders can acquire new privileges by calling programs with " this, false, "allow-new-privileges",
"setuid/setgid bits or with file capabilities."}; "Whether builders can acquire new privileges by calling programs with "
"setuid/setgid bits or with file capabilities."};
#endif #endif
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", Setting<Strings> hashedMirrors{
"A list of servers used by builtins.fetchurl to fetch files by hash."}; this,
{"http://tarballs.nixos.org/"},
"hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free", Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."}; "Automatically run the garbage collector when free "
"disk space drops below the specified amount."};
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(),
"Stop deleting garbage when free disk space is above the specified amount."}; "max-free",
"Stop deleting garbage when free disk space is "
"above the specified amount."};
Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", Setting<uint64_t> minFreeCheckInterval{
"Number of seconds between checking free disk space."}; this, 5, "min-free-check-interval",
"Number of seconds between checking free disk space."};
Setting<Paths> pluginFiles{this, {}, "plugin-files", Setting<Paths> pluginFiles{
"Plugins to dynamically load at nix initialization time."}; this,
{},
"plugin-files",
"Plugins to dynamically load at nix initialization time."};
}; };
// FIXME: don't use a global variable. // FIXME: don't use a global variable.
extern Settings settings; extern Settings settings;
@ -367,4 +481,4 @@ void loadConfFile();
extern const string nixVersion; extern const string nixVersion;
} } // namespace nix

267
third_party/nix/src/libstore/http-binary-cache-store.cc vendored
View file

@ -7,167 +7,150 @@ namespace nix {
MakeError(UploadToHTTP, Error); MakeError(UploadToHTTP, Error);
class HttpBinaryCacheStore : public BinaryCacheStore class HttpBinaryCacheStore : public BinaryCacheStore {
{ private:
private: Path cacheUri;
Path cacheUri; struct State {
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
struct State Sync<State> _state;
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state; public:
HttpBinaryCacheStore(const Params& params, const Path& _cacheUri)
: BinaryCacheStore(params), cacheUri(_cacheUri) {
if (cacheUri.back() == '/') cacheUri.pop_back();
public: diskCache = getNarInfoDiskCache();
}
HttpBinaryCacheStore( std::string getUri() override { return cacheUri; }
const Params & params, const Path & _cacheUri)
: BinaryCacheStore(params)
, cacheUri(_cacheUri)
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
diskCache = getNarInfoDiskCache(); void init() override {
// FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP&) {
throw Error("'%s' does not appear to be a binary cache", cacheUri);
}
diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
} }
}
std::string getUri() override protected:
{ void maybeDisable() {
return cacheUri; auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false;
state->disabledUntil =
std::chrono::steady_clock::now() + std::chrono::seconds(t);
} }
}
void init() override void checkEnabled() {
{ auto state(_state.lock());
// FIXME: do this lazily? if (state->enabled) return;
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { if (std::chrono::steady_clock::now() > state->disabledUntil) {
try { state->enabled = true;
BinaryCacheStore::init(); debug("re-enabling binary cache '%s'", getUri());
} catch (UploadToHTTP &) { return;
throw Error("'%s' does not appear to be a binary cache", cacheUri);
}
diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
}
} }
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
}
protected: bool fileExists(const std::string& path) override {
checkEnabled();
void maybeDisable() try {
{ DownloadRequest request(cacheUri + "/" + path);
auto state(_state.lock()); request.head = true;
if (state->enabled && settings.tryFallback) { getDownloader()->download(request);
int t = 60; return true;
printError("disabling binary cache '%s' for %s seconds", getUri(), t); } catch (DownloadError& e) {
state->enabled = false; /* S3 buckets return 403 if a file doesn't exist and the
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); bucket is unlistable, so treat 403 as 404. */
} if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return false;
maybeDisable();
throw;
} }
}
void checkEnabled() void upsertFile(const std::string& path, const std::string& data,
{ const std::string& mimeType) override {
auto state(_state.lock()); auto req = DownloadRequest(cacheUri + "/" + path);
if (state->enabled) return; req.data = std::make_shared<string>(data); // FIXME: inefficient
if (std::chrono::steady_clock::now() > state->disabledUntil) { req.mimeType = mimeType;
state->enabled = true; try {
debug("re-enabling binary cache '%s'", getUri()); getDownloader()->download(req);
return; } catch (DownloadError& e) {
} throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s",
throw SubstituterDisabled("substituter '%s' is disabled", getUri()); cacheUri, e.msg());
} }
}
bool fileExists(const std::string & path) override DownloadRequest makeRequest(const std::string& path) {
{ DownloadRequest request(cacheUri + "/" + path);
checkEnabled(); return request;
}
try { void getFile(const std::string& path, Sink& sink) override {
DownloadRequest request(cacheUri + "/" + path); checkEnabled();
request.head = true; auto request(makeRequest(path));
getDownloader()->download(request); try {
return true; getDownloader()->download(std::move(request), sink);
} catch (DownloadError & e) { } catch (DownloadError& e) {
/* S3 buckets return 403 if a file doesn't exist and the if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
bucket is unlistable, so treat 403 as 404. */ throw NoSuchBinaryCacheFile(
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) "file '%s' does not exist in binary cache '%s'", path, getUri());
return false; maybeDisable();
throw;
}
}
void getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept override {
checkEnabled();
auto request(makeRequest(path));
auto callbackPtr =
std::make_shared<decltype(callback)>(std::move(callback));
getDownloader()->enqueueDownload(
request, {[callbackPtr, this](std::future<DownloadResult> result) {
try {
(*callbackPtr)(result.get().data);
} catch (DownloadError& e) {
if (e.error == Downloader::NotFound ||
e.error == Downloader::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable(); maybeDisable();
throw; callbackPtr->rethrow();
} } catch (...) {
} callbackPtr->rethrow();
}
void upsertFile(const std::string & path, }});
const std::string & data, }
const std::string & mimeType) override
{
auto req = DownloadRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType;
try {
getDownloader()->download(req);
} catch (DownloadError & e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
}
}
DownloadRequest makeRequest(const std::string & path)
{
DownloadRequest request(cacheUri + "/" + path);
return request;
}
void getFile(const std::string & path, Sink & sink) override
{
checkEnabled();
auto request(makeRequest(path));
try {
getDownloader()->download(std::move(request), sink);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
}
}
void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept override
{
checkEnabled();
auto request(makeRequest(path));
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getDownloader()->enqueueDownload(request,
{[callbackPtr, this](std::future<DownloadResult> result) {
try {
(*callbackPtr)(result.get().data);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable();
callbackPtr->rethrow();
} catch (...) {
callbackPtr->rethrow();
}
}});
}
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{ if (std::string(uri, 0, 7) != "http://" &&
if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" &&
std::string(uri, 0, 8) != "https://" && (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" ||
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") std::string(uri, 0, 7) != "file://"))
) return 0; return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init(); store->init();
return store; return store;
}); });
}
} // namespace nix

457
third_party/nix/src/libstore/legacy-ssh-store.cc vendored
View file

@ -1,293 +1,258 @@
#include "archive.hh" #include "archive.hh"
#include "derivations.hh"
#include "pool.hh" #include "pool.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
#include "ssh.hh"
#include "store-api.hh" #include "store-api.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
namespace nix { namespace nix {
static std::string uriScheme = "ssh://"; static std::string uriScheme = "ssh://";
struct LegacySSHStore : public Store struct LegacySSHStore : public Store {
{ const Setting<int> maxConnections{
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; this, 1, "max-connections",
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; "maximum number of concurrent SSH connections"};
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; const Setting<bool> compress{this, false, "compress",
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; "whether to compress the connection"};
const Setting<Path> remoteProgram{
this, "nix-store", "remote-program",
"path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{
this, "", "remote-store", "URI of the store on the remote system"};
// Hack for getting remote build log output. // Hack for getting remote build log output.
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; const Setting<int> logFD{
this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection struct Connection {
{ std::unique_ptr<SSHMaster::Connection> sshConn;
std::unique_ptr<SSHMaster::Connection> sshConn; FdSink to;
FdSink to; FdSource from;
FdSource from; int remoteVersion;
int remoteVersion; bool good = true;
bool good = true; };
};
std::string host; std::string host;
ref<Pool<Connection>> connections; ref<Pool<Connection>> connections;
SSHMaster master; SSHMaster master;
LegacySSHStore(const string & host, const Params & params) LegacySSHStore(const string& host, const Params& params)
: Store(params) : Store(params),
, host(host) host(host),
, connections(make_ref<Pool<Connection>>( connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections), std::max(1, (int)maxConnections),
[this]() { return openConnection(); }, [this]() { return openConnection(); },
[](const ref<Connection> & r) { return r->good; } [](const ref<Connection>& r) { return r->good; })),
)) master(host, sshKey,
, master( // Use SSH master only if using more than 1 connection.
host, connections->capacity() > 1, compress, logFD) {}
sshKey,
// Use SSH master only if using more than 1 connection. ref<Connection> openConnection() {
connections->capacity() > 1, auto conn = make_ref<Connection>();
compress, conn->sshConn = master.startCommand(
logFD) fmt("%s --serve --write", remoteProgram) +
{ (remoteStore.get() == ""
? ""
: " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
try {
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
conn->to.flush();
unsigned int magic = readInt(conn->from);
if (magic != SERVE_MAGIC_2)
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'",
host);
} catch (EndOfFile& e) {
throw Error("cannot connect to '%1%'", host);
} }
ref<Connection> openConnection() return conn;
{ };
auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
try { string getUri() override { return uriScheme + host; }
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
conn->to.flush();
unsigned int magic = readInt(conn->from); void queryPathInfoUncached(
if (magic != SERVE_MAGIC_2) const Path& path,
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override {
conn->remoteVersion = readInt(conn->from); try {
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) auto conn(connections->get());
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
} catch (EndOfFile & e) { debug("querying remote host '%s' for info on '%s'", host, path);
throw Error("cannot connect to '%1%'", host);
}
return conn; conn->to << cmdQueryPathInfos << PathSet{path};
}; conn->to.flush();
string getUri() override auto info = std::make_shared<ValidPathInfo>();
{ conn->from >> info->path;
return uriScheme + host; if (info->path.empty()) return callback(nullptr);
assert(path == info->path);
PathSet references;
conn->from >> info->deriver;
info->references = readStorePaths<PathSet>(*this, conn->from);
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
conn->from >> info->ca;
info->sigs = readStrings<StringSet>(conn->from);
}
auto s = readString(conn->from);
assert(s == "");
callback(std::move(info));
} catch (...) {
callback.rethrow();
}
}
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override {
debug("adding path '%s' to remote host '%s'", info.path, host);
auto conn(connections->get());
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
conn->to << cmdAddToStoreNar << info.path << info.deriver
<< info.narHash.to_string(Base16, false) << info.references
<< info.registrationTime << info.narSize << info.ultimate
<< info.sigs << info.ca;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to.flush();
} else {
conn->to << cmdImportPaths << 1;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to << exportMagic << info.path << info.references << info.deriver
<< 0 << 0;
conn->to.flush();
} }
void queryPathInfoUncached(const Path & path, if (readInt(conn->from) != 1)
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override throw Error(
{ "failed to add path '%s' to remote host '%s', info.path, host");
try { }
auto conn(connections->get());
debug("querying remote host '%s' for info on '%s'", host, path); void narFromPath(const Path& path, Sink& sink) override {
auto conn(connections->get());
conn->to << cmdQueryPathInfos << PathSet{path}; conn->to << cmdDumpStorePath << path;
conn->to.flush(); conn->to.flush();
copyNAR(conn->from, sink);
}
auto info = std::make_shared<ValidPathInfo>(); Path queryPathFromHashPart(const string& hashPart) override {
conn->from >> info->path; unsupported("queryPathFromHashPart");
if (info->path.empty()) return callback(nullptr); }
assert(path == info->path);
PathSet references; Path addToStore(const string& name, const Path& srcPath, bool recursive,
conn->from >> info->deriver; HashType hashAlgo, PathFilter& filter,
info->references = readStorePaths<PathSet>(*this, conn->from); RepairFlag repair) override {
readLongLong(conn->from); // download size unsupported("addToStore");
info->narSize = readLongLong(conn->from); }
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { Path addTextToStore(const string& name, const string& s,
auto s = readString(conn->from); const PathSet& references, RepairFlag repair) override {
info->narHash = s.empty() ? Hash() : Hash(s); unsupported("addTextToStore");
conn->from >> info->ca; }
info->sigs = readStrings<StringSet>(conn->from);
}
auto s = readString(conn->from); BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
assert(s == ""); BuildMode buildMode) override {
auto conn(connections->get());
callback(std::move(info)); conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime
} catch (...) { callback.rethrow(); } << settings.buildTimeout;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
conn->to << settings.maxLogSize;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->to << settings.buildRepeat << settings.enforceDeterminism;
conn->to.flush();
BuildResult status;
status.status = (BuildResult::Status)readInt(conn->from);
conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >>
status.startTime >> status.stopTime;
return status;
}
void ensurePath(const Path& path) override { unsupported("ensurePath"); }
void computeFSClosure(const PathSet& paths, PathSet& out,
bool flipDirection = false, bool includeOutputs = false,
bool includeDerivers = false) override {
if (flipDirection || includeDerivers) {
Store::computeFSClosure(paths, out, flipDirection, includeOutputs,
includeDerivers);
return;
} }
void addToStore(const ValidPathInfo & info, Source & source, auto conn(connections->get());
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
{
debug("adding path '%s' to remote host '%s'", info.path, host);
auto conn(connections->get()); conn->to << cmdQueryClosure << includeOutputs << paths;
conn->to.flush();
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { auto res = readStorePaths<PathSet>(*this, conn->from);
conn->to out.insert(res.begin(), res.end());
<< cmdAddToStoreNar }
<< info.path
<< info.deriver
<< info.narHash.to_string(Base16, false)
<< info.references
<< info.registrationTime
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to.flush();
} else { PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
NoSubstitute) override {
auto conn(connections->get());
conn->to conn->to << cmdQueryValidPaths << false // lock
<< cmdImportPaths << maybeSubstitute << paths;
<< 1; conn->to.flush();
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to
<< exportMagic
<< info.path
<< info.references
<< info.deriver
<< 0
<< 0;
conn->to.flush();
} return readStorePaths<PathSet>(*this, conn->from);
}
if (readInt(conn->from) != 1) void connect() override { auto conn(connections->get()); }
throw Error("failed to add path '%s' to remote host '%s', info.path, host");
}
void narFromPath(const Path & path, Sink & sink) override unsigned int getProtocol() override {
{ auto conn(connections->get());
auto conn(connections->get()); return conn->remoteVersion;
}
conn->to << cmdDumpStorePath << path;
conn->to.flush();
copyNAR(conn->from, sink);
}
Path queryPathFromHashPart(const string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
Path addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override
{ unsupported("addTextToStore"); }
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{
auto conn(connections->get());
conn->to
<< cmdBuildDerivation
<< drvPath
<< drv
<< settings.maxSilentTime
<< settings.buildTimeout;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
conn->to
<< settings.maxLogSize;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->to
<< settings.buildRepeat
<< settings.enforceDeterminism;
conn->to.flush();
BuildResult status;
status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
return status;
}
void ensurePath(const Path & path) override
{ unsupported("ensurePath"); }
void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
{
if (flipDirection || includeDerivers) {
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
return;
}
auto conn(connections->get());
conn->to
<< cmdQueryClosure
<< includeOutputs
<< paths;
conn->to.flush();
auto res = readStorePaths<PathSet>(*this, conn->from);
out.insert(res.begin(), res.end());
}
PathSet queryValidPaths(const PathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override
{
auto conn(connections->get());
conn->to
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute
<< paths;
conn->to.flush();
return readStorePaths<PathSet>(*this, conn->from);
}
void connect() override
{
auto conn(connections->get());
}
unsigned int getProtocol() override
{
auto conn(connections->get());
return conn->remoteVersion;
}
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{ if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; return std::make_shared<LegacySSHStore>(
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); std::string(uri, uriScheme.size()), params);
}); });
} } // namespace nix

142
third_party/nix/src/libstore/local-binary-cache-store.cc vendored
View file

@ -4,100 +4,82 @@
namespace nix { namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore class LocalBinaryCacheStore : public BinaryCacheStore {
{ private:
private: Path binaryCacheDir;
Path binaryCacheDir; public:
LocalBinaryCacheStore(const Params& params, const Path& binaryCacheDir)
: BinaryCacheStore(params), binaryCacheDir(binaryCacheDir) {}
public: void init() override;
LocalBinaryCacheStore( std::string getUri() override { return "file://" + binaryCacheDir; }
const Params & params, const Path & binaryCacheDir)
: BinaryCacheStore(params) protected:
, binaryCacheDir(binaryCacheDir) bool fileExists(const std::string& path) override;
{
} void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) override;
void init() override;
void getFile(const std::string& path, Sink& sink) override {
std::string getUri() override try {
{ readFile(binaryCacheDir + "/" + path, sink);
return "file://" + binaryCacheDir; } catch (SysError& e) {
} if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache",
protected: path);
}
bool fileExists(const std::string & path) override; }
void upsertFile(const std::string & path, PathSet queryAllValidPaths() override {
const std::string & data, PathSet paths;
const std::string & mimeType) override;
for (auto& entry : readDirectory(binaryCacheDir)) {
void getFile(const std::string & path, Sink & sink) override if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo"))
{ continue;
try { paths.insert(storeDir + "/" +
readFile(binaryCacheDir + "/" + path, sink); entry.name.substr(0, entry.name.size() - 8));
} catch (SysError & e) {
if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
}
}
PathSet queryAllValidPaths() override
{
PathSet paths;
for (auto & entry : readDirectory(binaryCacheDir)) {
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8));
}
return paths;
} }
return paths;
}
}; };
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init() {
{ createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + "/nar"); BinaryCacheStore::init();
BinaryCacheStore::init();
} }
static void atomicWrite(const Path & path, const std::string & s) static void atomicWrite(const Path& path, const std::string& s) {
{ Path tmp = path + ".tmp." + std::to_string(getpid());
Path tmp = path + ".tmp." + std::to_string(getpid()); AutoDelete del(tmp, false);
AutoDelete del(tmp, false); writeFile(tmp, s);
writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str()))
if (rename(tmp.c_str(), path.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); del.cancel();
del.cancel();
} }
bool LocalBinaryCacheStore::fileExists(const std::string & path) bool LocalBinaryCacheStore::fileExists(const std::string& path) {
{ return pathExists(binaryCacheDir + "/" + path);
return pathExists(binaryCacheDir + "/" + path);
} }
void LocalBinaryCacheStore::upsertFile(const std::string & path, void LocalBinaryCacheStore::upsertFile(const std::string& path,
const std::string & data, const std::string& data,
const std::string & mimeType) const std::string& mimeType) {
{ atomicWrite(binaryCacheDir + "/" + path, data);
atomicWrite(binaryCacheDir + "/" + path, data);
} }
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{ if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || std::string(uri, 0, 7) != "file://")
std::string(uri, 0, 7) != "file://")
return 0; return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); auto store =
store->init(); std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
return store; store->init();
}); return store;
});
} } // namespace nix

168
third_party/nix/src/libstore/local-fs-store.cc vendored
View file

@ -1,131 +1,113 @@
#include "archive.hh" #include "archive.hh"
#include "fs-accessor.hh"
#include "store-api.hh"
#include "globals.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "store-api.hh"
namespace nix { namespace nix {
LocalFSStore::LocalFSStore(const Params & params) LocalFSStore::LocalFSStore(const Params& params) : Store(params) {}
: Store(params)
{
}
struct LocalStoreAccessor : public FSAccessor struct LocalStoreAccessor : public FSAccessor {
{ ref<LocalFSStore> store;
ref<LocalFSStore> store;
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { } LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {}
Path toRealPath(const Path & path) Path toRealPath(const Path& path) {
{ Path storePath = store->toStorePath(path);
Path storePath = store->toStorePath(path); if (!store->isValidPath(storePath))
if (!store->isValidPath(storePath)) throw InvalidPath(format("path '%1%' is not a valid store path") %
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size()); return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
FSAccessor::Stat stat(const Path& path) override {
auto realPath = toRealPath(path);
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR)
return {Type::tMissing, 0, false};
throw SysError(format("getting status of '%1%'") % path);
} }
FSAccessor::Stat stat(const Path & path) override if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
{ throw Error(format("file '%1%' has unsupported type") % path);
auto realPath = toRealPath(path);
struct stat st; return {S_ISREG(st.st_mode)
if (lstat(realPath.c_str(), &st)) { ? Type::tRegular
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory,
throw SysError(format("getting status of '%1%'") % path); S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0,
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error(format("file '%1%' has unsupported type") % path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
} }
StringSet readDirectory(const Path & path) override StringSet readDirectory(const Path& path) override {
{ auto realPath = toRealPath(path);
auto realPath = toRealPath(path);
auto entries = nix::readDirectory(realPath); auto entries = nix::readDirectory(realPath);
StringSet res; StringSet res;
for (auto & entry : entries) for (auto& entry : entries) res.insert(entry.name);
res.insert(entry.name);
return res; return res;
} }
std::string readFile(const Path & path) override std::string readFile(const Path& path) override {
{ return nix::readFile(toRealPath(path));
return nix::readFile(toRealPath(path)); }
}
std::string readLink(const Path & path) override std::string readLink(const Path& path) override {
{ return nix::readLink(toRealPath(path));
return nix::readLink(toRealPath(path)); }
}
}; };
ref<FSAccessor> LocalFSStore::getFSAccessor() ref<FSAccessor> LocalFSStore::getFSAccessor() {
{ return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
} }
void LocalFSStore::narFromPath(const Path & path, Sink & sink) void LocalFSStore::narFromPath(const Path& path, Sink& sink) {
{ if (!isValidPath(path)) throw Error(format("path '%s' is not valid") % path);
if (!isValidPath(path)) dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
throw Error(format("path '%s' is not valid") % path);
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
} }
const string LocalFSStore::drvsLogDir = "drvs"; const string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) {
auto path(path_);
assertStorePath(path);
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) if (!isDerivation(path)) {
{ try {
auto path(path_); path = queryPathInfo(path)->deriver;
} catch (InvalidPath&) {
assertStorePath(path); return nullptr;
if (!isDerivation(path)) {
try {
path = queryPathInfo(path)->deriver;
} catch (InvalidPath &) {
return nullptr;
}
if (path == "") return nullptr;
} }
if (path == "") return nullptr;
}
string baseName = baseNameOf(path); string baseName = baseNameOf(path);
for (int j = 0; j < 2; j++) { for (int j = 0; j < 2; j++) {
Path logPath = j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir,
string(baseName, 0, 2), string(baseName, 2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
Path logPath = if (pathExists(logPath))
j == 0 return std::make_shared<std::string>(readFile(logPath));
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
if (pathExists(logPath))
return std::make_shared<std::string>(readFile(logPath));
else if (pathExists(logBz2Path)) {
try {
return decompress("bzip2", readFile(logBz2Path));
} catch (Error &) { }
}
else if (pathExists(logBz2Path)) {
try {
return decompress("bzip2", readFile(logBz2Path));
} catch (Error&) {
}
} }
}
return nullptr; return nullptr;
} }
} } // namespace nix

2219
third_party/nix/src/libstore/local-store.cc vendored
View file

File diff suppressed because it is too large Load diff

393
third_party/nix/src/libstore/local-store.hh vendored
View file

@ -1,309 +1,295 @@
#pragma once #pragma once
#include "sqlite.hh"
#include "pathlocks.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include "pathlocks.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
namespace nix { namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <= /* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
const int nixSchemaVersion = 10; const int nixSchemaVersion = 10;
struct Derivation; struct Derivation;
struct OptimiseStats {
struct OptimiseStats unsigned long filesLinked = 0;
{ unsigned long long bytesFreed = 0;
unsigned long filesLinked = 0; unsigned long long blocksFreed = 0;
unsigned long long bytesFreed = 0;
unsigned long long blocksFreed = 0;
}; };
class LocalStore : public LocalFSStore {
private:
/* Lock file used for upgrading. */
AutoCloseFD globalLock;
class LocalStore : public LocalFSStore struct State {
{ /* The SQLite database object. */
private: SQLite db;
/* Lock file used for upgrading. */ /* Some precompiled SQLite statements. */
AutoCloseFD globalLock; SQLiteStmt stmtRegisterValidPath;
SQLiteStmt stmtUpdatePathInfo;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
struct State /* The file to which we write our temporary roots. */
{ AutoCloseFD fdTempRoots;
/* The SQLite database object. */
SQLite db;
/* Some precompiled SQLite statements. */ /* The last time we checked whether to do an auto-GC, or an
SQLiteStmt stmtRegisterValidPath; auto-GC finished. */
SQLiteStmt stmtUpdatePathInfo; std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
/* The file to which we write our temporary roots. */ /* Whether auto-GC is running. If so, get gcFuture to wait for
AutoCloseFD fdTempRoots; the GC to finish. */
bool gcRunning = false;
std::shared_future<void> gcFuture;
/* The last time we checked whether to do an auto-GC, or an /* How much disk space was available after the previous
auto-GC finished. */ auto-GC. If the current available disk space is below
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; minFree but not much below availAfterGC, then there is no
point in starting a new GC. */
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
/* Whether auto-GC is running. If so, get gcFuture to wait for std::unique_ptr<PublicKeys> publicKeys;
the GC to finish. */ };
bool gcRunning = false;
std::shared_future<void> gcFuture;
/* How much disk space was available after the previous Sync<State, std::recursive_mutex> _state;
auto-GC. If the current available disk space is below
minFree but not much below availAfterGC, then there is no
point in starting a new GC. */
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
std::unique_ptr<PublicKeys> publicKeys; public:
}; PathSetting realStoreDir_;
Sync<State, std::recursive_mutex> _state; const Path realStoreDir;
const Path dbDir;
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
public: private:
Setting<bool> requireSigs{
(Store*)this, settings.requireSigs, "require-sigs",
"whether store paths should have a trusted signature on import"};
PathSetting realStoreDir_; const PublicKeys& getPublicKeys();
const Path realStoreDir; public:
const Path dbDir; // Hack for build-remote.cc.
const Path linksDir; PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
const Path reservedPath;
const Path schemaPath;
const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
private: /* Initialise the local store, upgrading the schema if
necessary. */
LocalStore(const Params& params);
Setting<bool> requireSigs{(Store*) this, ~LocalStore();
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
const PublicKeys & getPublicKeys(); /* Implementations of abstract store API methods. */
public: std::string getUri() override;
// Hack for build-remote.cc. bool isValidPathUncached(const Path& path) override;
PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
/* Initialise the local store, upgrading the schema if PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
necessary. */ NoSubstitute) override;
LocalStore(const Params & params);
~LocalStore(); PathSet queryAllValidPaths() override;
/* Implementations of abstract store API methods. */ void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
std::string getUri() override; void queryReferrers(const Path& path, PathSet& referrers) override;
bool isValidPathUncached(const Path & path) override; PathSet queryValidDerivers(const Path& path) override;
PathSet queryValidPaths(const PathSet & paths, PathSet queryDerivationOutputs(const Path& path) override;
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryAllValidPaths() override; StringSet queryDerivationOutputNames(const Path& path) override;
void queryPathInfoUncached(const Path & path, Path queryPathFromHashPart(const string& hashPart) override;
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path & path, PathSet & referrers) override; PathSet querySubstitutablePaths(const PathSet& paths) override;
PathSet queryValidDerivers(const Path & path) override; void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override;
PathSet queryDerivationOutputs(const Path & path) override; void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
StringSet queryDerivationOutputNames(const Path & path) override; Path addToStore(const string& name, const Path& srcPath, bool recursive,
HashType hashAlgo, PathFilter& filter,
RepairFlag repair) override;
Path queryPathFromHashPart(const string & hashPart) override; /* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
Path addToStoreFromDump(const string& dump, const string& name,
bool recursive = true, HashType hashAlgo = htSHA256,
RepairFlag repair = NoRepair);
PathSet querySubstitutablePaths(const PathSet & paths) override; Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override;
void querySubstitutablePathInfos(const PathSet & paths, void buildPaths(const PathSet& paths, BuildMode buildMode) override;
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source, BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
RepairFlag repair, CheckSigsFlag checkSigs, BuildMode buildMode) override;
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath, void ensurePath(const Path& path) override;
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained void addTempRoot(const Path& path) override;
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
Path addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair);
Path addTextToStore(const string & name, const string & s, void addIndirectRoot(const Path& path) override;
const PathSet & references, RepairFlag repair) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override; void syncWithGC() override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, private:
BuildMode buildMode) override; typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
void ensurePath(const Path & path) override; void findTempRoots(FDs& fds, Roots& roots, bool censor);
void addTempRoot(const Path & path) override; public:
Roots findRoots(bool censor) override;
void addIndirectRoot(const Path & path) override; void collectGarbage(const GCOptions& options, GCResults& results) override;
void syncWithGC() override; /* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */
void optimiseStore(OptimiseStats& stats);
private: void optimiseStore() override;
typedef std::shared_ptr<AutoCloseFD> FDPtr; /* Optimise a single store path. */
typedef list<FDPtr> FDs; void optimisePath(const Path& path);
void findTempRoots(FDs & fds, Roots & roots, bool censor); bool verifyStore(bool checkContents, RepairFlag repair) override;
public: /* Register the validity of a path, i.e., that `path' exists, that
the paths referenced by it exists, and in the case of an output
path of a derivation, that it has been produced by a successful
execution of the derivation (or something equivalent). Also
register the hash of the file system contents of the path. The
hash must be a SHA-256 hash. */
void registerValidPath(const ValidPathInfo& info);
Roots findRoots(bool censor) override; void registerValidPaths(const ValidPathInfos& infos);
void collectGarbage(const GCOptions & options, GCResults & results) override; unsigned int getProtocol() override;
/* Optimise the disk space usage of the Nix store by hard-linking void vacuumDB();
files with the same contents. */
void optimiseStore(OptimiseStats & stats);
void optimiseStore() override; /* Repair the contents of the given path by redownloading it using
a substituter (if available). */
void repairPath(const Path& path);
/* Optimise a single store path. */ void addSignatures(const Path& storePath, const StringSet& sigs) override;
void optimisePath(const Path & path);
bool verifyStore(bool checkContents, RepairFlag repair) override; /* If free disk space in /nix/store if below minFree, delete
garbage until it exceeds maxFree. */
void autoGC(bool sync = true);
/* Register the validity of a path, i.e., that `path' exists, that private:
the paths referenced by it exists, and in the case of an output int getSchema();
path of a derivation, that it has been produced by a successful
execution of the derivation (or something equivalent). Also
register the hash of the file system contents of the path. The
hash must be a SHA-256 hash. */
void registerValidPath(const ValidPathInfo & info);
void registerValidPaths(const ValidPathInfos & infos); void openDB(State& state, bool create);
unsigned int getProtocol() override; void makeStoreWritable();
void vacuumDB(); uint64_t queryValidPathId(State& state, const Path& path);
/* Repair the contents of the given path by redownloading it using uint64_t addValidPath(State& state, const ValidPathInfo& info,
a substituter (if available). */ bool checkOutputs = true);
void repairPath(const Path & path);
void addSignatures(const Path & storePath, const StringSet & sigs) override; void invalidatePath(State& state, const Path& path);
/* If free disk space in /nix/store if below minFree, delete /* Delete a path from the Nix store. */
garbage until it exceeds maxFree. */ void invalidatePathChecked(const Path& path);
void autoGC(bool sync = true);
private: void verifyPath(const Path& path, const PathSet& store, PathSet& done,
PathSet& validPaths, RepairFlag repair, bool& errors);
int getSchema(); void updatePathInfo(State& state, const ValidPathInfo& info);
void openDB(State & state, bool create); void upgradeStore6();
void upgradeStore7();
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path& path);
void makeStoreWritable(); struct GCState;
uint64_t queryValidPathId(State & state, const Path & path); void deleteGarbage(GCState& state, const Path& path);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); void tryToDelete(GCState& state, const Path& path);
void invalidatePath(State & state, const Path & path); bool canReachRoot(GCState& state, PathSet& visited, const Path& path);
/* Delete a path from the Nix store. */ void deletePathRecursive(GCState& state, const Path& path);
void invalidatePathChecked(const Path & path);
void verifyPath(const Path & path, const PathSet & store, bool isActiveTempFile(const GCState& state, const Path& path,
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); const string& suffix);
void updatePathInfo(State & state, const ValidPathInfo & info); AutoCloseFD openGCLock(LockType lockType);
void upgradeStore6(); void findRoots(const Path& path, unsigned char type, Roots& roots);
void upgradeStore7();
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
struct GCState; void findRootsNoTemp(Roots& roots, bool censor);
void deleteGarbage(GCState & state, const Path & path); void findRuntimeRoots(Roots& roots, bool censor);
void tryToDelete(GCState & state, const Path & path); void removeUnusedLinks(const GCState& state);
bool canReachRoot(GCState & state, PathSet & visited, const Path & path); Path createTempDirInStore();
void deletePathRecursive(GCState & state, const Path & path); void checkDerivationOutputs(const Path& drvPath, const Derivation& drv);
bool isActiveTempFile(const GCState & state, typedef std::unordered_set<ino_t> InodeHash;
const Path & path, const string & suffix);
AutoCloseFD openGCLock(LockType lockType); InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path& path,
const InodeHash& inodeHash);
void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path,
InodeHash& inodeHash);
void findRoots(const Path & path, unsigned char type, Roots & roots); // Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State& state, const Path& path);
void queryReferrers(State& state, const Path& path, PathSet& referrers);
void findRootsNoTemp(Roots & roots, bool censor); /* Add signatures to a ValidPathInfo using the secret keys
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo& info);
void findRuntimeRoots(Roots & roots, bool censor); Path getRealStoreDir() override { return realStoreDir; }
void removeUnusedLinks(const GCState & state); void createUser(const std::string& userName, uid_t userId) override;
Path createTempDirInStore(); friend class DerivationGoal;
friend class SubstitutionGoal;
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State & state, const Path & path);
void queryReferrers(State & state, const Path & path, PathSet & referrers);
/* Add signatures to a ValidPathInfo using the secret keys
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
friend class DerivationGoal;
friend class SubstitutionGoal;
}; };
typedef std::pair<dev_t, ino_t> Inode; typedef std::pair<dev_t, ino_t> Inode;
typedef set<Inode> InodesSeen; typedef set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path /* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular: after it has been built. In particular:
- the last modification date on each file is set to 1 (i.e., - the last modification date on each file is set to 1 (i.e.,
@ -312,11 +298,12 @@ typedef set<Inode> InodesSeen;
without execute permission; setuid bits etc. are cleared) without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're - the owner and group are set to the Nix user and group, if we're
running as root. */ running as root. */
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); void canonicalisePathMetaData(const Path& path, uid_t fromUid,
void canonicalisePathMetaData(const Path & path, uid_t fromUid); InodesSeen& inodesSeen);
void canonicalisePathMetaData(const Path& path, uid_t fromUid);
void canonicaliseTimestampAndPermissions(const Path & path); void canonicaliseTimestampAndPermissions(const Path& path);
MakeError(PathInUse, Error); MakeError(PathInUse, Error);
} } // namespace nix

159
third_party/nix/src/libstore/machines.cc vendored
View file

@ -1,100 +1,93 @@
#include "machines.hh" #include "machines.hh"
#include "util.hh"
#include "globals.hh"
#include <algorithm> #include <algorithm>
#include "globals.hh"
#include "util.hh"
namespace nix { namespace nix {
Machine::Machine(decltype(storeUri) storeUri, Machine::Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
decltype(sshKey) sshKey, decltype(speedFactor) speedFactor,
decltype(maxJobs) maxJobs, decltype(supportedFeatures) supportedFeatures,
decltype(speedFactor) speedFactor, decltype(mandatoryFeatures) mandatoryFeatures,
decltype(supportedFeatures) supportedFeatures, decltype(sshPublicHostKey) sshPublicHostKey)
decltype(mandatoryFeatures) mandatoryFeatures, : storeUri(
decltype(sshPublicHostKey) sshPublicHostKey) : // Backwards compatibility: if the URI is a hostname,
storeUri( // prepend ssh://.
// Backwards compatibility: if the URI is a hostname, storeUri.find("://") != std::string::npos ||
// prepend ssh://. hasPrefix(storeUri, "local") ||
storeUri.find("://") != std::string::npos hasPrefix(storeUri, "remote") ||
|| hasPrefix(storeUri, "local") hasPrefix(storeUri, "auto") || hasPrefix(storeUri, "/")
|| hasPrefix(storeUri, "remote") ? storeUri
|| hasPrefix(storeUri, "auto") : "ssh://" + storeUri),
|| hasPrefix(storeUri, "/") systemTypes(systemTypes),
? storeUri sshKey(sshKey),
: "ssh://" + storeUri), maxJobs(maxJobs),
systemTypes(systemTypes), speedFactor(std::max(1U, speedFactor)),
sshKey(sshKey), supportedFeatures(supportedFeatures),
maxJobs(maxJobs), mandatoryFeatures(mandatoryFeatures),
speedFactor(std::max(1U, speedFactor)), sshPublicHostKey(sshPublicHostKey) {}
supportedFeatures(supportedFeatures),
mandatoryFeatures(mandatoryFeatures),
sshPublicHostKey(sshPublicHostKey)
{}
bool Machine::allSupported(const std::set<string> & features) const { bool Machine::allSupported(const std::set<string>& features) const {
return std::all_of(features.begin(), features.end(), return std::all_of(features.begin(), features.end(),
[&](const string & feature) { [&](const string& feature) {
return supportedFeatures.count(feature) || return supportedFeatures.count(feature) ||
mandatoryFeatures.count(feature); mandatoryFeatures.count(feature);
}); });
} }
bool Machine::mandatoryMet(const std::set<string> & features) const { bool Machine::mandatoryMet(const std::set<string>& features) const {
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), return std::all_of(
[&](const string & feature) { mandatoryFeatures.begin(), mandatoryFeatures.end(),
return features.count(feature); [&](const string& feature) { return features.count(feature); });
});
} }
void parseMachines(const std::string & s, Machines & machines) void parseMachines(const std::string& s, Machines& machines) {
{ for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { trim(line);
trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end());
line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue;
if (line.empty()) continue;
if (line[0] == '@') { if (line[0] == '@') {
auto file = trim(std::string(line, 1)); auto file = trim(std::string(line, 1));
try { try {
parseMachines(readFile(file), machines); parseMachines(readFile(file), machines);
} catch (const SysError & e) { } catch (const SysError& e) {
if (e.errNo != ENOENT) if (e.errNo != ENOENT) throw;
throw; debug("cannot find machines file '%s'", file);
debug("cannot find machines file '%s'", file); }
} continue;
continue;
}
auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size();
if (sz < 1)
throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
};
machines.emplace_back(tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(7) ? tokens[7] : "");
} }
auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size();
if (sz < 1) throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
};
machines.emplace_back(
tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",")
: std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",")
: std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",")
: std::set<string>{},
isSet(7) ? tokens[7] : "");
}
} }
Machines getMachines() Machines getMachines() {
{ static auto machines = [&]() {
static auto machines = [&]() { Machines machines;
Machines machines; parseMachines(settings.builders, machines);
parseMachines(settings.builders, machines);
return machines;
}();
return machines; return machines;
}();
return machines;
} }
} } // namespace nix

41
third_party/nix/src/libstore/machines.hh vendored
View file

@ -5,35 +5,32 @@
namespace nix { namespace nix {
struct Machine { struct Machine {
const string storeUri;
const std::vector<string> systemTypes;
const string sshKey;
const unsigned int maxJobs;
const unsigned int speedFactor;
const std::set<string> supportedFeatures;
const std::set<string> mandatoryFeatures;
const std::string sshPublicHostKey;
bool enabled = true;
const string storeUri; bool allSupported(const std::set<string>& features) const;
const std::vector<string> systemTypes;
const string sshKey;
const unsigned int maxJobs;
const unsigned int speedFactor;
const std::set<string> supportedFeatures;
const std::set<string> mandatoryFeatures;
const std::string sshPublicHostKey;
bool enabled = true;
bool allSupported(const std::set<string> & features) const; bool mandatoryMet(const std::set<string>& features) const;
bool mandatoryMet(const std::set<string> & features) const; Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
Machine(decltype(storeUri) storeUri, decltype(speedFactor) speedFactor,
decltype(systemTypes) systemTypes, decltype(supportedFeatures) supportedFeatures,
decltype(sshKey) sshKey, decltype(mandatoryFeatures) mandatoryFeatures,
decltype(maxJobs) maxJobs, decltype(sshPublicHostKey) sshPublicHostKey);
decltype(speedFactor) speedFactor,
decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey);
}; };
typedef std::vector<Machine> Machines; typedef std::vector<Machine> Machines;
void parseMachines(const std::string & s, Machines & machines); void parseMachines(const std::string& s, Machines& machines);
Machines getMachines(); Machines getMachines();
} } // namespace nix

470
third_party/nix/src/libstore/misc.cc vendored
View file

@ -1,282 +1,266 @@
#include "derivations.hh" #include "derivations.hh"
#include "parsed-derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "local-store.hh" #include "local-store.hh"
#include "parsed-derivations.hh"
#include "store-api.hh" #include "store-api.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
namespace nix { namespace nix {
void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_,
bool flipDirection, bool includeOutputs,
bool includeDerivers) {
struct State {
size_t pending;
PathSet& paths;
std::exception_ptr exc;
};
void Store::computeFSClosure(const PathSet & startPaths, Sync<State> state_(State{0, paths_, 0});
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{ std::function<void(const Path&)> enqueue;
struct State
std::condition_variable done;
enqueue = [&](const Path& path) -> void {
{ {
size_t pending; auto state(state_.lock());
PathSet & paths; if (state->exc) return;
std::exception_ptr exc; if (state->paths.count(path)) return;
}; state->paths.insert(path);
state->pending++;
Sync<State> state_(State{0, paths_, 0});
std::function<void(const Path &)> enqueue;
std::condition_variable done;
enqueue = [&](const Path & path) -> void {
{
auto state(state_.lock());
if (state->exc) return;
if (state->paths.count(path)) return;
state->paths.insert(path);
state->pending++;
}
queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
if (flipDirection) {
PathSet referrers;
queryReferrers(path, referrers);
for (auto & ref : referrers)
if (ref != path)
enqueue(ref);
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
enqueue(i);
if (includeDerivers && isDerivation(path))
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(i);
} else {
for (auto & ref : info->references)
if (ref != path)
enqueue(ref);
if (includeOutputs && isDerivation(path))
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i)) enqueue(i);
if (includeDerivers && isValidPath(info->deriver))
enqueue(info->deriver);
}
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending) done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
}});
};
for (auto & startPath : startPaths)
enqueue(startPath);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
} }
}
queryPathInfo(
path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
void Store::computeFSClosure(const Path & startPath, try {
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) auto info = fut.get();
{
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
}
if (flipDirection) {
PathSet referrers;
queryReferrers(path, referrers);
for (auto& ref : referrers)
if (ref != path) enqueue(ref);
void Store::queryMissing(const PathSet & targets, if (includeOutputs)
PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, for (auto& i : queryValidDerivers(path)) enqueue(i);
unsigned long long & downloadSize_, unsigned long long & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
downloadSize_ = narSize_ = 0; if (includeDerivers && isDerivation(path))
for (auto& i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(i);
ThreadPool pool; } else {
for (auto& ref : info->references)
if (ref != path) enqueue(ref);
struct State if (includeOutputs && isDerivation(path))
{ for (auto& i : queryDerivationOutputs(path))
PathSet done; if (isValidPath(i)) enqueue(i);
PathSet & unknown, & willSubstitute, & willBuild;
unsigned long long & downloadSize;
unsigned long long & narSize;
};
struct DrvState if (includeDerivers && isValidPath(info->deriver))
{ enqueue(info->deriver);
size_t left;
bool done = false;
PathSet outPaths;
DrvState(size_t left) : left(left) { }
};
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
std::function<void(Path)> doPath;
auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) {
{
auto state(state_.lock());
state->willBuild.insert(drvPath);
}
for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
};
auto checkOutput = [&](
const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_)
{
if (drvState_->lock()->done) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({outPath}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
mustBuildDrv(drvPath, *drv);
} else {
{
auto drvState(drvState_->lock());
if (drvState->done) return;
assert(drvState->left);
drvState->left--;
drvState->outPaths.insert(outPath);
if (!drvState->left) {
for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, path));
}
} }
}
};
doPath = [&](const Path & path) {
{
auto state(state_.lock());
if (state->done.count(path)) return;
state->done.insert(path);
}
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path);
if (isDerivation(i2.first)) {
if (!isValidPath(i2.first)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(path);
return;
}
Derivation drv = derivationFromPath(i2.first);
ParsedDerivation parsedDrv(i2.first, drv);
PathSet invalid;
for (auto & j : drv.outputs)
if (wantOutput(j.first, i2.second)
&& !isValidPath(j.second.path))
invalid.insert(j.second.path);
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
} else
mustBuildDrv(i2.first, drv);
} else {
if (isValidPath(path)) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({path}, infos);
if (infos.empty()) {
auto state(state_.lock());
state->unknown.insert(path);
return;
}
auto info = infos.find(path);
assert(info != infos.end());
{ {
auto state(state_.lock()); auto state(state_.lock());
state->willSubstitute.insert(path); assert(state->pending);
state->downloadSize += info->second.downloadSize; if (!--state->pending) done.notify_one();
state->narSize += info->second.narSize;
} }
for (auto & ref : info->second.references) } catch (...) {
pool.enqueue(std::bind(doPath, ref)); auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
}});
};
for (auto& startPath : startPaths) enqueue(startPath);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
}
}
void Store::computeFSClosure(const Path& startPath, PathSet& paths_,
bool flipDirection, bool includeOutputs,
bool includeDerivers) {
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs,
includeDerivers);
}
void Store::queryMissing(const PathSet& targets, PathSet& willBuild_,
PathSet& willSubstitute_, PathSet& unknown_,
unsigned long long& downloadSize_,
unsigned long long& narSize_) {
Activity act(*logger, lvlDebug, actUnknown,
"querying info about missing paths");
downloadSize_ = narSize_ = 0;
ThreadPool pool;
struct State {
PathSet done;
PathSet &unknown, &willSubstitute, &willBuild;
unsigned long long& downloadSize;
unsigned long long& narSize;
};
struct DrvState {
size_t left;
bool done = false;
PathSet outPaths;
DrvState(size_t left) : left(left) {}
};
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_,
downloadSize_, narSize_});
std::function<void(Path)> doPath;
auto mustBuildDrv = [&](const Path& drvPath, const Derivation& drv) {
{
auto state(state_.lock());
state->willBuild.insert(drvPath);
}
for (auto& i : drv.inputDrvs)
pool.enqueue(
std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
};
auto checkOutput = [&](const Path& drvPath, ref<Derivation> drv,
const Path& outPath, ref<Sync<DrvState>> drvState_) {
if (drvState_->lock()->done) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({outPath}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
mustBuildDrv(drvPath, *drv);
} else {
{
auto drvState(drvState_->lock());
if (drvState->done) return;
assert(drvState->left);
drvState->left--;
drvState->outPaths.insert(outPath);
if (!drvState->left) {
for (auto& path : drvState->outPaths)
pool.enqueue(std::bind(doPath, path));
} }
}; }
}
};
for (auto & path : targets) doPath = [&](const Path& path) {
pool.enqueue(std::bind(doPath, path)); {
auto state(state_.lock());
if (state->done.count(path)) return;
state->done.insert(path);
}
pool.process(); DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path);
if (isDerivation(i2.first)) {
if (!isValidPath(i2.first)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(path);
return;
}
Derivation drv = derivationFromPath(i2.first);
ParsedDerivation parsedDrv(i2.first, drv);
PathSet invalid;
for (auto& j : drv.outputs)
if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path))
invalid.insert(j.second.path);
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto& output : invalid)
pool.enqueue(std::bind(checkOutput, i2.first,
make_ref<Derivation>(drv), output, drvState));
} else
mustBuildDrv(i2.first, drv);
} else {
if (isValidPath(path)) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({path}, infos);
if (infos.empty()) {
auto state(state_.lock());
state->unknown.insert(path);
return;
}
auto info = infos.find(path);
assert(info != infos.end());
{
auto state(state_.lock());
state->willSubstitute.insert(path);
state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize;
}
for (auto& ref : info->second.references)
pool.enqueue(std::bind(doPath, ref));
}
};
for (auto& path : targets) pool.enqueue(std::bind(doPath, path));
pool.process();
} }
Paths Store::topoSortPaths(const PathSet& paths) {
Paths sorted;
PathSet visited, parents;
Paths Store::topoSortPaths(const PathSet & paths) std::function<void(const Path& path, const Path* parent)> dfsVisit;
{
Paths sorted;
PathSet visited, parents;
std::function<void(const Path & path, const Path * parent)> dfsVisit; dfsVisit = [&](const Path& path, const Path* parent) {
if (parents.find(path) != parents.end())
throw BuildError(
format("cycle detected in the references of '%1%' from '%2%'") %
path % *parent);
dfsVisit = [&](const Path & path, const Path * parent) { if (visited.find(path) != visited.end()) return;
if (parents.find(path) != parents.end()) visited.insert(path);
throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); parents.insert(path);
if (visited.find(path) != visited.end()) return; PathSet references;
visited.insert(path); try {
parents.insert(path); references = queryPathInfo(path)->references;
} catch (InvalidPath&) {
}
PathSet references; for (auto& i : references)
try { /* Don't traverse into paths that don't exist. That can
references = queryPathInfo(path)->references; happen due to substitutes for non-existent paths. */
} catch (InvalidPath &) { if (i != path && paths.find(i) != paths.end()) dfsVisit(i, &path);
}
for (auto & i : references) sorted.push_front(path);
/* Don't traverse into paths that don't exist. That can parents.erase(path);
happen due to substitutes for non-existent paths. */ };
if (i != path && paths.find(i) != paths.end())
dfsVisit(i, &path);
sorted.push_front(path); for (auto& i : paths) dfsVisit(i, nullptr);
parents.erase(path);
};
for (auto & i : paths) return sorted;
dfsVisit(i, nullptr);
return sorted;
} }
} // namespace nix
}

408
third_party/nix/src/libstore/nar-accessor.cc vendored
View file

@ -1,266 +1,242 @@
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include <algorithm>
#include <map>
#include <nlohmann/json.hpp>
#include <stack>
#include "archive.hh" #include "archive.hh"
#include "json.hh" #include "json.hh"
#include <map>
#include <stack>
#include <algorithm>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
struct NarMember struct NarMember {
{ FSAccessor::Type type = FSAccessor::Type::tMissing;
FSAccessor::Type type = FSAccessor::Type::tMissing;
bool isExecutable = false; bool isExecutable = false;
/* If this is a regular file, position of the contents of this /* If this is a regular file, position of the contents of this
file in the NAR. */ file in the NAR. */
size_t start = 0, size = 0; size_t start = 0, size = 0;
std::string target; std::string target;
/* If this is a directory, all the children of the directory. */ /* If this is a directory, all the children of the directory. */
std::map<std::string, NarMember> children; std::map<std::string, NarMember> children;
}; };
struct NarAccessor : public FSAccessor struct NarAccessor : public FSAccessor {
{ std::shared_ptr<const std::string> nar;
std::shared_ptr<const std::string> nar;
GetNarBytes getNarBytes; GetNarBytes getNarBytes;
NarMember root; NarMember root;
struct NarIndexer : ParseSink, StringSource struct NarIndexer : ParseSink, StringSource {
{ NarAccessor& acc;
NarAccessor & acc;
std::stack<NarMember *> parents; std::stack<NarMember*> parents;
std::string currentStart; std::string currentStart;
bool isExec = false; bool isExec = false;
NarIndexer(NarAccessor & acc, const std::string & nar) NarIndexer(NarAccessor& acc, const std::string& nar)
: StringSource(nar), acc(acc) : StringSource(nar), acc(acc) {}
{ }
void createMember(const Path & path, NarMember member) { void createMember(const Path& path, NarMember member) {
size_t level = std::count(path.begin(), path.end(), '/'); size_t level = std::count(path.begin(), path.end(), '/');
while (parents.size() > level) parents.pop(); while (parents.size() > level) parents.pop();
if (parents.empty()) { if (parents.empty()) {
acc.root = std::move(member); acc.root = std::move(member);
parents.push(&acc.root); parents.push(&acc.root);
} else { } else {
if (parents.top()->type != FSAccessor::Type::tDirectory) if (parents.top()->type != FSAccessor::Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path); throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); auto result = parents.top()->children.emplace(baseNameOf(path),
parents.push(&result.first->second); std::move(member));
} parents.push(&result.first->second);
} }
}
void createDirectory(const Path & path) override
{ void createDirectory(const Path& path) override {
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
} }
void createRegularFile(const Path & path) override void createRegularFile(const Path& path) override {
{ createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); }
}
void isExecutable() override { parents.top()->isExecutable = true; }
void isExecutable() override
{ void preallocateContents(unsigned long long size) override {
parents.top()->isExecutable = true; currentStart = string(s, pos, 16);
} assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
void preallocateContents(unsigned long long size) override parents.top()->start = pos;
{ }
currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max()); void receiveContents(unsigned char* data, unsigned int len) override {
parents.top()->size = (size_t)size; // Sanity check
parents.top()->start = pos; if (!currentStart.empty()) {
} assert(len < 16 || currentStart == string((char*)data, 16));
currentStart.clear();
void receiveContents(unsigned char * data, unsigned int len) override }
{ }
// Sanity check
if (!currentStart.empty()) { void createSymlink(const Path& path, const string& target) override {
assert(len < 16 || currentStart == string((char *) data, 16)); createMember(path,
currentStart.clear(); NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
} }
} };
void createSymlink(const Path & path, const string & target) override NarAccessor(ref<const std::string> nar) : nar(nar) {
{ NarIndexer indexer(*this, *nar);
createMember(path, parseDump(indexer, indexer);
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); }
NarAccessor(const std::string& listing, GetNarBytes getNarBytes)
: getNarBytes(getNarBytes) {
using json = nlohmann::json;
std::function<void(NarMember&, json&)> recurse;
recurse = [&](NarMember& member, json& v) {
std::string type = v["type"];
if (type == "directory") {
member.type = FSAccessor::Type::tDirectory;
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
std::string name = i.key();
recurse(member.children[name], i.value());
} }
} else if (type == "regular") {
member.type = FSAccessor::Type::tRegular;
member.size = v["size"];
member.isExecutable = v.value("executable", false);
member.start = v["narOffset"];
} else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink;
member.target = v.value("target", "");
} else
return;
}; };
NarAccessor(ref<const std::string> nar) : nar(nar) json v = json::parse(listing);
{ recurse(root, v);
NarIndexer indexer(*this, *nar); }
parseDump(indexer, indexer);
NarMember* find(const Path& path) {
Path canon = path == "" ? "" : canonPath(path);
NarMember* current = &root;
auto end = path.end();
for (auto it = path.begin(); it != end;) {
// because it != end, the remaining component is non-empty so we need
// a directory
if (current->type != FSAccessor::Type::tDirectory) return nullptr;
// skip slash (canonPath above ensures that this is always a slash)
assert(*it == '/');
it += 1;
// lookup current component
auto next = std::find(it, end, '/');
auto child = current->children.find(std::string(it, next));
if (child == current->children.end()) return nullptr;
current = &child->second;
it = next;
} }
NarAccessor(const std::string & listing, GetNarBytes getNarBytes) return current;
: getNarBytes(getNarBytes) }
{
using json = nlohmann::json;
std::function<void(NarMember &, json &)> recurse; NarMember& get(const Path& path) {
auto result = find(path);
if (result == nullptr)
throw Error("NAR file does not contain path '%1%'", path);
return *result;
}
recurse = [&](NarMember & member, json & v) { Stat stat(const Path& path) override {
std::string type = v["type"]; auto i = find(path);
if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false};
return {i->type, i->size, i->isExecutable, i->start};
}
if (type == "directory") { StringSet readDirectory(const Path& path) override {
member.type = FSAccessor::Type::tDirectory; auto i = get(path);
for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
std::string name = i.key();
recurse(member.children[name], i.value());
}
} else if (type == "regular") {
member.type = FSAccessor::Type::tRegular;
member.size = v["size"];
member.isExecutable = v.value("executable", false);
member.start = v["narOffset"];
} else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink;
member.target = v.value("target", "");
} else return;
};
json v = json::parse(listing); if (i.type != FSAccessor::Type::tDirectory)
recurse(root, v); throw Error(format("path '%1%' inside NAR file is not a directory") %
} path);
NarMember * find(const Path & path) StringSet res;
{ for (auto& child : i.children) res.insert(child.first);
Path canon = path == "" ? "" : canonPath(path);
NarMember * current = &root;
auto end = path.end();
for (auto it = path.begin(); it != end; ) {
// because it != end, the remaining component is non-empty so we need
// a directory
if (current->type != FSAccessor::Type::tDirectory) return nullptr;
// skip slash (canonPath above ensures that this is always a slash) return res;
assert(*it == '/'); }
it += 1;
// lookup current component std::string readFile(const Path& path) override {
auto next = std::find(it, end, '/'); auto i = get(path);
auto child = current->children.find(std::string(it, next)); if (i.type != FSAccessor::Type::tRegular)
if (child == current->children.end()) return nullptr; throw Error(format("path '%1%' inside NAR file is not a regular file") %
current = &child->second; path);
it = next; if (getNarBytes) return getNarBytes(i.start, i.size);
}
return current; assert(nar);
} return std::string(*nar, i.start, i.size);
}
NarMember & get(const Path & path) { std::string readLink(const Path& path) override {
auto result = find(path); auto i = get(path);
if (result == nullptr) if (i.type != FSAccessor::Type::tSymlink)
throw Error("NAR file does not contain path '%1%'", path); throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
return *result; return i.target;
} }
Stat stat(const Path & path) override
{
auto i = find(path);
if (i == nullptr)
return {FSAccessor::Type::tMissing, 0, false};
return {i->type, i->size, i->isExecutable, i->start};
}
StringSet readDirectory(const Path & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory)
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
StringSet res;
for (auto & child : i.children)
res.insert(child.first);
return res;
}
std::string readFile(const Path & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tRegular)
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
if (getNarBytes) return getNarBytes(i.start, i.size);
assert(nar);
return std::string(*nar, i.start, i.size);
}
std::string readLink(const Path & path) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink)
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
return i.target;
}
}; };
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) {
{ return make_ref<NarAccessor>(nar);
return make_ref<NarAccessor>(nar);
} }
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
GetNarBytes getNarBytes) GetNarBytes getNarBytes) {
{ return make_ref<NarAccessor>(listing, getNarBytes);
return make_ref<NarAccessor>(listing, getNarBytes);
} }
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
const Path & path, bool recurse) bool recurse) {
{ auto st = accessor->stat(path);
auto st = accessor->stat(path);
auto obj = res.object(); auto obj = res.object();
switch (st.type) { switch (st.type) {
case FSAccessor::Type::tRegular: case FSAccessor::Type::tRegular:
obj.attr("type", "regular"); obj.attr("type", "regular");
obj.attr("size", st.fileSize); obj.attr("size", st.fileSize);
if (st.isExecutable) if (st.isExecutable) obj.attr("executable", true);
obj.attr("executable", true); if (st.narOffset) obj.attr("narOffset", st.narOffset);
if (st.narOffset) break;
obj.attr("narOffset", st.narOffset);
break;
case FSAccessor::Type::tDirectory: case FSAccessor::Type::tDirectory:
obj.attr("type", "directory"); obj.attr("type", "directory");
{ {
auto res2 = obj.object("entries"); auto res2 = obj.object("entries");
for (auto & name : accessor->readDirectory(path)) { for (auto& name : accessor->readDirectory(path)) {
if (recurse) { if (recurse) {
auto res3 = res2.placeholder(name); auto res3 = res2.placeholder(name);
listNar(res3, accessor, path + "/" + name, true); listNar(res3, accessor, path + "/" + name, true);
} else } else
res2.object(name); res2.object(name);
}
} }
break; }
break;
case FSAccessor::Type::tSymlink: case FSAccessor::Type::tSymlink:
obj.attr("type", "symlink"); obj.attr("type", "symlink");
obj.attr("target", accessor->readLink(path)); obj.attr("target", accessor->readLink(path));
break; break;
default: default:
throw Error("path '%s' does not exist in NAR", path); throw Error("path '%s' does not exist in NAR", path);
} }
} }
} } // namespace nix

12
third_party/nix/src/libstore/nar-accessor.hh vendored
View file

@ -1,7 +1,6 @@
#pragma once #pragma once
#include <functional> #include <functional>
#include "fs-accessor.hh" #include "fs-accessor.hh"
namespace nix { namespace nix {
@ -16,15 +15,14 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
inside the NAR. */ inside the NAR. */
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor( ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
const std::string & listing, GetNarBytes getNarBytes);
GetNarBytes getNarBytes);
class JSONPlaceholder; class JSONPlaceholder;
/* Write a JSON representation of the contents of a NAR (except file /* Write a JSON representation of the contents of a NAR (except file
contents). */ contents). */
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
const Path & path, bool recurse); bool recurse);
} } // namespace nix

386
third_party/nix/src/libstore/nar-info-disk-cache.cc vendored
View file

@ -1,13 +1,12 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"
#include <sqlite3.h> #include <sqlite3.h>
#include "globals.hh"
#include "sqlite.hh"
#include "sync.hh"
namespace nix { namespace nix {
static const char * schema = R"sql( static const char* schema = R"sql(
create table if not exists BinaryCaches ( create table if not exists BinaryCaches (
id integer primary key autoincrement not null, id integer primary key autoincrement not null,
@ -45,223 +44,222 @@ create table if not exists LastPurge (
)sql"; )sql";
class NarInfoDiskCacheImpl : public NarInfoDiskCache class NarInfoDiskCacheImpl : public NarInfoDiskCache {
{ public:
public: /* How often to purge expired entries from the cache. */
const int purgeInterval = 24 * 3600;
/* How often to purge expired entries from the cache. */ struct Cache {
const int purgeInterval = 24 * 3600; int id;
Path storeDir;
bool wantMassQuery;
int priority;
};
struct Cache struct State {
{ SQLite db;
int id; SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR,
Path storeDir; purgeCache;
bool wantMassQuery; std::map<std::string, Cache> caches;
int priority; };
};
struct State Sync<State> _state;
{
SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
std::map<std::string, Cache> caches;
};
Sync<State> _state; NarInfoDiskCacheImpl() {
auto state(_state.lock());
NarInfoDiskCacheImpl() Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
{ createDirs(dirOf(dbPath));
auto state(_state.lock());
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; state->db = SQLite(dbPath);
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath); if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) // We can always reproduce the cache.
throwSQLiteError(state->db, "setting timeout"); state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
// We can always reproduce the cache. state->db.exec(schema);
state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
state->db.exec(schema); state->insertCache.create(
state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, "
"wantMassQuery, priority) values (?, ?, ?, ?, ?)");
state->insertCache.create(state->db, state->queryCache.create(state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); "select id, storeDir, wantMassQuery, priority "
"from BinaryCaches where url = ?");
state->queryCache.create(state->db, state->insertNAR.create(
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); state->db,
"insert or replace into NARs(cache, hashPart, namePart, url, "
"compression, fileHash, fileSize, narHash, "
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, "
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
state->insertNAR.create(state->db, state->insertMissingNAR.create(
"insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " state->db,
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); "insert or replace into NARs(cache, hashPart, timestamp, present) "
"values (?, ?, ?, 0)");
state->insertMissingNAR.create(state->db, state->queryNAR.create(
"insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->db,
"select present, namePart, url, compression, fileHash, fileSize, "
"narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? "
"and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 "
"and timestamp > ?))");
state->queryNAR.create(state->db, /* Periodically purge expired entries from the database. */
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); retrySQLite<void>([&]() {
auto now = time(0);
/* Periodically purge expired entries from the database. */ SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
retrySQLite<void>([&]() { auto queryLastPurge_(queryLastPurge.use());
auto now = time(0);
SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); if (!queryLastPurge_.next() ||
auto queryLastPurge_(queryLastPurge.use()); queryLastPurge_.getInt(0) < now - purgeInterval) {
SQLiteStmt(state->db,
"delete from NARs where ((present = 0 and timestamp < ?) or "
"(present = 1 and timestamp < ?))")
.use()(now - settings.ttlNegativeNarInfoCache)(
now - settings.ttlPositiveNarInfoCache)
.exec();
if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { debug("deleted %d entries from the NAR info disk cache",
SQLiteStmt(state->db, sqlite3_changes(state->db));
"delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
.use()
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache)
.exec();
debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); SQLiteStmt(
state->db,
"insert or replace into LastPurge(dummy, value) values ('', ?)")
.use()(now)
.exec();
}
});
}
SQLiteStmt(state->db, Cache& getCache(State& state, const std::string& uri) {
"insert or replace into LastPurge(dummy, value) values ('', ?)") auto i = state.caches.find(uri);
.use()(now).exec(); if (i == state.caches.end()) abort();
} return i->second;
}
void createCache(const std::string& uri, const Path& storeDir,
bool wantMassQuery, int priority) override {
retrySQLite<void>([&]() {
auto state(_state.lock());
// FIXME: race
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)
.exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int)sqlite3_last_insert_rowid(state->db),
storeDir, wantMassQuery, priority};
});
}
bool cacheExists(const std::string& uri, bool& wantMassQuery,
int& priority) override {
return retrySQLite<bool>([&]() {
auto state(_state.lock());
auto i = state->caches.find(uri);
if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri));
if (!queryCache.next()) return false;
state->caches.emplace(
uri, Cache{(int)queryCache.getInt(0), queryCache.getStr(1),
queryCache.getInt(2) != 0, (int)queryCache.getInt(3)});
}
auto& cache(getCache(*state, uri));
wantMassQuery = cache.wantMassQuery;
priority = cache.priority;
return true;
});
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string& uri, const std::string& hashPart) override {
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
auto state(_state.lock());
auto& cache(getCache(*state, uri));
auto now = time(0);
auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)(
now - settings.ttlNegativeNarInfoCache)(
now - settings.ttlPositiveNarInfoCache));
if (!queryNAR.next()) return {oUnknown, 0};
if (!queryNAR.getInt(0)) return {oInvalid, 0};
auto narInfo = make_ref<NarInfo>();
auto namePart = queryNAR.getStr(1);
narInfo->path = cache.storeDir + "/" + hashPart +
(namePart.empty() ? "" : "-" + namePart);
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4)) narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto& r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(cache.storeDir + "/" + r);
if (!queryNAR.isNull(9))
narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
for (auto& sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
narInfo->ca = queryNAR.getStr(11);
return {oValid, narInfo};
}); });
} }
Cache & getCache(State & state, const std::string & uri) void upsertNarInfo(const std::string& uri, const std::string& hashPart,
{ std::shared_ptr<ValidPathInfo> info) override {
auto i = state.caches.find(uri); retrySQLite<void>([&]() {
if (i == state.caches.end()) abort(); auto state(_state.lock());
return i->second;
}
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override auto& cache(getCache(*state, uri));
{
retrySQLite<void>([&]() {
auto state(_state.lock());
// FIXME: race if (info) {
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); assert(hashPart == storePathToHash(info->path));
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
});
}
bool cacheExists(const std::string & uri, state->insertNAR
bool & wantMassQuery, int & priority) override .use()(cache.id)(hashPart)(storePathToName(info->path))(
{ narInfo ? narInfo->url : "", narInfo != 0)(
return retrySQLite<bool>([&]() { narInfo ? narInfo->compression : "", narInfo != 0)(
auto state(_state.lock()); narInfo && narInfo->fileHash ? narInfo->fileHash.to_string()
: "",
narInfo && narInfo->fileHash)(
narInfo ? narInfo->fileSize : 0,
narInfo != 0 && narInfo->fileSize)(info->narHash.to_string())(
info->narSize)(concatStringsSep(" ", info->shortRefs()))(
info->deriver != "" ? baseNameOf(info->deriver) : "",
info->deriver !=
"")(concatStringsSep(" ", info->sigs))(info->ca)(time(0))
.exec();
auto i = state->caches.find(uri); } else {
if (i == state->caches.end()) { state->insertMissingNAR.use()(cache.id)(hashPart)(time(0)).exec();
auto queryCache(state->queryCache.use()(uri)); }
if (!queryCache.next()) return false; });
state->caches.emplace(uri, }
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
}
auto & cache(getCache(*state, uri));
wantMassQuery = cache.wantMassQuery;
priority = cache.priority;
return true;
});
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) override
{
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
auto now = time(0);
auto queryNAR(state->queryNAR.use()
(cache.id)
(hashPart)
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache));
if (!queryNAR.next())
return {oUnknown, 0};
if (!queryNAR.getInt(0))
return {oInvalid, 0};
auto narInfo = make_ref<NarInfo>();
auto namePart = queryNAR.getStr(1);
narInfo->path = cache.storeDir + "/" +
hashPart + (namePart.empty() ? "" : "-" + namePart);
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(cache.storeDir + "/" + r);
if (!queryNAR.isNull(9))
narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
narInfo->ca = queryNAR.getStr(11);
return {oValid, narInfo};
});
}
void upsertNarInfo(
const std::string & uri, const std::string & hashPart,
std::shared_ptr<ValidPathInfo> info) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
if (info) {
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
assert(hashPart == storePathToHash(info->path));
state->insertNAR.use()
(cache.id)
(hashPart)
(storePathToName(info->path))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string())
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
(concatStringsSep(" ", info->sigs))
(info->ca)
(time(0)).exec();
} else {
state->insertMissingNAR.use()
(cache.id)
(hashPart)
(time(0)).exec();
}
});
}
}; };
ref<NarInfoDiskCache> getNarInfoDiskCache() ref<NarInfoDiskCache> getNarInfoDiskCache() {
{ static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>(); return cache;
return cache;
} }
} } // namespace nix

29
third_party/nix/src/libstore/nar-info-disk-cache.hh vendored
View file

@ -1,31 +1,30 @@
#pragma once #pragma once
#include "ref.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "ref.hh"
namespace nix { namespace nix {
class NarInfoDiskCache class NarInfoDiskCache {
{ public:
public: typedef enum { oValid, oInvalid, oUnknown } Outcome;
typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual void createCache(const std::string & uri, const Path & storeDir, virtual void createCache(const std::string& uri, const Path& storeDir,
bool wantMassQuery, int priority) = 0; bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri, virtual bool cacheExists(const std::string& uri, bool& wantMassQuery,
bool & wantMassQuery, int & priority) = 0; int& priority) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0; const std::string& uri, const std::string& hashPart) = 0;
virtual void upsertNarInfo( virtual void upsertNarInfo(const std::string& uri,
const std::string & uri, const std::string & hashPart, const std::string& hashPart,
std::shared_ptr<ValidPathInfo> info) = 0; std::shared_ptr<ValidPathInfo> info) = 0;
}; };
/* Return a singleton cache object that can be used concurrently by /* Return a singleton cache object that can be used concurrently by
multiple threads. */ multiple threads. */
ref<NarInfoDiskCache> getNarInfoDiskCache(); ref<NarInfoDiskCache> getNarInfoDiskCache();
} } // namespace nix

175
third_party/nix/src/libstore/nar-info.cc vendored
View file

@ -1,116 +1,105 @@
#include "globals.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "globals.hh"
namespace nix { namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) NarInfo::NarInfo(const Store& store, const std::string& s,
{ const std::string& whence) {
auto corrupt = [&]() { auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence); throw Error(format("NAR info file '%1%' is corrupt") % whence);
}; };
auto parseHashField = [&](const string & s) { auto parseHashField = [&](const string& s) {
try { try {
return Hash(s); return Hash(s);
} catch (BadHash &) { } catch (BadHash&) {
corrupt(); corrupt();
return Hash(); // never reached return Hash(); // never reached
} }
}; };
size_t pos = 0; size_t pos = 0;
while (pos < s.size()) { while (pos < s.size()) {
size_t colon = s.find(':', pos);
if (colon == std::string::npos) corrupt();
size_t colon = s.find(':', pos); std::string name(s, pos, colon - pos);
if (colon == std::string::npos) corrupt();
std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2);
if (eol == std::string::npos) corrupt();
size_t eol = s.find('\n', colon + 2); std::string value(s, colon + 2, eol - colon - 2);
if (eol == std::string::npos) corrupt();
std::string value(s, colon + 2, eol - colon - 2); if (name == "StorePath") {
if (!store.isStorePath(value)) corrupt();
if (name == "StorePath") { path = value;
if (!store.isStorePath(value)) corrupt(); } else if (name == "URL")
path = value; url = value;
} else if (name == "Compression")
else if (name == "URL") compression = value;
url = value; else if (name == "FileHash")
else if (name == "Compression") fileHash = parseHashField(value);
compression = value; else if (name == "FileSize") {
else if (name == "FileHash") if (!string2Int(value, fileSize)) corrupt();
fileHash = parseHashField(value); } else if (name == "NarHash")
else if (name == "FileSize") { narHash = parseHashField(value);
if (!string2Int(value, fileSize)) corrupt(); else if (name == "NarSize") {
} if (!string2Int(value, narSize)) corrupt();
else if (name == "NarHash") } else if (name == "References") {
narHash = parseHashField(value); auto refs = tokenizeString<Strings>(value, " ");
else if (name == "NarSize") { if (!references.empty()) corrupt();
if (!string2Int(value, narSize)) corrupt(); for (auto& r : refs) {
} auto r2 = store.storeDir + "/" + r;
else if (name == "References") { if (!store.isStorePath(r2)) corrupt();
auto refs = tokenizeString<Strings>(value, " "); references.insert(r2);
if (!references.empty()) corrupt(); }
for (auto & r : refs) { } else if (name == "Deriver") {
auto r2 = store.storeDir + "/" + r; if (value != "unknown-deriver") {
if (!store.isStorePath(r2)) corrupt(); auto p = store.storeDir + "/" + value;
references.insert(r2); if (!store.isStorePath(p)) corrupt();
} deriver = p;
} }
else if (name == "Deriver") { } else if (name == "System")
if (value != "unknown-deriver") { system = value;
auto p = store.storeDir + "/" + value; else if (name == "Sig")
if (!store.isStorePath(p)) corrupt(); sigs.insert(value);
deriver = p; else if (name == "CA") {
} if (!ca.empty()) corrupt();
} ca = value;
else if (name == "System")
system = value;
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
if (!ca.empty()) corrupt();
ca = value;
}
pos = eol + 1;
} }
if (compression == "") compression = "bzip2"; pos = eol + 1;
}
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); if (compression == "") compression = "bzip2";
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
} }
std::string NarInfo::to_string() const std::string NarInfo::to_string() const {
{ std::string res;
std::string res; res += "StorePath: " + path + "\n";
res += "StorePath: " + path + "\n"; res += "URL: " + url + "\n";
res += "URL: " + url + "\n"; assert(compression != "");
assert(compression != ""); res += "Compression: " + compression + "\n";
res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256);
assert(fileHash.type == htSHA256); res += "FileHash: " + fileHash.to_string(Base32) + "\n";
res += "FileHash: " + fileHash.to_string(Base32) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256);
assert(narHash.type == htSHA256); res += "NarHash: " + narHash.to_string(Base32) + "\n";
res += "NarHash: " + narHash.to_string(Base32) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
if (!deriver.empty()) if (!deriver.empty()) res += "Deriver: " + baseNameOf(deriver) + "\n";
res += "Deriver: " + baseNameOf(deriver) + "\n";
if (!system.empty()) if (!system.empty()) res += "System: " + system + "\n";
res += "System: " + system + "\n";
for (auto sig : sigs) for (auto sig : sigs) res += "Sig: " + sig + "\n";
res += "Sig: " + sig + "\n";
if (!ca.empty()) if (!ca.empty()) res += "CA: " + ca + "\n";
res += "CA: " + ca + "\n";
return res; return res;
} }
} } // namespace nix

25
third_party/nix/src/libstore/nar-info.hh vendored
View file

@ -1,24 +1,23 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "store-api.hh" #include "store-api.hh"
#include "types.hh"
namespace nix { namespace nix {
struct NarInfo : ValidPathInfo struct NarInfo : ValidPathInfo {
{ std::string url;
std::string url; std::string compression;
std::string compression; Hash fileHash;
Hash fileHash; uint64_t fileSize = 0;
uint64_t fileSize = 0; std::string system;
std::string system;
NarInfo() { } NarInfo() {}
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const ValidPathInfo& info) : ValidPathInfo(info) {}
NarInfo(const Store & store, const std::string & s, const std::string & whence); NarInfo(const Store& store, const std::string& s, const std::string& whence);
std::string to_string() const; std::string to_string() const;
}; };
} } // namespace nix

475
third_party/nix/src/libstore/optimise-store.cc vendored
View file

@ -1,302 +1,285 @@
#include "util.hh"
#include "local-store.hh"
#include "globals.hh"
#include <cstdlib>
#include <cstring>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h> #include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <cstring>
#include <regex> #include <regex>
#include "globals.hh"
#include "local-store.hh"
#include "util.hh"
namespace nix { namespace nix {
static void makeWritable(const Path& path) {
static void makeWritable(const Path & path) struct stat st;
{ if (lstat(path.c_str(), &st))
struct stat st; throw SysError(format("getting attributes of path '%1%'") % path);
if (lstat(path.c_str(), &st)) if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("getting attributes of path '%1%'") % path); throw SysError(format("changing writability of '%1%'") % path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("changing writability of '%1%'") % path);
} }
struct MakeReadOnly {
struct MakeReadOnly Path path;
{ MakeReadOnly(const Path& path) : path(path) {}
Path path; ~MakeReadOnly() {
MakeReadOnly(const Path & path) : path(path) { } try {
~MakeReadOnly() /* This will make the path read-only. */
{ if (path != "") canonicaliseTimestampAndPermissions(path);
try { } catch (...) {
/* This will make the path read-only. */ ignoreException();
if (path != "") canonicaliseTimestampAndPermissions(path);
} catch (...) {
ignoreException();
}
} }
}
}; };
LocalStore::InodeHash LocalStore::loadInodeHash() {
debug("loading hash inodes in memory");
InodeHash inodeHash;
LocalStore::InodeHash LocalStore::loadInodeHash() AutoCloseDir dir(opendir(linksDir.c_str()));
{ if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
debug("loading hash inodes in memory");
InodeHash inodeHash;
AutoCloseDir dir(opendir(linksDir.c_str())); struct dirent* dirent;
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt();
// We don't care if we hit non-hash files, anything goes
inodeHash.insert(dirent->d_ino);
}
if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
struct dirent * dirent; printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt();
// We don't care if we hit non-hash files, anything goes
inodeHash.insert(dirent->d_ino);
}
if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); return inodeHash;
return inodeHash;
} }
Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
const InodeHash& inodeHash) {
Strings names;
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) AutoCloseDir dir(opendir(path.c_str()));
{ if (!dir) throw SysError(format("opening directory '%1%'") % path);
Strings names;
AutoCloseDir dir(opendir(path.c_str())); struct dirent* dirent;
if (!dir) throw SysError(format("opening directory '%1%'") % path); while (errno = 0, dirent = readdir(dir.get())) { /* sic */
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt();
if (inodeHash.count(dirent->d_ino)) {
debug(format("'%1%' is already linked") % dirent->d_name);
continue;
}
string name = dirent->d_name;
if (name == "." || name == "..") continue;
names.push_back(name);
}
if (errno) throw SysError(format("reading directory '%1%'") % path);
return names;
}
void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
const Path & path, InodeHash & inodeHash)
{
checkInterrupt(); checkInterrupt();
struct stat st; if (inodeHash.count(dirent->d_ino)) {
if (lstat(path.c_str(), &st)) debug(format("'%1%' is already linked") % dirent->d_name);
throw SysError(format("getting attributes of path '%1%'") % path); continue;
}
string name = dirent->d_name;
if (name == "." || name == "..") continue;
names.push_back(name);
}
if (errno) throw SysError(format("reading directory '%1%'") % path);
return names;
}
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
const Path& path, InodeHash& inodeHash) {
checkInterrupt();
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
#if __APPLE__ #if __APPLE__
/* HFS/macOS has some undocumented security feature disabling hardlinking for /* HFS/macOS has some undocumented security feature disabling hardlinking for
special files within .app dirs. *.app/Contents/PkgInfo and special files within .app dirs. *.app/Contents/PkgInfo and
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
https://github.com/NixOS/nix/issues/1443 for more discussion. */ https://github.com/NixOS/nix/issues/1443 for more discussion. */
if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) {
{ debug(format("'%1%' is not allowed to be linked in macOS") % path);
debug(format("'%1%' is not allowed to be linked in macOS") % path); return;
return; }
}
#endif #endif
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash); Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names) for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash);
optimisePath_(act, stats, path + "/" + i, inodeHash); return;
return; }
}
/* We can hard link regular files and maybe symlinks. */ /* We can hard link regular files and maybe symlinks. */
if (!S_ISREG(st.st_mode) if (!S_ISREG(st.st_mode)
#if CAN_LINK_SYMLINK #if CAN_LINK_SYMLINK
&& !S_ISLNK(st.st_mode) && !S_ISLNK(st.st_mode)
#endif #endif
) return; )
return;
/* Sometimes SNAFUs can cause files in the Nix store to be /* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under modified, in particular when running programs as root under
NixOS (example: $fontconfig/var/cache being modified). Skip NixOS (example: $fontconfig/var/cache being modified). Skip
those files. FIXME: check the modification time. */ those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printError(format("skipping suspicious writable file '%1%'") % path); printError(format("skipping suspicious writable file '%1%'") % path);
return;
}
/* This can still happen on top-level files. */
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
debug(format("'%1%' is already linked, with %2% other file(s)") % path %
(st.st_nlink - 2));
return;
}
/* Hash the file. Note that hashPath() returns the hash over the
NAR serialisation, which includes the execute bit on the file.
Thus, executable and non-executable files with the same
contents *won't* be linked (which is good because otherwise the
permissions would be screwed up).
Also note that if `path' is a symlink, then we're hashing the
contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first;
debug(format("'%1%' has hash '%2%'") % path % hash.to_string());
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
retry:
if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
if (link(path.c_str(), linkPath.c_str()) == 0) {
inodeHash.insert(st.st_ino);
return;
}
switch (errno) {
case EEXIST:
/* Fall through if another process created ‘linkPath’ before
we did. */
break;
case ENOSPC:
/* On ext4, that probably means the directory index is
full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this
file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path,
strerror(errno));
return; return;
default:
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
} }
}
/* This can still happen on top-level files. */ /* Yes! We've seen a file with the same contents. Replace the
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { current file with a hard link to that file. */
debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); struct stat stLink;
return; if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path '%1%'") % linkPath);
if (st.st_ino == stLink.st_ino) {
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
return;
}
if (st.st_size != stLink.st_size) {
printError(format("removing corrupted link '%1%'") % linkPath);
unlink(linkPath.c_str());
goto retry;
}
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = dirOf(path) != realStoreDir;
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
Path tempLink =
(format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random())
.str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files.
Just shrug and ignore. */
if (st.st_size)
printInfo(format("'%1%' has maximum number of links") % linkPath);
return;
} }
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
}
/* Hash the file. Note that hashPath() returns the hash over the /* Atomically replace the old file with the new hard link. */
NAR serialisation, which includes the execute bit on the file. if (rename(tempLink.c_str(), path.c_str()) == -1) {
Thus, executable and non-executable files with the same if (unlink(tempLink.c_str()) == -1)
contents *won't* be linked (which is good because otherwise the printError(format("unable to unlink '%1%'") % tempLink);
permissions would be screwed up). if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
Also note that if `path' is a symlink, then we're hashing the rather than on the original link. (Probably it
contents of the symlink (i.e. the result of readlink()), not temporarily increases the st_nlink field before
the contents of the target (which may not even exist). */ decreasing it again.) */
Hash hash = hashPath(htSHA256, path).first; debug("'%s' has reached maximum number of links", linkPath);
debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); return;
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
retry:
if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
if (link(path.c_str(), linkPath.c_str()) == 0) {
inodeHash.insert(st.st_ino);
return;
}
switch (errno) {
case EEXIST:
/* Fall through if another process created ‘linkPath’ before
we did. */
break;
case ENOSPC:
/* On ext4, that probably means the directory index is
full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this
file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
return;
default:
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
}
} }
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
}
/* Yes! We've seen a file with the same contents. Replace the stats.filesLinked++;
current file with a hard link to that file. */ stats.bytesFreed += st.st_size;
struct stat stLink; stats.blocksFreed += st.st_blocks;
if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path '%1%'") % linkPath);
if (st.st_ino == stLink.st_ino) { if (act) act->result(resFileLinked, st.st_size, st.st_blocks);
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
return;
}
if (st.st_size != stLink.st_size) {
printError(format("removing corrupted link '%1%'") % linkPath);
unlink(linkPath.c_str());
goto retry;
}
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = dirOf(path) != realStoreDir;
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% realStoreDir % getpid() % random()).str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files.
Just shrug and ignore. */
if (st.st_size)
printInfo(format("'%1%' has maximum number of links") % linkPath);
return;
}
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1)
printError(format("unable to unlink '%1%'") % tempLink);
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
temporarily increases the st_nlink field before
decreasing it again.) */
debug("'%s' has reached maximum number of links", linkPath);
return;
}
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
}
stats.filesLinked++;
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
if (act)
act->result(resFileLinked, st.st_size, st.st_blocks);
} }
void LocalStore::optimiseStore(OptimiseStats& stats) {
Activity act(*logger, actOptimiseStore);
void LocalStore::optimiseStore(OptimiseStats & stats) PathSet paths = queryAllValidPaths();
{ InodeHash inodeHash = loadInodeHash();
Activity act(*logger, actOptimiseStore);
PathSet paths = queryAllValidPaths(); act.progress(0, paths.size());
InodeHash inodeHash = loadInodeHash();
act.progress(0, paths.size()); uint64_t done = 0;
uint64_t done = 0; for (auto& i : paths) {
addTempRoot(i);
for (auto & i : paths) { if (!isValidPath(i)) continue; /* path was GC'ed, probably */
addTempRoot(i); {
if (!isValidPath(i)) continue; /* path was GC'ed, probably */ Activity act(*logger, lvlTalkative, actUnknown,
{ fmt("optimising path '%s'", i));
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
}
done++;
act.progress(done, paths.size());
} }
done++;
act.progress(done, paths.size());
}
} }
static string showBytes(unsigned long long bytes) static string showBytes(unsigned long long bytes) {
{ return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
} }
void LocalStore::optimiseStore() void LocalStore::optimiseStore() {
{ OptimiseStats stats;
OptimiseStats stats;
optimiseStore(stats); optimiseStore(stats);
printInfo( printInfo(format("%1% freed by hard-linking %2% files") %
format("%1% freed by hard-linking %2% files") showBytes(stats.bytesFreed) % stats.filesLinked);
% showBytes(stats.bytesFreed)
% stats.filesLinked);
} }
void LocalStore::optimisePath(const Path & path) void LocalStore::optimisePath(const Path& path) {
{ OptimiseStats stats;
OptimiseStats stats; InodeHash inodeHash;
InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); if (settings.autoOptimiseStore)
optimisePath_(nullptr, stats, path, inodeHash);
} }
} // namespace nix
}

180
third_party/nix/src/libstore/parsed-derivations.cc vendored
View file

@ -2,115 +2,115 @@
namespace nix { namespace nix {
ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
: drvPath(drvPath), drv(drv) : drvPath(drvPath), drv(drv) {
{ /* Parse the __json attribute, if any. */
/* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json");
auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) {
if (jsonAttr != drv.env.end()) { try {
try { structuredAttrs = nlohmann::json::parse(jsonAttr->second);
structuredAttrs = nlohmann::json::parse(jsonAttr->second); } catch (std::exception& e) {
} catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath,
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); e.what());
}
} }
}
} }
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const std::optional<std::string> ParsedDerivation::getStringAttr(
{ const std::string& name) const {
if (structuredAttrs) { if (structuredAttrs) {
auto i = structuredAttrs->find(name); auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end()) if (i == structuredAttrs->end())
return {}; return {};
else { else {
if (!i->is_string()) if (!i->is_string())
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); throw Error("attribute '%s' of derivation '%s' must be a string", name,
return i->get<std::string>(); drvPath);
} return i->get<std::string>();
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return {};
else
return i->second;
} }
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return {};
else
return i->second;
}
} }
bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const {
{ if (structuredAttrs) {
if (structuredAttrs) { auto i = structuredAttrs->find(name);
auto i = structuredAttrs->find(name); if (i == structuredAttrs->end())
if (i == structuredAttrs->end()) return def;
return def; else {
else { if (!i->is_boolean())
if (!i->is_boolean()) throw Error("attribute '%s' of derivation '%s' must be a Boolean", name,
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); drvPath);
return i->get<bool>(); return i->get<bool>();
}
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return def;
else
return i->second == "1";
} }
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return def;
else
return i->second == "1";
}
} }
std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const std::optional<Strings> ParsedDerivation::getStringsAttr(
{ const std::string& name) const {
if (structuredAttrs) { if (structuredAttrs) {
auto i = structuredAttrs->find(name); auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end()) if (i == structuredAttrs->end())
return {}; return {};
else { else {
if (!i->is_array()) if (!i->is_array())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); throw Error(
Strings res; "attribute '%s' of derivation '%s' must be a list of strings", name,
for (auto j = i->begin(); j != i->end(); ++j) { drvPath);
if (!j->is_string()) Strings res;
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); for (auto j = i->begin(); j != i->end(); ++j) {
res.push_back(j->get<std::string>()); if (!j->is_string())
} throw Error(
return res; "attribute '%s' of derivation '%s' must be a list of strings",
} name, drvPath);
} else { res.push_back(j->get<std::string>());
auto i = drv.env.find(name); }
if (i == drv.env.end()) return res;
return {};
else
return tokenizeString<Strings>(i->second);
} }
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return {};
else
return tokenizeString<Strings>(i->second);
}
} }
StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet ParsedDerivation::getRequiredSystemFeatures() const {
{ StringSet res;
StringSet res; for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i);
res.insert(i); return res;
return res;
} }
bool ParsedDerivation::canBuildLocally() const bool ParsedDerivation::canBuildLocally() const {
{ if (drv.platform != settings.thisSystem.get() &&
if (drv.platform != settings.thisSystem.get() !settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin())
&& !settings.extraPlatforms.get().count(drv.platform) return false;
&& !drv.isBuiltin())
return false;
for (auto & feature : getRequiredSystemFeatures()) for (auto& feature : getRequiredSystemFeatures())
if (!settings.systemFeatures.get().count(feature)) return false; if (!settings.systemFeatures.get().count(feature)) return false;
return true; return true;
} }
bool ParsedDerivation::willBuildLocally() const bool ParsedDerivation::willBuildLocally() const {
{ return getBoolAttr("preferLocalBuild") && canBuildLocally();
return getBoolAttr("preferLocalBuild") && canBuildLocally();
} }
bool ParsedDerivation::substitutesAllowed() const bool ParsedDerivation::substitutesAllowed() const {
{ return getBoolAttr("allowSubstitutes", true);
return getBoolAttr("allowSubstitutes", true);
} }
} } // namespace nix

40
third_party/nix/src/libstore/parsed-derivations.hh vendored
View file

@ -1,37 +1,33 @@
#include "derivations.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "derivations.hh"
namespace nix { namespace nix {
class ParsedDerivation class ParsedDerivation {
{ Path drvPath;
Path drvPath; BasicDerivation& drv;
BasicDerivation & drv; std::optional<nlohmann::json> structuredAttrs;
std::optional<nlohmann::json> structuredAttrs;
public: public:
ParsedDerivation(const Path& drvPath, BasicDerivation& drv);
ParsedDerivation(const Path & drvPath, BasicDerivation & drv); const std::optional<nlohmann::json>& getStructuredAttrs() const {
return structuredAttrs;
}
const std::optional<nlohmann::json> & getStructuredAttrs() const std::optional<std::string> getStringAttr(const std::string& name) const;
{
return structuredAttrs;
}
std::optional<std::string> getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string& name, bool def = false) const;
bool getBoolAttr(const std::string & name, bool def = false) const; std::optional<Strings> getStringsAttr(const std::string& name) const;
std::optional<Strings> getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const;
StringSet getRequiredSystemFeatures() const; bool canBuildLocally() const;
bool canBuildLocally() const; bool willBuildLocally() const;
bool willBuildLocally() const; bool substitutesAllowed() const;
bool substitutesAllowed() const;
}; };
} } // namespace nix

290
third_party/nix/src/libstore/pathlocks.cc vendored
View file

@ -1,178 +1,156 @@
#include "pathlocks.hh" #include "pathlocks.hh"
#include "util.hh" #include <fcntl.h>
#include "sync.hh" #include <sys/file.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <cerrno> #include <cerrno>
#include <cstdlib> #include <cstdlib>
#include "sync.hh"
#include <fcntl.h> #include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/file.h>
namespace nix { namespace nix {
AutoCloseFD openLockFile(const Path& path, bool create) {
AutoCloseFD fd;
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
if (!fd && (create || errno != ENOENT))
throw SysError(format("opening lock file '%1%'") % path);
return fd;
}
void deleteLockFile(const Path& path, int fd) {
/* Get rid of the lock file. Have to be careful not to introduce
races. Write a (meaningless) token to the file to indicate to
other processes waiting on this lock that the lock is stale
(deleted). */
unlink(path.c_str());
writeFull(fd, "d");
/* Note that the result of unlink() is ignored; removing the lock
file is an optimisation, not a necessity. */
}
bool lockFile(int fd, LockType lockType, bool wait) {
int type;
if (lockType == ltRead)
type = LOCK_SH;
else if (lockType == ltWrite)
type = LOCK_EX;
else if (lockType == ltNone)
type = LOCK_UN;
else
abort();
if (wait) {
while (flock(fd, type) != 0) {
checkInterrupt();
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
else
return false;
}
} else {
while (flock(fd, type | LOCK_NB) != 0) {
checkInterrupt();
if (errno == EWOULDBLOCK) return false;
if (errno != EINTR) throw SysError(format("acquiring/releasing lock"));
}
}
return true;
}
PathLocks::PathLocks() : deletePaths(false) {}
PathLocks::PathLocks(const PathSet& paths, const string& waitMsg)
: deletePaths(false) {
lockPaths(paths, waitMsg);
}
bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
bool wait) {
assert(fds.empty());
/* Note that `fds' is built incrementally so that the destructor
will only release those locks that we have already acquired. */
/* Acquire the lock for each path in sorted order. This ensures
that locks are always acquired in the same order, thus
preventing deadlocks. */
for (auto& path : paths) {
checkInterrupt();
Path lockPath = path + ".lock";
debug(format("locking path '%1%'") % path);
AutoCloseFD openLockFile(const Path & path, bool create)
{
AutoCloseFD fd; AutoCloseFD fd;
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); while (1) {
if (!fd && (create || errno != ENOENT)) /* Open/create the lock file. */
throw SysError(format("opening lock file '%1%'") % path); fd = openLockFile(lockPath, true);
return fd; /* Acquire an exclusive lock. */
} if (!lockFile(fd.get(), ltWrite, false)) {
if (wait) {
if (waitMsg != "") printError(waitMsg);
void deleteLockFile(const Path & path, int fd) lockFile(fd.get(), ltWrite, true);
{ } else {
/* Get rid of the lock file. Have to be careful not to introduce /* Failed to lock this path; release all other
races. Write a (meaningless) token to the file to indicate to locks. */
other processes waiting on this lock that the lock is stale unlock();
(deleted). */ return false;
unlink(path.c_str());
writeFull(fd, "d");
/* Note that the result of unlink() is ignored; removing the lock
file is an optimisation, not a necessity. */
}
bool lockFile(int fd, LockType lockType, bool wait)
{
int type;
if (lockType == ltRead) type = LOCK_SH;
else if (lockType == ltWrite) type = LOCK_EX;
else if (lockType == ltNone) type = LOCK_UN;
else abort();
if (wait) {
while (flock(fd, type) != 0) {
checkInterrupt();
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
else
return false;
}
} else {
while (flock(fd, type | LOCK_NB) != 0) {
checkInterrupt();
if (errno == EWOULDBLOCK) return false;
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
} }
}
debug(format("lock acquired on '%1%'") % lockPath);
/* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError(format("statting lock file '%1%'") % lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */
debug(format("open lock file '%1%' has become stale") % lockPath);
else
break;
} }
return true; /* Use borrow so that the descriptor isn't closed. */
fds.push_back(FDPair(fd.release(), lockPath));
}
return true;
} }
PathLocks::~PathLocks() {
PathLocks::PathLocks() try {
: deletePaths(false) unlock();
{ } catch (...) {
ignoreException();
}
} }
void PathLocks::unlock() {
for (auto& i : fds) {
if (deletePaths) deleteLockFile(i.second, i.first);
PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) if (close(i.first) == -1)
: deletePaths(false) printError(format("error (ignored): cannot close lock file on '%1%'") %
{ i.second);
lockPaths(paths, waitMsg);
debug(format("lock released on '%1%'") % i.second);
}
fds.clear();
} }
void PathLocks::setDeletion(bool deletePaths) {
bool PathLocks::lockPaths(const PathSet & paths, this->deletePaths = deletePaths;
const string & waitMsg, bool wait)
{
assert(fds.empty());
/* Note that `fds' is built incrementally so that the destructor
will only release those locks that we have already acquired. */
/* Acquire the lock for each path in sorted order. This ensures
that locks are always acquired in the same order, thus
preventing deadlocks. */
for (auto & path : paths) {
checkInterrupt();
Path lockPath = path + ".lock";
debug(format("locking path '%1%'") % path);
AutoCloseFD fd;
while (1) {
/* Open/create the lock file. */
fd = openLockFile(lockPath, true);
/* Acquire an exclusive lock. */
if (!lockFile(fd.get(), ltWrite, false)) {
if (wait) {
if (waitMsg != "") printError(waitMsg);
lockFile(fd.get(), ltWrite, true);
} else {
/* Failed to lock this path; release all other
locks. */
unlock();
return false;
}
}
debug(format("lock acquired on '%1%'") % lockPath);
/* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError(format("statting lock file '%1%'") % lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */
debug(format("open lock file '%1%' has become stale") % lockPath);
else
break;
}
/* Use borrow so that the descriptor isn't closed. */
fds.push_back(FDPair(fd.release(), lockPath));
}
return true;
} }
} // namespace nix
PathLocks::~PathLocks()
{
try {
unlock();
} catch (...) {
ignoreException();
}
}
void PathLocks::unlock()
{
for (auto & i : fds) {
if (deletePaths) deleteLockFile(i.second, i.first);
if (close(i.first) == -1)
printError(
format("error (ignored): cannot close lock file on '%1%'") % i.second);
debug(format("lock released on '%1%'") % i.second);
}
fds.clear();
}
void PathLocks::setDeletion(bool deletePaths)
{
this->deletePaths = deletePaths;
}
}

35
third_party/nix/src/libstore/pathlocks.hh vendored
View file

@ -7,32 +7,29 @@ namespace nix {
/* Open (possibly create) a lock file and return the file descriptor. /* Open (possibly create) a lock file and return the file descriptor.
-1 is returned if create is false and the lock could not be opened -1 is returned if create is false and the lock could not be opened
because it doesn't exist. Any other error throws an exception. */ because it doesn't exist. Any other error throws an exception. */
AutoCloseFD openLockFile(const Path & path, bool create); AutoCloseFD openLockFile(const Path& path, bool create);
/* Delete an open lock file. */ /* Delete an open lock file. */
void deleteLockFile(const Path & path, int fd); void deleteLockFile(const Path& path, int fd);
enum LockType { ltRead, ltWrite, ltNone }; enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(int fd, LockType lockType, bool wait); bool lockFile(int fd, LockType lockType, bool wait);
class PathLocks class PathLocks {
{ private:
private: typedef std::pair<int, Path> FDPair;
typedef std::pair<int, Path> FDPair; list<FDPair> fds;
list<FDPair> fds; bool deletePaths;
bool deletePaths;
public: public:
PathLocks(); PathLocks();
PathLocks(const PathSet & paths, PathLocks(const PathSet& paths, const string& waitMsg = "");
const string & waitMsg = ""); bool lockPaths(const PathSet& _paths, const string& waitMsg = "",
bool lockPaths(const PathSet & _paths, bool wait = true);
const string & waitMsg = "", ~PathLocks();
bool wait = true); void unlock();
~PathLocks(); void setDeletion(bool deletePaths);
void unlock();
void setDeletion(bool deletePaths);
}; };
} } // namespace nix

379
third_party/nix/src/libstore/profiles.cc vendored
View file

@ -1,259 +1,226 @@
#include "profiles.hh" #include "profiles.hh"
#include <errno.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
namespace nix { namespace nix {
static bool cmpGensByNumber(const Generation& a, const Generation& b) {
static bool cmpGensByNumber(const Generation & a, const Generation & b) return a.number < b.number;
{
return a.number < b.number;
} }
/* Parse a generation name of the format /* Parse a generation name of the format
`<profilename>-<number>-link'. */ `<profilename>-<number>-link'. */
static int parseName(const string & profileName, const string & name) static int parseName(const string& profileName, const string& name) {
{ if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; string s = string(name, profileName.size() + 1);
string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link");
string::size_type p = s.find("-link"); if (p == string::npos) return -1;
if (p == string::npos) return -1; int n;
if (string2Int(string(s, 0, p), n) && n >= 0)
return n;
else
return -1;
}
Generations findGenerations(Path profile, int& curGen) {
Generations gens;
Path profileDir = dirOf(profile);
string profileName = baseNameOf(profile);
for (auto& i : readDirectory(profileDir)) {
int n; int n;
if (string2Int(string(s, 0, p), n) && n >= 0) if ((n = parseName(profileName, i.name)) != -1) {
return n; Generation gen;
else gen.path = profileDir + "/" + i.name;
return -1; gen.number = n;
struct stat st;
if (lstat(gen.path.c_str(), &st) != 0)
throw SysError(format("statting '%1%'") % gen.path);
gen.creationTime = st.st_mtime;
gens.push_back(gen);
}
}
gens.sort(cmpGensByNumber);
curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1;
return gens;
} }
static void makeName(const Path& profile, unsigned int num, Path& outLink) {
Path prefix = (format("%1%-%2%") % profile % num).str();
outLink = prefix + "-link";
}
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) {
/* The new generation number should be higher than old the
previous ones. */
int dummy;
Generations gens = findGenerations(profile, dummy);
Generations findGenerations(Path profile, int & curGen) unsigned int num;
{ if (gens.size() > 0) {
Generations gens; Generation last = gens.back();
Path profileDir = dirOf(profile); if (readLink(last.path) == outPath) {
string profileName = baseNameOf(profile); /* We only create a new generation symlink if it differs
from the last one.
for (auto & i : readDirectory(profileDir)) { This helps keeping gratuitous installs/rebuilds from piling
int n; up uncontrolled numbers of generations, cluttering up the
if ((n = parseName(profileName, i.name)) != -1) { UI like grub. */
Generation gen; return last.path;
gen.path = profileDir + "/" + i.name;
gen.number = n;
struct stat st;
if (lstat(gen.path.c_str(), &st) != 0)
throw SysError(format("statting '%1%'") % gen.path);
gen.creationTime = st.st_mtime;
gens.push_back(gen);
}
} }
gens.sort(cmpGensByNumber); num = gens.back().number;
} else {
num = 0;
}
curGen = pathExists(profile) /* Create the new generation. Note that addPermRoot() blocks if
? parseName(profileName, readLink(profile)) the garbage collector is running to prevent the stuff we've
: -1; built from moving from the temporary roots (which the GC knows)
to the permanent roots (of which the GC would have a stale
view). If we didn't do it this way, the GC might remove the
user environment etc. we've just built. */
Path generation;
makeName(profile, num + 1, generation);
store->addPermRoot(outPath, generation, false, true);
return gens; return generation;
} }
static void removeFile(const Path& path) {
static void makeName(const Path & profile, unsigned int num, if (remove(path.c_str()) == -1)
Path & outLink) throw SysError(format("cannot unlink '%1%'") % path);
{
Path prefix = (format("%1%-%2%") % profile % num).str();
outLink = prefix + "-link";
} }
void deleteGeneration(const Path& profile, unsigned int gen) {
Path generation;
makeName(profile, gen, generation);
removeFile(generation);
}
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) static void deleteGeneration2(const Path& profile, unsigned int gen,
{ bool dryRun) {
/* The new generation number should be higher than old the if (dryRun)
previous ones. */ printInfo(format("would remove generation %1%") % gen);
int dummy; else {
Generations gens = findGenerations(profile, dummy); printInfo(format("removing generation %1%") % gen);
deleteGeneration(profile, gen);
}
}
unsigned int num; void deleteGenerations(const Path& profile,
if (gens.size() > 0) { const std::set<unsigned int>& gensToDelete,
Generation last = gens.back(); bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
if (readLink(last.path) == outPath) { int curGen;
/* We only create a new generation symlink if it differs Generations gens = findGenerations(profile, curGen);
from the last one.
This helps keeping gratuitous installs/rebuilds from piling if (gensToDelete.find(curGen) != gensToDelete.end())
up uncontrolled numbers of generations, cluttering up the throw Error(format("cannot delete current generation of profile %1%'") %
UI like grub. */ profile);
return last.path;
}
num = gens.back().number; for (auto& i : gens) {
} else { if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
num = 0; deleteGeneration2(profile, i.number, dryRun);
}
}
void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
int curGen;
bool fromCurGen = false;
Generations gens = findGenerations(profile, curGen);
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
if (i->number == curGen) {
fromCurGen = true;
max--;
continue;
} }
if (fromCurGen) {
/* Create the new generation. Note that addPermRoot() blocks if if (max) {
the garbage collector is running to prevent the stuff we've max--;
built from moving from the temporary roots (which the GC knows) continue;
to the permanent roots (of which the GC would have a stale }
view). If we didn't do it this way, the GC might remove the deleteGeneration2(profile, i->number, dryRun);
user environment etc. we've just built. */ }
Path generation; }
makeName(profile, num + 1, generation);
store->addPermRoot(outPath, generation, false, true);
return generation;
} }
void deleteOldGenerations(const Path& profile, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
static void removeFile(const Path & path) int curGen;
{ Generations gens = findGenerations(profile, curGen);
if (remove(path.c_str()) == -1)
throw SysError(format("cannot unlink '%1%'") % path); for (auto& i : gens)
if (i.number != curGen) deleteGeneration2(profile, i.number, dryRun);
} }
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
void deleteGeneration(const Path & profile, unsigned int gen) int curGen;
{ Generations gens = findGenerations(profile, curGen);
Path generation;
makeName(profile, gen, generation);
removeFile(generation);
}
bool canDelete = false;
static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) for (auto i = gens.rbegin(); i != gens.rend(); ++i)
{ if (canDelete) {
if (dryRun) assert(i->creationTime < t);
printInfo(format("would remove generation %1%") % gen); if (i->number != curGen) deleteGeneration2(profile, i->number, dryRun);
else { } else if (i->creationTime < t) {
printInfo(format("removing generation %1%") % gen); /* We may now start deleting generations, but we don't
deleteGeneration(profile, gen); delete this generation yet, because this generation was
still the one that was active at the requested point in
time. */
canDelete = true;
} }
} }
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
bool dryRun) {
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
int days;
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) if (!string2Int(strDays, days) || days < 1)
{ throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
PathLocks lock;
lockProfile(lock, profile);
int curGen; time_t oldTime = curTime - days * 24 * 3600;
Generations gens = findGenerations(profile, curGen);
if (gensToDelete.find(curGen) != gensToDelete.end()) deleteGenerationsOlderThan(profile, oldTime, dryRun);
throw Error(format("cannot delete current generation of profile %1%'") % profile);
for (auto & i : gens) {
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
deleteGeneration2(profile, i.number, dryRun);
}
} }
void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) void switchLink(Path link, Path target) {
{ /* Hacky. */
PathLocks lock; if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
lockProfile(lock, profile);
int curGen; replaceSymlink(target, link);
bool fromCurGen = false;
Generations gens = findGenerations(profile, curGen);
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
if (i->number == curGen) {
fromCurGen = true;
max--;
continue;
}
if (fromCurGen) {
if (max) {
max--;
continue;
}
deleteGeneration2(profile, i->number, dryRun);
}
}
} }
void deleteOldGenerations(const Path & profile, bool dryRun) void lockProfile(PathLocks& lock, const Path& profile) {
{ lock.lockPaths({profile},
PathLocks lock; (format("waiting for lock on profile '%1%'") % profile).str());
lockProfile(lock, profile); lock.setDeletion(true);
int curGen;
Generations gens = findGenerations(profile, curGen);
for (auto & i : gens)
if (i.number != curGen)
deleteGeneration2(profile, i.number, dryRun);
} }
string optimisticLockProfile(const Path& profile) {
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) return pathExists(profile) ? readLink(profile) : "";
{
PathLocks lock;
lockProfile(lock, profile);
int curGen;
Generations gens = findGenerations(profile, curGen);
bool canDelete = false;
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
if (canDelete) {
assert(i->creationTime < t);
if (i->number != curGen)
deleteGeneration2(profile, i->number, dryRun);
} else if (i->creationTime < t) {
/* We may now start deleting generations, but we don't
delete this generation yet, because this generation was
still the one that was active at the requested point in
time. */
canDelete = true;
}
} }
} // namespace nix
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
{
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
int days;
if (!string2Int(strDays, days) || days < 1)
throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
time_t oldTime = curTime - days * 24 * 3600;
deleteGenerationsOlderThan(profile, oldTime, dryRun);
}
void switchLink(Path link, Path target)
{
/* Hacky. */
if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
replaceSymlink(target, link);
}
void lockProfile(PathLocks & lock, const Path & profile)
{
lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());
lock.setDeletion(true);
}
string optimisticLockProfile(const Path & profile)
{
return pathExists(profile) ? readLink(profile) : "";
}
}

50
third_party/nix/src/libstore/profiles.hh vendored
View file

@ -1,57 +1,49 @@
#pragma once #pragma once
#include "types.hh"
#include "pathlocks.hh"
#include <time.h> #include <time.h>
#include "pathlocks.hh"
#include "types.hh"
namespace nix { namespace nix {
struct Generation {
struct Generation int number;
{ Path path;
int number; time_t creationTime;
Path path; Generation() { number = -1; }
time_t creationTime; operator bool() const { return number != -1; }
Generation()
{
number = -1;
}
operator bool() const
{
return number != -1;
}
}; };
typedef list<Generation> Generations; typedef list<Generation> Generations;
/* Returns the list of currently present generations for the specified /* Returns the list of currently present generations for the specified
profile, sorted by generation number. */ profile, sorted by generation number. */
Generations findGenerations(Path profile, int & curGen); Generations findGenerations(Path profile, int& curGen);
class LocalFSStore; class LocalFSStore;
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath); Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
void deleteGeneration(const Path & profile, unsigned int gen); void deleteGeneration(const Path& profile, unsigned int gen);
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); void deleteGenerations(const Path& profile,
const std::set<unsigned int>& gensToDelete, bool dryRun);
void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); void deleteGenerationsGreaterThan(const Path& profile, const int max,
bool dryRun);
void deleteOldGenerations(const Path & profile, bool dryRun); void deleteOldGenerations(const Path& profile, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun); void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
bool dryRun);
void switchLink(Path link, Path target); void switchLink(Path link, Path target);
/* Ensure exclusive access to a profile. Any command that modifies /* Ensure exclusive access to a profile. Any command that modifies
the profile first acquires this lock. */ the profile first acquires this lock. */
void lockProfile(PathLocks & lock, const Path & profile); void lockProfile(PathLocks& lock, const Path& profile);
/* Optimistic locking is used by long-running operations like `nix-env /* Optimistic locking is used by long-running operations like `nix-env
-i'. Instead of acquiring the exclusive lock for the entire -i'. Instead of acquiring the exclusive lock for the entire
@ -62,6 +54,6 @@ void lockProfile(PathLocks & lock, const Path & profile);
generally cheap, since the build results are still in the Nix generally cheap, since the build results are still in the Nix
store. Most of the time, only the user environment has to be store. Most of the time, only the user environment has to be
rebuilt. */ rebuilt. */
string optimisticLockProfile(const Path & profile); string optimisticLockProfile(const Path& profile);
} } // namespace nix

174
third_party/nix/src/libstore/references.cc vendored
View file

@ -1,122 +1,110 @@
#include "references.hh" #include "references.hh"
#include <cstdlib>
#include <map>
#include "archive.hh"
#include "hash.hh" #include "hash.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh"
#include <map>
#include <cstdlib>
namespace nix { namespace nix {
static unsigned int refLength = 32; /* characters */ static unsigned int refLength = 32; /* characters */
static void search(const unsigned char* s, size_t len, StringSet& hashes,
StringSet& seen) {
static bool initialised = false;
static bool isBase32[256];
if (!initialised) {
for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
for (unsigned int i = 0; i < base32Chars.size(); ++i)
isBase32[(unsigned char)base32Chars[i]] = true;
initialised = true;
}
static void search(const unsigned char * s, size_t len, for (size_t i = 0; i + refLength <= len;) {
StringSet & hashes, StringSet & seen) int j;
{ bool match = true;
static bool initialised = false; for (j = refLength - 1; j >= 0; --j)
static bool isBase32[256]; if (!isBase32[(unsigned char)s[i + j]]) {
if (!initialised) { i += j + 1;
for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; match = false;
for (unsigned int i = 0; i < base32Chars.size(); ++i) break;
isBase32[(unsigned char) base32Chars[i]] = true; }
initialised = true; if (!match) continue;
} string ref((const char*)s + i, refLength);
if (hashes.find(ref) != hashes.end()) {
for (size_t i = 0; i + refLength <= len; ) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i);
int j; seen.insert(ref);
bool match = true; hashes.erase(ref);
for (j = refLength - 1; j >= 0; --j)
if (!isBase32[(unsigned char) s[i + j]]) {
i += j + 1;
match = false;
break;
}
if (!match) continue;
string ref((const char *) s + i, refLength);
if (hashes.find(ref) != hashes.end()) {
debug(format("found reference to '%1%' at offset '%2%'")
% ref % i);
seen.insert(ref);
hashes.erase(ref);
}
++i;
} }
++i;
}
} }
struct RefScanSink : Sink {
HashSink hashSink;
StringSet hashes;
StringSet seen;
struct RefScanSink : Sink string tail;
{
HashSink hashSink;
StringSet hashes;
StringSet seen;
string tail; RefScanSink() : hashSink(htSHA256) {}
RefScanSink() : hashSink(htSHA256) { } void operator()(const unsigned char* data, size_t len);
void operator () (const unsigned char * data, size_t len);
}; };
void RefScanSink::operator()(const unsigned char* data, size_t len) {
hashSink(data, len);
void RefScanSink::operator () (const unsigned char * data, size_t len) /* It's possible that a reference spans the previous and current
{ fragment, so search in the concatenation of the tail of the
hashSink(data, len); previous fragment and the start of the current fragment. */
string s =
tail + string((const char*)data, len > refLength ? refLength : len);
search((const unsigned char*)s.data(), s.size(), hashes, seen);
/* It's possible that a reference spans the previous and current search(data, len, hashes, seen);
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + string((const char *) data, len > refLength ? refLength : len);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search(data, len, hashes, seen); size_t tailLen = len <= refLength ? len : refLength;
tail = string(tail, tail.size() < refLength - tailLen
size_t tailLen = len <= refLength ? len : refLength; ? 0
tail = : tail.size() - (refLength - tailLen)) +
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + string((const char*)data + len - tailLen, tailLen);
string((const char *) data + len - tailLen, tailLen);
} }
PathSet scanForReferences(const string& path, const PathSet& refs,
HashResult& hash) {
RefScanSink sink;
std::map<string, Path> backMap;
PathSet scanForReferences(const string & path, /* For efficiency (and a higher hit rate), just search for the
const PathSet & refs, HashResult & hash) hash part of the file name. (This assumes that all references
{ have the form `HASH-bla'). */
RefScanSink sink; for (auto& i : refs) {
std::map<string, Path> backMap; string baseName = baseNameOf(i);
string::size_type pos = baseName.find('-');
if (pos == string::npos) throw Error(format("bad reference '%1%'") % i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
sink.hashes.insert(s);
backMap[s] = i;
}
/* For efficiency (and a higher hit rate), just search for the /* Look for the hashes in the NAR dump of the path. */
hash part of the file name. (This assumes that all references dumpPath(path, sink);
have the form `HASH-bla'). */
for (auto & i : refs) {
string baseName = baseNameOf(i);
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error(format("bad reference '%1%'") % i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
sink.hashes.insert(s);
backMap[s] = i;
}
/* Look for the hashes in the NAR dump of the path. */ /* Map the hashes found back to their store paths. */
dumpPath(path, sink); PathSet found;
for (auto& i : sink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
found.insert(j->second);
}
/* Map the hashes found back to their store paths. */ hash = sink.hashSink.finish();
PathSet found;
for (auto & i : sink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
found.insert(j->second);
}
hash = sink.hashSink.finish(); return found;
return found;
} }
} // namespace nix
}

8
third_party/nix/src/libstore/references.hh vendored
View file

@ -1,11 +1,11 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "types.hh"
namespace nix { namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs, PathSet scanForReferences(const Path& path, const PathSet& refs,
HashResult & hash); HashResult& hash);
} }

173
third_party/nix/src/libstore/remote-fs-accessor.cc vendored
View file

@ -1,129 +1,120 @@
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "nar-accessor.hh"
#include "json.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "json.hh"
#include "nar-accessor.hh"
namespace nix { namespace nix {
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir)
: store(store) : store(store), cacheDir(cacheDir) {
, cacheDir(cacheDir) if (cacheDir != "") createDirs(cacheDir);
{
if (cacheDir != "")
createDirs(cacheDir);
} }
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) Path RemoteFSAccessor::makeCacheFile(const Path& storePath,
{ const std::string& ext) {
assert(cacheDir != ""); assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
} }
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar,
ref<FSAccessor> narAccessor) ref<FSAccessor> narAccessor) {
{ nars.emplace(storePath, narAccessor);
nars.emplace(storePath, narAccessor);
if (cacheDir != "") { if (cacheDir != "") {
try { try {
std::ostringstream str; std::ostringstream str;
JSONPlaceholder jsonRoot(str); JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true); listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(storePath, "ls"), str.str()); writeFile(makeCacheFile(storePath, "ls"), str.str());
/* FIXME: do this asynchronously. */ /* FIXME: do this asynchronously. */
writeFile(makeCacheFile(storePath, "nar"), nar); writeFile(makeCacheFile(storePath, "nar"), nar);
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
}
} }
}
} }
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) {
{ auto path = canonPath(path_);
auto path = canonPath(path_);
auto storePath = store->toStorePath(path); auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size()); std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath)) if (!store->isValidPath(storePath))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); throw InvalidPath(format("path '%1%' is not a valid store path") %
storePath);
auto i = nars.find(storePath); auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath}; if (i != nars.end()) return {i->second, restPath};
StringSink sink; StringSink sink;
std::string listing; std::string listing;
Path cacheFile; Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { if (cacheDir != "" &&
pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath, "ls"));
try { auto narAccessor = makeLazyNarAccessor(
listing = nix::readFile(makeCacheFile(storePath, "ls")); listing, [cacheFile](uint64_t offset, uint64_t length) {
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile);
auto narAccessor = makeLazyNarAccessor(listing, if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset)
[cacheFile](uint64_t offset, uint64_t length) { throw SysError("seeking in '%s'", cacheFile);
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); std::string buf(length, 0);
if (!fd) readFull(fd.get(), (unsigned char*)buf.data(), length);
throw SysError("opening NAR cache file '%s'", cacheFile);
if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) return buf;
throw SysError("seeking in '%s'", cacheFile); });
std::string buf(length, 0); nars.emplace(storePath, narAccessor);
readFull(fd.get(), (unsigned char *) buf.data(), length); return {narAccessor, restPath};
return buf; } catch (SysError&) {
});
nars.emplace(storePath, narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
try {
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
} }
store->narFromPath(storePath, sink); try {
auto narAccessor = makeNarAccessor(sink.s); *sink.s = nix::readFile(cacheFile);
addToCache(storePath, *sink.s, narAccessor);
return {narAccessor, restPath}; auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor);
return {narAccessor, restPath};
} catch (SysError&) {
}
}
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor);
return {narAccessor, restPath};
} }
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) {
{ auto res = fetch(path);
auto res = fetch(path); return res.first->stat(res.second);
return res.first->stat(res.second);
} }
StringSet RemoteFSAccessor::readDirectory(const Path & path) StringSet RemoteFSAccessor::readDirectory(const Path& path) {
{ auto res = fetch(path);
auto res = fetch(path); return res.first->readDirectory(res.second);
return res.first->readDirectory(res.second);
} }
std::string RemoteFSAccessor::readFile(const Path & path) std::string RemoteFSAccessor::readFile(const Path& path) {
{ auto res = fetch(path);
auto res = fetch(path); return res.first->readFile(res.second);
return res.first->readFile(res.second);
} }
std::string RemoteFSAccessor::readLink(const Path & path) std::string RemoteFSAccessor::readLink(const Path& path) {
{ auto res = fetch(path);
auto res = fetch(path); return res.first->readLink(res.second);
return res.first->readLink(res.second);
} }
} } // namespace nix

36
third_party/nix/src/libstore/remote-fs-accessor.hh vendored
View file

@ -6,35 +6,33 @@
namespace nix { namespace nix {
class RemoteFSAccessor : public FSAccessor class RemoteFSAccessor : public FSAccessor {
{ ref<Store> store;
ref<Store> store;
std::map<Path, ref<FSAccessor>> nars; std::map<Path, ref<FSAccessor>> nars;
Path cacheDir; Path cacheDir;
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); std::pair<ref<FSAccessor>, Path> fetch(const Path& path_);
friend class BinaryCacheStore; friend class BinaryCacheStore;
Path makeCacheFile(const Path & storePath, const std::string & ext); Path makeCacheFile(const Path& storePath, const std::string& ext);
void addToCache(const Path & storePath, const std::string & nar, void addToCache(const Path& storePath, const std::string& nar,
ref<FSAccessor> narAccessor); ref<FSAccessor> narAccessor);
public: public:
RemoteFSAccessor(ref<Store> store,
const /* FIXME: use std::optional */ Path& cacheDir = "");
RemoteFSAccessor(ref<Store> store, Stat stat(const Path& path) override;
const /* FIXME: use std::optional */ Path & cacheDir = "");
Stat stat(const Path & path) override; StringSet readDirectory(const Path& path) override;
StringSet readDirectory(const Path & path) override; std::string readFile(const Path& path) override;
std::string readFile(const Path & path) override; std::string readLink(const Path& path) override;
std::string readLink(const Path & path) override;
}; };
} } // namespace nix

1352
third_party/nix/src/libstore/remote-store.cc vendored
View file

File diff suppressed because it is too large Load diff

179
third_party/nix/src/libstore/remote-store.hh vendored
View file

@ -2,160 +2,151 @@
#include <limits> #include <limits>
#include <string> #include <string>
#include "store-api.hh" #include "store-api.hh"
namespace nix { namespace nix {
class Pipe; class Pipe;
class Pid; class Pid;
struct FdSink; struct FdSink;
struct FdSource; struct FdSource;
template<typename T> class Pool; template <typename T>
class Pool;
struct ConnectionHandle; struct ConnectionHandle;
/* FIXME: RemoteStore is a misnomer - should be something like /* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */ DaemonStore. */
class RemoteStore : public virtual Store class RemoteStore : public virtual Store {
{ public:
public: const Setting<int> maxConnections{
(Store*)this, 1, "max-connections",
"maximum number of concurrent connections to the Nix daemon"};
const Setting<int> maxConnections{(Store*) this, 1, const Setting<unsigned int> maxConnectionAge{
"max-connections", "maximum number of concurrent connections to the Nix daemon"}; (Store*)this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), virtual bool sameMachine() = 0;
"max-connection-age", "number of seconds to reuse a connection"};
virtual bool sameMachine() = 0; RemoteStore(const Params& params);
RemoteStore(const Params & params); /* Implementations of abstract store API methods. */
/* Implementations of abstract store API methods. */ bool isValidPathUncached(const Path& path) override;
bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
NoSubstitute) override;
PathSet queryValidPaths(const PathSet & paths, PathSet queryAllValidPaths() override;
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryAllValidPaths() override; void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryPathInfoUncached(const Path & path, void queryReferrers(const Path& path, PathSet& referrers) override;
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path & path, PathSet & referrers) override; PathSet queryValidDerivers(const Path& path) override;
PathSet queryValidDerivers(const Path & path) override; PathSet queryDerivationOutputs(const Path& path) override;
PathSet queryDerivationOutputs(const Path & path) override; StringSet queryDerivationOutputNames(const Path& path) override;
StringSet queryDerivationOutputNames(const Path & path) override; Path queryPathFromHashPart(const string& hashPart) override;
Path queryPathFromHashPart(const string & hashPart) override; PathSet querySubstitutablePaths(const PathSet& paths) override;
PathSet querySubstitutablePaths(const PathSet & paths) override; void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override;
void querySubstitutablePathInfos(const PathSet & paths, void addToStore(const ValidPathInfo& info, Source& nar, RepairFlag repair,
SubstitutablePathInfos & infos) override; CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
void addToStore(const ValidPathInfo & info, Source & nar, Path addToStore(const string& name, const Path& srcPath,
RepairFlag repair, CheckSigsFlag checkSigs, bool recursive = true, HashType hashAlgo = htSHA256,
std::shared_ptr<FSAccessor> accessor) override; PathFilter& filter = defaultPathFilter,
RepairFlag repair = NoRepair) override;
Path addToStore(const string & name, const Path & srcPath, Path addTextToStore(const string& name, const string& s,
bool recursive = true, HashType hashAlgo = htSHA256, const PathSet& references, RepairFlag repair) override;
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
Path addTextToStore(const string & name, const string & s, void buildPaths(const PathSet& paths, BuildMode buildMode) override;
const PathSet & references, RepairFlag repair) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override; BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, void ensurePath(const Path& path) override;
BuildMode buildMode) override;
void ensurePath(const Path & path) override; void addTempRoot(const Path& path) override;
void addTempRoot(const Path & path) override; void addIndirectRoot(const Path& path) override;
void addIndirectRoot(const Path & path) override; void syncWithGC() override;
void syncWithGC() override; Roots findRoots(bool censor) override;
Roots findRoots(bool censor) override; void collectGarbage(const GCOptions& options, GCResults& results) override;
void collectGarbage(const GCOptions & options, GCResults & results) override; void optimiseStore() override;
void optimiseStore() override; bool verifyStore(bool checkContents, RepairFlag repair) override;
bool verifyStore(bool checkContents, RepairFlag repair) override; void addSignatures(const Path& storePath, const StringSet& sigs) override;
void addSignatures(const Path & storePath, const StringSet & sigs) override; void queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet& willSubstitute, PathSet& unknown,
unsigned long long& downloadSize,
unsigned long long& narSize) override;
void queryMissing(const PathSet & targets, void connect() override;
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override;
void connect() override; unsigned int getProtocol() override;
unsigned int getProtocol() override; void flushBadConnections();
void flushBadConnections(); protected:
struct Connection {
AutoCloseFD fd;
FdSink to;
FdSource from;
unsigned int daemonVersion;
std::chrono::time_point<std::chrono::steady_clock> startTime;
protected: virtual ~Connection();
struct Connection std::exception_ptr processStderr(Sink* sink = 0, Source* source = 0);
{ };
AutoCloseFD fd;
FdSink to;
FdSource from;
unsigned int daemonVersion;
std::chrono::time_point<std::chrono::steady_clock> startTime;
virtual ~Connection(); ref<Connection> openConnectionWrapper();
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); virtual ref<Connection> openConnection() = 0;
};
ref<Connection> openConnectionWrapper(); void initConnection(Connection& conn);
virtual ref<Connection> openConnection() = 0; ref<Pool<Connection>> connections;
void initConnection(Connection & conn); virtual void setOptions(Connection& conn);
ref<Pool<Connection>> connections; ConnectionHandle getConnection();
virtual void setOptions(Connection & conn); friend struct ConnectionHandle;
ConnectionHandle getConnection();
friend struct ConnectionHandle;
private:
std::atomic_bool failed{false};
private:
std::atomic_bool failed{false};
}; };
class UDSRemoteStore : public LocalFSStore, public RemoteStore class UDSRemoteStore : public LocalFSStore, public RemoteStore {
{ public:
public: UDSRemoteStore(const Params& params);
UDSRemoteStore(std::string path, const Params& params);
UDSRemoteStore(const Params & params); std::string getUri() override;
UDSRemoteStore(std::string path, const Params & params);
std::string getUri() override; bool sameMachine() { return true; }
bool sameMachine() private:
{ return true; } ref<RemoteStore::Connection> openConnection() override;
std::optional<std::string> path;
private:
ref<RemoteStore::Connection> openConnection() override;
std::optional<std::string> path;
}; };
} // namespace nix
}

687
third_party/nix/src/libstore/s3-binary-cache-store.cc vendored
View file

@ -1,14 +1,6 @@
#if ENABLE_S3 #if ENABLE_S3
#include "s3.hh"
#include "s3-binary-cache-store.hh" #include "s3-binary-cache-store.hh"
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h> #include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h> #include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h> #include <aws/core/auth/AWSCredentialsProvider.h>
@ -24,408 +16,403 @@
#include <aws/s3/model/ListObjectsRequest.h> #include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h> #include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h> #include <aws/transfer/TransferManager.h>
#include "compression.hh"
#include "download.hh"
#include "globals.hh"
#include "istringstream_nocopy.hh"
#include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include "s3.hh"
using namespace Aws::Transfer; using namespace Aws::Transfer;
namespace nix { namespace nix {
struct S3Error : public Error struct S3Error : public Error {
{ Aws::S3::S3Errors err;
Aws::S3::S3Errors err; S3Error(Aws::S3::S3Errors err, const FormatOrString& fs)
S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) : Error(fs), err(err){};
: Error(fs), err(err) { };
}; };
/* Helper: given an Outcome<R, E>, return R in case of success, or /* Helper: given an Outcome<R, E>, return R in case of success, or
throw an exception in case of an error. */ throw an exception in case of an error. */
template<typename R, typename E> template <typename R, typename E>
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) {
{ if (!outcome.IsSuccess())
if (!outcome.IsSuccess()) throw S3Error(outcome.GetError().GetErrorType(),
throw S3Error( fs.s + ": " + outcome.GetError().GetMessage());
outcome.GetError().GetErrorType(), return outcome.GetResultWithOwnership();
fs.s + ": " + outcome.GetError().GetMessage());
return outcome.GetResultWithOwnership();
} }
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem {
{ using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
void ProcessFormattedStatement(Aws::String && statement) override void ProcessFormattedStatement(Aws::String&& statement) override {
{ debug("AWS: %s", chomp(statement));
debug("AWS: %s", chomp(statement)); }
}
}; };
static void initAWS() static void initAWS() {
{ static std::once_flag flag;
static std::once_flag flag; std::call_once(flag, []() {
std::call_once(flag, []() { Aws::SDKOptions options;
Aws::SDKOptions options;
/* We install our own OpenSSL locking function (see /* We install our own OpenSSL locking function (see
shared.cc), so don't let aws-sdk-cpp override it. */ shared.cc), so don't let aws-sdk-cpp override it. */
options.cryptoOptions.initAndCleanupOpenSSL = false; options.cryptoOptions.initAndCleanupOpenSSL = false;
if (verbosity >= lvlDebug) { if (verbosity >= lvlDebug) {
options.loggingOptions.logLevel = options.loggingOptions.logLevel =
verbosity == lvlDebug verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug
? Aws::Utils::Logging::LogLevel::Debug : Aws::Utils::Logging::LogLevel::Trace;
: Aws::Utils::Logging::LogLevel::Trace; options.loggingOptions.logger_create_fn = [options]() {
options.loggingOptions.logger_create_fn = [options]() { return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); };
}; }
}
Aws::InitAPI(options); Aws::InitAPI(options);
}); });
} }
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) S3Helper::S3Helper(const string& profile, const string& region,
: config(makeConfig(region, scheme, endpoint)) const string& scheme, const string& endpoint)
, client(make_ref<Aws::S3::S3Client>( : config(makeConfig(region, scheme, endpoint)),
profile == "" client(make_ref<Aws::S3::S3Client>(
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( profile == ""
std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>()) ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
: std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( std::make_shared<
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())), Aws::Auth::DefaultAWSCredentialsProviderChain>())
*config, : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 std::make_shared<
Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(
profile.c_str())),
*config,
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 #if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
false, false,
#else #else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif #endif
endpoint.empty())) endpoint.empty())) {
{
} }
/* Log AWS retries. */ /* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy class RetryStrategy : public Aws::Client::DefaultRetryStrategy {
{ bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override long attemptedRetries) const override {
{ auto retry =
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
if (retry) if (retry)
printError("AWS error '%s' (%s), will retry in %d ms", printError("AWS error '%s' (%s), will retry in %d ms",
error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); error.GetExceptionName(), error.GetMessage(),
return retry; CalculateDelayBeforeNextRetry(error, attemptedRetries));
} return retry;
}
}; };
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
{ const string& region, const string& scheme, const string& endpoint) {
initAWS(); initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>(); auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = region; res->region = region;
if (!scheme.empty()) { if (!scheme.empty()) {
res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str());
} }
if (!endpoint.empty()) { if (!endpoint.empty()) {
res->endpointOverride = endpoint; res->endpointOverride = endpoint;
} }
res->requestTimeoutMs = 600 * 1000; res->requestTimeoutMs = 600 * 1000;
res->connectTimeoutMs = 5 * 1000; res->connectTimeoutMs = 5 * 1000;
res->retryStrategy = std::make_shared<RetryStrategy>(); res->retryStrategy = std::make_shared<RetryStrategy>();
res->caFile = settings.caFile; res->caFile = settings.caFile;
return res; return res;
} }
S3Helper::DownloadResult S3Helper::getObject( S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName,
const std::string & bucketName, const std::string & key) const std::string& key) {
{ debug("fetching 's3://%s/%s'...", bucketName, key);
debug("fetching 's3://%s/%s'...", bucketName, key);
auto request = auto request =
Aws::S3::Model::GetObjectRequest() Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key);
.WithBucket(bucketName)
.WithKey(key);
request.SetResponseStreamFactory([&]() { request.SetResponseStreamFactory(
return Aws::New<std::stringstream>("STRINGSTREAM"); [&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); });
DownloadResult res;
auto now1 = std::chrono::steady_clock::now();
try {
auto result = checkAws(fmt("AWS error fetching '%s'", key),
client->GetObject(request));
res.data =
decompress(result.GetContentEncoding(),
dynamic_cast<std::stringstream&>(result.GetBody()).str());
} catch (S3Error& e) {
if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
}
auto now2 = std::chrono::steady_clock::now();
res.durationMs =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
return res;
}
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore {
const Setting<std::string> profile{
this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{
this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{
this, "", "scheme",
"The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{
this, "", "endpoint",
"An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression",
"compression method for .ls files"};
const Setting<std::string> logCompression{
this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{this, false, "multipart-upload",
"whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size",
"size (in bytes) of each part in multi-part uploads"};
std::string bucketName;
Stats stats;
S3Helper s3Helper;
S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName)
: S3BinaryCacheStore(params),
bucketName(bucketName),
s3Helper(profile, region, scheme, endpoint) {
diskCache = getNarInfoDiskCache();
}
std::string getUri() override { return "s3://" + bucketName; }
void init() override {
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
BinaryCacheStore::init();
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
}
}
const Stats& getS3Stats() override { return stats; }
/* This is a specialisation of isValidPath() that optimistically
fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing
a GET is unlikely to be slower than HEAD. */
bool isValidPathUncached(const Path& storePath) override {
try {
queryPathInfo(storePath);
return true;
} catch (InvalidPath& e) {
return false;
}
}
bool fileExists(const std::string& path) override {
stats.head++;
auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest()
.WithBucket(bucketName)
.WithKey(path));
if (!res.IsSuccess()) {
auto& error = res.GetError();
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND ||
error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
// If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
return false;
throw Error(format("AWS error fetching '%s': %s") % path %
error.GetMessage());
}
return true;
}
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string& path, const std::string& data,
const std::string& mimeType,
const std::string& contentEncoding) {
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor =
std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(
maxThreads);
std::call_once(transferManagerCreated, [&]() {
if (multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get());
transferConfig.s3Client = s3Helper.client;
transferConfig.bufferSize = bufferSize;
transferConfig.uploadProgressCallback =
[](const TransferManager* transferManager,
const std::shared_ptr<const TransferHandle>& transferHandle) {
// FIXME: find a way to properly abort the multipart upload.
// checkInterrupt();
debug("upload progress ('%s'): '%d' of '%d' bytes",
transferHandle->GetKey(),
transferHandle->GetBytesTransferred(),
transferHandle->GetBytesTotalSize());
};
transferManager = TransferManager::Create(transferConfig);
}
}); });
DownloadResult res;
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
try { if (transferManager) {
if (contentEncoding != "")
throw Error(
"setting a content encoding is not supported with S3 multi-part "
"uploads");
auto result = checkAws(fmt("AWS error fetching '%s'", key), std::shared_ptr<TransferHandle> transferHandle =
client->GetObject(request)); transferManager->UploadFile(stream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
res.data = decompress(result.GetContentEncoding(), transferHandle->WaitUntilFinished();
dynamic_cast<std::stringstream &>(result.GetBody()).str());
} catch (S3Error & e) { if (transferHandle->GetStatus() == TransferStatus::FAILED)
if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName,
path, transferHandle->GetLastError().GetMessage());
if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
throw Error(
"AWS error: transfer status of 's3://%s/%s' in unexpected state",
bucketName, path);
} else {
auto request = Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);
request.SetContentType(mimeType);
if (contentEncoding != "") request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
} }
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
return res; printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
} bucketName % path % data.size() % duration);
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore stats.putTimeMs += duration;
{ stats.putBytes += data.size();
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; stats.put++;
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; }
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{
this, false, "multipart-upload", "whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
std::string bucketName; void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) override {
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType,
narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType,
logCompression);
else
uploadFile(path, data, mimeType, "");
}
Stats stats; void getFile(const std::string& path, Sink& sink) override {
stats.get++;
S3Helper s3Helper; // FIXME: stream output to sink.
auto res = s3Helper.getObject(bucketName, path);
S3BinaryCacheStoreImpl( stats.getBytes += res.data ? res.data->size() : 0;
const Params & params, const std::string & bucketName) stats.getTimeMs += res.durationMs;
: S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint)
{
diskCache = getNarInfoDiskCache();
}
std::string getUri() override if (res.data) {
{ printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName,
return "s3://" + bucketName; path, res.data->size(), res.durationMs);
}
void init() override sink((unsigned char*)res.data->data(), res.data->size());
{ } else
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri());
}
BinaryCacheStore::init(); PathSet queryAllValidPaths() override {
PathSet paths;
std::string marker;
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); do {
} debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName %
} marker);
const Stats & getS3Stats() override auto res = checkAws(
{ format("AWS error listing bucket '%s'") % bucketName,
return stats; s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest()
} .WithBucket(bucketName)
.WithDelimiter("/")
.WithMarker(marker)));
/* This is a specialisation of isValidPath() that optimistically auto& contents = res.GetContents();
fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing
a GET is unlikely to be slower than HEAD. */
bool isValidPathUncached(const Path & storePath) override
{
try {
queryPathInfo(storePath);
return true;
} catch (InvalidPath & e) {
return false;
}
}
bool fileExists(const std::string & path) override debug(format("got %d keys, next marker '%s'") % contents.size() %
{ res.GetNextMarker());
stats.head++;
auto res = s3Helper.client->HeadObject( for (auto object : contents) {
Aws::S3::Model::HeadObjectRequest() auto& key = object.GetKey();
.WithBucket(bucketName) if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
.WithKey(path)); paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
}
if (!res.IsSuccess()) { marker = res.GetNextMarker();
auto & error = res.GetError(); } while (!marker.empty());
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
// If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
return false;
throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
}
return true;
}
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data,
const std::string & mimeType,
const std::string & contentEncoding)
{
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
std::call_once(transferManagerCreated, [&]()
{
if (multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get());
transferConfig.s3Client = s3Helper.client;
transferConfig.bufferSize = bufferSize;
transferConfig.uploadProgressCallback =
[](const TransferManager *transferManager,
const std::shared_ptr<const TransferHandle>
&transferHandle)
{
//FIXME: find a way to properly abort the multipart upload.
//checkInterrupt();
debug("upload progress ('%s'): '%d' of '%d' bytes",
transferHandle->GetKey(),
transferHandle->GetBytesTransferred(),
transferHandle->GetBytesTotalSize());
};
transferManager = TransferManager::Create(transferConfig);
}
});
auto now1 = std::chrono::steady_clock::now();
if (transferManager) {
if (contentEncoding != "")
throw Error("setting a content encoding is not supported with S3 multi-part uploads");
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
stream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
transferHandle->WaitUntilFinished();
if (transferHandle->GetStatus() == TransferStatus::FAILED)
throw Error("AWS error: failed to upload 's3://%s/%s': %s",
bucketName, path, transferHandle->GetLastError().GetMessage());
if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
bucketName, path);
} else {
auto request =
Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);
request.SetContentType(mimeType);
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
}
auto now2 = std::chrono::steady_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
bucketName % path % data.size() % duration);
stats.putTimeMs += duration;
stats.putBytes += data.size();
stats.put++;
}
void upsertFile(const std::string & path, const std::string & data,
const std::string & mimeType) override
{
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
else
uploadFile(path, data, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
{
stats.get++;
// FIXME: stream output to sink.
auto res = s3Helper.getObject(bucketName, path);
stats.getBytes += res.data ? res.data->size() : 0;
stats.getTimeMs += res.durationMs;
if (res.data) {
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
sink((unsigned char *) res.data->data(), res.data->size());
} else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
}
PathSet queryAllValidPaths() override
{
PathSet paths;
std::string marker;
do {
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);
auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
s3Helper.client->ListObjects(
Aws::S3::Model::ListObjectsRequest()
.WithBucket(bucketName)
.WithDelimiter("/")
.WithMarker(marker)));
auto & contents = res.GetContents();
debug(format("got %d keys, next marker '%s'")
% contents.size() % res.GetNextMarker());
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
}
marker = res.GetNextMarker();
} while (!marker.empty());
return paths;
}
return paths;
}
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{ if (std::string(uri, 0, 5) != "s3://") return 0;
if (std::string(uri, 0, 5) != "s3://") return 0; auto store =
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
store->init(); store->init();
return store; return store;
}); });
} } // namespace nix
#endif #endif

39
third_party/nix/src/libstore/s3-binary-cache-store.hh vendored
View file

@ -1,33 +1,26 @@
#pragma once #pragma once
#include "binary-cache-store.hh"
#include <atomic> #include <atomic>
#include "binary-cache-store.hh"
namespace nix { namespace nix {
class S3BinaryCacheStore : public BinaryCacheStore class S3BinaryCacheStore : public BinaryCacheStore {
{ protected:
protected: S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {}
S3BinaryCacheStore(const Params & params) public:
: BinaryCacheStore(params) struct Stats {
{ } std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};
std::atomic<uint64_t> getTimeMs{0};
std::atomic<uint64_t> head{0};
};
public: virtual const Stats& getS3Stats() = 0;
struct Stats
{
std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};
std::atomic<uint64_t> getTimeMs{0};
std::atomic<uint64_t> head{0};
};
virtual const Stats & getS3Stats() = 0;
}; };
} } // namespace nix

41
third_party/nix/src/libstore/s3.hh vendored
View file

@ -4,30 +4,39 @@
#include "ref.hh" #include "ref.hh"
namespace Aws { namespace Client { class ClientConfiguration; } } namespace Aws {
namespace Aws { namespace S3 { class S3Client; } } namespace Client {
class ClientConfiguration;
}
} // namespace Aws
namespace Aws {
namespace S3 {
class S3Client;
}
} // namespace Aws
namespace nix { namespace nix {
struct S3Helper struct S3Helper {
{ ref<Aws::Client::ClientConfiguration> config;
ref<Aws::Client::ClientConfiguration> config; ref<Aws::S3::S3Client> client;
ref<Aws::S3::S3Client> client;
S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); S3Helper(const std::string& profile, const std::string& region,
const std::string& scheme, const std::string& endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region,
const std::string& scheme,
const std::string& endpoint);
struct DownloadResult struct DownloadResult {
{ std::shared_ptr<std::string> data;
std::shared_ptr<std::string> data; unsigned int durationMs;
unsigned int durationMs; };
};
DownloadResult getObject( DownloadResult getObject(const std::string& bucketName,
const std::string & bucketName, const std::string & key); const std::string& key);
}; };
} } // namespace nix
#endif #endif

24
third_party/nix/src/libstore/serve-protocol.hh vendored
View file

@ -6,19 +6,19 @@ namespace nix {
#define SERVE_MAGIC_2 0x5452eecb #define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION 0x205 #define SERVE_PROTOCOL_VERSION 0x205
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
typedef enum { typedef enum {
cmdQueryValidPaths = 1, cmdQueryValidPaths = 1,
cmdQueryPathInfos = 2, cmdQueryPathInfos = 2,
cmdDumpStorePath = 3, cmdDumpStorePath = 3,
cmdImportPaths = 4, cmdImportPaths = 4,
cmdExportPaths = 5, cmdExportPaths = 5,
cmdBuildPaths = 6, cmdBuildPaths = 6,
cmdQueryClosure = 7, cmdQueryClosure = 7,
cmdBuildDerivation = 8, cmdBuildDerivation = 8,
cmdAddToStoreNar = 9, cmdAddToStoreNar = 9,
} ServeCommand; } ServeCommand;
} } // namespace nix

274
third_party/nix/src/libstore/sqlite.cc vendored
View file

@ -1,198 +1,172 @@
#include "sqlite.hh" #include "sqlite.hh"
#include "util.hh"
#include <sqlite3.h> #include <sqlite3.h>
#include <atomic> #include <atomic>
#include "util.hh"
namespace nix { namespace nix {
[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) [[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) {
{ int err = sqlite3_errcode(db);
int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db);
int exterr = sqlite3_extended_errcode(db);
auto path = sqlite3_db_filename(db, nullptr); auto path = sqlite3_db_filename(db, nullptr);
if (!path) path = "(in-memory)"; if (!path) path = "(in-memory)";
if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
throw SQLiteBusy( throw SQLiteBusy(
err == SQLITE_PROTOCOL err == SQLITE_PROTOCOL
? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
: fmt("SQLite database '%s' is busy", path)); : fmt("SQLite database '%s' is busy", path));
} } else
else throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
} }
SQLite::SQLite(const Path & path) SQLite::SQLite(const Path& path) {
{ if (sqlite3_open_v2(path.c_str(), &db,
if (sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path); throw Error(format("cannot open SQLite database '%s'") % path);
} }
SQLite::~SQLite() SQLite::~SQLite() {
{ try {
try { if (db && sqlite3_close(db) != SQLITE_OK)
if (db && sqlite3_close(db) != SQLITE_OK) throwSQLiteError(db, "closing database");
throwSQLiteError(db, "closing database"); } catch (...) {
} catch (...) { ignoreException();
ignoreException(); }
}
} }
void SQLite::exec(const std::string & stmt) void SQLite::exec(const std::string& stmt) {
{ retrySQLite<void>([&]() {
retrySQLite<void>([&]() { if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); });
});
} }
void SQLiteStmt::create(sqlite3 * db, const string & sql) void SQLiteStmt::create(sqlite3* db, const string& sql) {
{ checkInterrupt();
checkInterrupt(); assert(!stmt);
assert(!stmt); if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) throwSQLiteError(db, fmt("creating statement '%s'", sql));
throwSQLiteError(db, fmt("creating statement '%s'", sql)); this->db = db;
this->db = db; this->sql = sql;
this->sql = sql;
} }
SQLiteStmt::~SQLiteStmt() SQLiteStmt::~SQLiteStmt() {
{ try {
try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); } catch (...) {
} catch (...) { ignoreException();
ignoreException(); }
}
} }
SQLiteStmt::Use::Use(SQLiteStmt & stmt) SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) {
: stmt(stmt) assert(stmt.stmt);
{ /* Note: sqlite3_reset() returns the error code for the most
assert(stmt.stmt); recent call to sqlite3_step(). So ignore it. */
/* Note: sqlite3_reset() returns the error code for the most sqlite3_reset(stmt);
recent call to sqlite3_step(). So ignore it. */
sqlite3_reset(stmt);
} }
SQLiteStmt::Use::~Use() SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); }
{
sqlite3_reset(stmt); SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value,
bool notNull) {
if (notNull) {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1,
SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) {
{ if (notNull) {
if (notNull) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument");
throwSQLiteError(stmt.db, "binding argument"); } else
} else bind();
bind(); return *this;
return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) SQLiteStmt::Use& SQLiteStmt::Use::bind() {
{ if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
if (notNull) { throwSQLiteError(stmt.db, "binding argument");
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) return *this;
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::bind() int SQLiteStmt::Use::step() { return sqlite3_step(stmt); }
{
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) void SQLiteStmt::Use::exec() {
throwSQLiteError(stmt.db, "binding argument"); int r = step();
return *this; assert(r != SQLITE_ROW);
if (r != SQLITE_DONE)
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
} }
int SQLiteStmt::Use::step() bool SQLiteStmt::Use::next() {
{ int r = step();
return sqlite3_step(stmt); if (r != SQLITE_DONE && r != SQLITE_ROW)
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
return r == SQLITE_ROW;
} }
void SQLiteStmt::Use::exec() std::string SQLiteStmt::Use::getStr(int col) {
{ auto s = (const char*)sqlite3_column_text(stmt, col);
int r = step(); assert(s);
assert(r != SQLITE_ROW); return s;
if (r != SQLITE_DONE)
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
} }
bool SQLiteStmt::Use::next() int64_t SQLiteStmt::Use::getInt(int col) {
{ // FIXME: detect nulls?
int r = step(); return sqlite3_column_int64(stmt, col);
if (r != SQLITE_DONE && r != SQLITE_ROW)
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
return r == SQLITE_ROW;
} }
std::string SQLiteStmt::Use::getStr(int col) bool SQLiteStmt::Use::isNull(int col) {
{ return sqlite3_column_type(stmt, col) == SQLITE_NULL;
auto s = (const char *) sqlite3_column_text(stmt, col);
assert(s);
return s;
} }
int64_t SQLiteStmt::Use::getInt(int col) SQLiteTxn::SQLiteTxn(sqlite3* db) {
{ this->db = db;
// FIXME: detect nulls? if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
return sqlite3_column_int64(stmt, col); throwSQLiteError(db, "starting transaction");
active = true;
} }
bool SQLiteStmt::Use::isNull(int col) void SQLiteTxn::commit() {
{ if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
return sqlite3_column_type(stmt, col) == SQLITE_NULL; throwSQLiteError(db, "committing transaction");
active = false;
} }
SQLiteTxn::SQLiteTxn(sqlite3 * db) SQLiteTxn::~SQLiteTxn() {
{ try {
this->db = db; if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "aborting transaction");
throwSQLiteError(db, "starting transaction"); } catch (...) {
active = true; ignoreException();
}
} }
void SQLiteTxn::commit() void handleSQLiteBusy(const SQLiteBusy& e) {
{ static std::atomic<time_t> lastWarned{0};
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "committing transaction"); time_t now = time(0);
active = false;
if (now > lastWarned + 10) {
lastWarned = now;
printError("warning: %s", e.what());
}
/* Sleep for a while since retrying the transaction right away
is likely to fail again. */
checkInterrupt();
struct timespec t;
t.tv_sec = 0;
t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
nanosleep(&t, 0);
} }
SQLiteTxn::~SQLiteTxn() } // namespace nix
{
try {
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "aborting transaction");
} catch (...) {
ignoreException();
}
}
void handleSQLiteBusy(const SQLiteBusy & e)
{
static std::atomic<time_t> lastWarned{0};
time_t now = time(0);
if (now > lastWarned + 10) {
lastWarned = now;
printError("warning: %s", e.what());
}
/* Sleep for a while since retrying the transaction right away
is likely to fail again. */
checkInterrupt();
struct timespec t;
t.tv_sec = 0;
t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
nanosleep(&t, 0);
}
}

134
third_party/nix/src/libstore/sqlite.hh vendored
View file

@ -2,7 +2,6 @@
#include <functional> #include <functional>
#include <string> #include <string>
#include "types.hh" #include "types.hh"
class sqlite3; class sqlite3;
@ -11,104 +10,99 @@ class sqlite3_stmt;
namespace nix { namespace nix {
/* RAII wrapper to close a SQLite database automatically. */ /* RAII wrapper to close a SQLite database automatically. */
struct SQLite struct SQLite {
{ sqlite3* db = 0;
sqlite3 * db = 0; SQLite() {}
SQLite() { } SQLite(const Path& path);
SQLite(const Path & path); SQLite(const SQLite& from) = delete;
SQLite(const SQLite & from) = delete; SQLite& operator=(const SQLite& from) = delete;
SQLite& operator = (const SQLite & from) = delete; SQLite& operator=(SQLite&& from) {
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } db = from.db;
~SQLite(); from.db = 0;
operator sqlite3 * () { return db; } return *this;
}
~SQLite();
operator sqlite3*() { return db; }
void exec(const std::string & stmt); void exec(const std::string& stmt);
}; };
/* RAII wrapper to create and destroy SQLite prepared statements. */ /* RAII wrapper to create and destroy SQLite prepared statements. */
struct SQLiteStmt struct SQLiteStmt {
{ sqlite3* db = 0;
sqlite3 * db = 0; sqlite3_stmt* stmt = 0;
sqlite3_stmt * stmt = 0; std::string sql;
std::string sql; SQLiteStmt() {}
SQLiteStmt() { } SQLiteStmt(sqlite3* db, const std::string& sql) { create(db, sql); }
SQLiteStmt(sqlite3 * db, const std::string & sql) { create(db, sql); } void create(sqlite3* db, const std::string& s);
void create(sqlite3 * db, const std::string & s); ~SQLiteStmt();
~SQLiteStmt(); operator sqlite3_stmt*() { return stmt; }
operator sqlite3_stmt * () { return stmt; }
/* Helper for binding / executing statements. */ /* Helper for binding / executing statements. */
class Use class Use {
{ friend struct SQLiteStmt;
friend struct SQLiteStmt;
private:
SQLiteStmt & stmt;
unsigned int curArg = 1;
Use(SQLiteStmt & stmt);
public: private:
SQLiteStmt& stmt;
unsigned int curArg = 1;
Use(SQLiteStmt& stmt);
~Use(); public:
~Use();
/* Bind the next parameter. */ /* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true); Use& operator()(const std::string& value, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true); Use& operator()(int64_t value, bool notNull = true);
Use & bind(); // null Use& bind(); // null
int step(); int step();
/* Execute a statement that does not return rows. */ /* Execute a statement that does not return rows. */
void exec(); void exec();
/* For statements that return 0 or more rows. Returns true iff /* For statements that return 0 or more rows. Returns true iff
a row is available. */ a row is available. */
bool next(); bool next();
std::string getStr(int col); std::string getStr(int col);
int64_t getInt(int col); int64_t getInt(int col);
bool isNull(int col); bool isNull(int col);
}; };
Use use() Use use() { return Use(*this); }
{
return Use(*this);
}
}; };
/* RAII helper that ensures transactions are aborted unless explicitly /* RAII helper that ensures transactions are aborted unless explicitly
committed. */ committed. */
struct SQLiteTxn struct SQLiteTxn {
{ bool active = false;
bool active = false; sqlite3* db;
sqlite3 * db;
SQLiteTxn(sqlite3 * db); SQLiteTxn(sqlite3* db);
void commit(); void commit();
~SQLiteTxn(); ~SQLiteTxn();
}; };
MakeError(SQLiteError, Error); MakeError(SQLiteError, Error);
MakeError(SQLiteBusy, SQLiteError); MakeError(SQLiteBusy, SQLiteError);
[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); [[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs);
void handleSQLiteBusy(const SQLiteBusy & e); void handleSQLiteBusy(const SQLiteBusy& e);
/* Convenience function for retrying a SQLite transaction when the /* Convenience function for retrying a SQLite transaction when the
database is busy. */ database is busy. */
template<typename T> template <typename T>
T retrySQLite(std::function<T()> fun) T retrySQLite(std::function<T()> fun) {
{ while (true) {
while (true) { try {
try { return fun();
return fun(); } catch (SQLiteBusy& e) {
} catch (SQLiteBusy & e) { handleSQLiteBusy(e);
handleSQLiteBusy(e);
}
} }
}
} }
} } // namespace nix

128
third_party/nix/src/libstore/ssh-store.cc vendored
View file

@ -1,100 +1,84 @@
#include "store-api.hh"
#include "remote-store.hh"
#include "remote-fs-accessor.hh"
#include "archive.hh" #include "archive.hh"
#include "worker-protocol.hh"
#include "pool.hh" #include "pool.hh"
#include "remote-fs-accessor.hh"
#include "remote-store.hh"
#include "ssh.hh" #include "ssh.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
static std::string uriScheme = "ssh-ng://"; static std::string uriScheme = "ssh-ng://";
class SSHStore : public RemoteStore class SSHStore : public RemoteStore {
{ public:
public: const Setting<Path> sshKey{(Store*)this, "", "ssh-key",
"path to an SSH private key"};
const Setting<bool> compress{(Store*)this, false, "compress",
"whether to compress the connection"};
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; SSHStore(const std::string& host, const Params& params)
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; : Store(params),
RemoteStore(params),
host(host),
master(host, sshKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1, compress) {}
SSHStore(const std::string & host, const Params & params) std::string getUri() override { return uriScheme + host; }
: Store(params)
, RemoteStore(params)
, host(host)
, master(
host,
sshKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress)
{
}
std::string getUri() override bool sameMachine() { return false; }
{
return uriScheme + host;
}
bool sameMachine() void narFromPath(const Path& path, Sink& sink) override;
{ return false; }
void narFromPath(const Path & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override;
ref<FSAccessor> getFSAccessor() override; private:
struct Connection : RemoteStore::Connection {
std::unique_ptr<SSHMaster::Connection> sshConn;
};
private: ref<RemoteStore::Connection> openConnection() override;
struct Connection : RemoteStore::Connection std::string host;
{
std::unique_ptr<SSHMaster::Connection> sshConn;
};
ref<RemoteStore::Connection> openConnection() override; SSHMaster master;
std::string host; void setOptions(RemoteStore::Connection& conn) override{
/* TODO Add a way to explicitly ask for some options to be
SSHMaster master; forwarded. One option: A way to query the daemon for its
settings, and then a series of params to SSHStore like
void setOptions(RemoteStore::Connection & conn) override forward-cores or forward-overridden-cores that only
{ override the requested settings.
/* TODO Add a way to explicitly ask for some options to be */
forwarded. One option: A way to query the daemon for its };
settings, and then a series of params to SSHStore like
forward-cores or forward-overridden-cores that only
override the requested settings.
*/
};
}; };
void SSHStore::narFromPath(const Path & path, Sink & sink) void SSHStore::narFromPath(const Path& path, Sink& sink) {
{ auto conn(connections->get());
auto conn(connections->get()); conn->to << wopNarFromPath << path;
conn->to << wopNarFromPath << path; conn->processStderr();
conn->processStderr(); copyNAR(conn->from, sink);
copyNAR(conn->from, sink);
} }
ref<FSAccessor> SSHStore::getFSAccessor() ref<FSAccessor> SSHStore::getFSAccessor() {
{ return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
} }
ref<RemoteStore::Connection> SSHStore::openConnection() ref<RemoteStore::Connection> SSHStore::openConnection() {
{ auto conn = make_ref<Connection>();
auto conn = make_ref<Connection>(); conn->sshConn = master.startCommand("nix-daemon --stdio");
conn->sshConn = master.startCommand("nix-daemon --stdio"); conn->to = FdSink(conn->sshConn->in.get());
conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get());
conn->from = FdSource(conn->sshConn->out.get()); initConnection(*conn);
initConnection(*conn); return conn;
return conn;
} }
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore([](const std::string& uri,
const std::string & uri, const Store::Params & params) const Store::Params& params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store> {
{ if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
}); });
} } // namespace nix

148
third_party/nix/src/libstore/ssh.cc vendored
View file

@ -2,64 +2,60 @@
namespace nix { namespace nix {
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) SSHMaster::SSHMaster(const std::string& host, const std::string& keyFile,
: host(host) bool useMaster, bool compress, int logFD)
, fakeSSH(host == "localhost") : host(host),
, keyFile(keyFile) fakeSSH(host == "localhost"),
, useMaster(useMaster && !fakeSSH) keyFile(keyFile),
, compress(compress) useMaster(useMaster && !fakeSSH),
, logFD(logFD) compress(compress),
{ logFD(logFD) {
if (host == "" || hasPrefix(host, "-")) if (host == "" || hasPrefix(host, "-"))
throw Error("invalid SSH host name '%s'", host); throw Error("invalid SSH host name '%s'", host);
} }
void SSHMaster::addCommonSSHOpts(Strings & args) void SSHMaster::addCommonSSHOpts(Strings& args) {
{ for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) args.push_back(i);
args.push_back(i); if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile});
if (!keyFile.empty()) if (compress) args.push_back("-C");
args.insert(args.end(), {"-i", keyFile});
if (compress)
args.push_back("-C");
} }
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
{ const std::string& command) {
Path socketPath = startMaster(); Path socketPath = startMaster();
Pipe in, out; Pipe in, out;
in.create(); in.create();
out.create(); out.create();
auto conn = std::make_unique<Connection>(); auto conn = std::make_unique<Connection>();
ProcessOptions options; ProcessOptions options;
options.dieWithParent = false; options.dieWithParent = false;
conn->sshPid = startProcess([&]() { conn->sshPid = startProcess(
[&]() {
restoreSignals(); restoreSignals();
close(in.writeSide.get()); close(in.writeSide.get());
close(out.readSide.get()); close(out.readSide.get());
if (dup2(in.readSide.get(), STDIN_FILENO) == -1) if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
throw SysError("duping over stdin"); throw SysError("duping over stdin");
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("duping over stdout"); throw SysError("duping over stdout");
if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1)
throw SysError("duping over stderr"); throw SysError("duping over stderr");
Strings args; Strings args;
if (fakeSSH) { if (fakeSSH) {
args = { "bash", "-c" }; args = {"bash", "-c"};
} else { } else {
args = { "ssh", host.c_str(), "-x", "-a" }; args = {"ssh", host.c_str(), "-x", "-a"};
addCommonSSHOpts(args); addCommonSSHOpts(args);
if (socketPath != "") if (socketPath != "") args.insert(args.end(), {"-S", socketPath});
args.insert(args.end(), {"-S", socketPath}); if (verbosity >= lvlChatty) args.push_back("-v");
if (verbosity >= lvlChatty)
args.push_back("-v");
} }
args.push_back(command); args.push_back(command);
@ -67,68 +63,70 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
// could not exec ssh/bash // could not exec ssh/bash
throw SysError("unable to execute '%s'", args.front()); throw SysError("unable to execute '%s'", args.front());
}, options); },
options);
in.readSide = -1;
out.writeSide = -1;
in.readSide = -1; conn->out = std::move(out.readSide);
out.writeSide = -1; conn->in = std::move(in.writeSide);
conn->out = std::move(out.readSide); return conn;
conn->in = std::move(in.writeSide);
return conn;
} }
Path SSHMaster::startMaster() Path SSHMaster::startMaster() {
{ if (!useMaster) return "";
if (!useMaster) return "";
auto state(state_.lock()); auto state(state_.lock());
if (state->sshMaster != -1) return state->socketPath; if (state->sshMaster != -1) return state->socketPath;
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700)); state->tmpDir =
std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; state->socketPath = (Path)*state->tmpDir + "/ssh.sock";
Pipe out; Pipe out;
out.create(); out.create();
ProcessOptions options; ProcessOptions options;
options.dieWithParent = false; options.dieWithParent = false;
state->sshMaster = startProcess([&]() { state->sshMaster = startProcess(
[&]() {
restoreSignals(); restoreSignals();
close(out.readSide.get()); close(out.readSide.get());
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("duping over stdout"); throw SysError("duping over stdout");
Strings args = Strings args = {"ssh", host.c_str(),
{ "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath "-M", "-N",
, "-o", "LocalCommand=echo started" "-S", state->socketPath,
, "-o", "PermitLocalCommand=yes" "-o", "LocalCommand=echo started",
}; "-o", "PermitLocalCommand=yes"};
if (verbosity >= lvlChatty) if (verbosity >= lvlChatty) args.push_back("-v");
args.push_back("-v");
addCommonSSHOpts(args); addCommonSSHOpts(args);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
throw SysError("unable to execute '%s'", args.front()); throw SysError("unable to execute '%s'", args.front());
}, options); },
options);
out.writeSide = -1; out.writeSide = -1;
std::string reply; std::string reply;
try { try {
reply = readLine(out.readSide.get()); reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { } } catch (EndOfFile& e) {
}
if (reply != "started") if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host); throw Error("failed to start SSH master connection to '%s'", host);
return state->socketPath; return state->socketPath;
} }
} } // namespace nix

56
third_party/nix/src/libstore/ssh.hh vendored
View file

@ -1,45 +1,41 @@
#pragma once #pragma once
#include "util.hh"
#include "sync.hh" #include "sync.hh"
#include "util.hh"
namespace nix { namespace nix {
class SSHMaster class SSHMaster {
{ private:
private: const std::string host;
bool fakeSSH;
const std::string keyFile;
const bool useMaster;
const bool compress;
const int logFD;
const std::string host; struct State {
bool fakeSSH; Pid sshMaster;
const std::string keyFile; std::unique_ptr<AutoDelete> tmpDir;
const bool useMaster; Path socketPath;
const bool compress; };
const int logFD;
struct State Sync<State> state_;
{
Pid sshMaster;
std::unique_ptr<AutoDelete> tmpDir;
Path socketPath;
};
Sync<State> state_; void addCommonSSHOpts(Strings& args);
void addCommonSSHOpts(Strings & args); public:
SSHMaster(const std::string& host, const std::string& keyFile, bool useMaster,
bool compress, int logFD = -1);
public: struct Connection {
Pid sshPid;
AutoCloseFD out, in;
};
SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); std::unique_ptr<Connection> startCommand(const std::string& command);
struct Connection Path startMaster();
{
Pid sshPid;
AutoCloseFD out, in;
};
std::unique_ptr<Connection> startCommand(const std::string & command);
Path startMaster();
}; };
} } // namespace nix

1375
third_party/nix/src/libstore/store-api.cc vendored
View file

File diff suppressed because it is too large Load diff

1246
third_party/nix/src/libstore/store-api.hh vendored
View file

File diff suppressed because it is too large Load diff

102
third_party/nix/src/libstore/worker-protocol.hh vendored
View file

@ -2,68 +2,64 @@
namespace nix { namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x115 #define PROTOCOL_VERSION 0x115
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
typedef enum { typedef enum {
wopIsValidPath = 1, wopIsValidPath = 1,
wopHasSubstitutes = 3, wopHasSubstitutes = 3,
wopQueryPathHash = 4, // obsolete wopQueryPathHash = 4, // obsolete
wopQueryReferences = 5, // obsolete wopQueryReferences = 5, // obsolete
wopQueryReferrers = 6, wopQueryReferrers = 6,
wopAddToStore = 7, wopAddToStore = 7,
wopAddTextToStore = 8, wopAddTextToStore = 8,
wopBuildPaths = 9, wopBuildPaths = 9,
wopEnsurePath = 10, wopEnsurePath = 10,
wopAddTempRoot = 11, wopAddTempRoot = 11,
wopAddIndirectRoot = 12, wopAddIndirectRoot = 12,
wopSyncWithGC = 13, wopSyncWithGC = 13,
wopFindRoots = 14, wopFindRoots = 14,
wopExportPath = 16, // obsolete wopExportPath = 16, // obsolete
wopQueryDeriver = 18, // obsolete wopQueryDeriver = 18, // obsolete
wopSetOptions = 19, wopSetOptions = 19,
wopCollectGarbage = 20, wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21, wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22, wopQueryDerivationOutputs = 22,
wopQueryAllValidPaths = 23, wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24, wopQueryFailedPaths = 24,
wopClearFailedPaths = 25, wopClearFailedPaths = 25,
wopQueryPathInfo = 26, wopQueryPathInfo = 26,
wopImportPaths = 27, // obsolete wopImportPaths = 27, // obsolete
wopQueryDerivationOutputNames = 28, wopQueryDerivationOutputNames = 28,
wopQueryPathFromHashPart = 29, wopQueryPathFromHashPart = 29,
wopQuerySubstitutablePathInfos = 30, wopQuerySubstitutablePathInfos = 30,
wopQueryValidPaths = 31, wopQueryValidPaths = 31,
wopQuerySubstitutablePaths = 32, wopQuerySubstitutablePaths = 32,
wopQueryValidDerivers = 33, wopQueryValidDerivers = 33,
wopOptimiseStore = 34, wopOptimiseStore = 34,
wopVerifyStore = 35, wopVerifyStore = 35,
wopBuildDerivation = 36, wopBuildDerivation = 36,
wopAddSignatures = 37, wopAddSignatures = 37,
wopNarFromPath = 38, wopNarFromPath = 38,
wopAddToStoreNar = 39, wopAddToStoreNar = 39,
wopQueryMissing = 40, wopQueryMissing = 40,
} WorkerOp; } WorkerOp;
#define STDERR_NEXT 0x6f6c6d67
#define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source
#define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink
#define STDERR_WRITE 0x64617416 // data for sink #define STDERR_LAST 0x616c7473
#define STDERR_LAST 0x616c7473
#define STDERR_ERROR 0x63787470 #define STDERR_ERROR 0x63787470
#define STDERR_START_ACTIVITY 0x53545254 #define STDERR_START_ACTIVITY 0x53545254
#define STDERR_STOP_ACTIVITY 0x53544f50 #define STDERR_STOP_ACTIVITY 0x53544f50
#define STDERR_RESULT 0x52534c54 #define STDERR_RESULT 0x52534c54
Path readStorePath(Store& store, Source& from);
template <class T>
T readStorePaths(Store& store, Source& from);
Path readStorePath(Store & store, Source & from); } // namespace nix
template<class T> T readStorePaths(Store & store, Source & from);
}

48
third_party/nix/src/libutil/affinity.cc vendored
View file

@ -1,6 +1,6 @@
#include "affinity.hh"
#include "types.hh" #include "types.hh"
#include "util.hh" #include "util.hh"
#include "affinity.hh"
#if __linux__ #if __linux__
#include <sched.h> #include <sched.h>
@ -8,48 +8,40 @@
namespace nix { namespace nix {
#if __linux__ #if __linux__
static bool didSaveAffinity = false; static bool didSaveAffinity = false;
static cpu_set_t savedAffinity; static cpu_set_t savedAffinity;
#endif #endif
void setAffinityTo(int cpu) {
void setAffinityTo(int cpu)
{
#if __linux__ #if __linux__
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true; didSaveAffinity = true;
debug(format("locking this thread to CPU %1%") % cpu); debug(format("locking this thread to CPU %1%") % cpu);
cpu_set_t newAffinity; cpu_set_t newAffinity;
CPU_ZERO(&newAffinity); CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity); CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printError(format("failed to lock thread to CPU %1%") % cpu); printError(format("failed to lock thread to CPU %1%") % cpu);
#endif #endif
} }
int lockToCurrentCPU() {
int lockToCurrentCPU()
{
#if __linux__ #if __linux__
int cpu = sched_getcpu(); int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu); if (cpu != -1) setAffinityTo(cpu);
return cpu; return cpu;
#else #else
return -1; return -1;
#endif #endif
} }
void restoreAffinity() {
void restoreAffinity()
{
#if __linux__ #if __linux__
if (!didSaveAffinity) return; if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
printError("failed to restore affinity %1%"); printError("failed to restore affinity %1%");
#endif #endif
} }
} // namespace nix
}

2
third_party/nix/src/libutil/affinity.hh vendored
View file

@ -6,4 +6,4 @@ void setAffinityTo(int cpu);
int lockToCurrentCPU(); int lockToCurrentCPU();
void restoreAffinity(); void restoreAffinity();
} } // namespace nix

550
third_party/nix/src/libutil/archive.cc vendored
View file

@ -1,32 +1,31 @@
#include <cerrno> #include "archive.hh"
#include <algorithm>
#include <vector>
#include <map>
#include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h> #include <dirent.h>
#include <fcntl.h> #include <fcntl.h>
#include <strings.h> // for strcasecmp
#include "archive.hh" #include <sys/stat.h>
#include "util.hh" #include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <cerrno>
#include <map>
#include <vector>
#include "config.hh" #include "config.hh"
#include "util.hh"
namespace nix { namespace nix {
struct ArchiveSettings : Config struct ArchiveSettings : Config {
{ Setting<bool> useCaseHack {
Setting<bool> useCaseHack{this, this,
#if __APPLE__ #if __APPLE__
true, true,
#else #else
false, false,
#endif #endif
"use-case-hack", "use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."}; "Whether to enable a Darwin-specific hack for dealing with file name "
"collisions."
};
}; };
static ArchiveSettings archiveSettings; static ArchiveSettings archiveSettings;
@ -37,105 +36,105 @@ const std::string narVersionMagic1 = "nix-archive-1";
static string caseHackSuffix = "~nix~case~hack~"; static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; }; PathFilter defaultPathFilter = [](const Path&) { return true; };
static void dumpContents(const Path& path, size_t size, Sink& sink) {
sink << "contents" << size;
static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
Sink & sink) if (!fd) throw SysError(format("opening file '%1%'") % path);
{
sink << "contents" << size;
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); std::vector<unsigned char> buf(65536);
if (!fd) throw SysError(format("opening file '%1%'") % path); size_t left = size;
std::vector<unsigned char> buf(65536); while (left > 0) {
size_t left = size; auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink(buf.data(), n);
}
while (left > 0) { writePadding(size, sink);
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink(buf.data(), n);
}
writePadding(size, sink);
} }
static void dump(const Path& path, Sink& sink, PathFilter& filter) {
checkInterrupt();
static void dump(const Path & path, Sink & sink, PathFilter & filter) struct stat st;
{ if (lstat(path.c_str(), &st))
checkInterrupt(); throw SysError(format("getting attributes of path '%1%'") % path);
struct stat st; sink << "(";
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
sink << "("; if (S_ISREG(st.st_mode)) {
sink << "type"
<< "regular";
if (st.st_mode & S_IXUSR)
sink << "executable"
<< "";
dumpContents(path, (size_t)st.st_size, sink);
}
if (S_ISREG(st.st_mode)) { else if (S_ISDIR(st.st_mode)) {
sink << "type" << "regular"; sink << "type"
if (st.st_mode & S_IXUSR) << "directory";
sink << "executable" << "";
dumpContents(path, (size_t) st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) { /* If we're on a case-insensitive system like macOS, undo
sink << "type" << "directory"; the case hack applied by restorePath(). */
std::map<string, string> unhacked;
for (auto& i : readDirectory(path))
if (archiveSettings.useCaseHack) {
string name(i.name);
size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) {
debug(format("removing case hack suffix from '%1%'") %
(path + "/" + i.name));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'") %
(path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
/* If we're on a case-insensitive system like macOS, undo for (auto& i : unhacked)
the case hack applied by restorePath(). */ if (filter(path + "/" + i.first)) {
std::map<string, string> unhacked; sink << "entry"
for (auto & i : readDirectory(path)) << "("
if (archiveSettings.useCaseHack) { << "name" << i.first << "node";
string name(i.name); dump(path + "/" + i.second, sink, filter);
size_t pos = i.name.find(caseHackSuffix); sink << ")";
if (pos != string::npos) { }
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); }
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'")
% (path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
for (auto & i : unhacked) else if (S_ISLNK(st.st_mode))
if (filter(path + "/" + i.first)) { sink << "type"
sink << "entry" << "(" << "name" << i.first << "node"; << "symlink"
dump(path + "/" + i.second, sink, filter); << "target" << readLink(path);
sink << ")";
}
}
else if (S_ISLNK(st.st_mode)) else
sink << "type" << "symlink" << "target" << readLink(path); throw Error(format("file '%1%' has an unsupported type") % path);
else throw Error(format("file '%1%' has an unsupported type") % path); sink << ")";
sink << ")";
} }
void dumpPath(const Path& path, Sink& sink, PathFilter& filter) {
void dumpPath(const Path & path, Sink & sink, PathFilter & filter) sink << narVersionMagic1;
{ dump(path, sink, filter);
sink << narVersionMagic1;
dump(path, sink, filter);
} }
void dumpString(const std::string& s, Sink& sink) {
void dumpString(const std::string & s, Sink & sink) sink << narVersionMagic1 << "("
{ << "type"
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; << "regular"
<< "contents" << s << ")";
} }
static SerialisationError badArchive(string s) {
static SerialisationError badArchive(string s) return SerialisationError("bad archive: " + s);
{
return SerialisationError("bad archive: " + s);
} }
#if 0 #if 0
static void skipGeneric(Source & source) static void skipGeneric(Source & source)
{ {
@ -146,233 +145,212 @@ static void skipGeneric(Source & source)
} }
#endif #endif
static void parseContents(ParseSink& sink, Source& source, const Path& path) {
unsigned long long size = readLongLong(source);
static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size);
{
unsigned long long size = readLongLong(source);
sink.preallocateContents(size); unsigned long long left = size;
std::vector<unsigned char> buf(65536);
unsigned long long left = size; while (left) {
std::vector<unsigned char> buf(65536); checkInterrupt();
auto n = buf.size();
if ((unsigned long long)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
}
while (left) { readPadding(size, source);
checkInterrupt();
auto n = buf.size();
if ((unsigned long long)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
}
readPadding(size, source);
} }
struct CaseInsensitiveCompare {
struct CaseInsensitiveCompare bool operator()(const string& a, const string& b) const {
{ return strcasecmp(a.c_str(), b.c_str()) < 0;
bool operator() (const string & a, const string & b) const }
{
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
}; };
static void parse(ParseSink& sink, Source& source, const Path& path) {
string s;
static void parse(ParseSink & sink, Source & source, const Path & path) s = readString(source);
{ if (s != "(") throw badArchive("expected open tag");
string s;
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
std::map<Path, int, CaseInsensitiveCompare> names;
while (1) {
checkInterrupt();
s = readString(source); s = readString(source);
if (s != "(") throw badArchive("expected open tag");
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; if (s == ")") {
break;
}
std::map<Path, int, CaseInsensitiveCompare> names; else if (s == "type") {
if (type != tpUnknown) throw badArchive("multiple type fields");
string t = readString(source);
while (1) { if (t == "regular") {
type = tpRegular;
sink.createRegularFile(path);
}
else if (t == "directory") {
sink.createDirectory(path);
type = tpDirectory;
}
else if (t == "symlink") {
type = tpSymlink;
}
else
throw badArchive("unknown file type " + t);
}
else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path);
}
else if (s == "executable" && type == tpRegular) {
auto s = readString(source);
if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
else if (s == "entry" && type == tpDirectory) {
string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
while (1) {
checkInterrupt(); checkInterrupt();
s = readString(source); s = readString(source);
if (s == ")") { if (s == ")") {
break; break;
} } else if (s == "name") {
name = readString(source);
else if (s == "type") { if (name.empty() || name == "." || name == ".." ||
if (type != tpUnknown) name.find('/') != string::npos ||
throw badArchive("multiple type fields"); name.find((char)0) != string::npos)
string t = readString(source); throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName) throw Error("NAR directory is not sorted");
if (t == "regular") { prevName = name;
type = tpRegular; if (archiveSettings.useCaseHack) {
sink.createRegularFile(path); auto i = names.find(name);
} if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") %
else if (t == "directory") { i->first % name);
sink.createDirectory(path); name += caseHackSuffix;
type = tpDirectory; name += std::to_string(++i->second);
} } else
names[name] = 0;
else if (t == "symlink") { }
type = tpSymlink; } else if (s == "node") {
} if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
else throw badArchive("unknown file type " + t); } else
throw badArchive("unknown field " + s);
} }
else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path);
}
else if (s == "executable" && type == tpRegular) {
auto s = readString(source);
if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
else if (s == "entry" && type == tpDirectory) {
string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
while (1) {
checkInterrupt();
s = readString(source);
if (s == ")") {
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") % i->first % name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}
else if (s == "target" && type == tpSymlink) {
string target = readString(source);
sink.createSymlink(path, target);
}
else
throw badArchive("unknown field " + s);
} }
else if (s == "target" && type == tpSymlink) {
string target = readString(source);
sink.createSymlink(path, target);
}
else
throw badArchive("unknown field " + s);
}
} }
void parseDump(ParseSink& sink, Source& source) {
void parseDump(ParseSink & sink, Source & source) string version;
{ try {
string version; version = readString(source, narVersionMagic1.size());
try { } catch (SerialisationError& e) {
version = readString(source, narVersionMagic1.size()); /* This generally means the integer at the start couldn't be
} catch (SerialisationError & e) { decoded. Ignore and throw the exception below. */
/* This generally means the integer at the start couldn't be }
decoded. Ignore and throw the exception below. */ if (version != narVersionMagic1)
} throw badArchive("input doesn't look like a Nix archive");
if (version != narVersionMagic1) parse(sink, source, "");
throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, "");
} }
struct RestoreSink : ParseSink {
Path dstPath;
AutoCloseFD fd;
struct RestoreSink : ParseSink void createDirectory(const Path& path) {
{ Path p = dstPath + path;
Path dstPath; if (mkdir(p.c_str(), 0777) == -1)
AutoCloseFD fd; throw SysError(format("creating directory '%1%'") % p);
};
void createDirectory(const Path & path) void createRegularFile(const Path& path) {
{ Path p = dstPath + path;
Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (mkdir(p.c_str(), 0777) == -1) if (!fd) throw SysError(format("creating file '%1%'") % p);
throw SysError(format("creating directory '%1%'") % p); }
};
void createRegularFile(const Path & path) void isExecutable() {
{ struct stat st;
Path p = dstPath + path; if (fstat(fd.get(), &st) == -1) throw SysError("fstat");
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
if (!fd) throw SysError(format("creating file '%1%'") % p); throw SysError("fchmod");
} }
void isExecutable() void preallocateContents(unsigned long long len) {
{
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("fstat");
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
throw SysError("fchmod");
}
void preallocateContents(unsigned long long len)
{
#if HAVE_POSIX_FALLOCATE #if HAVE_POSIX_FALLOCATE
if (len) { if (len) {
errno = posix_fallocate(fd.get(), 0, len); errno = posix_fallocate(fd.get(), 0, len);
/* Note that EINVAL may indicate that the underlying /* Note that EINVAL may indicate that the underlying
filesystem doesn't support preallocation (e.g. on filesystem doesn't support preallocation (e.g. on
OpenSolaris). Since preallocation is just an OpenSolaris). Since preallocation is just an
optimisation, ignore it. */ optimisation, ignore it. */
if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
throw SysError(format("preallocating file of %1% bytes") % len); throw SysError(format("preallocating file of %1% bytes") % len);
} }
#endif #endif
} }
void receiveContents(unsigned char * data, unsigned int len) void receiveContents(unsigned char* data, unsigned int len) {
{ writeFull(fd.get(), data, len);
writeFull(fd.get(), data, len); }
}
void createSymlink(const Path & path, const string & target) void createSymlink(const Path& path, const string& target) {
{ Path p = dstPath + path;
Path p = dstPath + path; nix::createSymlink(target, p);
nix::createSymlink(target, p); }
}
}; };
void restorePath(const Path& path, Source& source) {
void restorePath(const Path & path, Source & source) RestoreSink sink;
{ sink.dstPath = path;
RestoreSink sink; parseDump(sink, source);
sink.dstPath = path;
parseDump(sink, source);
} }
void copyNAR(Source& source, Sink& sink) {
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
void copyNAR(Source & source, Sink & sink) ParseSink parseSink; /* null sink; just parse the NAR */
{
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
ParseSink parseSink; /* null sink; just parse the NAR */ LambdaSource wrapper([&](unsigned char* data, size_t len) {
auto n = source.read(data, len);
sink(data, n);
return n;
});
LambdaSource wrapper([&](unsigned char * data, size_t len) { parseDump(parseSink, wrapper);
auto n = source.read(data, len);
sink(data, n);
return n;
});
parseDump(parseSink, wrapper);
} }
} // namespace nix
}

43
third_party/nix/src/libutil/archive.hh vendored
View file

@ -1,12 +1,10 @@
#pragma once #pragma once
#include "types.hh"
#include "serialise.hh" #include "serialise.hh"
#include "types.hh"
namespace nix { namespace nix {
/* dumpPath creates a Nix archive of the specified path. The format /* dumpPath creates a Nix archive of the specified path. The format
is as follows: is as follows:
@ -44,41 +42,36 @@ namespace nix {
`+' denotes string concatenation. */ `+' denotes string concatenation. */
void dumpPath(const Path& path, Sink& sink,
PathFilter& filter = defaultPathFilter);
void dumpPath(const Path & path, Sink & sink, void dumpString(const std::string& s, Sink& sink);
PathFilter & filter = defaultPathFilter);
void dumpString(const std::string & s, Sink & sink);
/* FIXME: fix this API, it sucks. */ /* FIXME: fix this API, it sucks. */
struct ParseSink struct ParseSink {
{ virtual void createDirectory(const Path& path){};
virtual void createDirectory(const Path & path) { };
virtual void createRegularFile(const Path & path) { }; virtual void createRegularFile(const Path& path){};
virtual void isExecutable() { }; virtual void isExecutable(){};
virtual void preallocateContents(unsigned long long size) { }; virtual void preallocateContents(unsigned long long size){};
virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void receiveContents(unsigned char* data, unsigned int len){};
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path& path, const string& target){};
}; };
struct TeeSink : ParseSink struct TeeSink : ParseSink {
{ TeeSource source;
TeeSource source;
TeeSink(Source & source) : source(source) { } TeeSink(Source& source) : source(source) {}
}; };
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink& sink, Source& source);
void restorePath(const Path & path, Source & source); void restorePath(const Path& path, Source& source);
/* Read a NAR from 'source' and write it to 'sink'. */ /* Read a NAR from 'source' and write it to 'sink'. */
void copyNAR(Source & source, Sink & sink); void copyNAR(Source& source, Sink& sink);
extern const std::string narVersionMagic1; extern const std::string narVersionMagic1;
} // namespace nix
}

325
third_party/nix/src/libutil/args.cc vendored
View file

@ -3,201 +3,184 @@
namespace nix { namespace nix {
Args::FlagMaker Args::mkFlag() Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); }
{
return FlagMaker(*this); Args::FlagMaker::~FlagMaker() {
assert(flag->longName != "");
args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
} }
Args::FlagMaker::~FlagMaker() void Args::parseCmdline(const Strings& _cmdline) {
{ Strings pendingArgs;
assert(flag->longName != ""); bool dashDash = false;
args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
}
void Args::parseCmdline(const Strings & _cmdline) Strings cmdline(_cmdline);
{
Strings pendingArgs;
bool dashDash = false;
Strings cmdline(_cmdline); for (auto pos = cmdline.begin(); pos != cmdline.end();) {
auto arg = *pos;
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */
auto arg = *pos; if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' &&
isalpha(arg[1])) {
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', *pos = (string) "-" + arg[1];
`-j3` -> `-j 3`). */ auto next = pos;
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { ++next;
*pos = (string) "-" + arg[1]; for (unsigned int j = 2; j < arg.length(); j++)
auto next = pos; ++next; if (isalpha(arg[j]))
for (unsigned int j = 2; j < arg.length(); j++) cmdline.insert(next, (string) "-" + arg[j]);
if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]);
else {
cmdline.insert(next, string(arg, j));
break;
}
arg = *pos;
}
if (!dashDash && arg == "--") {
dashDash = true;
++pos;
}
else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg);
}
else { else {
pendingArgs.push_back(*pos++); cmdline.insert(next, string(arg, j));
if (processArgs(pendingArgs, false)) break;
pendingArgs.clear();
} }
arg = *pos;
} }
processArgs(pendingArgs, true); if (!dashDash && arg == "--") {
dashDash = true;
++pos;
} else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg);
} else {
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false)) pendingArgs.clear();
}
}
processArgs(pendingArgs, true);
} }
void Args::printHelp(const string & programName, std::ostream & out) void Args::printHelp(const string& programName, std::ostream& out) {
{ std::cout << "Usage: " << programName << " <FLAGS>...";
std::cout << "Usage: " << programName << " <FLAGS>..."; for (auto& exp : expectedArgs) {
for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label});
std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1
// FIXME: handle arity > 1 if (exp.arity == 0) std::cout << "...";
if (exp.arity == 0) std::cout << "..."; if (exp.optional) std::cout << "?";
if (exp.optional) std::cout << "?"; }
} std::cout << "\n";
auto s = description();
if (s != "") std::cout << "\nSummary: " << s << ".\n";
if (longFlags.size()) {
std::cout << "\n"; std::cout << "\n";
std::cout << "Flags:\n";
printFlags(out);
}
}
auto s = description(); void Args::printFlags(std::ostream& out) {
if (s != "") Table2 table;
std::cout << "\nSummary: " << s << ".\n"; for (auto& flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair(
(flag.second->shortName
? std::string("-") + flag.second->shortName + ", "
: " ") +
"--" + flag.first + renderLabels(flag.second->labels),
flag.second->description));
}
printTable(out, table);
}
if (longFlags.size()) { bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) {
std::cout << "\n"; assert(pos != end);
std::cout << "Flags:\n";
printFlags(out); auto process = [&](const std::string& name, const Flag& flag) -> bool {
++pos;
std::vector<std::string> args;
for (size_t n = 0; n < flag.arity; ++n) {
if (pos == end) {
if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)") % name %
flag.arity);
}
args.push_back(*pos++);
} }
flag.handler(std::move(args));
return true;
};
if (string(*pos, 0, 2) == "--") {
auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
}
if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
auto c = (*pos)[1];
auto i = shortFlags.find(c);
if (i == shortFlags.end()) return false;
return process(std::string("-") + c, *i->second);
}
return false;
} }
void Args::printFlags(std::ostream & out) bool Args::processArgs(const Strings& args, bool finish) {
{ if (expectedArgs.empty()) {
Table2 table; if (!args.empty())
for (auto & flag : longFlags) { throw UsageError(format("unexpected argument '%1%'") % args.front());
if (hiddenCategories.count(flag.second->category)) continue; return true;
table.push_back(std::make_pair( }
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
+ "--" + flag.first + renderLabels(flag.second->labels), auto& exp = expectedArgs.front();
flag.second->description));
} bool res = false;
printTable(out, table);
if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity)) {
std::vector<std::string> ss;
for (auto& s : args) ss.push_back(s);
exp.handler(std::move(ss));
expectedArgs.pop_front();
res = true;
}
if (finish && !expectedArgs.empty() && !expectedArgs.front().optional)
throw UsageError("more arguments are required");
return res;
} }
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) {
{ arity(1);
assert(pos != end); label("type");
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
auto process = [&](const std::string & name, const Flag & flag) -> bool { handler([ht](std::string s) {
++pos; *ht = parseHashType(s);
std::vector<std::string> args; if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s);
for (size_t n = 0 ; n < flag.arity; ++n) { });
if (pos == end) { return *this;
if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)")
% name % flag.arity);
}
args.push_back(*pos++);
}
flag.handler(std::move(args));
return true;
};
if (string(*pos, 0, 2) == "--") {
auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
}
if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
auto c = (*pos)[1];
auto i = shortFlags.find(c);
if (i == shortFlags.end()) return false;
return process(std::string("-") + c, *i->second);
}
return false;
} }
bool Args::processArgs(const Strings & args, bool finish) Strings argvToStrings(int argc, char** argv) {
{ Strings args;
if (expectedArgs.empty()) { argc--;
if (!args.empty()) argv++;
throw UsageError(format("unexpected argument '%1%'") % args.front()); while (argc--) args.push_back(*argv++);
return true; return args;
}
auto & exp = expectedArgs.front();
bool res = false;
if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity))
{
std::vector<std::string> ss;
for (auto & s : args) ss.push_back(s);
exp.handler(std::move(ss));
expectedArgs.pop_front();
res = true;
}
if (finish && !expectedArgs.empty() && !expectedArgs.front().optional)
throw UsageError("more arguments are required");
return res;
} }
Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) std::string renderLabels(const Strings& labels) {
{ std::string res;
arity(1); for (auto label : labels) {
label("type"); for (auto& c : label) c = std::toupper(c);
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); res += " <" + label + ">";
handler([ht](std::string s) { }
*ht = parseHashType(s); return res;
if (*ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
});
return *this;
} }
Strings argvToStrings(int argc, char * * argv) void printTable(std::ostream& out, const Table2& table) {
{ size_t max = 0;
Strings args; for (auto& row : table) max = std::max(max, row.first.size());
argc--; argv++; for (auto& row : table) {
while (argc--) args.push_back(*argv++); out << " " << row.first << std::string(max - row.first.size() + 2, ' ')
return args; << row.second << "\n";
}
} }
std::string renderLabels(const Strings & labels) } // namespace nix
{
std::string res;
for (auto label : labels) {
for (auto & c : label) c = std::toupper(c);
res += " <" + label + ">";
}
return res;
}
void printTable(std::ostream & out, const Table2 & table)
{
size_t max = 0;
for (auto & row : table)
max = std::max(max, row.first.size());
for (auto & row : table) {
out << " " << row.first
<< std::string(max - row.first.size() + 2, ' ')
<< row.second << "\n";
}
}
}

331
third_party/nix/src/libutil/args.hh vendored
View file

@ -3,7 +3,6 @@
#include <iostream> #include <iostream>
#include <map> #include <map>
#include <memory> #include <memory>
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
@ -12,190 +11,206 @@ MakeError(UsageError, Error);
enum HashType : char; enum HashType : char;
class Args class Args {
{ public:
public: /* Parse the command line, throwing a UsageError if something goes
wrong. */
void parseCmdline(const Strings& cmdline);
/* Parse the command line, throwing a UsageError if something goes virtual void printHelp(const string& programName, std::ostream& out);
wrong. */
void parseCmdline(const Strings & cmdline);
virtual void printHelp(const string & programName, std::ostream & out); virtual std::string description() { return ""; }
virtual std::string description() { return ""; } protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max();
protected: /* Flags. */
struct Flag {
typedef std::shared_ptr<Flag> ptr;
std::string longName;
char shortName = 0;
std::string description;
Strings labels;
size_t arity = 0;
std::function<void(std::vector<std::string>)> handler;
std::string category;
};
static const size_t ArityAny = std::numeric_limits<size_t>::max(); std::map<std::string, Flag::ptr> longFlags;
std::map<char, Flag::ptr> shortFlags;
/* Flags. */ virtual bool processFlag(Strings::iterator& pos, Strings::iterator end);
struct Flag
{ virtual void printFlags(std::ostream& out);
typedef std::shared_ptr<Flag> ptr;
std::string longName; /* Positional arguments. */
char shortName = 0; struct ExpectedArg {
std::string description; std::string label;
Strings labels; size_t arity; // 0 = any
size_t arity = 0; bool optional;
std::function<void(std::vector<std::string>)> handler; std::function<void(std::vector<std::string>)> handler;
std::string category; };
std::list<ExpectedArg> expectedArgs;
virtual bool processArgs(const Strings& args, bool finish);
std::set<std::string> hiddenCategories;
public:
class FlagMaker {
Args& args;
Flag::ptr flag;
friend class Args;
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
public:
~FlagMaker();
FlagMaker& longName(const std::string& s) {
flag->longName = s;
return *this;
};
FlagMaker& shortName(char s) {
flag->shortName = s;
return *this;
};
FlagMaker& description(const std::string& s) {
flag->description = s;
return *this;
};
FlagMaker& label(const std::string& l) {
flag->arity = 1;
flag->labels = {l};
return *this;
};
FlagMaker& labels(const Strings& ls) {
flag->arity = ls.size();
flag->labels = ls;
return *this;
};
FlagMaker& arity(size_t arity) {
flag->arity = arity;
return *this;
};
FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) {
flag->handler = handler;
return *this;
};
FlagMaker& handler(std::function<void()> handler) {
flag->handler = [handler](std::vector<std::string>) { handler(); };
return *this;
};
FlagMaker& handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) {
handler(std::move(ss[0]));
};
return *this;
};
FlagMaker& category(const std::string& s) {
flag->category = s;
return *this;
}; };
std::map<std::string, Flag::ptr> longFlags; template <class T>
std::map<char, Flag::ptr> shortFlags; FlagMaker& dest(T* dest) {
flag->arity = 1;
virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this;
virtual void printFlags(std::ostream & out);
/* Positional arguments. */
struct ExpectedArg
{
std::string label;
size_t arity; // 0 = any
bool optional;
std::function<void(std::vector<std::string>)> handler;
}; };
std::list<ExpectedArg> expectedArgs; template <class T>
FlagMaker& set(T* dest, const T& val) {
virtual bool processArgs(const Strings & args, bool finish); flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
std::set<std::string> hiddenCategories; return *this;
public:
class FlagMaker
{
Args & args;
Flag::ptr flag;
friend class Args;
FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { };
public:
~FlagMaker();
FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
FlagMaker & handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
return *this;
};
FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
template<class T>
FlagMaker & dest(T * dest)
{
flag->arity = 1;
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this;
};
template<class T>
FlagMaker & set(T * dest, const T & val)
{
flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
return *this;
};
FlagMaker & mkHashTypeFlag(HashType * ht);
}; };
FlagMaker mkFlag(); FlagMaker& mkHashTypeFlag(HashType* ht);
};
/* Helper functions for constructing flags / positional FlagMaker mkFlag();
arguments. */
void mkFlag1(char shortName, const std::string & longName, /* Helper functions for constructing flags / positional
const std::string & label, const std::string & description, arguments. */
std::function<void(std::string)> fun)
{
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({label})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) { fun(ss[0]); });
}
void mkFlag(char shortName, const std::string & name, void mkFlag1(char shortName, const std::string& longName,
const std::string & description, bool * dest) const std::string& label, const std::string& description,
{ std::function<void(std::string)> fun) {
mkFlag(shortName, name, description, dest, true); mkFlag()
} .shortName(shortName)
.longName(longName)
.labels({label})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) { fun(ss[0]); });
}
template<class T> void mkFlag(char shortName, const std::string& name,
void mkFlag(char shortName, const std::string & longName, const std::string & description, const std::string& description, bool* dest) {
T * dest, const T & value) mkFlag(shortName, name, description, dest, true);
{ }
mkFlag()
.shortName(shortName)
.longName(longName)
.description(description)
.handler([=](std::vector<std::string> ss) { *dest = value; });
}
template<class I> template <class T>
void mkIntFlag(char shortName, const std::string & longName, void mkFlag(char shortName, const std::string& longName,
const std::string & description, I * dest) const std::string& description, T* dest, const T& value) {
{ mkFlag()
mkFlag<I>(shortName, longName, description, [=](I n) { .shortName(shortName)
*dest = n; .longName(longName)
.description(description)
.handler([=](std::vector<std::string> ss) { *dest = value; });
}
template <class I>
void mkIntFlag(char shortName, const std::string& longName,
const std::string& description, I* dest) {
mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; });
}
template <class I>
void mkFlag(char shortName, const std::string& longName,
const std::string& description, std::function<void(I)> fun) {
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({"N"})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) {
I n;
if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument",
longName);
fun(n);
}); });
} }
template<class I> /* Expect a string argument. */
void mkFlag(char shortName, const std::string & longName, void expectArg(const std::string& label, string* dest,
const std::string & description, std::function<void(I)> fun) bool optional = false) {
{ expectedArgs.push_back(
mkFlag() ExpectedArg{label, 1, optional,
.shortName(shortName) [=](std::vector<std::string> ss) { *dest = ss[0]; }});
.longName(longName) }
.labels({"N"})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) {
I n;
if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument", longName);
fun(n);
});
}
/* Expect a string argument. */ /* Expect 0 or more arguments. */
void expectArg(const std::string & label, string * dest, bool optional = false) void expectArgs(const std::string& label, std::vector<std::string>* dest) {
{ expectedArgs.push_back(ExpectedArg{
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { label, 0, false,
*dest = ss[0]; [=](std::vector<std::string> ss) { *dest = std::move(ss); }});
}}); }
}
/* Expect 0 or more arguments. */ friend class MultiCommand;
void expectArgs(const std::string & label, std::vector<std::string> * dest)
{
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
*dest = std::move(ss);
}});
}
friend class MultiCommand;
}; };
Strings argvToStrings(int argc, char * * argv); Strings argvToStrings(int argc, char** argv);
/* Helper function for rendering argument labels. */ /* Helper function for rendering argument labels. */
std::string renderLabels(const Strings & labels); std::string renderLabels(const Strings& labels);
/* Helper function for printing 2-column tables. */ /* Helper function for printing 2-column tables. */
typedef std::vector<std::pair<std::string, std::string>> Table2; typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table); void printTable(std::ostream& out, const Table2& table);
} } // namespace nix

Some files were not shown because too many files have changed in this diff Show more