style(3p/nix): Reformat project in Google C++ style

Reformatted with:

    fd . -e hh -e cc | xargs clang-format -i
This commit is contained in:
Vincent Ambo 2020-05-17 16:31:57 +01:00
parent 65a1aae98c
commit 0f2cf531f7
175 changed files with 32126 additions and 34689 deletions

View file

@ -1,41 +1,39 @@
#include <algorithm>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <algorithm>
#include <set>
#include <memory>
#include <tuple>
#include <iomanip> #include <iomanip>
#include <memory>
#include <set>
#include <tuple>
#if __APPLE__ #if __APPLE__
#include <sys/time.h> #include <sys/time.h>
#endif #endif
#include "machines.hh"
#include "shared.hh"
#include "pathlocks.hh"
#include "globals.hh"
#include "serialise.hh"
#include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "local-store.hh" #include "globals.hh"
#include "legacy.hh" #include "legacy.hh"
#include "local-store.hh"
#include "machines.hh"
#include "pathlocks.hh"
#include "serialise.hh"
#include "shared.hh"
#include "store-api.hh"
using namespace nix; using namespace nix;
using std::cin; using std::cin;
static void handleAlarm(int sig) { static void handleAlarm(int sig) {}
}
std::string escapeUri(std::string uri) std::string escapeUri(std::string uri) {
{
std::replace(uri.begin(), uri.end(), '/', '_'); std::replace(uri.begin(), uri.end(), '/', '_');
return uri; return uri;
} }
static string currentLoad; static string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) {
{ return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot),
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); true);
} }
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
@ -44,8 +42,7 @@ static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
return true; return true;
} }
static int _main(int argc, char * * argv) static int _main(int argc, char** argv) {
{
{ {
logger = makeJSONLogger(*logger); logger = makeJSONLogger(*logger);
@ -53,8 +50,7 @@ static int _main(int argc, char * * argv)
unsetenv("DISPLAY"); unsetenv("DISPLAY");
unsetenv("SSH_ASKPASS"); unsetenv("SSH_ASKPASS");
if (argc != 2) if (argc != 2) throw UsageError("called without required arguments");
throw UsageError("called without required arguments");
verbosity = (Verbosity)std::stoll(argv[1]); verbosity = (Verbosity)std::stoll(argv[1]);
@ -92,21 +88,23 @@ static int _main(int argc, char * * argv)
string storeUri; string storeUri;
while (true) { while (true) {
try { try {
auto s = readString(source); auto s = readString(source);
if (s != "try") return 0; if (s != "try") return 0;
} catch (EndOfFile &) { return 0; } } catch (EndOfFile&) {
return 0;
}
auto amWilling = readInt(source); auto amWilling = readInt(source);
auto neededSystem = readString(source); auto neededSystem = readString(source);
source >> drvPath; source >> drvPath;
auto requiredFeatures = readStrings<std::set<std::string>>(source); auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally = amWilling auto canBuildLocally =
&& ( neededSystem == settings.thisSystem amWilling &&
|| settings.extraPlatforms.get().count(neededSystem) > 0) (neededSystem == settings.thisSystem ||
&& allSupportedLocally(requiredFeatures); settings.extraPlatforms.get().count(neededSystem) > 0) &&
allSupportedLocally(requiredFeatures);
/* Error ignored here, will be caught later */ /* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777); mkdir(currentLoad.c_str(), 0777);
@ -123,8 +121,8 @@ static int _main(int argc, char * * argv)
for (auto& m : machines) { for (auto& m : machines) {
debug("considering building on remote machine '%s'", m.storeUri); debug("considering building on remote machine '%s'", m.storeUri);
if (m.enabled && std::find(m.systemTypes.begin(), if (m.enabled &&
m.systemTypes.end(), std::find(m.systemTypes.begin(), m.systemTypes.end(),
neededSystem) != m.systemTypes.end() && neededSystem) != m.systemTypes.end() &&
m.allSupported(requiredFeatures) && m.allSupported(requiredFeatures) &&
m.mandatoryMet(requiredFeatures)) { m.mandatoryMet(requiredFeatures)) {
@ -147,9 +145,11 @@ static int _main(int argc, char * * argv)
bool best = false; bool best = false;
if (!bestSlotLock) { if (!bestSlotLock) {
best = true; best = true;
} else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) { } else if (load / m.speedFactor <
bestLoad / bestMachine->speedFactor) {
best = true; best = true;
} else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) { } else if (load / m.speedFactor ==
bestLoad / bestMachine->speedFactor) {
if (m.speedFactor > bestMachine->speedFactor) { if (m.speedFactor > bestMachine->speedFactor) {
best = true; best = true;
} else if (m.speedFactor == bestMachine->speedFactor) { } else if (m.speedFactor == bestMachine->speedFactor) {
@ -183,8 +183,8 @@ static int _main(int argc, char * * argv)
lock = -1; lock = -1;
try { try {
Activity act(*logger, lvlTalkative, actUnknown,
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); fmt("connecting to '%s'", bestMachine->storeUri));
Store::Params storeParams; Store::Params storeParams;
if (hasPrefix(bestMachine->storeUri, "ssh://")) { if (hasPrefix(bestMachine->storeUri, "ssh://")) {
@ -200,9 +200,8 @@ static int _main(int argc, char * * argv)
} catch (std::exception& e) { } catch (std::exception& e) {
auto msg = chomp(drainFD(5, false)); auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s", printError("cannot build on '%s': %s%s", bestMachine->storeUri,
bestMachine->storeUri, e.what(), e.what(), (msg.empty() ? "" : ": " + msg));
(msg.empty() ? "" : ": " + msg));
bestMachine->enabled = false; bestMachine->enabled = false;
continue; continue;
} }
@ -219,44 +218,54 @@ connected:
auto inputs = readStrings<PathSet>(source); auto inputs = readStrings<PathSet>(source);
auto outputs = readStrings<PathSet>(source); auto outputs = readStrings<PathSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); AutoCloseFD uploadLock = openLockFile(
currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown,
fmt("waiting for the upload lock to '%s'", storeUri));
auto old = signal(SIGALRM, handleAlarm); auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60); alarm(15 * 60);
if (!lockFile(uploadLock.get(), ltWrite, true)) if (!lockFile(uploadLock.get(), ltWrite, true))
printError("somebody is hogging the upload lock for '%s', continuing..."); printError(
"somebody is hogging the upload lock for '%s', continuing...");
alarm(0); alarm(0);
signal(SIGALRM, old); signal(SIGALRM, old);
} }
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; auto substitute =
settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown,
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute); fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs,
substitute);
} }
uploadLock = -1; uploadLock = -1;
BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); BasicDerivation drv(
readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
drv.inputSrcs = inputs; drv.inputSrcs = inputs;
auto result = sshStore->buildDerivation(drvPath, drv); auto result = sshStore->buildDerivation(drvPath, drv);
if (!result.success()) if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg); throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri,
result.errorMsg);
PathSet missing; PathSet missing;
for (auto& path : outputs) for (auto& path : outputs)
if (!store->isValidPath(path)) missing.insert(path); if (!store->isValidPath(path)) missing.insert(path);
if (!missing.empty()) { if (!missing.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown,
fmt("copying outputs from '%s'", storeUri));
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs,
NoSubstitute);
} }
return 0; return 0;

View file

@ -2,12 +2,9 @@
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
static Strings parseAttrPath(const string& s) {
static Strings parseAttrPath(const string & s)
{
Strings res; Strings res;
string cur; string cur;
string::const_iterator i = s.begin(); string::const_iterator i = s.begin();
@ -19,7 +16,8 @@ static Strings parseAttrPath(const string & s)
++i; ++i;
while (1) { while (1) {
if (i == s.end()) if (i == s.end())
throw Error(format("missing closing quote in selection path '%1%'") % s); throw Error(format("missing closing quote in selection path '%1%'") %
s);
if (*i == '"') break; if (*i == '"') break;
cur.push_back(*i++); cur.push_back(*i++);
} }
@ -31,19 +29,17 @@ static Strings parseAttrPath(const string & s)
return res; return res;
} }
Value* findAlongAttrPath(EvalState& state, const string& attrPath, Value* findAlongAttrPath(EvalState& state, const string& attrPath,
Bindings & autoArgs, Value & vIn) Bindings& autoArgs, Value& vIn) {
{
Strings tokens = parseAttrPath(attrPath); Strings tokens = parseAttrPath(attrPath);
Error attrError = Error attrError =
Error(format("attribute selection path '%1%' does not match expression") % attrPath); Error(format("attribute selection path '%1%' does not match expression") %
attrPath);
Value* v = &vIn; Value* v = &vIn;
for (auto& attr : tokens) { for (auto& attr : tokens) {
/* Is i an index (integer) or a normal attribute name? */ /* Is i an index (integer) or a normal attribute name? */
enum { apAttr, apIndex } apType = apAttr; enum { apAttr, apIndex } apType = apAttr;
unsigned int attrIndex; unsigned int attrIndex;
@ -59,38 +55,39 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
according to what is specified in the attrPath. */ according to what is specified in the attrPath. */
if (apType == apAttr) { if (apType == apAttr) {
if (v->type != tAttrs) if (v->type != tAttrs)
throw TypeError( throw TypeError(format("the expression selected by the selection path "
format("the expression selected by the selection path '%1%' should be a set but is %2%") "'%1%' should be a set but is %2%") %
% attrPath % showType(*v)); attrPath % showType(*v));
if (attr.empty()) if (attr.empty())
throw Error(format("empty attribute name in selection path '%1%'") % attrPath); throw Error(format("empty attribute name in selection path '%1%'") %
attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end()) if (a == v->attrs->end())
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); throw Error(
format("attribute '%1%' in selection path '%2%' not found") % attr %
attrPath);
v = &*a->value; v = &*a->value;
} }
else if (apType == apIndex) { else if (apType == apIndex) {
if (!v->isList()) if (!v->isList())
throw TypeError( throw TypeError(format("the expression selected by the selection path "
format("the expression selected by the selection path '%1%' should be a list but is %2%") "'%1%' should be a list but is %2%") %
% attrPath % showType(*v)); attrPath % showType(*v));
if (attrIndex >= v->listSize()) if (attrIndex >= v->listSize())
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); throw Error(
format("list index %1% in selection path '%2%' is out of range") %
attrIndex % attrPath);
v = v->listElems()[attrIndex]; v = v->listElems()[attrIndex];
} }
} }
return v; return v;
} }
} // namespace nix
}

View file

@ -1,9 +1,8 @@
#pragma once #pragma once
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
namespace nix { namespace nix {

View file

@ -1,25 +1,20 @@
#include "attr-set.hh" #include "attr-set.hh"
#include <algorithm>
#include "eval-inline.hh" #include "eval-inline.hh"
#include <algorithm>
namespace nix { namespace nix {
/* Allocate a new array of attributes for an attribute set with a specific /* Allocate a new array of attributes for an attribute set with a specific
capacity. The space is implicitly reserved after the Bindings capacity. The space is implicitly reserved after the Bindings
structure. */ structure. */
Bindings * EvalState::allocBindings(size_t capacity) Bindings* EvalState::allocBindings(size_t capacity) {
{
if (capacity > std::numeric_limits<Bindings::size_t>::max()) if (capacity > std::numeric_limits<Bindings::size_t>::max())
throw Error("attribute set of size %d is too big", capacity); throw Error("attribute set of size %d is too big", capacity);
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity))
Bindings((Bindings::size_t)capacity);
} }
void EvalState::mkAttrs(Value& v, size_t capacity) {
void EvalState::mkAttrs(Value & v, size_t capacity)
{
if (capacity == 0) { if (capacity == 0) {
v = vEmptySet; v = vEmptySet;
return; return;
@ -31,22 +26,15 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
nrAttrsInAttrsets += capacity; nrAttrsInAttrsets += capacity;
} }
/* Create a new attribute named 'name' on an existing attribute set stored /* Create a new attribute named 'name' on an existing attribute set stored
in 'vAttrs' and return the newly allocated Value which is associated with in 'vAttrs' and return the newly allocated Value which is associated with
this attribute. */ this attribute. */
Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) {
{
Value* v = allocValue(); Value* v = allocValue();
vAttrs.attrs->push_back(Attr(name, v)); vAttrs.attrs->push_back(Attr(name, v));
return v; return v;
} }
void Bindings::sort() { std::sort(begin(), end()); }
void Bindings::sort() } // namespace nix
{
std::sort(begin(), end());
}
}

View file

@ -1,37 +1,30 @@
#pragma once #pragma once
#include <algorithm>
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include <algorithm>
namespace nix { namespace nix {
class EvalState; class EvalState;
struct Value; struct Value;
/* Map one attribute name to its value. */ /* Map one attribute name to its value. */
struct Attr struct Attr {
{
Symbol name; Symbol name;
Value* value; Value* value;
Pos* pos; Pos* pos;
Attr(Symbol name, Value* value, Pos* pos = &noPos) Attr(Symbol name, Value* value, Pos* pos = &noPos)
: name(name), value(value), pos(pos){}; : name(name), value(value), pos(pos){};
Attr() : pos(&noPos){}; Attr() : pos(&noPos){};
bool operator < (const Attr & a) const bool operator<(const Attr& a) const { return name < a.name; }
{
return name < a.name;
}
}; };
/* Bindings contains all the attributes of an attribute set. It is defined /* Bindings contains all the attributes of an attribute set. It is defined
by its size and its capacity, the capacity being the number of Attr by its size and its capacity, the capacity being the number of Attr
elements allocated after this structure, while the size corresponds to elements allocated after this structure, while the size corresponds to
the number of elements already inserted in this structure. */ the number of elements already inserted in this structure. */
class Bindings class Bindings {
{
public: public:
typedef uint32_t size_t; typedef uint32_t size_t;
@ -49,14 +42,12 @@ public:
typedef Attr* iterator; typedef Attr* iterator;
void push_back(const Attr & attr) void push_back(const Attr& attr) {
{
assert(size_ < capacity_); assert(size_ < capacity_);
attrs[size_++] = attr; attrs[size_++] = attr;
} }
iterator find(const Symbol & name) iterator find(const Symbol& name) {
{
Attr key(name, 0); Attr key(name, 0);
iterator i = std::lower_bound(begin(), end(), key); iterator i = std::lower_bound(begin(), end(), key);
if (i != end() && i->name == name) return i; if (i != end() && i->name == name) return i;
@ -66,22 +57,17 @@ public:
iterator begin() { return &attrs[0]; } iterator begin() { return &attrs[0]; }
iterator end() { return &attrs[size_]; } iterator end() { return &attrs[size_]; }
Attr & operator[](size_t pos) Attr& operator[](size_t pos) { return attrs[pos]; }
{
return attrs[pos];
}
void sort(); void sort();
size_t capacity() { return capacity_; } size_t capacity() { return capacity_; }
/* Returns the attributes in lexicographically sorted order. */ /* Returns the attributes in lexicographically sorted order. */
std::vector<const Attr *> lexicographicOrder() const std::vector<const Attr*> lexicographicOrder() const {
{
std::vector<const Attr*> res; std::vector<const Attr*> res;
res.reserve(size_); res.reserve(size_);
for (size_t n = 0; n < size_; n++) for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]);
res.emplace_back(&attrs[n]);
std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) { std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) {
return (const string&)a->name < (const string&)b->name; return (const string&)a->name < (const string&)b->name;
}); });
@ -91,5 +77,4 @@ public:
friend class EvalState; friend class EvalState;
}; };
} // namespace nix
}

View file

@ -1,40 +1,43 @@
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "shared.hh"
#include "download.hh" #include "download.hh"
#include "util.hh"
#include "eval.hh" #include "eval.hh"
#include "shared.hh"
#include "util.hh"
namespace nix { namespace nix {
MixEvalArgs::MixEvalArgs() MixEvalArgs::MixEvalArgs() {
{
mkFlag() mkFlag()
.longName("arg") .longName("arg")
.description("argument to be passed to Nix functions") .description("argument to be passed to Nix functions")
.labels({"name", "expr"}) .labels({"name", "expr"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); .handler(
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
mkFlag() mkFlag()
.longName("argstr") .longName("argstr")
.description("string-valued argument to be passed to Nix functions") .description("string-valued argument to be passed to Nix functions")
.labels({"name", "string"}) .labels({"name", "string"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); .handler(
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
mkFlag() mkFlag()
.shortName('I') .shortName('I')
.longName("include") .longName("include")
.description("add a path to the list of locations used to look up <...> file names") .description(
"add a path to the list of locations used to look up <...> file "
"names")
.label("path") .label("path")
.handler([&](std::string s) { searchPath.push_back(s); }); .handler([&](std::string s) { searchPath.push_back(s); });
} }
Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Bindings* MixEvalArgs::getAutoArgs(EvalState& state) {
{
Bindings* res = state.allocBindings(autoArgs.size()); Bindings* res = state.allocBindings(autoArgs.size());
for (auto& i : autoArgs) { for (auto& i : autoArgs) {
Value* v = state.allocValue(); Value* v = state.allocValue();
if (i.second[0] == 'E') if (i.second[0] == 'E')
state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); state.mkThunk_(
*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
else else
mkString(*v, string(i.second, 1)); mkString(*v, string(i.second, 1));
res->push_back(Attr(state.symbols.create(i.first), v)); res->push_back(Attr(state.symbols.create(i.first), v));
@ -43,8 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
return res; return res;
} }
Path lookupFileArg(EvalState & state, string s) Path lookupFileArg(EvalState& state, string s) {
{
if (isUri(s)) { if (isUri(s)) {
CachedDownloadRequest request(s); CachedDownloadRequest request(s);
request.unpack = true; request.unpack = true;
@ -56,4 +58,4 @@ Path lookupFileArg(EvalState & state, string s)
return absPath(s); return absPath(s);
} }
} } // namespace nix

View file

@ -8,8 +8,7 @@ class Store;
class EvalState; class EvalState;
class Bindings; class Bindings;
struct MixEvalArgs : virtual Args struct MixEvalArgs : virtual Args {
{
MixEvalArgs(); MixEvalArgs();
Bindings* getAutoArgs(EvalState& state); Bindings* getAutoArgs(EvalState& state);
@ -17,10 +16,9 @@ struct MixEvalArgs : virtual Args
Strings searchPath; Strings searchPath;
private: private:
std::map<std::string, std::string> autoArgs; std::map<std::string, std::string> autoArgs;
}; };
Path lookupFileArg(EvalState& state, string s); Path lookupFileArg(EvalState& state, string s);
} } // namespace nix

View file

@ -2,30 +2,29 @@
#include "eval.hh" #include "eval.hh"
#define LocalNoInline(f) static f __attribute__((noinline)); f #define LocalNoInline(f) \
#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f static f __attribute__((noinline)); \
f
#define LocalNoInlineNoReturn(f) \
static f __attribute__((noinline, noreturn)); \
f
namespace nix { namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) {
{
throw EvalError(format(s) % pos); throw EvalError(format(s) % pos);
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) {
{
throw TypeError(format(s) % showType(v)); throw TypeError(format(s) % showType(v));
} }
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v,
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) const Pos& pos)) {
{
throw TypeError(format(s) % showType(v) % pos); throw TypeError(format(s) % showType(v) % pos);
} }
void EvalState::forceValue(Value& v, const Pos& pos) {
void EvalState::forceValue(Value & v, const Pos & pos)
{
if (v.type == tThunk) { if (v.type == tThunk) {
Env* env = v.thunk.env; Env* env = v.thunk.env;
Expr* expr = v.thunk.expr; Expr* expr = v.thunk.expr;
@ -39,48 +38,37 @@ void EvalState::forceValue(Value & v, const Pos & pos)
v.thunk.expr = expr; v.thunk.expr = expr;
throw; throw;
} }
} } else if (v.type == tApp)
else if (v.type == tApp)
callFunction(*v.app.left, *v.app.right, v, noPos); callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole) else if (v.type == tBlackhole)
throwEvalError("infinite recursion encountered, at %1%", pos); throwEvalError("infinite recursion encountered, at %1%", pos);
} }
inline void EvalState::forceAttrs(Value& v) {
inline void EvalState::forceAttrs(Value & v)
{
forceValue(v); forceValue(v);
if (v.type != tAttrs) if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected", v); throwTypeError("value is %1% while a set was expected", v);
} }
inline void EvalState::forceAttrs(Value& v, const Pos& pos) {
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
forceValue(v); forceValue(v);
if (v.type != tAttrs) if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected, at %2%", v, pos); throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
} }
inline void EvalState::forceList(Value& v) {
inline void EvalState::forceList(Value & v)
{
forceValue(v); forceValue(v);
if (!v.isList()) if (!v.isList()) throwTypeError("value is %1% while a list was expected", v);
throwTypeError("value is %1% while a list was expected", v);
} }
inline void EvalState::forceList(Value& v, const Pos& pos) {
inline void EvalState::forceList(Value & v, const Pos & pos)
{
forceValue(v); forceValue(v);
if (!v.isList()) if (!v.isList())
throwTypeError("value is %1% while a list was expected, at %2%", v, pos); throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
} }
/* Note: Various places expect the allocated memory to be zeroed. */ /* Note: Various places expect the allocated memory to be zeroed. */
inline void * allocBytes(size_t n) inline void* allocBytes(size_t n) {
{
void* p; void* p;
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
p = GC_MALLOC(n); p = GC_MALLOC(n);
@ -91,5 +79,4 @@ inline void * allocBytes(size_t n)
return p; return p;
} }
} // namespace nix
}

File diff suppressed because it is too large Load diff

View file

@ -1,30 +1,25 @@
#pragma once #pragma once
#include "attr-set.hh"
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "hash.hh"
#include "config.hh"
#include <map> #include <map>
#include <optional> #include <optional>
#include <unordered_map> #include <unordered_map>
#include "attr-set.hh"
#include "config.hh"
#include "hash.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "value.hh"
namespace nix { namespace nix {
class Store; class Store;
class EvalState; class EvalState;
enum RepairFlag : bool; enum RepairFlag : bool;
typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args,
Value& v);
typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); struct PrimOp {
struct PrimOp
{
PrimOpFun fun; PrimOpFun fun;
size_t arity; size_t arity;
Symbol name; Symbol name;
@ -32,9 +27,7 @@ struct PrimOp
: fun(fun), arity(arity), name(name) {} : fun(fun), arity(arity), name(name) {}
}; };
struct Env {
struct Env
{
Env* up; Env* up;
unsigned short size; // used by valueSize unsigned short size; // used by valueSize
unsigned short prevWith : 14; // nr of levels up to next `with' environment unsigned short prevWith : 14; // nr of levels up to next `with' environment
@ -42,37 +35,29 @@ struct Env
Value* values[0]; Value* values[0];
}; };
Value& mkString(Value& v, const string& s, const PathSet& context = PathSet()); Value& mkString(Value& v, const string& s, const PathSet& context = PathSet());
void copyContext(const Value& v, PathSet& context); void copyContext(const Value& v, PathSet& context);
/* Cache for calls to addToStore(); maps source paths to the store /* Cache for calls to addToStore(); maps source paths to the store
paths. */ paths. */
typedef std::map<Path, Path> SrcToStore; typedef std::map<Path, Path> SrcToStore;
std::ostream& operator<<(std::ostream& str, const Value& v); std::ostream& operator<<(std::ostream& str, const Value& v);
typedef std::pair<std::string, std::string> SearchPathElem; typedef std::pair<std::string, std::string> SearchPathElem;
typedef std::list<SearchPathElem> SearchPath; typedef std::list<SearchPathElem> SearchPath;
/* Initialise the Boehm GC, if applicable. */ /* Initialise the Boehm GC, if applicable. */
void initGC(); void initGC();
class EvalState {
class EvalState
{
public: public:
SymbolTable symbols; SymbolTable symbols;
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem,
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn,
sFile, sLine, sColumn, sFunctor, sToString, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode; sOutputHash, sOutputHashAlgo, sOutputHashMode;
Symbol sDerivationNix; Symbol sDerivationNix;
@ -93,7 +78,9 @@ private:
/* A cache from path names to parse trees. */ /* A cache from path names to parse trees. */
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache; typedef std::map<Path, Expr*, std::less<Path>,
traceable_allocator<std::pair<const Path, Expr*>>>
FileParseCache;
#else #else
typedef std::map<Path, Expr*> FileParseCache; typedef std::map<Path, Expr*> FileParseCache;
#endif #endif
@ -101,7 +88,9 @@ private:
/* A cache from path names to values. */ /* A cache from path names to values. */
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache; typedef std::map<Path, Value, std::less<Path>,
traceable_allocator<std::pair<const Path, Value>>>
FileEvalCache;
#else #else
typedef std::map<Path, Value> FileEvalCache; typedef std::map<Path, Value> FileEvalCache;
#endif #endif
@ -115,7 +104,6 @@ private:
std::unordered_map<Path, Path> resolvedPaths; std::unordered_map<Path, Path> resolvedPaths;
public: public:
EvalState(const Strings& _searchPath, ref<Store> store); EvalState(const Strings& _searchPath, ref<Store> store);
~EvalState(); ~EvalState();
@ -141,7 +129,8 @@ public:
Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv); Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv);
/* Parse a Nix expression from the specified string. */ /* Parse a Nix expression from the specified string. */
Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); Expr* parseExprFromString(const string& s, const Path& basePath,
StaticEnv& staticEnv);
Expr* parseExprFromString(const string& s, const Path& basePath); Expr* parseExprFromString(const string& s, const Path& basePath);
Expr* parseStdin(); Expr* parseStdin();
@ -154,10 +143,12 @@ public:
/* Look up a file in the search path. */ /* Look up a file in the search path. */
Path findFile(const string& path); Path findFile(const string& path);
Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); Path findFile(SearchPath& searchPath, const string& path,
const Pos& pos = noPos);
/* If the specified search path element is a URI, download it. */ /* If the specified search path element is a URI, download it. */
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); std::pair<bool, std::string> resolveSearchPathElem(
const SearchPathElem& elem);
/* Evaluate an expression to normal form, storing the result in /* Evaluate an expression to normal form, storing the result in
value `v'. */ value `v'. */
@ -197,7 +188,9 @@ public:
bool isDerivation(Value& v); bool isDerivation(Value& v);
std::optional<string> tryAttrsToString(const Pos& pos, Value& v, std::optional<string> tryAttrsToString(const Pos& pos, Value& v,
PathSet & context, bool coerceMore = false, bool copyToStore = true); PathSet& context,
bool coerceMore = false,
bool copyToStore = true);
/* String coercion. Converts strings, paths and derivations to a /* String coercion. Converts strings, paths and derivations to a
string. If `coerceMore' is set, also converts nulls, integers, string. If `coerceMore' is set, also converts nulls, integers,
@ -214,7 +207,6 @@ public:
Path coerceToPath(const Pos& pos, Value& v, PathSet& context); Path coerceToPath(const Pos& pos, Value& v, PathSet& context);
public: public:
/* The base environment, containing the builtin functions and /* The base environment, containing the builtin functions and
values. */ values. */
Env& baseEnv; Env& baseEnv;
@ -223,33 +215,28 @@ public:
StaticEnv staticBaseEnv; // !!! should be private StaticEnv staticBaseEnv; // !!! should be private
private: private:
unsigned int baseEnvDispl = 0; unsigned int baseEnvDispl = 0;
void createBaseEnv(); void createBaseEnv();
Value* addConstant(const string& name, Value& v); Value* addConstant(const string& name, Value& v);
Value * addPrimOp(const string & name, Value* addPrimOp(const string& name, size_t arity, PrimOpFun primOp);
size_t arity, PrimOpFun primOp);
public: public:
Value& getBuiltin(const string& name); Value& getBuiltin(const string& name);
private: private:
inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval); inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval);
friend struct ExprVar; friend struct ExprVar;
friend struct ExprAttrs; friend struct ExprAttrs;
friend struct ExprLet; friend struct ExprLet;
Expr * parse(const char * text, const Path & path, Expr* parse(const char* text, const Path& path, const Path& basePath,
const Path & basePath, StaticEnv & staticEnv); StaticEnv& staticEnv);
public: public:
/* Do a deep equality test between two values. That is, list /* Do a deep equality test between two values. That is, list
elements and attributes are compared recursively. */ elements and attributes are compared recursively. */
bool eqValues(Value& v1, Value& v2); bool eqValues(Value& v1, Value& v2);
@ -284,7 +271,6 @@ public:
void realiseContext(const PathSet& context); void realiseContext(const PathSet& context);
private: private:
unsigned long nrEnvs = 0; unsigned long nrEnvs = 0;
unsigned long nrValuesInEnvs = 0; unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0; unsigned long nrValues = 0;
@ -313,10 +299,10 @@ private:
friend struct ExprOpUpdate; friend struct ExprOpUpdate;
friend struct ExprOpConcatLists; friend struct ExprOpConcatLists;
friend struct ExprSelect; friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args,
Value& v);
}; };
/* Return a string representing the type of the value `v'. */ /* Return a string representing the type of the value `v'. */
string showType(const Value& v); string showType(const Value& v);
@ -327,8 +313,7 @@ std::pair<string, string> decodeContext(const string & s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);
struct InvalidPathError : EvalError struct InvalidPathError : EvalError {
{
Path path; Path path;
InvalidPathError(const Path& path); InvalidPathError(const Path& path);
#ifdef EXCEPTION_NEEDS_THROW_SPEC #ifdef EXCEPTION_NEEDS_THROW_SPEC
@ -336,28 +321,37 @@ struct InvalidPathError : EvalError
#endif #endif
}; };
struct EvalSettings : Config struct EvalSettings : Config {
{ Setting<bool> enableNativeCode{this, false,
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."}; "Whether builtin functions that allow "
"executing native code should be enabled."};
Setting<bool> restrictEval{this, false, "restrict-eval", Setting<bool> restrictEval{
this, false, "restrict-eval",
"Whether to restrict file system access to paths in $NIX_PATH, " "Whether to restrict file system access to paths in $NIX_PATH, "
"and network access to the URI prefixes listed in 'allowed-uris'."}; "and network access to the URI prefixes listed in 'allowed-uris'."};
Setting<bool> pureEval{this, false, "pure-eval", Setting<bool> pureEval{this, false, "pure-eval",
"Whether to restrict file system and network access to files specified by cryptographic hash."}; "Whether to restrict file system and network access "
"to files specified by cryptographic hash."};
Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
"Whether the evaluator allows importing the result of a derivation."}; "Whether the evaluator allows importing the result of a derivation."};
Setting<Strings> allowedUris{this, {}, "allowed-uris", Setting<Strings> allowedUris{
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; this,
{},
"allowed-uris",
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit "
"are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; "Emit log messages for each function entry "
"and exit at the 'vomit' log level (-vvvv)"};
}; };
extern EvalSettings evalSettings; extern EvalSettings evalSettings;
} } // namespace nix

View file

@ -14,4 +14,4 @@ FunctionCallTrace::~FunctionCallTrace() {
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
} }
} } // namespace nix

View file

@ -1,15 +1,13 @@
#pragma once #pragma once
#include "eval.hh"
#include <chrono> #include <chrono>
#include "eval.hh"
namespace nix { namespace nix {
struct FunctionCallTrace struct FunctionCallTrace {
{
const Pos& pos; const Pos& pos;
FunctionCallTrace(const Pos& pos); FunctionCallTrace(const Pos& pos);
~FunctionCallTrace(); ~FunctionCallTrace();
}; };
} } // namespace nix

View file

@ -1,24 +1,18 @@
#include "get-drvs.hh" #include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include <cstring> #include <cstring>
#include <regex> #include <regex>
#include "derivations.hh"
#include "eval-inline.hh"
#include "util.hh"
namespace nix { namespace nix {
DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs) DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs)
: state(&state), attrs(attrs), attrPath(attrPath) : state(&state), attrs(attrs), attrPath(attrPath) {}
{
}
DrvInfo::DrvInfo(EvalState& state, ref<Store> store,
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) const std::string& drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("") : state(&state), attrs(nullptr), attrPath("") {
{
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
drvPath = spec.first; drvPath = spec.first;
@ -28,23 +22,22 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
name = storePathToName(drvPath); name = storePathToName(drvPath);
if (spec.second.size() > 1) if (spec.second.size() > 1)
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs); throw Error(
"building more than one derivation output is not supported, in '%s'",
drvPathWithOutputs);
outputName = outputName = spec.second.empty() ? get(drv.env, "outputName", "out")
spec.second.empty()
? get(drv.env, "outputName", "out")
: *spec.second.begin(); : *spec.second.begin();
auto i = drv.outputs.find(outputName); auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end()) if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", drvPath, outputName); throw Error("derivation '%s' does not have output '%s'", drvPath,
outputName);
outPath = i->second.path; outPath = i->second.path;
} }
string DrvInfo::queryName() const {
string DrvInfo::queryName() const
{
if (name == "" && attrs) { if (name == "" && attrs) {
auto i = attrs->find(state->sName); auto i = attrs->find(state->sName);
if (i == attrs->end()) throw TypeError("derivation name missing"); if (i == attrs->end()) throw TypeError("derivation name missing");
@ -53,41 +46,38 @@ string DrvInfo::queryName() const
return name; return name;
} }
string DrvInfo::querySystem() const {
string DrvInfo::querySystem() const
{
if (system == "" && attrs) { if (system == "" && attrs) {
auto i = attrs->find(state->sSystem); auto i = attrs->find(state->sSystem);
system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); system = i == attrs->end() ? "unknown"
: state->forceStringNoCtx(*i->value, *i->pos);
} }
return system; return system;
} }
string DrvInfo::queryDrvPath() const {
string DrvInfo::queryDrvPath() const
{
if (drvPath == "" && attrs) { if (drvPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sDrvPath); Bindings::iterator i = attrs->find(state->sDrvPath);
PathSet context; PathSet context;
drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; drvPath = i != attrs->end()
? state->coerceToPath(*i->pos, *i->value, context)
: "";
} }
return drvPath; return drvPath;
} }
string DrvInfo::queryOutPath() const {
string DrvInfo::queryOutPath() const
{
if (outPath == "" && attrs) { if (outPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath); Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context; PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; outPath = i != attrs->end()
? state->coerceToPath(*i->pos, *i->value, context)
: "";
} }
return outPath; return outPath;
} }
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) {
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
{
if (outputs.empty()) { if (outputs.empty()) {
/* Get the outputs list. */ /* Get the outputs list. */
Bindings::iterator i; Bindings::iterator i;
@ -97,22 +87,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
/* For each output... */ /* For each output... */
for (unsigned int j = 0; j < i->value->listSize(); ++j) { for (unsigned int j = 0; j < i->value->listSize(); ++j) {
/* Evaluate the corresponding set. */ /* Evaluate the corresponding set. */
string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); string name =
state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
Bindings::iterator out = attrs->find(state->symbols.create(name)); Bindings::iterator out = attrs->find(state->symbols.create(name));
if (out == attrs->end()) continue; // FIXME: throw error? if (out == attrs->end()) continue; // FIXME: throw error?
state->forceAttrs(*out->value); state->forceAttrs(*out->value);
/* And evaluate its outPath attribute. */ /* And evaluate its outPath attribute. */
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? if (outPath == out->value->attrs->end())
continue; // FIXME: throw error?
PathSet context; PathSet context;
outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); outputs[name] =
state->coerceToPath(*outPath->pos, *outPath->value, context);
} }
} else } else
outputs["out"] = queryOutPath(); outputs["out"] = queryOutPath();
} }
if (!onlyOutputsToInstall || !attrs) if (!onlyOutputsToInstall || !attrs) return outputs;
return outputs;
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
const Value* outTI = queryMeta("outputsToInstall"); const Value* outTI = queryMeta("outputsToInstall");
@ -121,7 +113,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
/* ^ this shows during `nix-env -i` right under the bad derivation */ /* ^ this shows during `nix-env -i` right under the bad derivation */
if (!outTI->isList()) throw errMsg; if (!outTI->isList()) throw errMsg;
Outputs result; Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize();
++i) {
if ((*i)->type != tString) throw errMsg; if ((*i)->type != tString) throw errMsg;
auto out = outputs.find((*i)->string.s); auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg; if (out == outputs.end()) throw errMsg;
@ -130,9 +123,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
return result; return result;
} }
string DrvInfo::queryOutputName() const {
string DrvInfo::queryOutputName() const
{
if (outputName == "" && attrs) { if (outputName == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutputName); Bindings::iterator i = attrs->find(state->sOutputName);
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
@ -140,9 +131,7 @@ string DrvInfo::queryOutputName() const
return outputName; return outputName;
} }
Bindings* DrvInfo::getMeta() {
Bindings * DrvInfo::getMeta()
{
if (meta) return meta; if (meta) return meta;
if (!attrs) return 0; if (!attrs) return 0;
Bindings::iterator a = attrs->find(state->sMeta); Bindings::iterator a = attrs->find(state->sMeta);
@ -152,56 +141,44 @@ Bindings * DrvInfo::getMeta()
return meta; return meta;
} }
StringSet DrvInfo::queryMetaNames() {
StringSet DrvInfo::queryMetaNames()
{
StringSet res; StringSet res;
if (!getMeta()) return res; if (!getMeta()) return res;
for (auto & i : *meta) for (auto& i : *meta) res.insert(i.name);
res.insert(i.name);
return res; return res;
} }
bool DrvInfo::checkMeta(Value& v) {
bool DrvInfo::checkMeta(Value & v)
{
state->forceValue(v); state->forceValue(v);
if (v.isList()) { if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n) for (unsigned int n = 0; n < v.listSize(); ++n)
if (!checkMeta(*v.listElems()[n])) return false; if (!checkMeta(*v.listElems()[n])) return false;
return true; return true;
} } else if (v.type == tAttrs) {
else if (v.type == tAttrs) {
Bindings::iterator i = v.attrs->find(state->sOutPath); Bindings::iterator i = v.attrs->find(state->sOutPath);
if (i != v.attrs->end()) return false; if (i != v.attrs->end()) return false;
for (auto& i : *v.attrs) for (auto& i : *v.attrs)
if (!checkMeta(*i.value)) return false; if (!checkMeta(*i.value)) return false;
return true; return true;
} } else
else return v.type == tInt || v.type == tBool || v.type == tString || return v.type == tInt || v.type == tBool || v.type == tString ||
v.type == tFloat; v.type == tFloat;
} }
Value* DrvInfo::queryMeta(const string& name) {
Value * DrvInfo::queryMeta(const string & name)
{
if (!getMeta()) return 0; if (!getMeta()) return 0;
Bindings::iterator a = meta->find(state->symbols.create(name)); Bindings::iterator a = meta->find(state->symbols.create(name));
if (a == meta->end() || !checkMeta(*a->value)) return 0; if (a == meta->end() || !checkMeta(*a->value)) return 0;
return a->value; return a->value;
} }
string DrvInfo::queryMetaString(const string& name) {
string DrvInfo::queryMetaString(const string & name)
{
Value* v = queryMeta(name); Value* v = queryMeta(name);
if (!v || v->type != tString) return ""; if (!v || v->type != tString) return "";
return v->string.s; return v->string.s;
} }
NixInt DrvInfo::queryMetaInt(const string& name, NixInt def) {
NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
{
Value* v = queryMeta(name); Value* v = queryMeta(name);
if (!v) return def; if (!v) return def;
if (v->type == tInt) return v->integer; if (v->type == tInt) return v->integer;
@ -214,8 +191,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
return def; return def;
} }
NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) NixFloat DrvInfo::queryMetaFloat(const string& name, NixFloat def) {
{
Value* v = queryMeta(name); Value* v = queryMeta(name);
if (!v) return def; if (!v) return def;
if (v->type == tFloat) return v->fpoint; if (v->type == tFloat) return v->fpoint;
@ -228,9 +204,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
return def; return def;
} }
bool DrvInfo::queryMetaBool(const string& name, bool def) {
bool DrvInfo::queryMetaBool(const string & name, bool def)
{
Value* v = queryMeta(name); Value* v = queryMeta(name);
if (!v) return def; if (!v) return def;
if (v->type == tBool) return v->boolean; if (v->type == tBool) return v->boolean;
@ -243,34 +217,28 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
return def; return def;
} }
void DrvInfo::setMeta(const string& name, Value* v) {
void DrvInfo::setMeta(const string & name, Value * v)
{
getMeta(); getMeta();
Bindings* old = meta; Bindings* old = meta;
meta = state->allocBindings(1 + (old ? old->size() : 0)); meta = state->allocBindings(1 + (old ? old->size() : 0));
Symbol sym = state->symbols.create(name); Symbol sym = state->symbols.create(name);
if (old) if (old)
for (auto i : *old) for (auto i : *old)
if (i.name != sym) if (i.name != sym) meta->push_back(i);
meta->push_back(i);
if (v) meta->push_back(Attr(sym, v)); if (v) meta->push_back(Attr(sym, v));
meta->sort(); meta->sort();
} }
/* Cache for already considered attrsets. */ /* Cache for already considered attrsets. */
typedef set<Bindings*> Done; typedef set<Bindings*> Done;
/* Evaluate value `v'. If it evaluates to a set of type `derivation', /* Evaluate value `v'. If it evaluates to a set of type `derivation',
then put information about it in `drvs' (unless it's already in `done'). then put information about it in `drvs' (unless it's already in `done').
The result boolean indicates whether it makes sense The result boolean indicates whether it makes sense
for the caller to recursively search for derivations in `v'. */ for the caller to recursively search for derivations in `v'. */
static bool getDerivation(EvalState & state, Value & v, static bool getDerivation(EvalState& state, Value& v, const string& attrPath,
const string & attrPath, DrvInfos & drvs, Done & done, DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures) bool ignoreAssertionFailures) {
{
try { try {
state.forceValue(v); state.forceValue(v);
if (!state.isDerivation(v)) return true; if (!state.isDerivation(v)) return true;
@ -294,10 +262,8 @@ static bool getDerivation(EvalState & state, Value & v,
} }
} }
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
bool ignoreAssertionFailures) bool ignoreAssertionFailures) {
{
Done done; Done done;
DrvInfos drvs; DrvInfos drvs;
getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
@ -305,32 +271,29 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
return std::move(drvs.front()); return std::move(drvs.front());
} }
static string addToPath(const string& s1, const string& s2) {
static string addToPath(const string & s1, const string & s2)
{
return s1.empty() ? s2 : s1 + "." + s2; return s1.empty() ? s2 : s1 + "." + s2;
} }
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
static void getDerivations(EvalState& state, Value& vIn, static void getDerivations(EvalState& state, Value& vIn,
const string& pathPrefix, Bindings& autoArgs, const string& pathPrefix, Bindings& autoArgs,
DrvInfos& drvs, Done& done, DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures) bool ignoreAssertionFailures) {
{
Value v; Value v;
state.autoCallFunction(autoArgs, vIn, v); state.autoCallFunction(autoArgs, vIn, v);
/* Process the expression. */ /* Process the expression. */
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures))
;
else if (v.type == tAttrs) { else if (v.type == tAttrs) {
/* !!! undocumented hackery to support combining channels in /* !!! undocumented hackery to support combining channels in
nix-env.cc. */ nix-env.cc. */
bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); bool combineChannels =
v.attrs->find(state.symbols.create("_combineChannels")) !=
v.attrs->end();
/* Consider the attributes in sorted order to get more /* Consider the attributes in sorted order to get more
deterministic behaviour in nix-env operations (e.g. when deterministic behaviour in nix-env operations (e.g. when
@ -339,19 +302,23 @@ static void getDerivations(EvalState & state, Value & vIn,
precedence). */ precedence). */
for (auto& i : v.attrs->lexicographicOrder()) { for (auto& i : v.attrs->lexicographicOrder()) {
debug("evaluating attribute '%1%'", i->name); debug("evaluating attribute '%1%'", i->name);
if (!std::regex_match(std::string(i->name), attrRegex)) if (!std::regex_match(std::string(i->name), attrRegex)) continue;
continue;
string pathPrefix2 = addToPath(pathPrefix, i->name); string pathPrefix2 = addToPath(pathPrefix, i->name);
if (combineChannels) if (combineChannels)
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { ignoreAssertionFailures);
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done,
ignoreAssertionFailures)) {
/* If the value of this attribute is itself a set, /* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */ `recurseForDerivations = true' attribute. */
if (i->value->type == tAttrs) { if (i->value->type == tAttrs) {
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); Bindings::iterator j = i->value->attrs->find(
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) state.symbols.create("recurseForDerivations"));
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); if (j != i->value->attrs->end() &&
state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
ignoreAssertionFailures);
} }
} }
} }
@ -360,21 +327,25 @@ static void getDerivations(EvalState & state, Value & vIn,
else if (v.isList()) { else if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n) { for (unsigned int n = 0; n < v.listSize(); ++n) {
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done,
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); ignoreAssertionFailures))
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs,
done, ignoreAssertionFailures);
} }
} }
else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); else
throw TypeError(
"expression does not evaluate to a derivation (or a set or list of "
"those)");
} }
void getDerivations(EvalState& state, Value& v, const string& pathPrefix, void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) Bindings& autoArgs, DrvInfos& drvs,
{ bool ignoreAssertionFailures) {
Done done; Done done;
getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); getDerivations(state, v, pathPrefix, autoArgs, drvs, done,
ignoreAssertionFailures);
} }
} // namespace nix
}

View file

@ -1,16 +1,12 @@
#pragma once #pragma once
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
namespace nix { namespace nix {
struct DrvInfo {
struct DrvInfo
{
public: public:
typedef std::map<string, Path> Outputs; typedef std::map<string, Path> Outputs;
@ -37,14 +33,16 @@ public:
DrvInfo(EvalState& state) : state(&state){}; DrvInfo(EvalState& state) : state(&state){};
DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs); DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs);
DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs); DrvInfo(EvalState& state, ref<Store> store,
const std::string& drvPathWithOutputs);
string queryName() const; string queryName() const;
string querySystem() const; string querySystem() const;
string queryDrvPath() const; string queryDrvPath() const;
string queryOutPath() const; string queryOutPath() const;
string queryOutputName() const; string queryOutputName() const;
/** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ /** Return the list of outputs. The "outputs to install" are determined by
* `meta.outputsToInstall`. */
Outputs queryOutputs(bool onlyOutputsToInstall = false); Outputs queryOutputs(bool onlyOutputsToInstall = false);
StringSet queryMetaNames(); StringSet queryMetaNames();
@ -68,22 +66,19 @@ public:
bool hasFailed() { return failed; }; bool hasFailed() { return failed; };
}; };
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
#else #else
typedef list<DrvInfo> DrvInfos; typedef list<DrvInfo> DrvInfos;
#endif #endif
/* If value `v' denotes a derivation, return a DrvInfo object /* If value `v' denotes a derivation, return a DrvInfo object
describing it. Otherwise return nothing. */ describing it. Otherwise return nothing. */
std::optional<DrvInfo> getDerivation(EvalState & state, std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
Value & v, bool ignoreAssertionFailures); bool ignoreAssertionFailures);
void getDerivations(EvalState& state, Value& v, const string& pathPrefix, void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
Bindings& autoArgs, DrvInfos& drvs, Bindings& autoArgs, DrvInfos& drvs,
bool ignoreAssertionFailures); bool ignoreAssertionFailures);
} // namespace nix
}

View file

@ -1,35 +1,42 @@
#include "json-to-value.hh" #include "json-to-value.hh"
#include <cstring> #include <cstring>
namespace nix { namespace nix {
static void skipWhitespace(const char*& s) {
static void skipWhitespace(const char * & s)
{
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
} }
static string parseJSONString(const char*& s) {
static string parseJSONString(const char * & s)
{
string res; string res;
if (*s++ != '"') throw JSONParseError("expected JSON string"); if (*s++ != '"') throw JSONParseError("expected JSON string");
while (*s != '"') { while (*s != '"') {
if (!*s) throw JSONParseError("got end-of-string in JSON string"); if (!*s) throw JSONParseError("got end-of-string in JSON string");
if (*s == '\\') { if (*s == '\\') {
s++; s++;
if (*s == '"') res += '"'; if (*s == '"')
else if (*s == '\\') res += '\\'; res += '"';
else if (*s == '/') res += '/'; else if (*s == '\\')
else if (*s == '/') res += '/'; res += '\\';
else if (*s == 'b') res += '\b'; else if (*s == '/')
else if (*s == 'f') res += '\f'; res += '/';
else if (*s == 'n') res += '\n'; else if (*s == '/')
else if (*s == 'r') res += '\r'; res += '/';
else if (*s == 't') res += '\t'; else if (*s == 'b')
else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); res += '\b';
else throw JSONParseError("invalid escaped character in JSON string"); else if (*s == 'f')
res += '\f';
else if (*s == 'n')
res += '\n';
else if (*s == 'r')
res += '\r';
else if (*s == 't')
res += '\t';
else if (*s == 'u')
throw JSONParseError(
"\\u characters in JSON strings are currently not supported");
else
throw JSONParseError("invalid escaped character in JSON string");
s++; s++;
} else } else
res += *s++; res += *s++;
@ -38,9 +45,7 @@ static string parseJSONString(const char * & s)
return res; return res;
} }
static void parseJSON(EvalState& state, const char*& s, Value& v) {
static void parseJSON(EvalState & state, const char * & s, Value & v)
{
skipWhitespace(s); skipWhitespace(s);
if (!*s) throw JSONParseError("expected JSON value"); if (!*s) throw JSONParseError("expected JSON value");
@ -57,13 +62,13 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
values.push_back(v2); values.push_back(v2);
skipWhitespace(s); skipWhitespace(s);
if (*s == ']') break; if (*s == ']') break;
if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element"); if (*s != ',')
throw JSONParseError("expected ',' or ']' after JSON array element");
s++; s++;
} }
s++; s++;
state.mkList(v, values.size()); state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n) for (size_t n = 0; n < values.size(); ++n) v.listElems()[n] = values[n];
v.listElems()[n] = values[n];
} }
else if (*s == '{') { else if (*s == '{') {
@ -81,12 +86,12 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
attrs[state.symbols.create(name)] = v2; attrs[state.symbols.create(name)] = v2;
skipWhitespace(s); skipWhitespace(s);
if (*s == '}') break; if (*s == '}') break;
if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member"); if (*s != ',')
throw JSONParseError("expected ',' or '}' after JSON member");
s++; s++;
} }
state.mkAttrs(v, attrs.size()); state.mkAttrs(v, attrs.size());
for (auto & i : attrs) for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second));
v.attrs->push_back(Attr(i.first, i.second));
v.attrs->sort(); v.attrs->sort();
s++; s++;
} }
@ -101,8 +106,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
ValueType number_type = tInt; ValueType number_type = tInt;
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
if (*s == '.' || *s == 'e' || *s == 'E') if (*s == '.' || *s == 'e' || *s == 'E') number_type = tFloat;
number_type = tFloat;
tmp_number += *s++; tmp_number += *s++;
} }
@ -133,17 +137,17 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
mkNull(v); mkNull(v);
} }
else throw JSONParseError("unrecognised JSON value"); else
throw JSONParseError("unrecognised JSON value");
} }
void parseJSON(EvalState& state, const string& s_, Value& v) {
void parseJSON(EvalState & state, const string & s_, Value & v)
{
const char* s = s_.c_str(); const char* s = s_.c_str();
parseJSON(state, s, v); parseJSON(state, s, v);
skipWhitespace(s); skipWhitespace(s);
if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); if (*s)
throw JSONParseError(
format("expected end-of-string while parsing JSON value: %1%") % s);
} }
} // namespace nix
}

View file

@ -1,8 +1,7 @@
#pragma once #pragma once
#include "eval.hh"
#include <string> #include <string>
#include "eval.hh"
namespace nix { namespace nix {

View file

@ -1,23 +1,16 @@
#include "names.hh" #include "names.hh"
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
DrvName::DrvName() { name = ""; }
DrvName::DrvName()
{
name = "";
}
/* Parse a derivation name. The `name' part of a derivation name is /* Parse a derivation name. The `name' part of a derivation name is
everything up to but not including the first dash *not* followed by everything up to but not including the first dash *not* followed by
a letter. The `version' part is the rest (excluding the separating a letter. The `version' part is the rest (excluding the separating
dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
'2.0.48'). */ '2.0.48'). */
DrvName::DrvName(const string & s) : hits(0) DrvName::DrvName(const string& s) : hits(0) {
{
name = fullName = s; name = fullName = s;
for (unsigned int i = 0; i < s.size(); ++i) { for (unsigned int i = 0; i < s.size(); ++i) {
/* !!! isalpha/isdigit are affected by the locale. */ /* !!! isalpha/isdigit are affected by the locale. */
@ -29,21 +22,19 @@ DrvName::DrvName(const string & s) : hits(0)
} }
} }
bool DrvName::matches(DrvName& n) {
bool DrvName::matches(DrvName & n)
{
if (name != "*") { if (name != "*") {
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); if (!regex)
regex = std::unique_ptr<std::regex>(
new std::regex(name, std::regex::extended));
if (!std::regex_match(n.name, *regex)) return false; if (!std::regex_match(n.name, *regex)) return false;
} }
if (version != "" && version != n.version) return false; if (version != "" && version != n.version) return false;
return true; return true;
} }
string nextComponent(string::const_iterator& p, string nextComponent(string::const_iterator& p,
const string::const_iterator end) const string::const_iterator end) {
{
/* Skip any dots and dashes (component separators). */ /* Skip any dots and dashes (component separators). */
while (p != end && (*p == '.' || *p == '-')) ++p; while (p != end && (*p == '.' || *p == '-')) ++p;
@ -56,52 +47,52 @@ string nextComponent(string::const_iterator & p,
if (isdigit(*p)) if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++; while (p != end && isdigit(*p)) s += *p++;
else else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) s += *p++;
s += *p++;
return s; return s;
} }
static bool componentsLT(const string& c1, const string& c2) {
static bool componentsLT(const string & c1, const string & c2)
{
int n1, n2; int n1, n2;
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
if (c1Num && c2Num) return n1 < n2; if (c1Num && c2Num)
else if (c1 == "" && c2Num) return true; return n1 < n2;
else if (c1 == "pre" && c2 != "pre") return true; else if (c1 == "" && c2Num)
else if (c2 == "pre") return false; return true;
else if (c1 == "pre" && c2 != "pre")
return true;
else if (c2 == "pre")
return false;
/* Assume that `2.3a' < `2.3.1'. */ /* Assume that `2.3a' < `2.3.1'. */
else if (c2Num) return true; else if (c2Num)
else if (c1Num) return false; return true;
else return c1 < c2; else if (c1Num)
return false;
else
return c1 < c2;
} }
int compareVersions(const string& v1, const string& v2) {
int compareVersions(const string & v1, const string & v2)
{
string::const_iterator p1 = v1.begin(); string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin(); string::const_iterator p2 = v2.begin();
while (p1 != v1.end() || p2 != v2.end()) { while (p1 != v1.end() || p2 != v2.end()) {
string c1 = nextComponent(p1, v1.end()); string c1 = nextComponent(p1, v1.end());
string c2 = nextComponent(p2, v2.end()); string c2 = nextComponent(p2, v2.end());
if (componentsLT(c1, c2)) return -1; if (componentsLT(c1, c2))
else if (componentsLT(c2, c1)) return 1; return -1;
else if (componentsLT(c2, c1))
return 1;
} }
return 0; return 0;
} }
DrvNames drvNamesFromArgs(const Strings& opArgs) {
DrvNames drvNamesFromArgs(const Strings & opArgs)
{
DrvNames result; DrvNames result;
for (auto & i : opArgs) for (auto& i : opArgs) result.push_back(DrvName(i));
result.push_back(DrvName(i));
return result; return result;
} }
} // namespace nix
}

View file

@ -1,14 +1,12 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include "types.hh"
#include <regex> #include <regex>
#include "types.hh"
namespace nix { namespace nix {
struct DrvName struct DrvName {
{
string fullName; string fullName;
string name; string name;
string version; string version;
@ -29,4 +27,4 @@ string nextComponent(string::const_iterator & p,
int compareVersions(const string& v1, const string& v2); int compareVersions(const string& v1, const string& v2);
DrvNames drvNamesFromArgs(const Strings& opArgs); DrvNames drvNamesFromArgs(const Strings& opArgs);
} } // namespace nix

View file

@ -1,35 +1,34 @@
#include "nixexpr.hh" #include "nixexpr.hh"
#include <cstdlib>
#include "derivations.hh" #include "derivations.hh"
#include "util.hh" #include "util.hh"
#include <cstdlib>
namespace nix { namespace nix {
/* Displaying abstract syntax trees. */ /* Displaying abstract syntax trees. */
std::ostream & operator << (std::ostream & str, const Expr & e) std::ostream& operator<<(std::ostream& str, const Expr& e) {
{
e.show(str); e.show(str);
return str; return str;
} }
static void showString(std::ostream & str, const string & s) static void showString(std::ostream& str, const string& s) {
{
str << '"'; str << '"';
for (auto c : (string)s) for (auto c : (string)s)
if (c == '"' || c == '\\' || c == '$') str << "\\" << c; if (c == '"' || c == '\\' || c == '$')
else if (c == '\n') str << "\\n"; str << "\\" << c;
else if (c == '\r') str << "\\r"; else if (c == '\n')
else if (c == '\t') str << "\\t"; str << "\\n";
else str << c; else if (c == '\r')
str << "\\r";
else if (c == '\t')
str << "\\t";
else
str << c;
str << '"'; str << '"';
} }
static void showId(std::ostream & str, const string & s) static void showId(std::ostream& str, const string& s) {
{
if (s.empty()) if (s.empty())
str << "\"\""; str << "\"\"";
else if (s == "if") // FIXME: handle other keywords else if (s == "if") // FIXME: handle other keywords
@ -41,10 +40,8 @@ static void showId(std::ostream & str, const string & s)
return; return;
} }
for (auto c : s) for (auto c : s)
if (!((c >= 'a' && c <= 'z') || if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) {
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
showString(str, s); showString(str, s);
return; return;
} }
@ -52,60 +49,39 @@ static void showId(std::ostream & str, const string & s)
} }
} }
std::ostream & operator << (std::ostream & str, const Symbol & sym) std::ostream& operator<<(std::ostream& str, const Symbol& sym) {
{
showId(str, *sym.s); showId(str, *sym.s);
return str; return str;
} }
void Expr::show(std::ostream & str) const void Expr::show(std::ostream& str) const { abort(); }
{
abort();
}
void ExprInt::show(std::ostream & str) const void ExprInt::show(std::ostream& str) const { str << n; }
{
str << n;
}
void ExprFloat::show(std::ostream & str) const void ExprFloat::show(std::ostream& str) const { str << nf; }
{
str << nf;
}
void ExprString::show(std::ostream & str) const void ExprString::show(std::ostream& str) const { showString(str, s); }
{
showString(str, s);
}
void ExprPath::show(std::ostream & str) const void ExprPath::show(std::ostream& str) const { str << s; }
{
str << s;
}
void ExprVar::show(std::ostream & str) const void ExprVar::show(std::ostream& str) const { str << name; }
{
str << name;
}
void ExprSelect::show(std::ostream & str) const void ExprSelect::show(std::ostream& str) const {
{
str << "(" << *e << ")." << showAttrPath(attrPath); str << "(" << *e << ")." << showAttrPath(attrPath);
if (def) str << " or (" << *def << ")"; if (def) str << " or (" << *def << ")";
} }
void ExprOpHasAttr::show(std::ostream & str) const void ExprOpHasAttr::show(std::ostream& str) const {
{
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
} }
void ExprAttrs::show(std::ostream & str) const void ExprAttrs::show(std::ostream& str) const {
{
if (recursive) str << "rec "; if (recursive) str << "rec ";
str << "{ "; str << "{ ";
for (auto& i : attrs) for (auto& i : attrs)
if (i.second.inherited) if (i.second.inherited)
str << "inherit " << i.first << " " << "; "; str << "inherit " << i.first << " "
<< "; ";
else else
str << i.first << " = " << *i.second.e << "; "; str << i.first << " = " << *i.second.e << "; ";
for (auto& i : dynamicAttrs) for (auto& i : dynamicAttrs)
@ -113,22 +89,22 @@ void ExprAttrs::show(std::ostream & str) const
str << "}"; str << "}";
} }
void ExprList::show(std::ostream & str) const void ExprList::show(std::ostream& str) const {
{
str << "[ "; str << "[ ";
for (auto & i : elems) for (auto& i : elems) str << "(" << *i << ") ";
str << "(" << *i << ") ";
str << "]"; str << "]";
} }
void ExprLambda::show(std::ostream & str) const void ExprLambda::show(std::ostream& str) const {
{
str << "("; str << "(";
if (matchAttrs) { if (matchAttrs) {
str << "{ "; str << "{ ";
bool first = true; bool first = true;
for (auto& i : formals->formals) { for (auto& i : formals->formals) {
if (first) first = false; else str << ", "; if (first)
first = false;
else
str << ", ";
str << i.name; str << i.name;
if (i.def) str << " ? " << *i.def; if (i.def) str << " ? " << *i.def;
} }
@ -143,71 +119,63 @@ void ExprLambda::show(std::ostream & str) const
str << ": " << *body << ")"; str << ": " << *body << ")";
} }
void ExprLet::show(std::ostream & str) const void ExprLet::show(std::ostream& str) const {
{
str << "(let "; str << "(let ";
for (auto& i : attrs->attrs) for (auto& i : attrs->attrs)
if (i.second.inherited) { if (i.second.inherited) {
str << "inherit " << i.first << "; "; str << "inherit " << i.first << "; ";
} } else
else
str << i.first << " = " << *i.second.e << "; "; str << i.first << " = " << *i.second.e << "; ";
str << "in " << *body << ")"; str << "in " << *body << ")";
} }
void ExprWith::show(std::ostream & str) const void ExprWith::show(std::ostream& str) const {
{
str << "(with " << *attrs << "; " << *body << ")"; str << "(with " << *attrs << "; " << *body << ")";
} }
void ExprIf::show(std::ostream & str) const void ExprIf::show(std::ostream& str) const {
{
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
} }
void ExprAssert::show(std::ostream & str) const void ExprAssert::show(std::ostream& str) const {
{
str << "assert " << *cond << "; " << *body; str << "assert " << *cond << "; " << *body;
} }
void ExprOpNot::show(std::ostream & str) const void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; }
{
str << "(! " << *e << ")";
}
void ExprConcatStrings::show(std::ostream & str) const void ExprConcatStrings::show(std::ostream& str) const {
{
bool first = true; bool first = true;
str << "("; str << "(";
for (auto& i : *es) { for (auto& i : *es) {
if (first) first = false; else str << " + "; if (first)
first = false;
else
str << " + ";
str << *i; str << *i;
} }
str << ")"; str << ")";
} }
void ExprPos::show(std::ostream & str) const void ExprPos::show(std::ostream& str) const { str << "__curPos"; }
{
str << "__curPos";
}
std::ostream& operator<<(std::ostream& str, const Pos& pos) {
std::ostream & operator << (std::ostream & str, const Pos & pos)
{
if (!pos) if (!pos)
str << "undefined position"; str << "undefined position";
else else
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str(); str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string)pos.file %
pos.line % pos.column)
.str();
return str; return str;
} }
string showAttrPath(const AttrPath& attrPath) {
string showAttrPath(const AttrPath & attrPath)
{
std::ostringstream out; std::ostringstream out;
bool first = true; bool first = true;
for (auto& i : attrPath) { for (auto& i : attrPath) {
if (!first) out << '.'; else first = false; if (!first)
out << '.';
else
first = false;
if (i.symbol.set()) if (i.symbol.set())
out << i.symbol; out << i.symbol;
else else
@ -216,35 +184,21 @@ string showAttrPath(const AttrPath & attrPath)
return out.str(); return out.str();
} }
Pos noPos; Pos noPos;
/* Computing levels/displacements for variables. */ /* Computing levels/displacements for variables. */
void Expr::bindVars(const StaticEnv & env) void Expr::bindVars(const StaticEnv& env) { abort(); }
{
abort();
}
void ExprInt::bindVars(const StaticEnv & env) void ExprInt::bindVars(const StaticEnv& env) {}
{
}
void ExprFloat::bindVars(const StaticEnv & env) void ExprFloat::bindVars(const StaticEnv& env) {}
{
}
void ExprString::bindVars(const StaticEnv & env) void ExprString::bindVars(const StaticEnv& env) {}
{
}
void ExprPath::bindVars(const StaticEnv & env) void ExprPath::bindVars(const StaticEnv& env) {}
{
}
void ExprVar::bindVars(const StaticEnv & env) void ExprVar::bindVars(const StaticEnv& env) {
{
/* Check whether the variable appears in the environment. If so, /* Check whether the variable appears in the environment. If so,
set its level and displacement. */ set its level and displacement. */
const StaticEnv* curEnv; const StaticEnv* curEnv;
@ -267,31 +221,28 @@ void ExprVar::bindVars(const StaticEnv & env)
/* Otherwise, the variable must be obtained from the nearest /* Otherwise, the variable must be obtained from the nearest
enclosing `with'. If there is no `with', then we can issue an enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */ "undefined variable" error now. */
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); if (withLevel == -1)
throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name %
pos);
fromWith = true; fromWith = true;
this->level = withLevel; this->level = withLevel;
} }
void ExprSelect::bindVars(const StaticEnv & env) void ExprSelect::bindVars(const StaticEnv& env) {
{
e->bindVars(env); e->bindVars(env);
if (def) def->bindVars(env); if (def) def->bindVars(env);
for (auto& i : attrPath) for (auto& i : attrPath)
if (!i.symbol.set()) if (!i.symbol.set()) i.expr->bindVars(env);
i.expr->bindVars(env);
} }
void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprOpHasAttr::bindVars(const StaticEnv& env) {
{
e->bindVars(env); e->bindVars(env);
for (auto& i : attrPath) for (auto& i : attrPath)
if (!i.symbol.set()) if (!i.symbol.set()) i.expr->bindVars(env);
i.expr->bindVars(env);
} }
void ExprAttrs::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv& env) {
{
const StaticEnv* dynamicEnv = &env; const StaticEnv* dynamicEnv = &env;
StaticEnv newEnv(false, &env); StaticEnv newEnv(false, &env);
@ -299,16 +250,14 @@ void ExprAttrs::bindVars(const StaticEnv & env)
dynamicEnv = &newEnv; dynamicEnv = &newEnv;
unsigned int displ = 0; unsigned int displ = 0;
for (auto & i : attrs) for (auto& i : attrs) newEnv.vars[i.first] = i.second.displ = displ++;
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs) for (auto& i : attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv); i.second.e->bindVars(i.second.inherited ? env : newEnv);
} }
else else
for (auto & i : attrs) for (auto& i : attrs) i.second.e->bindVars(env);
i.second.e->bindVars(env);
for (auto& i : dynamicAttrs) { for (auto& i : dynamicAttrs) {
i.nameExpr->bindVars(*dynamicEnv); i.nameExpr->bindVars(*dynamicEnv);
@ -316,14 +265,11 @@ void ExprAttrs::bindVars(const StaticEnv & env)
} }
} }
void ExprList::bindVars(const StaticEnv & env) void ExprList::bindVars(const StaticEnv& env) {
{ for (auto& i : elems) i->bindVars(env);
for (auto & i : elems)
i->bindVars(env);
} }
void ExprLambda::bindVars(const StaticEnv & env) void ExprLambda::bindVars(const StaticEnv& env) {
{
StaticEnv newEnv(false, &env); StaticEnv newEnv(false, &env);
unsigned int displ = 0; unsigned int displ = 0;
@ -331,8 +277,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
if (!arg.empty()) newEnv.vars[arg] = displ++; if (!arg.empty()) newEnv.vars[arg] = displ++;
if (matchAttrs) { if (matchAttrs) {
for (auto & i : formals->formals) for (auto& i : formals->formals) newEnv.vars[i.name] = displ++;
newEnv.vars[i.name] = displ++;
for (auto& i : formals->formals) for (auto& i : formals->formals)
if (i.def) i.def->bindVars(newEnv); if (i.def) i.def->bindVars(newEnv);
@ -341,13 +286,11 @@ void ExprLambda::bindVars(const StaticEnv & env)
body->bindVars(newEnv); body->bindVars(newEnv);
} }
void ExprLet::bindVars(const StaticEnv & env) void ExprLet::bindVars(const StaticEnv& env) {
{
StaticEnv newEnv(false, &env); StaticEnv newEnv(false, &env);
unsigned int displ = 0; unsigned int displ = 0;
for (auto & i : attrs->attrs) for (auto& i : attrs->attrs) newEnv.vars[i.first] = i.second.displ = displ++;
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs->attrs) for (auto& i : attrs->attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv); i.second.e->bindVars(i.second.inherited ? env : newEnv);
@ -355,8 +298,7 @@ void ExprLet::bindVars(const StaticEnv & env)
body->bindVars(newEnv); body->bindVars(newEnv);
} }
void ExprWith::bindVars(const StaticEnv & env) void ExprWith::bindVars(const StaticEnv& env) {
{
/* Does this `with' have an enclosing `with'? If so, record its /* Does this `with' have an enclosing `with'? If so, record its
level so that `lookupVar' can look up variables in the previous level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */ `with' if this one doesn't contain the desired attribute. */
@ -374,65 +316,46 @@ void ExprWith::bindVars(const StaticEnv & env)
body->bindVars(newEnv); body->bindVars(newEnv);
} }
void ExprIf::bindVars(const StaticEnv & env) void ExprIf::bindVars(const StaticEnv& env) {
{
cond->bindVars(env); cond->bindVars(env);
then->bindVars(env); then->bindVars(env);
else_->bindVars(env); else_->bindVars(env);
} }
void ExprAssert::bindVars(const StaticEnv & env) void ExprAssert::bindVars(const StaticEnv& env) {
{
cond->bindVars(env); cond->bindVars(env);
body->bindVars(env); body->bindVars(env);
} }
void ExprOpNot::bindVars(const StaticEnv & env) void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); }
{
e->bindVars(env); void ExprConcatStrings::bindVars(const StaticEnv& env) {
} for (auto& i : *es) i->bindVars(env);
void ExprConcatStrings::bindVars(const StaticEnv & env)
{
for (auto & i : *es)
i->bindVars(env);
}
void ExprPos::bindVars(const StaticEnv & env)
{
} }
void ExprPos::bindVars(const StaticEnv& env) {}
/* Storing function names. */ /* Storing function names. */
void Expr::setName(Symbol & name) void Expr::setName(Symbol& name) {}
{
}
void ExprLambda::setName(Symbol& name) {
void ExprLambda::setName(Symbol & name)
{
this->name = name; this->name = name;
body->setName(name); body->setName(name);
} }
string ExprLambda::showNamePos() const {
string ExprLambda::showNamePos() const return (format("%1% at %2%") %
{ (name.set() ? "'" + (string)name + "'" : "anonymous function") % pos)
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); .str();
} }
/* Symbol table. */ /* Symbol table. */
size_t SymbolTable::totalSize() const size_t SymbolTable::totalSize() const {
{
size_t n = 0; size_t n = 0;
for (auto & i : symbols) for (auto& i : symbols) n += i.size();
n += i.size();
return n; return n;
} }
} // namespace nix
}

View file

@ -1,39 +1,27 @@
#pragma once #pragma once
#include "value.hh"
#include "symbol-table.hh"
#include <map> #include <map>
#include "symbol-table.hh"
#include "value.hh"
namespace nix { namespace nix {
MakeError(EvalError, Error) MakeError(ParseError, Error)
MakeError(EvalError, Error) MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError)
MakeError(ParseError, Error) MakeError(Abort, EvalError) MakeError(TypeError, EvalError)
MakeError(AssertionError, EvalError)
MakeError(ThrownError, AssertionError)
MakeError(Abort, EvalError)
MakeError(TypeError, EvalError)
MakeError(UndefinedVarError, Error) MakeError(UndefinedVarError, Error)
MakeError(RestrictedPathError, Error) MakeError(RestrictedPathError, Error)
/* Position objects. */ /* Position objects. */
struct Pos struct Pos {
{
Symbol file; Symbol file;
unsigned int line, column; unsigned int line, column;
Pos() : line(0), column(0){}; Pos() : line(0), column(0){};
Pos(const Symbol& file, unsigned int line, unsigned int column) Pos(const Symbol& file, unsigned int line, unsigned int column)
: file(file), line(line), column(column){}; : file(file), line(line), column(column){};
operator bool() const operator bool() const { return line != 0; }
{ bool operator<(const Pos& p2) const {
return line != 0;
}
bool operator < (const Pos & p2) const
{
if (!line) return p2.line; if (!line) return p2.line;
if (!p2.line) return false; if (!p2.line) return false;
int d = ((string)file).compare((string)p2.file); int d = ((string)file).compare((string)p2.file);
@ -49,16 +37,13 @@ extern Pos noPos;
std::ostream& operator<<(std::ostream& str, const Pos& pos); std::ostream& operator<<(std::ostream& str, const Pos& pos);
struct Env; struct Env;
struct Value; struct Value;
class EvalState; class EvalState;
struct StaticEnv; struct StaticEnv;
/* An attribute path is a sequence of attribute names. */ /* An attribute path is a sequence of attribute names. */
struct AttrName struct AttrName {
{
Symbol symbol; Symbol symbol;
Expr* expr; Expr* expr;
AttrName(const Symbol& s) : symbol(s){}; AttrName(const Symbol& s) : symbol(s){};
@ -69,11 +54,9 @@ typedef std::vector<AttrName> AttrPath;
string showAttrPath(const AttrPath& attrPath); string showAttrPath(const AttrPath& attrPath);
/* Abstract syntax of Nix expressions. */ /* Abstract syntax of Nix expressions. */
struct Expr struct Expr {
{
virtual ~Expr(){}; virtual ~Expr(){};
virtual void show(std::ostream& str) const; virtual void show(std::ostream& str) const;
virtual void bindVars(const StaticEnv& env); virtual void bindVars(const StaticEnv& env);
@ -89,8 +72,7 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
void eval(EvalState& state, Env& env, Value& v); \ void eval(EvalState& state, Env& env, Value& v); \
void bindVars(const StaticEnv& env); void bindVars(const StaticEnv& env);
struct ExprInt : Expr struct ExprInt : Expr {
{
NixInt n; NixInt n;
Value v; Value v;
ExprInt(NixInt n) : n(n) { mkInt(v, n); }; ExprInt(NixInt n) : n(n) { mkInt(v, n); };
@ -98,8 +80,7 @@ struct ExprInt : Expr
Value* maybeThunk(EvalState& state, Env& env); Value* maybeThunk(EvalState& state, Env& env);
}; };
struct ExprFloat : Expr struct ExprFloat : Expr {
{
NixFloat nf; NixFloat nf;
Value v; Value v;
ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
@ -107,8 +88,7 @@ struct ExprFloat : Expr
Value* maybeThunk(EvalState& state, Env& env); Value* maybeThunk(EvalState& state, Env& env);
}; };
struct ExprString : Expr struct ExprString : Expr {
{
Symbol s; Symbol s;
Value v; Value v;
ExprString(const Symbol& s) : s(s) { mkString(v, s); }; ExprString(const Symbol& s) : s(s) { mkString(v, s); };
@ -117,14 +97,12 @@ struct ExprString : Expr
}; };
/* Temporary class used during parsing of indented strings. */ /* Temporary class used during parsing of indented strings. */
struct ExprIndStr : Expr struct ExprIndStr : Expr {
{
string s; string s;
ExprIndStr(const string& s) : s(s){}; ExprIndStr(const string& s) : s(s){};
}; };
struct ExprPath : Expr struct ExprPath : Expr {
{
string s; string s;
Value v; Value v;
ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
@ -132,8 +110,7 @@ struct ExprPath : Expr
Value* maybeThunk(EvalState& state, Env& env); Value* maybeThunk(EvalState& state, Env& env);
}; };
struct ExprVar : Expr struct ExprVar : Expr {
{
Pos pos; Pos pos;
Symbol name; Symbol name;
@ -156,26 +133,27 @@ struct ExprVar : Expr
Value* maybeThunk(EvalState& state, Env& env); Value* maybeThunk(EvalState& state, Env& env);
}; };
struct ExprSelect : Expr struct ExprSelect : Expr {
{
Pos pos; Pos pos;
Expr *e, *def; Expr *e, *def;
AttrPath attrPath; AttrPath attrPath;
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def)
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; : pos(pos), e(e), def(def), attrPath(attrPath){};
ExprSelect(const Pos& pos, Expr* e, const Symbol& name)
: pos(pos), e(e), def(0) {
attrPath.push_back(AttrName(name));
};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprOpHasAttr : Expr struct ExprOpHasAttr : Expr {
{
Expr* e; Expr* e;
AttrPath attrPath; AttrPath attrPath;
ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){}; ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprAttrs : Expr struct ExprAttrs : Expr {
{
bool recursive; bool recursive;
struct AttrDef { struct AttrDef {
bool inherited; bool inherited;
@ -200,116 +178,109 @@ struct ExprAttrs : Expr
COMMON_METHODS COMMON_METHODS
}; };
struct ExprList : Expr struct ExprList : Expr {
{
std::vector<Expr*> elems; std::vector<Expr*> elems;
ExprList(){}; ExprList(){};
COMMON_METHODS COMMON_METHODS
}; };
struct Formal struct Formal {
{
Symbol name; Symbol name;
Expr* def; Expr* def;
Formal(const Symbol& name, Expr* def) : name(name), def(def){}; Formal(const Symbol& name, Expr* def) : name(name), def(def){};
}; };
struct Formals struct Formals {
{
typedef std::list<Formal> Formals_; typedef std::list<Formal> Formals_;
Formals_ formals; Formals_ formals;
std::set<Symbol> argNames; // used during parsing std::set<Symbol> argNames; // used during parsing
bool ellipsis; bool ellipsis;
}; };
struct ExprLambda : Expr struct ExprLambda : Expr {
{
Pos pos; Pos pos;
Symbol name; Symbol name;
Symbol arg; Symbol arg;
bool matchAttrs; bool matchAttrs;
Formals* formals; Formals* formals;
Expr* body; Expr* body;
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs,
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) Formals* formals, Expr* body)
{ : pos(pos),
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) arg(arg),
throw ParseError(format("duplicate formal function argument '%1%' at %2%") matchAttrs(matchAttrs),
% arg % pos); formals(formals),
body(body) {
if (!arg.empty() && formals &&
formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(
format("duplicate formal function argument '%1%' at %2%") % arg %
pos);
}; };
void setName(Symbol& name); void setName(Symbol& name);
string showNamePos() const; string showNamePos() const;
COMMON_METHODS COMMON_METHODS
}; };
struct ExprLet : Expr struct ExprLet : Expr {
{
ExprAttrs* attrs; ExprAttrs* attrs;
Expr* body; Expr* body;
ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){}; ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprWith : Expr struct ExprWith : Expr {
{
Pos pos; Pos pos;
Expr *attrs, *body; Expr *attrs, *body;
size_t prevWith; size_t prevWith;
ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; ExprWith(const Pos& pos, Expr* attrs, Expr* body)
: pos(pos), attrs(attrs), body(body){};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprIf : Expr struct ExprIf : Expr {
{
Expr *cond, *then, *else_; Expr *cond, *then, *else_;
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; ExprIf(Expr* cond, Expr* then, Expr* else_)
: cond(cond), then(then), else_(else_){};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprAssert : Expr struct ExprAssert : Expr {
{
Pos pos; Pos pos;
Expr *cond, *body; Expr *cond, *body;
ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; ExprAssert(const Pos& pos, Expr* cond, Expr* body)
: pos(pos), cond(cond), body(body){};
COMMON_METHODS COMMON_METHODS
}; };
struct ExprOpNot : Expr struct ExprOpNot : Expr {
{
Expr* e; Expr* e;
ExprOpNot(Expr* e) : e(e){}; ExprOpNot(Expr* e) : e(e){};
COMMON_METHODS COMMON_METHODS
}; };
#define MakeBinOp(name, s) \ #define MakeBinOp(name, s) \
struct name : Expr \ struct name : Expr { \
{ \
Pos pos; \ Pos pos; \
Expr *e1, *e2; \ Expr *e1, *e2; \
name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \ name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \
name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \ name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \
void show(std::ostream & str) const \ void show(std::ostream& str) const { \
{ \
str << "(" << *e1 << " " s " " << *e2 << ")"; \ str << "(" << *e1 << " " s " " << *e2 << ")"; \
} \ } \
void bindVars(const StaticEnv & env) \ void bindVars(const StaticEnv& env) { \
{ \ e1->bindVars(env); \
e1->bindVars(env); e2->bindVars(env); \ e2->bindVars(env); \
} \ } \
void eval(EvalState& state, Env& env, Value& v); \ void eval(EvalState& state, Env& env, Value& v); \
}; };
MakeBinOp(ExprApp, "") MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprOpAnd, "&&")
MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpImpl, "->")
MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprOpConcatLists, "++") MakeBinOp(ExprOpConcatLists, "++")
struct ExprConcatStrings : Expr struct ExprConcatStrings : Expr {
{
Pos pos; Pos pos;
bool forceString; bool forceString;
vector<Expr*>* es; vector<Expr*>* es;
@ -318,19 +289,16 @@ struct ExprConcatStrings : Expr
COMMON_METHODS COMMON_METHODS
}; };
struct ExprPos : Expr struct ExprPos : Expr {
{
Pos pos; Pos pos;
ExprPos(const Pos& pos) : pos(pos){}; ExprPos(const Pos& pos) : pos(pos){};
COMMON_METHODS COMMON_METHODS
}; };
/* Static environments are used to map variable names onto (level, /* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at displacement) pairs used to obtain the value of the variable at
runtime. */ runtime. */
struct StaticEnv struct StaticEnv {
{
bool isWith; bool isWith;
const StaticEnv* up; const StaticEnv* up;
typedef std::map<Symbol, unsigned int> Vars; typedef std::map<Symbol, unsigned int> Vars;
@ -338,5 +306,4 @@ struct StaticEnv
StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){}; StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){};
}; };
} // namespace nix
}

File diff suppressed because it is too large Load diff

View file

@ -1,12 +1,10 @@
#include "eval.hh"
#include <tuple> #include <tuple>
#include <vector> #include <vector>
#include "eval.hh"
namespace nix { namespace nix {
struct RegisterPrimOp struct RegisterPrimOp {
{
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
static PrimOps* primOps; static PrimOps* primOps;
/* You can register a constant by passing an arity of 0. fun /* You can register a constant by passing an arity of 0. fun
@ -19,8 +17,9 @@ struct RegisterPrimOp
may wish to use them in limited contexts without globally enabling may wish to use them in limited contexts without globally enabling
them. */ them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */ /* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); void prim_importNative(EvalState& state, const Pos& pos, Value** args,
Value& v);
/* Execute a program and parse its output */ /* Execute a program and parse its output */
void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v); void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v);
} } // namespace nix

View file

@ -1,21 +1,21 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "derivations.hh" #include "derivations.hh"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix { namespace nix {
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos,
{ Value** args, Value& v) {
PathSet context; PathSet context;
string s = state.coerceToString(pos, *args[0], context); string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet()); mkString(v, s, PathSet());
} }
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static RegisterPrimOp r1("__unsafeDiscardStringContext", 1,
prim_unsafeDiscardStringContext);
static void prim_hasContext(EvalState& state, const Pos& pos, Value** args,
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) Value& v) {
{
PathSet context; PathSet context;
state.forceString(*args[0], context, pos); state.forceString(*args[0], context, pos);
mkBool(v, !context.empty()); mkBool(v, !context.empty());
@ -23,27 +23,25 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance, builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */ drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos,
{ Value** args, Value& v) {
PathSet context; PathSet context;
string s = state.coerceToString(pos, *args[0], context); string s = state.coerceToString(pos, *args[0], context);
PathSet context2; PathSet context2;
for (auto & p : context) for (auto& p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p);
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
mkString(v, s, context2); mkString(v, s, context2);
} }
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1,
prim_unsafeDiscardOutputDependency);
/* Extract the context of a string as a structured Nix value. /* Extract the context of a string as a structured Nix value.
@ -64,8 +62,8 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar
Note that for a given path any combination of the above attributes Note that for a given path any combination of the above attributes
may be present. may be present.
*/ */
static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_getContext(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
struct ContextInfo { struct ContextInfo {
bool path = false; bool path = false;
bool allOutputs = false; bool allOutputs = false;
@ -92,7 +90,10 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
auto iter = contextInfos.find(*path); auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) { if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); contextInfos.emplace(
*path,
ContextInfo{isPath, isAllOutputs,
output.empty() ? Strings{} : Strings{std::move(output)}});
} else { } else {
if (isPath) if (isPath)
iter->second.path = true; iter->second.path = true;
@ -110,8 +111,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
for (const auto& info : contextInfos) { for (const auto& info : contextInfos) {
auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first)); auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first));
state.mkAttrs(infoVal, 3); state.mkAttrs(infoVal, 3);
if (info.second.path) if (info.second.path) mkBool(*state.allocAttr(infoVal, sPath), true);
mkBool(*state.allocAttr(infoVal, sPath), true);
if (info.second.allOutputs) if (info.second.allOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true); mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
if (!info.second.outputs.empty()) { if (!info.second.outputs.empty()) {
@ -129,14 +129,13 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
static RegisterPrimOp r4("__getContext", 1, prim_getContext); static RegisterPrimOp r4("__getContext", 1, prim_getContext);
/* Append the given context to a given string. /* Append the given context to a given string.
See the commentary above unsafeGetContext for details of the See the commentary above unsafeGetContext for details of the
context representation. context representation.
*/ */
static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_appendContext(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
PathSet context; PathSet context;
auto orig = state.forceString(*args[0], context, pos); auto orig = state.forceString(*args[0], context, pos);
@ -146,21 +145,23 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
auto sAllOutputs = state.symbols.create("allOutputs"); auto sAllOutputs = state.symbols.create("allOutputs");
for (auto& i : *args[1]->attrs) { for (auto& i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name)) if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); throw EvalError("Context key '%s' is not a store path, at %s", i.name,
if (!settings.readOnlyMode) i.pos);
state.store->ensurePath(i.name); if (!settings.readOnlyMode) state.store->ensurePath(i.name);
state.forceAttrs(*i.value, *i.pos); state.forceAttrs(*i.value, *i.pos);
auto iter = i.value->attrs->find(sPath); auto iter = i.value->attrs->find(sPath);
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) if (state.forceBool(*iter->value, *iter->pos)) context.insert(i.name);
context.insert(i.name);
} }
iter = i.value->attrs->find(sAllOutputs); iter = i.value->attrs->find(sAllOutputs);
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) { if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) { if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); throw EvalError(
"Tried to add all-outputs context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
} }
context.insert("=" + string(i.name)); context.insert("=" + string(i.name));
} }
@ -170,10 +171,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos); state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) { if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); throw EvalError(
"Tried to add derivation output context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
} }
for (unsigned int n = 0; n < iter->value->listSize(); ++n) { for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); auto name =
state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name)); context.insert("!" + name + "!" + string(i.name));
} }
} }
@ -184,4 +189,4 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
} } // namespace nix

View file

@ -1,22 +1,18 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"
#include <sys/time.h> #include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "hash.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals; using namespace std::string_literals;
namespace nix { namespace nix {
struct GitInfo struct GitInfo {
{
Path storePath; Path storePath;
std::string rev; std::string rev;
std::string shortRev; std::string shortRev;
@ -27,24 +23,22 @@ std::regex revRegex("^[0-9a-fA-F]{40}$");
GitInfo exportGit(ref<Store> store, const std::string& uri, GitInfo exportGit(ref<Store> store, const std::string& uri,
std::optional<std::string> ref, std::string rev, std::optional<std::string> ref, std::string rev,
const std::string & name) const std::string& name) {
{
if (evalSettings.pureEval && rev == "") if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
bool clean = true; bool clean = true;
try { try {
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); runProgram("git", true,
{"-C", uri, "diff-index", "--quiet", "HEAD", "--"});
} catch (ExecError& e) { } catch (ExecError& e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
clean = false; clean = false;
} }
if (!clean) { if (!clean) {
/* This is an unclean working tree. So copy all tracked /* This is an unclean working tree. So copy all tracked
files. */ files. */
@ -70,7 +64,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
return files.count(file); return files.count(file);
}; };
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); gitInfo.storePath =
store->addToStore("source", uri, true, htSHA256, filter);
return gitInfo; return gitInfo;
} }
@ -87,7 +82,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
deletePath(getCacheDir() + "/nix/git"); deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); Path cacheDir = getCacheDir() + "/nix/gitv2/" +
hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) { if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir)); createDirs(dirOf(cacheDir));
@ -122,13 +118,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
doFetch = stat(localRefFile.c_str(), &st) != 0 || doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now; (uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now;
} }
if (doFetch) if (doFetch) {
{ Activity act(*logger, lvlTalkative, actUnknown,
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so // FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr. // we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); runProgram("git", true,
{"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri,
fmt("%s:%s", *ref, *ref)});
struct timeval times[2]; struct timeval times[2];
times[0].tv_sec = now; times[0].tv_sec = now;
@ -146,9 +144,12 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev)
.to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link"; Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...",
storeLink)); // FIXME: broken
try { try {
auto json = nlohmann::json::parse(readFile(storeLink)); auto json = nlohmann::json::parse(readFile(storeLink));
@ -177,7 +178,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); gitInfo.revCount = std::stoull(runProgram(
"git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev}));
nlohmann::json json; nlohmann::json json;
json["storePath"] = gitInfo.storePath; json["storePath"] = gitInfo.storePath;
@ -191,8 +193,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
return gitInfo; return gitInfo;
} }
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
std::string url; std::string url;
std::optional<std::string> ref; std::optional<std::string> ref;
std::string rev; std::string rev;
@ -202,13 +204,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
state.forceValue(*args[0]); state.forceValue(*args[0]);
if (args[0]->type == tAttrs) { if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos); state.forceAttrs(*args[0], pos);
for (auto& attr : *args[0]->attrs) { for (auto& attr : *args[0]->attrs) {
string n(attr.name); string n(attr.name);
if (n == "url") if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false); url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "ref") else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos); ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev") else if (n == "rev")
@ -216,7 +218,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
else if (n == "name") else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos); name = state.forceStringNoCtx(*attr.value, *attr.pos);
else else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); throw EvalError("unsupported argument '%s' to 'fetchGit', at %s",
attr.name, *attr.pos);
} }
if (url.empty()) if (url.empty())
@ -232,10 +235,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
auto gitInfo = exportGit(state.store, url, ref, rev, name); auto gitInfo = exportGit(state.store, url, ref, rev, name);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath,
PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
gitInfo.revCount);
v.attrs->sort(); v.attrs->sort();
if (state.allowedPaths) if (state.allowedPaths)
@ -244,4 +250,4 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
} } // namespace nix

View file

@ -1,21 +1,17 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include <sys/time.h> #include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals; using namespace std::string_literals;
namespace nix { namespace nix {
struct HgInfo struct HgInfo {
{
Path storePath; Path storePath;
std::string branch; std::string branch;
std::string rev; std::string rev;
@ -25,17 +21,18 @@ struct HgInfo
std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string& uri, HgInfo exportMercurial(ref<Store> store, const std::string& uri,
std::string rev, const std::string & name) std::string rev, const std::string& name) {
{
if (evalSettings.pureEval && rev == "") if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); throw Error(
"in pure evaluation mode, 'fetchMercurial' requires a Mercurial "
"revision");
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
bool clean = runProgram("hg", true,
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; {"status", "-R", uri, "--modified", "--added",
"--removed"}) == "";
if (!clean) { if (!clean) {
/* This is an unclean working tree. So copy all tracked /* This is an unclean working tree. So copy all tracked
files. */ files. */
@ -46,7 +43,10 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri})); hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri}));
auto files = tokenizeString<std::set<std::string>>( auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); runProgram("hg", true,
{"status", "-R", uri, "--clean", "--modified", "--added",
"--no-status", "--print0"}),
"\0"s);
PathFilter filter = [&](const Path& p) -> bool { PathFilter filter = [&](const Path& p) -> bool {
assert(hasPrefix(p, uri)); assert(hasPrefix(p, uri));
@ -63,7 +63,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
return files.count(file); return files.count(file);
}; };
hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); hgInfo.storePath =
store->addToStore("source", uri, true, htSHA256, filter);
return hgInfo; return hgInfo;
} }
@ -71,39 +72,40 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
if (rev == "") rev = "default"; if (rev == "") rev = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(),
hashString(htSHA256, uri).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir,
hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than tarball-ttl seconds, /* If we haven't pulled this repo less than tarball-ttl seconds,
do so now. */ do so now. */
time_t now = time(0); time_t now = time(0);
struct stat st; struct stat st;
if (stat(stampFile.c_str(), &st) != 0 || if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) (uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now) {
{
/* Except that if this is a commit hash that we already have, /* Except that if this is a commit hash that we already have,
we don't have to pull again. */ we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex) if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) &&
&& pathExists(cacheDir) runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev,
&& runProgram( "--template", "1"})
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) .killStderr(true))
.killStderr(true)).second == "1")) .second == "1")) {
{ Activity act(*logger, lvlTalkative, actUnknown,
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) { if (pathExists(cacheDir)) {
try { try {
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
} } catch (ExecError& e) {
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal"; string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */ /* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) { if (pathExists(transJournal)) {
runProgram("hg", true, {"recover", "-R", cacheDir}); runProgram("hg", true, {"recover", "-R", cacheDir});
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
} else { } else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); throw ExecError(e.status,
fmt("'hg pull' %s", statusToString(e.status)));
} }
} }
} else { } else {
@ -116,7 +118,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
} }
auto tokens = tokenizeString<std::vector<std::string>>( auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); runProgram("hg", true,
{"log", "-R", cacheDir, "-r", rev, "--template",
"{node} {rev} {branch}"}));
assert(tokens.size() == 3); assert(tokens.size() == 3);
HgInfo hgInfo; HgInfo hgInfo;
@ -124,7 +128,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.revCount = std::stoull(tokens[1]); hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2]; hgInfo.branch = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev)
.to_string(Base32, false);
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
try { try {
@ -135,7 +141,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.storePath = json["storePath"]; hgInfo.storePath = json["storePath"];
if (store->isValidPath(hgInfo.storePath)) { if (store->isValidPath(hgInfo.storePath)) {
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); printTalkative("using cached Mercurial store path '%s'",
hgInfo.storePath);
return hgInfo; return hgInfo;
} }
@ -165,8 +172,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
return hgInfo; return hgInfo;
} }
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
std::string url; std::string url;
std::string rev; std::string rev;
std::string name = "source"; std::string name = "source";
@ -175,19 +182,20 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
state.forceValue(*args[0]); state.forceValue(*args[0]);
if (args[0]->type == tAttrs) { if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos); state.forceAttrs(*args[0], pos);
for (auto& attr : *args[0]->attrs) { for (auto& attr : *args[0]->attrs) {
string n(attr.name); string n(attr.name);
if (n == "url") if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false); url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev") else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos); rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name") else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos); name = state.forceStringNoCtx(*attr.value, *attr.pos);
else else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s",
attr.name, *attr.pos);
} }
if (url.empty()) if (url.empty())
@ -203,10 +211,12 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
auto hgInfo = exportMercurial(state.store, url, rev, name); auto hgInfo = exportMercurial(state.store, url, rev, name);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath,
PathSet({hgInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
std::string(hgInfo.rev, 0, 12));
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
v.attrs->sort(); v.attrs->sort();
@ -216,4 +226,4 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
} } // namespace nix

View file

@ -1,12 +1,11 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "cpptoml/cpptoml.h" #include "cpptoml/cpptoml.h"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix { namespace nix {
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args,
{ Value& v) {
using namespace cpptoml; using namespace cpptoml;
auto toml = state.forceStringNoCtx(*args[0], pos); auto toml = state.forceStringNoCtx(*args[0], pos);
@ -16,11 +15,12 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
std::function<void(Value&, std::shared_ptr<base>)> visit; std::function<void(Value&, std::shared_ptr<base>)> visit;
visit = [&](Value& v, std::shared_ptr<base> t) { visit = [&](Value& v, std::shared_ptr<base> t) {
if (auto t2 = t->as_table()) { if (auto t2 = t->as_table()) {
size_t size = 0; size_t size = 0;
for (auto & i : *t2) { (void) i; size++; } for (auto& i : *t2) {
(void)i;
size++;
}
state.mkAttrs(v, size); state.mkAttrs(v, size);
@ -32,8 +32,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
state.mkList(v2, size2); state.mkList(v2, size2);
for (size_t j = 0; j < size2; ++j) for (size_t j = 0; j < size2; ++j)
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
} } else
else
visit(v2, i.second); visit(v2, i.second);
} }
@ -75,7 +74,8 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
throw EvalError("unsupported value type in TOML"); throw EvalError("unsupported value type in TOML");
} }
else abort(); else
abort();
}; };
try { try {
@ -87,4 +87,4 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
} } // namespace nix

View file

@ -2,7 +2,6 @@
#include <map> #include <map>
#include <unordered_set> #include <unordered_set>
#include "types.hh" #include "types.hh"
namespace nix { namespace nix {
@ -13,8 +12,7 @@ namespace nix {
they can be compared efficiently (using a pointer equality test), they can be compared efficiently (using a pointer equality test),
because the symbol table stores only one copy of each string. */ because the symbol table stores only one copy of each string. */
class Symbol class Symbol {
{
private: private:
const string* s; // pointer into SymbolTable const string* s; // pointer into SymbolTable
Symbol(const string* s) : s(s){}; Symbol(const string* s) : s(s){};
@ -23,65 +21,40 @@ private:
public: public:
Symbol() : s(0){}; Symbol() : s(0){};
bool operator == (const Symbol & s2) const bool operator==(const Symbol& s2) const { return s == s2.s; }
{
return s == s2.s;
}
bool operator != (const Symbol & s2) const bool operator!=(const Symbol& s2) const { return s != s2.s; }
{
return s != s2.s;
}
bool operator < (const Symbol & s2) const bool operator<(const Symbol& s2) const { return s < s2.s; }
{
return s < s2.s;
}
operator const string & () const operator const string&() const { return *s; }
{
return *s;
}
bool set() const bool set() const { return s; }
{
return s;
}
bool empty() const bool empty() const { return s->empty(); }
{
return s->empty();
}
friend std::ostream& operator<<(std::ostream& str, const Symbol& sym); friend std::ostream& operator<<(std::ostream& str, const Symbol& sym);
}; };
class SymbolTable class SymbolTable {
{
private: private:
typedef std::unordered_set<string> Symbols; typedef std::unordered_set<string> Symbols;
Symbols symbols; Symbols symbols;
public: public:
Symbol create(const string & s) Symbol create(const string& s) {
{
std::pair<Symbols::iterator, bool> res = symbols.insert(s); std::pair<Symbols::iterator, bool> res = symbols.insert(s);
return Symbol(&*res.first); return Symbol(&*res.first);
} }
size_t size() const size_t size() const { return symbols.size(); }
{
return symbols.size();
}
size_t totalSize() const; size_t totalSize() const;
template <typename T> template <typename T>
void dump(T callback) void dump(T callback) {
{ for (auto& s : symbols) callback(s);
for (auto & s : symbols)
callback(s);
} }
}; };
} } // namespace nix

View file

@ -1,23 +1,19 @@
#include "value-to-json.hh" #include "value-to-json.hh"
#include "json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
#include "eval-inline.hh"
#include "json.hh"
#include "util.hh"
namespace nix { namespace nix {
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, JSONPlaceholder & out, PathSet & context) JSONPlaceholder& out, PathSet& context) {
{
checkInterrupt(); checkInterrupt();
if (strict) state.forceValue(v); if (strict) state.forceValue(v);
switch (v.type) { switch (v.type) {
case tInt: case tInt:
out.write(v.integer); out.write(v.integer);
break; break;
@ -40,7 +36,8 @@ void printValueAsJSON(EvalState & state, bool strict,
break; break;
case tAttrs: { case tAttrs: {
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); auto maybeString =
state.tryAttrsToString(noPos, v, context, false, false);
if (maybeString) { if (maybeString) {
out.write(*maybeString); out.write(*maybeString);
break; break;
@ -49,8 +46,7 @@ void printValueAsJSON(EvalState & state, bool strict,
if (i == v.attrs->end()) { if (i == v.attrs->end()) {
auto obj(out.object()); auto obj(out.object());
StringSet names; StringSet names;
for (auto & j : *v.attrs) for (auto& j : *v.attrs) names.insert(j.name);
names.insert(j.name);
for (auto& j : names) { for (auto& j : names) {
Attr& a(*v.attrs->find(state.symbols.create(j))); Attr& a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j)); auto placeholder(obj.placeholder(j));
@ -61,11 +57,14 @@ void printValueAsJSON(EvalState & state, bool strict,
break; break;
} }
case tList1: case tList2: case tListN: { case tList1:
case tList2:
case tListN: {
auto list(out.list()); auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) { for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder()); auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); printValueAsJSON(state, strict, *v.listElems()[n], placeholder,
context);
} }
break; break;
} }
@ -83,18 +82,16 @@ void printValueAsJSON(EvalState & state, bool strict,
} }
} }
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, std::ostream & str, PathSet & context) std::ostream& str, PathSet& context) {
{
JSONPlaceholder out(str); JSONPlaceholder out(str);
printValueAsJSON(state, strict, v, out, context); printValueAsJSON(state, strict, v, out, context);
} }
void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict,
JSONPlaceholder & out, PathSet & context) const JSONPlaceholder& out,
{ PathSet& context) const {
throw TypeError(format("cannot convert %1% to JSON") % showType()); throw TypeError(format("cannot convert %1% to JSON") % showType());
} }
} // namespace nix
}

View file

@ -1,19 +1,18 @@
#pragma once #pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix { namespace nix {
class JSONPlaceholder; class JSONPlaceholder;
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, JSONPlaceholder & out, PathSet & context); JSONPlaceholder& out, PathSet& context);
void printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState& state, bool strict, Value& v,
Value & v, std::ostream & str, PathSet & context); std::ostream& str, PathSet& context);
} } // namespace nix

View file

@ -1,41 +1,33 @@
#include "value-to-xml.hh" #include "value-to-xml.hh"
#include "xml-writer.hh" #include <cstdlib>
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
#include "xml-writer.hh"
#include <cstdlib>
namespace nix { namespace nix {
static XMLAttrs singletonAttrs(const string& name, const string& value) {
static XMLAttrs singletonAttrs(const string & name, const string & value)
{
XMLAttrs attrs; XMLAttrs attrs;
attrs[name] = value; attrs[name] = value;
return attrs; return attrs;
} }
static void printValueAsXML(EvalState& state, bool strict, bool location, static void printValueAsXML(EvalState& state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); Value& v, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen);
static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) {
static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
{
xmlAttrs["path"] = pos.file; xmlAttrs["path"] = pos.file;
xmlAttrs["line"] = (format("%1%") % pos.line).str(); xmlAttrs["line"] = (format("%1%") % pos.line).str();
xmlAttrs["column"] = (format("%1%") % pos.column).str(); xmlAttrs["column"] = (format("%1%") % pos.column).str();
} }
static void showAttrs(EvalState& state, bool strict, bool location, static void showAttrs(EvalState& state, bool strict, bool location,
Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) Bindings& attrs, XMLWriter& doc, PathSet& context,
{ PathSet& drvsSeen) {
StringSet names; StringSet names;
for (auto & i : attrs) for (auto& i : attrs) names.insert(i.name);
names.insert(i.name);
for (auto& i : names) { for (auto& i : names) {
Attr& a(*attrs.find(state.symbols.create(i))); Attr& a(*attrs.find(state.symbols.create(i)));
@ -45,27 +37,26 @@ static void showAttrs(EvalState & state, bool strict, bool location,
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
XMLOpenElement _(doc, "attr", xmlAttrs); XMLOpenElement _(doc, "attr", xmlAttrs);
printValueAsXML(state, strict, location, printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen);
*a.value, doc, context, drvsSeen);
} }
} }
static void printValueAsXML(EvalState& state, bool strict, bool location, static void printValueAsXML(EvalState& state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) Value& v, XMLWriter& doc, PathSet& context,
{ PathSet& drvsSeen) {
checkInterrupt(); checkInterrupt();
if (strict) state.forceValue(v); if (strict) state.forceValue(v);
switch (v.type) { switch (v.type) {
case tInt: case tInt:
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); doc.writeEmptyElement(
"int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break; break;
case tBool: case tBool:
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); doc.writeEmptyElement(
"bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break; break;
case tString: case tString:
@ -86,7 +77,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (state.isDerivation(v)) { if (state.isDerivation(v)) {
XMLAttrs xmlAttrs; XMLAttrs xmlAttrs;
Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); Bindings::iterator a =
v.attrs->find(state.symbols.create("derivation"));
Path drvPath; Path drvPath;
a = v.attrs->find(state.sDrvPath); a = v.attrs->find(state.sDrvPath);
@ -119,10 +111,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break; break;
case tList1: case tList2: case tListN: { case tList1:
case tList2:
case tListN: {
XMLOpenElement _(doc, "list"); XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n) for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); printValueAsXML(state, strict, location, *v.listElems()[n], doc,
context, drvsSeen);
break; break;
} }
@ -139,17 +134,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
for (auto& i : v.lambda.fun->formals->formals) for (auto& i : v.lambda.fun->formals->formals)
doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
} else } else
doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); doc.writeEmptyElement("varpat",
singletonAttrs("name", v.lambda.fun->arg));
break; break;
} }
case tExternal: case tExternal:
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); v.external->printValueAsXML(state, strict, location, doc, context,
drvsSeen);
break; break;
case tFloat: case tFloat:
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); doc.writeEmptyElement(
"float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
break; break;
default: default:
@ -157,22 +155,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
} }
} }
void ExternalValueBase::printValueAsXML(EvalState& state, bool strict, void ExternalValueBase::printValueAsXML(EvalState& state, bool strict,
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const bool location, XMLWriter& doc,
{ PathSet& context,
PathSet& drvsSeen) const {
doc.writeEmptyElement("unevaluated"); doc.writeEmptyElement("unevaluated");
} }
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
void printValueAsXML(EvalState & state, bool strict, bool location, std::ostream& out, PathSet& context) {
Value & v, std::ostream & out, PathSet & context)
{
XMLWriter doc(true, out); XMLWriter doc(true, out);
XMLOpenElement root(doc, "expr"); XMLOpenElement root(doc, "expr");
PathSet drvsSeen; PathSet drvsSeen;
printValueAsXML(state, strict, location, v, doc, context, drvsSeen); printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
} }
} // namespace nix
}

View file

@ -1,14 +1,13 @@
#pragma once #pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map> #include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix { namespace nix {
void printValueAsXML(EvalState & state, bool strict, bool location, void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
Value & v, std::ostream & out, PathSet & context); std::ostream& out, PathSet& context);
} }

View file

@ -8,7 +8,6 @@
namespace nix { namespace nix {
typedef enum { typedef enum {
tInt = 1, tInt = 1,
tBool, tBool,
@ -29,7 +28,6 @@ typedef enum {
tFloat tFloat
} ValueType; } ValueType;
class Bindings; class Bindings;
struct Env; struct Env;
struct Expr; struct Expr;
@ -42,16 +40,16 @@ class EvalState;
class XMLWriter; class XMLWriter;
class JSONPlaceholder; class JSONPlaceholder;
typedef int64_t NixInt; typedef int64_t NixInt;
typedef double NixFloat; typedef double NixFloat;
/* External values must descend from ExternalValueBase, so that /* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented * type-agnostic nix functions (e.g. showType) can be implemented
*/ */
class ExternalValueBase class ExternalValueBase {
{ friend std::ostream& operator<<(std::ostream& str,
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); const ExternalValueBase& v);
protected: protected:
/* Print out the value */ /* Print out the value */
virtual std::ostream& print(std::ostream& str) const = 0; virtual std::ostream& print(std::ostream& str) const = 0;
@ -69,7 +67,8 @@ class ExternalValueBase
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* error * error
*/ */
virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; virtual string coerceToString(const Pos& pos, PathSet& context, bool copyMore,
bool copyToStore) const;
/* Compare to another value of the same type. Defaults to uncomparable, /* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false. * i.e. always false.
@ -82,21 +81,17 @@ class ExternalValueBase
/* Print the value as XML. Defaults to unevaluated */ /* Print the value as XML. Defaults to unevaluated */
virtual void printValueAsXML(EvalState& state, bool strict, bool location, virtual void printValueAsXML(EvalState& state, bool strict, bool location,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) const;
virtual ~ExternalValueBase() virtual ~ExternalValueBase(){};
{
};
}; };
std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v); std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v);
struct Value {
struct Value
{
ValueType type; ValueType type;
union union {
{
NixInt integer; NixInt integer;
bool boolean; bool boolean;
@ -151,124 +146,95 @@ struct Value
NixFloat fpoint; NixFloat fpoint;
}; };
bool isList() const bool isList() const {
{
return type == tList1 || type == tList2 || type == tListN; return type == tList1 || type == tList2 || type == tListN;
} }
Value * * listElems() Value** listElems() {
{
return type == tList1 || type == tList2 ? smallList : bigList.elems; return type == tList1 || type == tList2 ? smallList : bigList.elems;
} }
const Value * const * listElems() const const Value* const* listElems() const {
{
return type == tList1 || type == tList2 ? smallList : bigList.elems; return type == tList1 || type == tList2 ? smallList : bigList.elems;
} }
size_t listSize() const size_t listSize() const {
{
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
} }
}; };
/* After overwriting an app node, be sure to clear pointers in the /* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */ Value to ensure that the target isn't kept alive unnecessarily. */
static inline void clearValue(Value & v) static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; }
{
v.app.left = v.app.right = 0;
}
static inline void mkInt(Value& v, NixInt n) {
static inline void mkInt(Value & v, NixInt n)
{
clearValue(v); clearValue(v);
v.type = tInt; v.type = tInt;
v.integer = n; v.integer = n;
} }
static inline void mkFloat(Value& v, NixFloat n) {
static inline void mkFloat(Value & v, NixFloat n)
{
clearValue(v); clearValue(v);
v.type = tFloat; v.type = tFloat;
v.fpoint = n; v.fpoint = n;
} }
static inline void mkBool(Value& v, bool b) {
static inline void mkBool(Value & v, bool b)
{
clearValue(v); clearValue(v);
v.type = tBool; v.type = tBool;
v.boolean = b; v.boolean = b;
} }
static inline void mkNull(Value& v) {
static inline void mkNull(Value & v)
{
clearValue(v); clearValue(v);
v.type = tNull; v.type = tNull;
} }
static inline void mkApp(Value& v, Value& left, Value& right) {
static inline void mkApp(Value & v, Value & left, Value & right)
{
v.type = tApp; v.type = tApp;
v.app.left = &left; v.app.left = &left;
v.app.right = &right; v.app.right = &right;
} }
static inline void mkPrimOpApp(Value& v, Value& left, Value& right) {
static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
{
v.type = tPrimOpApp; v.type = tPrimOpApp;
v.app.left = &left; v.app.left = &left;
v.app.right = &right; v.app.right = &right;
} }
static inline void mkStringNoCopy(Value& v, const char* s) {
static inline void mkStringNoCopy(Value & v, const char * s)
{
v.type = tString; v.type = tString;
v.string.s = s; v.string.s = s;
v.string.context = 0; v.string.context = 0;
} }
static inline void mkString(Value& v, const Symbol& s) {
static inline void mkString(Value & v, const Symbol & s)
{
mkStringNoCopy(v, ((const string&)s).c_str()); mkStringNoCopy(v, ((const string&)s).c_str());
} }
void mkString(Value& v, const char* s); void mkString(Value& v, const char* s);
static inline void mkPathNoCopy(Value& v, const char* s) {
static inline void mkPathNoCopy(Value & v, const char * s)
{
clearValue(v); clearValue(v);
v.type = tPath; v.type = tPath;
v.path = s; v.path = s;
} }
void mkPath(Value& v, const char* s); void mkPath(Value& v, const char* s);
/* Compute the size in bytes of the given value, including all values /* Compute the size in bytes of the given value, including all values
and environments reachable from it. Static expressions (Exprs) are and environments reachable from it. Static expressions (Exprs) are
not included. */ not included. */
size_t valueSize(Value& v); size_t valueSize(Value& v);
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::vector<Value*, gc_allocator<Value*> > ValueVector; typedef std::vector<Value*, gc_allocator<Value*> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap; typedef std::map<Symbol, Value*, std::less<Symbol>,
gc_allocator<std::pair<const Symbol, Value*> > >
ValueMap;
#else #else
typedef std::vector<Value*> ValueVector; typedef std::vector<Value*> ValueVector;
typedef std::map<Symbol, Value*> ValueMap; typedef std::map<Symbol, Value*> ValueMap;
#endif #endif
} // namespace nix
}

View file

@ -4,8 +4,7 @@
namespace nix { namespace nix {
MixCommonArgs::MixCommonArgs(const string& programName) MixCommonArgs::MixCommonArgs(const string& programName)
: programName(programName) : programName(programName) {
{
mkFlag() mkFlag()
.longName("verbose") .longName("verbose")
.shortName('v') .shortName('v')
@ -15,12 +14,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
mkFlag() mkFlag()
.longName("quiet") .longName("quiet")
.description("decrease verbosity level") .description("decrease verbosity level")
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); .handler([]() {
verbosity =
verbosity > lvlError ? (Verbosity)(verbosity - 1) : lvlError;
});
mkFlag() mkFlag().longName("debug").description("enable debug output").handler([]() {
.longName("debug") verbosity = lvlDebug;
.description("enable debug output") });
.handler([]() { verbosity = lvlDebug; });
mkFlag() mkFlag()
.longName("option") .longName("option")
@ -40,9 +41,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
.shortName('j') .shortName('j')
.label("jobs") .label("jobs")
.description("maximum number of parallel builds") .description("maximum number of parallel builds")
.handler([=](std::string s) { .handler([=](std::string s) { settings.set("max-jobs", s); });
settings.set("max-jobs", s);
});
std::string cat = "config"; std::string cat = "config";
globalConfig.convertToArgs(*this, cat); globalConfig.convertToArgs(*this, cat);
@ -53,4 +52,4 @@ MixCommonArgs::MixCommonArgs(const string & programName)
hiddenCategories.insert(cat); hiddenCategories.insert(cat);
} }
} } // namespace nix

View file

@ -4,30 +4,24 @@
namespace nix { namespace nix {
struct MixCommonArgs : virtual Args struct MixCommonArgs : virtual Args {
{
string programName; string programName;
MixCommonArgs(const string& programName); MixCommonArgs(const string& programName);
}; };
struct MixDryRun : virtual Args struct MixDryRun : virtual Args {
{
bool dryRun = false; bool dryRun = false;
MixDryRun() MixDryRun() {
{ mkFlag(0, "dry-run", "show what this command would do without doing it",
mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); &dryRun);
} }
}; };
struct MixJSON : virtual Args struct MixJSON : virtual Args {
{
bool json = false; bool json = false;
MixJSON() MixJSON() { mkFlag(0, "json", "produce JSON output", &json); }
{
mkFlag(0, "json", "produce JSON output", &json);
}
}; };
} } // namespace nix

View file

@ -1,30 +1,24 @@
#include "globals.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include <openssl/crypto.h>
#include "util.hh" #include <signal.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm> #include <algorithm>
#include <cctype> #include <cctype>
#include <cstdlib>
#include <exception> #include <exception>
#include <iostream> #include <iostream>
#include <mutex> #include <mutex>
#include "globals.hh"
#include <cstdlib> #include "store-api.hh"
#include <sys/time.h> #include "util.hh"
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#include <openssl/crypto.h>
namespace nix { namespace nix {
static bool gcWarning = true; static bool gcWarning = true;
void printGCWarning() void printGCWarning() {
{
if (!gcWarning) return; if (!gcWarning) return;
static bool haveWarned = false; static bool haveWarned = false;
warnOnce(haveWarned, warnOnce(haveWarned,
@ -32,62 +26,58 @@ void printGCWarning()
"the result might be removed by the garbage collector"); "the result might be removed by the garbage collector");
} }
void printMissing(ref<Store> store, const PathSet& paths, Verbosity lvl) {
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
{
unsigned long long downloadSize, narSize; unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown; PathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize,
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize,
lvl);
} }
void printMissing(ref<Store> store, const PathSet& willBuild, void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet& willSubstitute, const PathSet& unknown, const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) unsigned long long downloadSize, unsigned long long narSize,
{ Verbosity lvl) {
if (!willBuild.empty()) { if (!willBuild.empty()) {
printMsg(lvl, "these derivations will be built:"); printMsg(lvl, "these derivations will be built:");
Paths sorted = store->topoSortPaths(willBuild); Paths sorted = store->topoSortPaths(willBuild);
reverse(sorted.begin(), sorted.end()); reverse(sorted.begin(), sorted.end());
for (auto & i : sorted) for (auto& i : sorted) printMsg(lvl, fmt(" %s", i));
printMsg(lvl, fmt(" %s", i));
} }
if (!willSubstitute.empty()) { if (!willSubstitute.empty()) {
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f "
"MiB unpacked):",
downloadSize / (1024.0 * 1024.0), downloadSize / (1024.0 * 1024.0),
narSize / (1024.0 * 1024.0))); narSize / (1024.0 * 1024.0)));
for (auto & i : willSubstitute) for (auto& i : willSubstitute) printMsg(lvl, fmt(" %s", i));
printMsg(lvl, fmt(" %s", i));
} }
if (!unknown.empty()) { if (!unknown.empty()) {
printMsg(lvl, fmt("don't know how to build these paths%s:", printMsg(lvl, fmt("don't know how to build these paths%s:",
(settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); (settings.readOnlyMode
for (auto & i : unknown) ? " (may be caused by read-only store access)"
printMsg(lvl, fmt(" %s", i)); : "")));
for (auto& i : unknown) printMsg(lvl, fmt(" %s", i));
} }
} }
string getArg(const string& opt, Strings::iterator& i,
string getArg(const string & opt, const Strings::iterator& end) {
Strings::iterator & i, const Strings::iterator & end)
{
++i; ++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
return *i; return *i;
} }
#if OPENSSL_VERSION_NUMBER < 0x10101000L #if OPENSSL_VERSION_NUMBER < 0x10101000L
/* OpenSSL is not thread-safe by default - it will randomly crash /* OpenSSL is not thread-safe by default - it will randomly crash
unless the user supplies a mutex locking function. So let's do unless the user supplies a mutex locking function. So let's do
that. */ that. */
static std::vector<std::mutex> opensslLocks; static std::vector<std::mutex> opensslLocks;
static void opensslLockCallback(int mode, int type, const char * file, int line) static void opensslLockCallback(int mode, int type, const char* file,
{ int line) {
if (mode & CRYPTO_LOCK) if (mode & CRYPTO_LOCK)
opensslLocks[type].lock(); opensslLocks[type].lock();
else else
@ -95,12 +85,9 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
} }
#endif #endif
static void sigHandler(int signo) {} static void sigHandler(int signo) {}
void initNix() {
void initNix()
{
/* Turn on buffering for cerr. */ /* Turn on buffering for cerr. */
#if HAVE_PUBSETBUF #if HAVE_PUBSETBUF
static char buf[1024]; static char buf[1024];
@ -122,8 +109,7 @@ void initNix()
sigemptyset(&act.sa_mask); sigemptyset(&act.sa_mask);
act.sa_handler = SIG_DFL; act.sa_handler = SIG_DFL;
act.sa_flags = 0; act.sa_flags = 0;
if (sigaction(SIGCHLD, &act, 0)) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD");
throw SysError("resetting SIGCHLD");
/* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
act.sa_handler = sigHandler; act.sa_handler = sigHandler;
@ -160,11 +146,11 @@ void initNix()
#endif #endif
} }
LegacyArgs::LegacyArgs(
LegacyArgs::LegacyArgs(const std::string & programName, const std::string& programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
: MixCommonArgs(programName), parseArg(parseArg) parseArg)
{ : MixCommonArgs(programName), parseArg(parseArg) {
mkFlag() mkFlag()
.longName("no-build-output") .longName("no-build-output")
.shortName('Q') .shortName('Q')
@ -189,15 +175,20 @@ LegacyArgs::LegacyArgs(const std::string & programName,
.set(&(bool&)settings.tryFallback, true); .set(&(bool&)settings.tryFallback, true);
auto intSettingAlias = [&](char shortName, const std::string& longName, auto intSettingAlias = [&](char shortName, const std::string& longName,
const std::string & description, const std::string & dest) { const std::string& description,
const std::string& dest) {
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
settings.set(dest, std::to_string(n)); settings.set(dest, std::to_string(n));
}); });
}; };
intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); intSettingAlias(0, "cores",
intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); "maximum number of CPU cores to use inside a build", "cores");
intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); intSettingAlias(0, "max-silent-time",
"number of seconds of silence before a build is killed",
"max-silent-time");
intSettingAlias(0, "timeout", "number of seconds before a build is killed",
"timeout");
mkFlag(0, "readonly-mode", "do not write to the Nix store", mkFlag(0, "readonly-mode", "do not write to the Nix store",
&settings.readOnlyMode); &settings.readOnlyMode);
@ -212,18 +203,14 @@ LegacyArgs::LegacyArgs(const std::string & programName,
.dest(&(std::string&)settings.storeUri); .dest(&(std::string&)settings.storeUri);
} }
bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) {
bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end)
{
if (MixCommonArgs::processFlag(pos, end)) return true; if (MixCommonArgs::processFlag(pos, end)) return true;
bool res = parseArg(pos, end); bool res = parseArg(pos, end);
if (res) ++pos; if (res) ++pos;
return res; return res;
} }
bool LegacyArgs::processArgs(const Strings& args, bool finish) {
bool LegacyArgs::processArgs(const Strings & args, bool finish)
{
if (args.empty()) return true; if (args.empty()) return true;
assert(args.size() == 1); assert(args.size() == 1);
Strings ss(args); Strings ss(args);
@ -233,23 +220,21 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
return true; return true;
} }
void parseCmdLine(
void parseCmdLine(int argc, char * * argv, int argc, char** argv,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
{ parseArg) {
parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
} }
void parseCmdLine(
void parseCmdLine(const string & programName, const Strings & args, const string& programName, const Strings& args,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
{ parseArg) {
LegacyArgs(programName, parseArg).parseCmdline(args); LegacyArgs(programName, parseArg).parseCmdline(args);
} }
void printVersion(const string& programName) {
void printVersion(const string & programName)
{
std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
if (verbosity > lvlInfo) { if (verbosity > lvlInfo) {
Strings cfg; Strings cfg;
@ -260,25 +245,22 @@ void printVersion(const string & programName)
cfg.push_back("signed-caches"); cfg.push_back("signed-caches");
#endif #endif
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf"
<< "\n";
std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n";
} }
throw Exit(); throw Exit();
} }
void showManPage(const string& name) {
void showManPage(const string & name)
{
restoreSignals(); restoreSignals();
setenv("MANPATH", settings.nixManDir.c_str(), 1); setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr); execlp("man", "man", name.c_str(), nullptr);
throw SysError(format("command 'man %1%' failed") % name.c_str()); throw SysError(format("command 'man %1%' failed") % name.c_str());
} }
int handleExceptions(const string& programName, std::function<void()> fun) {
int handleExceptions(const string & programName, std::function<void()> fun)
{
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
string error = ANSI_RED "error:" ANSI_NORMAL " "; string error = ANSI_RED "error:" ANSI_NORMAL " ";
@ -296,12 +278,12 @@ int handleExceptions(const string & programName, std::function<void()> fun)
} catch (Exit& e) { } catch (Exit& e) {
return e.status; return e.status;
} catch (UsageError& e) { } catch (UsageError& e) {
printError( printError(format(error + "%1%\nTry '%2% --help' for more information.") %
format(error + "%1%\nTry '%2% --help' for more information.") e.what() % programName);
% e.what() % programName);
return 1; return 1;
} catch (BaseError& e) { } catch (BaseError& e) {
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); printError(format(error + "%1%%2%") %
(settings.showTrace ? e.prefix() : "") % e.msg());
if (e.prefix() != "" && !settings.showTrace) if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)"); printError("(use '--show-trace' to show detailed location information)");
return e.status; return e.status;
@ -316,9 +298,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
return 0; return 0;
} }
RunPager::RunPager() {
RunPager::RunPager()
{
if (!isatty(STDOUT_FILENO)) return; if (!isatty(STDOUT_FILENO)) return;
char* pager = getenv("NIX_PAGER"); char* pager = getenv("NIX_PAGER");
if (!pager) pager = getenv("PAGER"); if (!pager) pager = getenv("PAGER");
@ -330,11 +310,9 @@ RunPager::RunPager()
pid = startProcess([&]() { pid = startProcess([&]() {
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin"); throw SysError("dupping stdin");
if (!getenv("LESS")) if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1);
setenv("LESS", "FRSXMK", 1);
restoreSignals(); restoreSignals();
if (pager) if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr);
execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr); execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr); execlp("less", "less", nullptr);
execlp("more", "more", nullptr); execlp("more", "more", nullptr);
@ -347,9 +325,7 @@ RunPager::RunPager()
throw SysError("dupping stdout"); throw SysError("dupping stdout");
} }
RunPager::~RunPager() {
RunPager::~RunPager()
{
try { try {
if (pid != -1) { if (pid != -1) {
std::cout.flush(); std::cout.flush();
@ -361,21 +337,16 @@ RunPager::~RunPager()
} }
} }
string showBytes(unsigned long long bytes) {
string showBytes(unsigned long long bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
} }
PrintFreed::~PrintFreed() {
PrintFreed::~PrintFreed()
{
if (show) if (show)
std::cout << format("%1% store paths deleted, %2% freed\n") std::cout << format("%1% store paths deleted, %2% freed\n") %
% results.paths.size() results.paths.size() % showBytes(results.bytesFreed);
% showBytes(results.bytesFreed);
} }
Exit::~Exit() {} Exit::~Exit() {}
} } // namespace nix

View file

@ -1,18 +1,14 @@
#pragma once #pragma once
#include "util.hh" #include <signal.h>
#include <locale>
#include "args.hh" #include "args.hh"
#include "common-args.hh" #include "common-args.hh"
#include "util.hh"
#include <signal.h>
#include <locale>
namespace nix { namespace nix {
class Exit : public std::exception class Exit : public std::exception {
{
public: public:
int status; int status;
Exit() : status(0) {} Exit() : status(0) {}
@ -25,11 +21,15 @@ int handleExceptions(const string & programName, std::function<void()> fun);
/* Don't forget to call initPlugins() after settings are initialized! */ /* Don't forget to call initPlugins() after settings are initialized! */
void initNix(); void initNix();
void parseCmdLine(int argc, char * * argv, void parseCmdLine(
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); int argc, char** argv,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void parseCmdLine(const string & programName, const Strings & args, void parseCmdLine(
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); const string& programName, const Strings& args,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void printVersion(const string& programName); void printVersion(const string& programName);
@ -38,18 +38,20 @@ void printGCWarning();
class Store; class Store;
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo); void printMissing(ref<Store> store, const PathSet& paths,
Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const PathSet& willBuild, void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet& willSubstitute, const PathSet& unknown, const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); unsigned long long downloadSize, unsigned long long narSize,
Verbosity lvl = lvlInfo);
string getArg(const string & opt, string getArg(const string& opt, Strings::iterator& i,
Strings::iterator & i, const Strings::iterator & end); const Strings::iterator& end);
template<class N> N getIntArg(const string & opt, template <class N>
Strings::iterator & i, const Strings::iterator & end, bool allowUnit) N getIntArg(const string& opt, Strings::iterator& i,
{ const Strings::iterator& end, bool allowUnit) {
++i; ++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
string s = *i; string s = *i;
@ -57,11 +59,16 @@ template<class N> N getIntArg(const string & opt,
if (allowUnit && !s.empty()) { if (allowUnit && !s.empty()) {
char u = std::toupper(*s.rbegin()); char u = std::toupper(*s.rbegin());
if (std::isalpha(u)) { if (std::isalpha(u)) {
if (u == 'K') multiplier = 1ULL << 10; if (u == 'K')
else if (u == 'M') multiplier = 1ULL << 20; multiplier = 1ULL << 10;
else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'M')
else if (u == 'T') multiplier = 1ULL << 40; multiplier = 1ULL << 20;
else throw UsageError(format("invalid unit specifier '%1%'") % u); else if (u == 'G')
multiplier = 1ULL << 30;
else if (u == 'T')
multiplier = 1ULL << 40;
else
throw UsageError(format("invalid unit specifier '%1%'") % u);
s.resize(s.size() - 1); s.resize(s.size() - 1);
} }
} }
@ -71,27 +78,26 @@ template<class N> N getIntArg(const string & opt,
return n * multiplier; return n * multiplier;
} }
struct LegacyArgs : public MixCommonArgs {
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg;
struct LegacyArgs : public MixCommonArgs LegacyArgs(
{ const std::string& programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
LegacyArgs(const std::string & programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
bool processFlag(Strings::iterator& pos, Strings::iterator end) override; bool processFlag(Strings::iterator& pos, Strings::iterator end) override;
bool processArgs(const Strings& args, bool finish) override; bool processArgs(const Strings& args, bool finish) override;
}; };
/* Show the manual page for the specified program. */ /* Show the manual page for the specified program. */
void showManPage(const string& name); void showManPage(const string& name);
/* The constructor of this class starts a pager if stdout is a /* The constructor of this class starts a pager if stdout is a
terminal and $PAGER is set. Stdout is redirected to the pager. */ terminal and $PAGER is set. Stdout is redirected to the pager. */
class RunPager class RunPager {
{
public: public:
RunPager(); RunPager();
~RunPager(); ~RunPager();
@ -102,15 +108,13 @@ private:
extern volatile ::sig_atomic_t blockInt; extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */ /* GC helpers. */
string showBytes(unsigned long long bytes); string showBytes(unsigned long long bytes);
struct GCResults; struct GCResults;
struct PrintFreed struct PrintFreed {
{
bool show; bool show;
const GCResults& results; const GCResults& results;
PrintFreed(bool show, const GCResults& results) PrintFreed(bool show, const GCResults& results)
@ -118,9 +122,7 @@ struct PrintFreed
~PrintFreed(); ~PrintFreed();
}; };
/* Install a SIGSEGV handler to detect stack overflows. */ /* Install a SIGSEGV handler to detect stack overflows. */
void detectStackOverflow(); void detectStackOverflow();
} // namespace nix
}

View file

@ -1,17 +1,13 @@
#include "types.hh" #include <signal.h>
#include <unistd.h>
#include <cstring>
#include <cstddef> #include <cstddef>
#include <cstdlib> #include <cstdlib>
#include <cstring>
#include <unistd.h> #include "types.hh"
#include <signal.h>
namespace nix { namespace nix {
static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) {
static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
{
/* Detect stack overflows by comparing the faulting address with /* Detect stack overflows by comparing the faulting address with
the stack pointer. Unfortunately, getting the stack pointer is the stack pointer. Unfortunately, getting the stack pointer is
not portable. */ not portable. */
@ -43,9 +39,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
if (sigaction(SIGSEGV, &act, 0)) abort(); if (sigaction(SIGSEGV, &act, 0)) abort();
} }
void detectStackOverflow() {
void detectStackOverflow()
{
#if defined(SA_SIGINFO) && defined(SA_ONSTACK) #if defined(SA_SIGINFO) && defined(SA_ONSTACK)
/* Install a SIGSEGV handler to detect stack overflows. This /* Install a SIGSEGV handler to detect stack overflows. This
requires an alternative stack, otherwise the signal cannot be requires an alternative stack, otherwise the signal cannot be
@ -56,16 +50,15 @@ void detectStackOverflow()
stack.ss_sp = stackBuf->data(); stack.ss_sp = stackBuf->data();
if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
stack.ss_flags = 0; stack.ss_flags = 0;
if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); if (sigaltstack(&stack, 0) == -1)
throw SysError("cannot set alternative stack");
struct sigaction act; struct sigaction act;
sigfillset(&act.sa_mask); sigfillset(&act.sa_mask);
act.sa_sigaction = sigsegvHandler; act.sa_sigaction = sigsegvHandler;
act.sa_flags = SA_SIGINFO | SA_ONSTACK; act.sa_flags = SA_SIGINFO | SA_ONSTACK;
if (sigaction(SIGSEGV, &act, 0)) if (sigaction(SIGSEGV, &act, 0)) throw SysError("resetting SIGSEGV");
throw SysError("resetting SIGSEGV");
#endif #endif
} }
} // namespace nix
}

View file

@ -1,40 +1,37 @@
#include "archive.hh"
#include "binary-cache-store.hh" #include "binary-cache-store.hh"
#include <chrono>
#include <future>
#include "archive.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "nar-info.hh"
#include "sync.hh"
#include "remote-fs-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "nar-accessor.hh"
#include "json.hh" #include "json.hh"
#include "nar-accessor.hh"
#include <chrono> #include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include <future> #include "remote-fs-accessor.hh"
#include "sync.hh"
namespace nix { namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params) BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) {
: Store(params)
{
if (secretKeyFile != "") if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); secretKey =
std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
StringSink sink; StringSink sink;
sink << narVersionMagic1; sink << narVersionMagic1;
narMagic = *sink.s; narMagic = *sink.s;
} }
void BinaryCacheStore::init() void BinaryCacheStore::init() {
{
std::string cacheInfoFile = "nix-cache-info"; std::string cacheInfoFile = "nix-cache-info";
auto cacheInfo = getFile(cacheInfoFile); auto cacheInfo = getFile(cacheInfoFile);
if (!cacheInfo) { if (!cacheInfo) {
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n",
"text/x-nix-cache-info");
} else { } else {
for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) { for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
size_t colon = line.find(':'); size_t colon = line.find(':');
@ -43,8 +40,9 @@ void BinaryCacheStore::init()
auto value = trim(line.substr(colon + 1, std::string::npos)); auto value = trim(line.substr(colon + 1, std::string::npos));
if (name == "StoreDir") { if (name == "StoreDir") {
if (value != storeDir) if (value != storeDir)
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") throw Error(format("binary cache '%s' is for Nix stores with prefix "
% getUri() % value % storeDir); "'%s', not '%s'") %
getUri() % value % storeDir);
} else if (name == "WantMassQuery") { } else if (name == "WantMassQuery") {
wantMassQuery_ = value == "1"; wantMassQuery_ = value == "1";
} else if (name == "Priority") { } else if (name == "Priority") {
@ -54,19 +52,19 @@ void BinaryCacheStore::init()
} }
} }
void BinaryCacheStore::getFile(const std::string & path, void BinaryCacheStore::getFile(
Callback<std::shared_ptr<std::string>> callback) noexcept const std::string& path,
{ Callback<std::shared_ptr<std::string>> callback) noexcept {
try { try {
callback(getFile(path)); callback(getFile(path));
} catch (...) { callback.rethrow(); } } catch (...) {
callback.rethrow();
}
} }
void BinaryCacheStore::getFile(const std::string & path, Sink & sink) void BinaryCacheStore::getFile(const std::string& path, Sink& sink) {
{
std::promise<std::shared_ptr<std::string>> promise; std::promise<std::shared_ptr<std::string>> promise;
getFile(path, getFile(path, {[&](std::future<std::shared_ptr<std::string>> result) {
{[&](std::future<std::shared_ptr<std::string>> result) {
try { try {
promise.set_value(result.get()); promise.set_value(result.get());
} catch (...) { } catch (...) {
@ -77,8 +75,8 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
sink((unsigned char*)data->data(), data->size()); sink((unsigned char*)data->data(), data->size());
} }
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) std::shared_ptr<std::string> BinaryCacheStore::getFile(
{ const std::string& path) {
StringSink sink; StringSink sink;
try { try {
getFile(path, sink); getFile(path, sink);
@ -88,14 +86,12 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
return sink.s; return sink.s;
} }
Path BinaryCacheStore::narInfoFileFor(const Path & storePath) Path BinaryCacheStore::narInfoFileFor(const Path& storePath) {
{
assertStorePath(storePath); assertStorePath(storePath);
return storePathToHash(storePath) + ".narinfo"; return storePathToHash(storePath) + ".narinfo";
} }
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) {
{
auto narInfoFile = narInfoFileFor(narInfo->path); auto narInfoFile = narInfoFileFor(narInfo->path);
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
@ -108,23 +104,24 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
} }
if (diskCache) if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); diskCache->upsertNarInfo(getUri(), hashPart,
std::shared_ptr<NarInfo>(narInfo));
} }
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, void BinaryCacheStore::addToStore(const ValidPathInfo& info,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) const ref<std::string>& nar,
{ RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) {
if (!repair && isValidPath(info.path)) return; if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo /* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */ reads, but typically they'll already be cached. */
for (auto & ref : info.references) for (auto& ref : info.references) try {
try { if (ref != info.path) queryPathInfo(ref);
if (ref != info.path)
queryPathInfo(ref);
} catch (InvalidPath&) { } catch (InvalidPath&) {
throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid") throw Error(format("cannot add '%s' to the binary cache because the "
% info.path % ref); "reference '%s' is not valid") %
info.path % ref);
} }
assert(nar->compare(0, narMagic.size(), narMagic) == 0); assert(nar->compare(0, narMagic.size(), narMagic) == 0);
@ -135,7 +132,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->narHash = hashString(htSHA256, *nar); narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash) if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); throw Error(
format("refusing to copy corrupted path '%1%' to binary cache") %
info.path);
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
@ -150,8 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
auto narAccessor = makeNarAccessor(nar); auto narAccessor = makeNarAccessor(nar);
if (accessor_) if (accessor_) accessor_->addToCache(info.path, *nar, narAccessor);
accessor_->addToCache(info.path, *nar, narAccessor);
{ {
auto res = jsonRoot.placeholder("root"); auto res = jsonRoot.placeholder("root");
@ -159,12 +157,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
} }
} }
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(),
"application/json");
} }
else { else {
if (accessor_) if (accessor_) accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
} }
/* Compress the NAR. */ /* Compress the NAR. */
@ -175,18 +173,22 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size(); narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); auto duration =
printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
% narInfo->path % narInfo->narSize .count();
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) printMsg(lvlTalkative,
% duration); format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% "
"ms) to binary cache") %
narInfo->path % narInfo->narSize %
((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) %
duration);
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" +
+ (compression == "xz" ? ".xz" : (compression == "xz" ? ".xz"
compression == "bzip2" ? ".bz2" : : compression == "bzip2"
compression == "br" ? ".br" : ? ".bz2"
""); : compression == "br" ? ".br" : "");
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
@ -205,16 +207,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
stats.narInfoWrite++; stats.narInfoWrite++;
} }
bool BinaryCacheStore::isValidPathUncached(const Path & storePath) bool BinaryCacheStore::isValidPathUncached(const Path& storePath) {
{
// FIXME: this only checks whether a .narinfo with a matching hash // FIXME: this only checks whether a .narinfo with a matching hash
// part exists. So f4kb...-foo matches f4kb...-bar, even // part exists. So f4kb...-foo matches f4kb...-bar, even
// though they shouldn't. Not easily fixed. // though they shouldn't. Not easily fixed.
return fileExists(narInfoFileFor(storePath)); return fileExists(narInfoFileFor(storePath));
} }
void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) {
{
auto info = queryPathInfo(storePath).cast<const NarInfo>(); auto info = queryPathInfo(storePath).cast<const NarInfo>();
uint64_t narSize = 0; uint64_t narSize = 0;
@ -239,20 +239,22 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
stats.narReadBytes += narSize; stats.narReadBytes += narSize;
} }
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, void BinaryCacheStore::queryPathInfoUncached(
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept const Path& storePath,
{ Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
auto uri = getUri(); auto uri = getUri();
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, auto act = std::make_shared<Activity>(
fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri}); *logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri),
Logger::Fields{storePath, uri});
PushActivity pact(act->id); PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath); auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile, getFile(
{[=](std::future<std::shared_ptr<std::string>> fut) { narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) {
try { try {
auto data = fut.get(); auto data = fut.get();
@ -260,10 +262,12 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
stats.narInfoRead++; stats.narInfoRead++;
(*callbackPtr)((std::shared_ptr<ValidPathInfo>) (*callbackPtr)(
std::make_shared<NarInfo>(*this, *data, narInfoFile)); (std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>(
*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive (void)
act; // force Activity into this lambda to ensure it stays alive
} catch (...) { } catch (...) {
callbackPtr->rethrow(); callbackPtr->rethrow();
} }
@ -271,8 +275,8 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
} }
Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath, Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) bool recursive, HashType hashAlgo,
{ PathFilter& filter, RepairFlag repair) {
// FIXME: some cut&paste from LocalStore::addToStore(). // FIXME: some cut&paste from LocalStore::addToStore().
/* Read the whole path into memory. This is not a very scalable /* Read the whole path into memory. This is not a very scalable
@ -298,8 +302,8 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
} }
Path BinaryCacheStore::addTextToStore(const string& name, const string& s, Path BinaryCacheStore::addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair) const PathSet& references,
{ RepairFlag repair) {
ValidPathInfo info; ValidPathInfo info;
info.path = computeStorePathForText(name, s, references); info.path = computeStorePathForText(name, s, references);
info.references = references; info.references = references;
@ -313,13 +317,13 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
return info.path; return info.path;
} }
ref<FSAccessor> BinaryCacheStore::getFSAccessor() ref<FSAccessor> BinaryCacheStore::getFSAccessor() {
{ return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()),
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); localNarCache);
} }
void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs) void BinaryCacheStore::addSignatures(const Path& storePath,
{ const StringSet& sigs) {
/* Note: this is inherently racy since there is no locking on /* Note: this is inherently racy since there is no locking on
binary caches. In particular, with S3 this unreliable, even binary caches. In particular, with S3 this unreliable, even
when addSignatures() is called sequentially on a path, because when addSignatures() is called sequentially on a path, because
@ -334,8 +338,7 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
writeNarInfo(narInfo); writeNarInfo(narInfo);
} }
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) {
{
Path drvPath; Path drvPath;
if (isDerivation(path)) if (isDerivation(path))
@ -358,4 +361,4 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
return getFile(logPath); return getFile(logPath);
} }
} } // namespace nix

View file

@ -1,41 +1,41 @@
#pragma once #pragma once
#include "crypto.hh"
#include "store-api.hh"
#include "pool.hh"
#include <atomic> #include <atomic>
#include "crypto.hh"
#include "pool.hh"
#include "store-api.hh"
namespace nix { namespace nix {
struct NarInfo; struct NarInfo;
class BinaryCacheStore : public Store class BinaryCacheStore : public Store {
{
public: public:
const Setting<std::string> compression{
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; this, "xz", "compression",
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<bool> writeNARListing{
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; this, false, "write-nar-listing",
const Setting<bool> parallelCompression{this, false, "parallel-compression", "whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{
this, "", "secret-key",
"path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"path to a local cache of NARs"};
const Setting<bool> parallelCompression{
this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"}; "enable multi-threading compression, available for xz only currently"};
private: private:
std::unique_ptr<SecretKey> secretKey; std::unique_ptr<SecretKey> secretKey;
protected: protected:
BinaryCacheStore(const Params& params); BinaryCacheStore(const Params& params);
public: public:
virtual bool fileExists(const std::string& path) = 0; virtual bool fileExists(const std::string& path) = 0;
virtual void upsertFile(const std::string & path, virtual void upsertFile(const std::string& path, const std::string& data,
const std::string & data,
const std::string& mimeType) = 0; const std::string& mimeType) = 0;
/* Note: subclasses must implement at least one of the two /* Note: subclasses must implement at least one of the two
@ -46,22 +46,20 @@ public:
/* Fetch the specified file and call the specified callback with /* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */ the result. A subclass may implement this asynchronously. */
virtual void getFile(const std::string & path, virtual void getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept; Callback<std::shared_ptr<std::string>> callback) noexcept;
std::shared_ptr<std::string> getFile(const std::string& path); std::shared_ptr<std::string> getFile(const std::string& path);
protected: protected:
bool wantMassQuery_ = false; bool wantMassQuery_ = false;
int priority = 50; int priority = 50;
public: public:
virtual void init(); virtual void init();
private: private:
std::string narMagic; std::string narMagic;
std::string narInfoFileFor(const Path& storePath); std::string narInfoFileFor(const Path& storePath);
@ -69,14 +67,15 @@ private:
void writeNarInfo(ref<NarInfo> narInfo); void writeNarInfo(ref<NarInfo> narInfo);
public: public:
bool isValidPathUncached(const Path& path) override; bool isValidPathUncached(const Path& path) override;
void queryPathInfoUncached(const Path & path, void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
Path queryPathFromHashPart(const string & hashPart) override Path queryPathFromHashPart(const string& hashPart) override {
{ unsupported("queryPathFromHashPart"); } unsupported("queryPathFromHashPart");
}
bool wantMassQuery() override { return wantMassQuery_; } bool wantMassQuery() override { return wantMassQuery_; }
@ -84,9 +83,9 @@ public:
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath, Path addToStore(const string& name, const Path& srcPath, bool recursive,
bool recursive, HashType hashAlgo, HashType hashAlgo, PathFilter& filter,
PathFilter & filter, RepairFlag repair) override; RepairFlag repair) override;
Path addTextToStore(const string& name, const string& s, Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override; const PathSet& references, RepairFlag repair) override;
@ -94,11 +93,11 @@ public:
void narFromPath(const Path& path, Sink& sink) override; void narFromPath(const Path& path, Sink& sink) override;
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override BuildMode buildMode) override {
{ unsupported("buildDerivation"); } unsupported("buildDerivation");
}
void ensurePath(const Path & path) override void ensurePath(const Path& path) override { unsupported("ensurePath"); }
{ unsupported("ensurePath"); }
ref<FSAccessor> getFSAccessor() override; ref<FSAccessor> getFSAccessor() override;
@ -107,9 +106,8 @@ public:
std::shared_ptr<std::string> getBuildLog(const Path& path) override; std::shared_ptr<std::string> getBuildLog(const Path& path) override;
int getPriority() override { return priority; } int getPriority() override { return priority; }
}; };
MakeError(NoSuchBinaryCacheFile, Error); MakeError(NoSuchBinaryCacheFile, Error);
} } // namespace nix

File diff suppressed because it is too large Load diff

View file

@ -8,4 +8,4 @@ namespace nix {
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData); void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData);
void builtinBuildenv(const BasicDerivation& drv); void builtinBuildenv(const BasicDerivation& drv);
} } // namespace nix

View file

@ -1,9 +1,8 @@
#include "builtins.hh" #include <fcntl.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <fcntl.h>
#include <algorithm> #include <algorithm>
#include "builtins.hh"
namespace nix { namespace nix {
@ -16,23 +15,24 @@ static Priorities priorities;
static unsigned long symlinks; static unsigned long symlinks;
/* For each activated package, create symlinks */ /* For each activated package, create symlinks */
static void createLinks(const Path & srcDir, const Path & dstDir, int priority) static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
{
DirEntries srcFiles; DirEntries srcFiles;
try { try {
srcFiles = readDirectory(srcDir); srcFiles = readDirectory(srcDir);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOTDIR) { if (e.errNo == ENOTDIR) {
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); printError(
"warning: not including '%s' in the user environment because it's "
"not a directory",
srcDir);
return; return;
} }
throw; throw;
} }
for (const auto& ent : srcFiles) { for (const auto& ent : srcFiles) {
if (ent.name[0] == '.') if (ent.name[0] == '.') /* not matched by glob */
/* not matched by glob */
continue; continue;
auto srcFile = srcDir + "/" + ent.name; auto srcFile = srcDir + "/" + ent.name;
auto dstFile = dstDir + "/" + ent.name; auto dstFile = dstDir + "/" + ent.name;
@ -58,8 +58,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
if (hasSuffix(srcFile, "/propagated-build-inputs") || if (hasSuffix(srcFile, "/propagated-build-inputs") ||
hasSuffix(srcFile, "/nix-support") || hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") || hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/log"))
hasSuffix(srcFile, "/log"))
continue; continue;
else if (S_ISDIR(srcSt.st_mode)) { else if (S_ISDIR(srcSt.st_mode)) {
@ -72,7 +71,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
} else if (S_ISLNK(dstSt.st_mode)) { } else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true); auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode)) if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); throw Error("collision between '%1%' and non-directory '%2%'",
srcFile, target);
if (unlink(dstFile.c_str()) == -1) if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile); throw SysError(format("unlinking '%1%'") % dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1) if (mkdir(dstFile.c_str(), 0755) == -1)
@ -98,12 +98,13 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
"to change the priority of one of the conflicting packages" "to change the priority of one of the conflicting packages"
" (0 being the highest priority)", " (0 being the highest priority)",
srcFile, readLink(dstFile), priority); srcFile, readLink(dstFile), priority);
if (prevPriority < priority) if (prevPriority < priority) continue;
continue;
if (unlink(dstFile.c_str()) == -1) if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile); throw SysError(format("unlinking '%1%'") % dstFile);
} else if (S_ISDIR(dstSt.st_mode)) } else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); throw Error(
"collision between non-directory '%1%' and directory '%2%'",
srcFile, dstFile);
} else if (errno != ENOENT) } else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile); throw SysError(format("getting status of '%1%'") % dstFile);
} }
@ -121,17 +122,16 @@ static FileProp postponed = FileProp{};
static Path out; static Path out;
static void addPkg(const Path & pkgDir, int priority) static void addPkg(const Path& pkgDir, int priority) {
{
if (done.count(pkgDir)) return; if (done.count(pkgDir)) return;
done.insert(pkgDir); done.insert(pkgDir);
createLinks(pkgDir, out, priority); createLinks(pkgDir, out, priority);
try { try {
for (const auto& p : tokenizeString<std::vector<string>>( for (const auto& p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) readFile(pkgDir + "/nix-support/propagated-user-env-packages"),
if (!done.count(p)) " \n"))
postponed.insert(p); if (!done.count(p)) postponed.insert(p);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
} }
@ -141,13 +141,13 @@ struct Package {
Path path; Path path;
bool active; bool active;
int priority; int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} Package(Path path, bool active, int priority)
: path{path}, active{active}, priority{priority} {}
}; };
typedef std::vector<Package> Packages; typedef std::vector<Package> Packages;
void builtinBuildenv(const BasicDerivation & drv) void builtinBuildenv(const BasicDerivation& drv) {
{
auto getAttr = [&](const string& name) { auto getAttr = [&](const string& name) {
auto i = drv.env.find(name); auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
@ -162,12 +162,17 @@ void builtinBuildenv(const BasicDerivation & drv)
Packages pkgs; Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations")); auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) { while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */ /* !!! We're trusting the caller to structure derivations env var correctly
auto active = derivations.front(); derivations.pop_front(); */
auto priority = stoi(derivations.front()); derivations.pop_front(); auto active = derivations.front();
auto outputs = stoi(derivations.front()); derivations.pop_front(); derivations.pop_front();
auto priority = stoi(derivations.front());
derivations.pop_front();
auto outputs = stoi(derivations.front());
derivations.pop_front();
for (auto n = 0; n < outputs; n++) { for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front(); auto path = derivations.front();
derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority); pkgs.emplace_back(path, active != "false", priority);
} }
} }
@ -177,11 +182,11 @@ void builtinBuildenv(const BasicDerivation & drv)
* symlink/unlink steps. * symlink/unlink steps.
*/ */
std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) { std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) {
return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); return a.priority < b.priority ||
(a.priority == b.priority && a.path < b.path);
}); });
for (const auto& pkg : pkgs) for (const auto& pkg : pkgs)
if (pkg.active) if (pkg.active) addPkg(pkg.path, pkg.priority);
addPkg(pkg.path, pkg.priority);
/* Symlink to the packages that have been "propagated" by packages /* Symlink to the packages that have been "propagated" by packages
* installed by the user (i.e., package X declares that it wants Y * installed by the user (i.e., package X declares that it wants Y
@ -192,8 +197,7 @@ void builtinBuildenv(const BasicDerivation & drv)
while (!postponed.empty()) { while (!postponed.empty()) {
auto pkgDirs = postponed; auto pkgDirs = postponed;
postponed = FileProp{}; postponed = FileProp{};
for (const auto & pkgDir : pkgDirs) for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++);
addPkg(pkgDir, priorityCounter++);
} }
printError("created %d symlinks in user environment", symlinks); printError("created %d symlinks in user environment", symlinks);
@ -201,4 +205,4 @@ void builtinBuildenv(const BasicDerivation & drv)
createSymlink(getAttr("manifest"), out + "/manifest.nix"); createSymlink(getAttr("manifest"), out + "/manifest.nix");
} }
} } // namespace nix

View file

@ -1,13 +1,12 @@
#include "archive.hh"
#include "builtins.hh" #include "builtins.hh"
#include "compression.hh"
#include "download.hh" #include "download.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
namespace nix { namespace nix {
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) {
{
/* Make the host's netrc data available. Too bad curl requires /* Make the host's netrc data available. Too bad curl requires
this to be stored in a file. It would be nice if we could just this to be stored in a file. It would be nice if we could just
pass a pointer to the data. */ pass a pointer to the data. */
@ -18,7 +17,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto getAttr = [&](const string& name) { auto getAttr = [&](const string& name) {
auto i = drv.env.find(name); auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); if (i == drv.env.end())
throw Error(format("attribute '%s' missing") % name);
return i->second; return i->second;
}; };
@ -31,9 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto downloader = makeDownloader(); auto downloader = makeDownloader();
auto fetch = [&](const std::string& url) { auto fetch = [&](const std::string& url) {
auto source = sinkToSource([&](Sink& sink) { auto source = sinkToSource([&](Sink& sink) {
/* No need to do TLS verification, because we check the hash of /* No need to do TLS verification, because we check the hash of
the result anyway. */ the result anyway. */
DownloadRequest request(url); DownloadRequest request(url);
@ -60,12 +58,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
/* Try the hashed mirrors first. */ /* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat") if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get()) for (auto hashedMirror : settings.hashedMirrors.get()) try {
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashType(getAttr("outputHashAlgo")); auto ht = parseHashType(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht); auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); fetch(hashedMirror + printHashType(h.type) + "/" +
h.to_string(Base16, false));
return; return;
} catch (Error& e) { } catch (Error& e) {
debug(e.what()); debug(e.what());
@ -75,4 +73,4 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
fetch(mainUrl); fetch(mainUrl);
} }
} } // namespace nix

View file

@ -1,6 +1,6 @@
#include "crypto.hh" #include "crypto.hh"
#include "util.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh"
#if HAVE_SODIUM #if HAVE_SODIUM
#include <sodium.h> #include <sodium.h>
@ -8,30 +8,24 @@
namespace nix { namespace nix {
static std::pair<std::string, std::string> split(const string & s) static std::pair<std::string, std::string> split(const string& s) {
{
size_t colon = s.find(':'); size_t colon = s.find(':');
if (colon == std::string::npos || colon == 0) if (colon == std::string::npos || colon == 0) return {"", ""};
return {"", ""};
return {std::string(s, 0, colon), std::string(s, colon + 1)}; return {std::string(s, 0, colon), std::string(s, colon + 1)};
} }
Key::Key(const string & s) Key::Key(const string& s) {
{
auto ss = split(s); auto ss = split(s);
name = ss.first; name = ss.first;
key = ss.second; key = ss.second;
if (name == "" || key == "") if (name == "" || key == "") throw Error("secret key is corrupt");
throw Error("secret key is corrupt");
key = base64Decode(key); key = base64Decode(key);
} }
SecretKey::SecretKey(const string & s) SecretKey::SecretKey(const string& s) : Key(s) {
: Key(s)
{
#if HAVE_SODIUM #if HAVE_SODIUM
if (key.size() != crypto_sign_SECRETKEYBYTES) if (key.size() != crypto_sign_SECRETKEYBYTES)
throw Error("secret key is not valid"); throw Error("secret key is not valid");
@ -39,14 +33,14 @@ SecretKey::SecretKey(const string & s)
} }
#if !HAVE_SODIUM #if !HAVE_SODIUM
[[noreturn]] static void noSodium() [[noreturn]] static void noSodium() {
{ throw Error(
throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); "Nix was not compiled with libsodium, required for signed binary cache "
"support");
} }
#endif #endif
std::string SecretKey::signDetached(const std::string & data) const std::string SecretKey::signDetached(const std::string& data) const {
{
#if HAVE_SODIUM #if HAVE_SODIUM
unsigned char sig[crypto_sign_BYTES]; unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen; unsigned long long sigLen;
@ -58,8 +52,7 @@ std::string SecretKey::signDetached(const std::string & data) const
#endif #endif
} }
PublicKey SecretKey::toPublicKey() const PublicKey SecretKey::toPublicKey() const {
{
#if HAVE_SODIUM #if HAVE_SODIUM
unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data()); crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data());
@ -69,9 +62,7 @@ PublicKey SecretKey::toPublicKey() const
#endif #endif
} }
PublicKey::PublicKey(const string & s) PublicKey::PublicKey(const string& s) : Key(s) {
: Key(s)
{
#if HAVE_SODIUM #if HAVE_SODIUM
if (key.size() != crypto_sign_PUBLICKEYBYTES) if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid"); throw Error("public key is not valid");
@ -79,8 +70,7 @@ PublicKey::PublicKey(const string & s)
} }
bool verifyDetached(const std::string& data, const std::string& sig, bool verifyDetached(const std::string& data, const std::string& sig,
const PublicKeys & publicKeys) const PublicKeys& publicKeys) {
{
#if HAVE_SODIUM #if HAVE_SODIUM
auto ss = split(sig); auto ss = split(sig);
@ -88,19 +78,17 @@ bool verifyDetached(const std::string & data, const std::string & sig,
if (key == publicKeys.end()) return false; if (key == publicKeys.end()) return false;
auto sig2 = base64Decode(ss.second); auto sig2 = base64Decode(ss.second);
if (sig2.size() != crypto_sign_BYTES) if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid");
throw Error("signature is not valid");
return crypto_sign_verify_detached((unsigned char *) sig2.data(), return crypto_sign_verify_detached(
(unsigned char *) data.data(), data.size(), (unsigned char*)sig2.data(), (unsigned char*)data.data(),
(unsigned char *) key->second.key.data()) == 0; data.size(), (unsigned char*)key->second.key.data()) == 0;
#else #else
noSodium(); noSodium();
#endif #endif
} }
PublicKeys getDefaultPublicKeys() PublicKeys getDefaultPublicKeys() {
{
PublicKeys publicKeys; PublicKeys publicKeys;
// FIXME: filter duplicates // FIXME: filter duplicates
@ -123,4 +111,4 @@ PublicKeys getDefaultPublicKeys()
return publicKeys; return publicKeys;
} }
} } // namespace nix

View file

@ -1,13 +1,11 @@
#pragma once #pragma once
#include "types.hh"
#include <map> #include <map>
#include "types.hh"
namespace nix { namespace nix {
struct Key struct Key {
{
std::string name; std::string name;
std::string key; std::string key;
@ -16,14 +14,12 @@ struct Key
Key(const std::string& s); Key(const std::string& s);
protected: protected:
Key(const std::string & name, const std::string & key) Key(const std::string& name, const std::string& key) : name(name), key(key) {}
: name(name), key(key) { }
}; };
struct PublicKey; struct PublicKey;
struct SecretKey : Key struct SecretKey : Key {
{
SecretKey(const std::string& s); SecretKey(const std::string& s);
/* Return a detached signature of the given string. */ /* Return a detached signature of the given string. */
@ -32,13 +28,11 @@ struct SecretKey : Key
PublicKey toPublicKey() const; PublicKey toPublicKey() const;
}; };
struct PublicKey : Key struct PublicKey : Key {
{
PublicKey(const std::string& data); PublicKey(const std::string& data);
private: private:
PublicKey(const std::string & name, const std::string & key) PublicKey(const std::string& name, const std::string& key) : Key(name, key) {}
: Key(name, key) { }
friend struct SecretKey; friend struct SecretKey;
}; };
@ -51,4 +45,4 @@ bool verifyDetached(const std::string & data, const std::string & sig,
PublicKeys getDefaultPublicKeys(); PublicKeys getDefaultPublicKeys();
} } // namespace nix

View file

@ -1,16 +1,14 @@
#include "derivations.hh" #include "derivations.hh"
#include "store-api.hh" #include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "istringstream_nocopy.hh"
#include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix { namespace nix {
void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
{
recursive = false; recursive = false;
string algo = hashAlgo; string algo = hashAlgo;
@ -26,29 +24,22 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
hash = Hash(this->hash, hashType); hash = Hash(this->hash, hashType);
} }
Path BasicDerivation::findOutput(const string& id) const {
Path BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id); auto i = outputs.find(id);
if (i == outputs.end()) if (i == outputs.end())
throw Error(format("derivation has no output '%1%'") % id); throw Error(format("derivation has no output '%1%'") % id);
return i->second.path; return i->second.path;
} }
bool BasicDerivation::isBuiltin() const {
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:"; return string(builder, 0, 8) == "builtin:";
} }
Path writeDerivation(ref<Store> store, const Derivation& drv,
Path writeDerivation(ref<Store> store, const string& name, RepairFlag repair) {
const Derivation & drv, const string & name, RepairFlag repair)
{
PathSet references; PathSet references;
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
for (auto & i : drv.inputDrvs) for (auto& i : drv.inputDrvs) references.insert(i.first);
references.insert(i.first);
/* Note that the outputs of a derivation are *not* references /* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be (that can be missing (of course) and should not necessarily be
held during a garbage collection). */ held during a garbage collection). */
@ -59,47 +50,43 @@ Path writeDerivation(ref<Store> store,
: store->addTextToStore(suffix, contents, references, repair); : store->addTextToStore(suffix, contents, references, repair);
} }
/* Read string `s' from stream `str'. */ /* Read string `s' from stream `str'. */
static void expect(std::istream & str, const string & s) static void expect(std::istream& str, const string& s) {
{
char s2[s.size()]; char s2[s.size()];
str.read(s2, s.size()); str.read(s2, s.size());
if (string(s2, s.size()) != s) if (string(s2, s.size()) != s)
throw FormatError(format("expected string '%1%'") % s); throw FormatError(format("expected string '%1%'") % s);
} }
/* Read a C-style string from stream `str'. */ /* Read a C-style string from stream `str'. */
static string parseString(std::istream & str) static string parseString(std::istream& str) {
{
string res; string res;
expect(str, "\""); expect(str, "\"");
int c; int c;
while ((c = str.get()) != '"') while ((c = str.get()) != '"')
if (c == '\\') { if (c == '\\') {
c = str.get(); c = str.get();
if (c == 'n') res += '\n'; if (c == 'n')
else if (c == 'r') res += '\r'; res += '\n';
else if (c == 't') res += '\t'; else if (c == 'r')
else res += c; res += '\r';
} else if (c == 't')
else res += c; res += '\t';
else
res += c;
} else
res += c;
return res; return res;
} }
static Path parsePath(std::istream& str) {
static Path parsePath(std::istream & str)
{
string s = parseString(str); string s = parseString(str);
if (s.size() == 0 || s[0] != '/') if (s.size() == 0 || s[0] != '/')
throw FormatError(format("bad path '%1%' in derivation") % s); throw FormatError(format("bad path '%1%' in derivation") % s);
return s; return s;
} }
static bool endOfList(std::istream& str) {
static bool endOfList(std::istream & str)
{
if (str.peek() == ',') { if (str.peek() == ',') {
str.get(); str.get();
return false; return false;
@ -111,18 +98,14 @@ static bool endOfList(std::istream & str)
return false; return false;
} }
static StringSet parseStrings(std::istream& str, bool arePaths) {
static StringSet parseStrings(std::istream & str, bool arePaths)
{
StringSet res; StringSet res;
while (!endOfList(str)) while (!endOfList(str))
res.insert(arePaths ? parsePath(str) : parseString(str)); res.insert(arePaths ? parsePath(str) : parseString(str));
return res; return res;
} }
static Derivation parseDerivation(const string& s) {
static Derivation parseDerivation(const string & s)
{
Derivation drv; Derivation drv;
istringstream_nocopy str(s); istringstream_nocopy str(s);
expect(str, "Derive(["); expect(str, "Derive([");
@ -130,10 +113,14 @@ static Derivation parseDerivation(const string & s)
/* Parse the list of outputs. */ /* Parse the list of outputs. */
while (!endOfList(str)) { while (!endOfList(str)) {
DerivationOutput out; DerivationOutput out;
expect(str, "("); string id = parseString(str); expect(str, "(");
expect(str, ","); out.path = parsePath(str); string id = parseString(str);
expect(str, ","); out.hashAlgo = parseString(str); expect(str, ",");
expect(str, ","); out.hash = parseString(str); out.path = parsePath(str);
expect(str, ",");
out.hashAlgo = parseString(str);
expect(str, ",");
out.hash = parseString(str);
expect(str, ")"); expect(str, ")");
drv.outputs[id] = out; drv.outputs[id] = out;
} }
@ -148,20 +135,24 @@ static Derivation parseDerivation(const string & s)
expect(str, ")"); expect(str, ")");
} }
expect(str, ",["); drv.inputSrcs = parseStrings(str, true); expect(str, ",[");
expect(str, ","); drv.platform = parseString(str); drv.inputSrcs = parseStrings(str, true);
expect(str, ","); drv.builder = parseString(str); expect(str, ",");
drv.platform = parseString(str);
expect(str, ",");
drv.builder = parseString(str);
/* Parse the builder arguments. */ /* Parse the builder arguments. */
expect(str, ",["); expect(str, ",[");
while (!endOfList(str)) while (!endOfList(str)) drv.args.push_back(parseString(str));
drv.args.push_back(parseString(str));
/* Parse the environment variables. */ /* Parse the environment variables. */
expect(str, ",["); expect(str, ",[");
while (!endOfList(str)) { while (!endOfList(str)) {
expect(str, "("); string name = parseString(str); expect(str, "(");
expect(str, ","); string value = parseString(str); string name = parseString(str);
expect(str, ",");
string value = parseString(str);
expect(str, ")"); expect(str, ")");
drv.env[name] = value; drv.env[name] = value;
} }
@ -170,94 +161,115 @@ static Derivation parseDerivation(const string & s)
return drv; return drv;
} }
Derivation readDerivation(const Path& drvPath) {
Derivation readDerivation(const Path & drvPath)
{
try { try {
return parseDerivation(readFile(drvPath)); return parseDerivation(readFile(drvPath));
} catch (FormatError& e) { } catch (FormatError& e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
e.msg());
} }
} }
Derivation Store::derivationFromPath(const Path& drvPath) {
Derivation Store::derivationFromPath(const Path & drvPath)
{
assertStorePath(drvPath); assertStorePath(drvPath);
ensurePath(drvPath); ensurePath(drvPath);
auto accessor = getFSAccessor(); auto accessor = getFSAccessor();
try { try {
return parseDerivation(accessor->readFile(drvPath)); return parseDerivation(accessor->readFile(drvPath));
} catch (FormatError& e) { } catch (FormatError& e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
e.msg());
} }
} }
static void printString(string& res, const string& s) {
static void printString(string & res, const string & s)
{
res += '"'; res += '"';
for (const char* i = s.c_str(); *i; i++) for (const char* i = s.c_str(); *i; i++)
if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } if (*i == '\"' || *i == '\\') {
else if (*i == '\n') res += "\\n"; res += "\\";
else if (*i == '\r') res += "\\r"; res += *i;
else if (*i == '\t') res += "\\t"; } else if (*i == '\n')
else res += *i; res += "\\n";
else if (*i == '\r')
res += "\\r";
else if (*i == '\t')
res += "\\t";
else
res += *i;
res += '"'; res += '"';
} }
template <class ForwardIterator> template <class ForwardIterator>
static void printStrings(string & res, ForwardIterator i, ForwardIterator j) static void printStrings(string& res, ForwardIterator i, ForwardIterator j) {
{
res += '['; res += '[';
bool first = true; bool first = true;
for (; i != j; ++i) { for (; i != j; ++i) {
if (first) first = false; else res += ','; if (first)
first = false;
else
res += ',';
printString(res, *i); printString(res, *i);
} }
res += ']'; res += ']';
} }
string Derivation::unparse() const {
string Derivation::unparse() const
{
string s; string s;
s.reserve(65536); s.reserve(65536);
s += "Derive(["; s += "Derive([";
bool first = true; bool first = true;
for (auto& i : outputs) { for (auto& i : outputs) {
if (first) first = false; else s += ','; if (first)
s += '('; printString(s, i.first); first = false;
s += ','; printString(s, i.second.path); else
s += ','; printString(s, i.second.hashAlgo); s += ',';
s += ','; printString(s, i.second.hash); s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second.path);
s += ',';
printString(s, i.second.hashAlgo);
s += ',';
printString(s, i.second.hash);
s += ')'; s += ')';
} }
s += "],["; s += "],[";
first = true; first = true;
for (auto& i : inputDrvs) { for (auto& i : inputDrvs) {
if (first) first = false; else s += ','; if (first)
s += '('; printString(s, i.first); first = false;
s += ','; printStrings(s, i.second.begin(), i.second.end()); else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printStrings(s, i.second.begin(), i.second.end());
s += ')'; s += ')';
} }
s += "],"; s += "],";
printStrings(s, inputSrcs.begin(), inputSrcs.end()); printStrings(s, inputSrcs.begin(), inputSrcs.end());
s += ','; printString(s, platform); s += ',';
s += ','; printString(s, builder); printString(s, platform);
s += ','; printStrings(s, args.begin(), args.end()); s += ',';
printString(s, builder);
s += ',';
printStrings(s, args.begin(), args.end());
s += ",["; s += ",[";
first = true; first = true;
for (auto& i : env) { for (auto& i : env) {
if (first) first = false; else s += ','; if (first)
s += '('; printString(s, i.first); first = false;
s += ','; printString(s, i.second); else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second);
s += ')'; s += ')';
} }
@ -266,24 +278,17 @@ string Derivation::unparse() const
return s; return s;
} }
bool isDerivation(const string& fileName) {
bool isDerivation(const string & fileName)
{
return hasSuffix(fileName, drvExtension); return hasSuffix(fileName, drvExtension);
} }
bool BasicDerivation::isFixedOutput() const {
bool BasicDerivation::isFixedOutput() const return outputs.size() == 1 && outputs.begin()->first == "out" &&
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != ""; outputs.begin()->second.hash != "";
} }
DrvHashes drvHashes; DrvHashes drvHashes;
/* Returns the hash of a derivation modulo fixed-output /* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are output (`out') for which an expected hash and hash algorithm are
@ -304,15 +309,12 @@ DrvHashes drvHashes;
paths have been replaced by the result of a recursive call to this paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of function, and that for fixed-output derivations we return a hash of
its output path. */ its output path. */
Hash hashDerivationModulo(Store & store, Derivation drv) Hash hashDerivationModulo(Store& store, Derivation drv) {
{
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) { if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin(); DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:" return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" +
+ i->second.hashAlgo + ":" i->second.hash + ":" + i->second.path);
+ i->second.hash + ":"
+ i->second.path);
} }
/* For other derivations, replace the inputs paths with recursive /* For other derivations, replace the inputs paths with recursive
@ -333,41 +335,31 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
return hashString(htSHA256, drv.unparse()); return hashString(htSHA256, drv.unparse());
} }
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s) {
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
{
size_t n = s.find("!"); size_t n = s.find("!");
return n == s.npos return n == s.npos ? DrvPathWithOutputs(s, std::set<string>())
? DrvPathWithOutputs(s, std::set<string>()) : DrvPathWithOutputs(string(s, 0, n),
: DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); tokenizeString<std::set<string> >(
string(s, n + 1), ","));
} }
Path makeDrvPathWithOutputs(const Path& drvPath,
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) const std::set<string>& outputs) {
{ return outputs.empty() ? drvPath
return outputs.empty()
? drvPath
: drvPath + "!" + concatStringsSep(",", outputs); : drvPath + "!" + concatStringsSep(",", outputs);
} }
bool wantOutput(const string& output, const std::set<string>& wanted) {
bool wantOutput(const string & output, const std::set<string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end(); return wanted.empty() || wanted.find(output) != wanted.end();
} }
PathSet BasicDerivation::outputPaths() const {
PathSet BasicDerivation::outputPaths() const
{
PathSet paths; PathSet paths;
for (auto & i : outputs) for (auto& i : outputs) paths.insert(i.second.path);
paths.insert(i.second.path);
return paths; return paths;
} }
Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) {
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
{
drv.outputs.clear(); drv.outputs.clear();
auto nr = readNum<size_t>(in); auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) { for (size_t n = 0; n < nr; n++) {
@ -392,25 +384,20 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
return in; return in;
} }
Sink& operator<<(Sink& out, const BasicDerivation& drv) {
Sink & operator << (Sink & out, const BasicDerivation & drv)
{
out << drv.outputs.size(); out << drv.outputs.size();
for (auto& i : drv.outputs) for (auto& i : drv.outputs)
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
out << drv.inputSrcs << drv.platform << drv.builder << drv.args; out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
out << drv.env.size(); out << drv.env.size();
for (auto & i : drv.env) for (auto& i : drv.env) out << i.first << i.second;
out << i.first << i.second;
return out; return out;
} }
std::string hashPlaceholder(const std::string& outputName) {
std::string hashPlaceholder(const std::string & outputName)
{
// FIXME: memoize? // FIXME: memoize?
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); return "/" + hashString(htSHA256, "nix-output:" + outputName)
.to_string(Base32, false);
} }
} // namespace nix
}

View file

@ -1,31 +1,23 @@
#pragma once #pragma once
#include "types.hh" #include <map>
#include "hash.hh" #include "hash.hh"
#include "store-api.hh" #include "store-api.hh"
#include "types.hh"
#include <map>
namespace nix { namespace nix {
/* Extension of derivations in the Nix store. */ /* Extension of derivations in the Nix store. */
const string drvExtension = ".drv"; const string drvExtension = ".drv";
/* Abstract syntax of derivations. */ /* Abstract syntax of derivations. */
struct DerivationOutput struct DerivationOutput {
{
Path path; Path path;
string hashAlgo; /* hash used for expected hash computation */ string hashAlgo; /* hash used for expected hash computation */
string hash; /* expected hash, may be null */ string hash; /* expected hash, may be null */
DerivationOutput() DerivationOutput() {}
{ DerivationOutput(Path path, string hashAlgo, string hash) {
}
DerivationOutput(Path path, string hashAlgo, string hash)
{
this->path = path; this->path = path;
this->hashAlgo = hashAlgo; this->hashAlgo = hashAlgo;
this->hash = hash; this->hash = hash;
@ -41,8 +33,7 @@ typedef std::map<Path, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs; typedef std::map<string, string> StringPairs;
struct BasicDerivation struct BasicDerivation {
{
DerivationOutputs outputs; /* keyed on symbolic IDs */ DerivationOutputs outputs; /* keyed on symbolic IDs */
PathSet inputSrcs; /* inputs that are sources */ PathSet inputSrcs; /* inputs that are sources */
string platform; string platform;
@ -63,24 +54,20 @@ struct BasicDerivation
/* Return the output paths of a derivation. */ /* Return the output paths of a derivation. */
PathSet outputPaths() const; PathSet outputPaths() const;
}; };
struct Derivation : BasicDerivation struct Derivation : BasicDerivation {
{
DerivationInputs inputDrvs; /* inputs that are sub-derivations */ DerivationInputs inputDrvs; /* inputs that are sub-derivations */
/* Print a derivation. */ /* Print a derivation. */
std::string unparse() const; std::string unparse() const;
}; };
class Store; class Store;
/* Write a derivation to the Nix store, and return its path. */ /* Write a derivation to the Nix store, and return its path. */
Path writeDerivation(ref<Store> store, Path writeDerivation(ref<Store> store, const Derivation& drv,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair); const string& name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */ /* Read a derivation from a file. */
Derivation readDerivation(const Path& drvPath); Derivation readDerivation(const Path& drvPath);
@ -102,7 +89,8 @@ extern DrvHashes drvHashes; // FIXME: global, not thread-safe
typedef std::pair<string, std::set<string> > DrvPathWithOutputs; typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s); DrvPathWithOutputs parseDrvPathWithOutputs(const string& s);
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); Path makeDrvPathWithOutputs(const Path& drvPath,
const std::set<string>& outputs);
bool wantOutput(const string& output, const std::set<string>& wanted); bool wantOutput(const string& output, const std::set<string>& wanted);
@ -114,4 +102,4 @@ Sink & operator << (Sink & out, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string& outputName); std::string hashPlaceholder(const std::string& outputName);
} } // namespace nix

View file

@ -1,23 +1,21 @@
#include "download.hh" #include "download.hh"
#include "util.hh" #include "archive.hh"
#include "compression.hh"
#include "finally.hh"
#include "globals.hh" #include "globals.hh"
#include "hash.hh" #include "hash.hh"
#include "store-api.hh"
#include "archive.hh"
#include "s3.hh"
#include "compression.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "finally.hh" #include "s3.hh"
#include "store-api.hh"
#include "util.hh"
#ifdef ENABLE_S3 #ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/ClientConfiguration.h>
#endif #endif
#include <unistd.h>
#include <fcntl.h>
#include <curl/curl.h> #include <curl/curl.h>
#include <fcntl.h>
#include <unistd.h>
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
#include <cstring> #include <cstring>
@ -34,31 +32,31 @@ DownloadSettings downloadSettings;
static GlobalConfig::Register r1(&downloadSettings); static GlobalConfig::Register r1(&downloadSettings);
std::string resolveUri(const std::string & uri) std::string resolveUri(const std::string& uri) {
{
if (uri.compare(0, 8, "channel:") == 0) if (uri.compare(0, 8, "channel:") == 0)
return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; return "https://nixos.org/channels/" + std::string(uri, 8) +
"/nixexprs.tar.xz";
else else
return uri; return uri;
} }
struct CurlDownloader : public Downloader struct CurlDownloader : public Downloader {
{
CURLM* curlm = 0; CURLM* curlm = 0;
std::random_device rd; std::random_device rd;
std::mt19937 mt19937; std::mt19937 mt19937;
struct DownloadItem : public std::enable_shared_from_this<DownloadItem> struct DownloadItem : public std::enable_shared_from_this<DownloadItem> {
{
CurlDownloader& downloader; CurlDownloader& downloader;
DownloadRequest request; DownloadRequest request;
DownloadResult result; DownloadResult result;
Activity act; Activity act;
bool done = false; // whether either the success or failure function has been called bool done = false; // whether either the success or failure function has
// been called
Callback<DownloadResult> callback; Callback<DownloadResult> callback;
CURL* req = 0; CURL* req = 0;
bool active = false; // whether the handle has been added to the multi object bool active =
false; // whether the handle has been added to the multi object
std::string status; std::string status;
unsigned int attempt = 0; unsigned int attempt = 0;
@ -75,62 +73,62 @@ struct CurlDownloader : public Downloader
curl_off_t writtenToSink = 0; curl_off_t writtenToSink = 0;
DownloadItem(CurlDownloader & downloader, DownloadItem(CurlDownloader& downloader, const DownloadRequest& request,
const DownloadRequest & request,
Callback<DownloadResult>&& callback) Callback<DownloadResult>&& callback)
: downloader(downloader) : downloader(downloader),
, request(request) request(request),
, act(*logger, lvlTalkative, actDownload, act(*logger, lvlTalkative, actDownload,
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), fmt(request.data ? "uploading '%s'" : "downloading '%s'",
{request.uri}, request.parentAct) request.uri),
, callback(std::move(callback)) {request.uri}, request.parentAct),
, finalSink([this](const unsigned char * data, size_t len) { callback(std::move(callback)),
finalSink([this](const unsigned char* data, size_t len) {
if (this->request.dataCallback) { if (this->request.dataCallback) {
long httpStatus = 0; long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
/* Only write data to the sink if this is a /* Only write data to the sink if this is a
successful response. */ successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 ||
httpStatus == 206) {
writtenToSink += len; writtenToSink += len;
this->request.dataCallback((char*)data, len); this->request.dataCallback((char*)data, len);
} }
} else } else
this->result.data->append((char*)data, len); this->result.data->append((char*)data, len);
}) }) {
{
if (!request.expectedETag.empty()) if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); requestHeaders = curl_slist_append(
requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
if (!request.mimeType.empty()) if (!request.mimeType.empty())
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); requestHeaders = curl_slist_append(
requestHeaders, ("Content-Type: " + request.mimeType).c_str());
} }
~DownloadItem() ~DownloadItem() {
{
if (req) { if (req) {
if (active) if (active) curl_multi_remove_handle(downloader.curlm, req);
curl_multi_remove_handle(downloader.curlm, req);
curl_easy_cleanup(req); curl_easy_cleanup(req);
} }
if (requestHeaders) curl_slist_free_all(requestHeaders); if (requestHeaders) curl_slist_free_all(requestHeaders);
try { try {
if (!done) if (!done)
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); fail(DownloadError(
Interrupted,
format("download of '%s' was interrupted") % request.uri));
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
} }
void failEx(std::exception_ptr ex) void failEx(std::exception_ptr ex) {
{
assert(!done); assert(!done);
done = true; done = true;
callback.rethrow(ex); callback.rethrow(ex);
} }
template <class T> template <class T>
void fail(const T & e) void fail(const T& e) {
{
failEx(std::make_exception_ptr(e)); failEx(std::make_exception_ptr(e));
} }
@ -139,8 +137,7 @@ struct CurlDownloader : public Downloader
std::exception_ptr writeException; std::exception_ptr writeException;
size_t writeCallback(void * contents, size_t size, size_t nmemb) size_t writeCallback(void* contents, size_t size, size_t nmemb) {
{
try { try {
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
result.bodySize += realSize; result.bodySize += realSize;
@ -157,16 +154,16 @@ struct CurlDownloader : public Downloader
} }
} }
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t writeCallbackWrapper(void* contents, size_t size,
{ size_t nmemb, void* userp) {
return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb); return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb);
} }
size_t headerCallback(void * contents, size_t size, size_t nmemb) size_t headerCallback(void* contents, size_t size, size_t nmemb) {
{
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
std::string line((char*)contents, realSize); std::string line((char*)contents, realSize);
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); printMsg(lvlVomit,
format("got header for '%s': %s") % request.uri % trim(line));
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
result.etag = ""; result.etag = "";
auto ss = tokenizeString<vector<string>>(line, " "); auto ss = tokenizeString<vector<string>>(line, " ");
@ -187,25 +184,26 @@ struct CurlDownloader : public Downloader
down the connection because we already have the down the connection because we already have the
data. */ data. */
if (result.etag == request.expectedETag && status == "200") { if (result.etag == request.expectedETag && status == "200") {
debug(format("shutting down on 200 HTTP response with expected ETag")); debug(format(
"shutting down on 200 HTTP response with expected ETag"));
return 0; return 0;
} }
} else if (name == "content-encoding") } else if (name == "content-encoding")
encoding = trim(string(line, i + 1)); encoding = trim(string(line, i + 1));
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") else if (name == "accept-ranges" &&
toLower(trim(std::string(line, i + 1))) == "bytes")
acceptRanges = true; acceptRanges = true;
} }
} }
return realSize; return realSize;
} }
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t headerCallbackWrapper(void* contents, size_t size,
{ size_t nmemb, void* userp) {
return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb); return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb);
} }
int progressCallback(double dltotal, double dlnow) int progressCallback(double dltotal, double dlnow) {
{
try { try {
act.progress(dlnow, dltotal); act.progress(dlnow, dltotal);
} catch (nix::Interrupted&) { } catch (nix::Interrupted&) {
@ -214,23 +212,22 @@ struct CurlDownloader : public Downloader
return _isInterrupted; return _isInterrupted;
} }
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) static int progressCallbackWrapper(void* userp, double dltotal,
{ double dlnow, double ultotal,
double ulnow) {
return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow); return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow);
} }
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) static int debugCallback(CURL* handle, curl_infotype type, char* data,
{ size_t size, void* userptr) {
if (type == CURLINFO_TEXT) if (type == CURLINFO_TEXT)
vomit("curl: %s", chomp(std::string(data, size))); vomit("curl: %s", chomp(std::string(data, size)));
return 0; return 0;
} }
size_t readOffset = 0; size_t readOffset = 0;
size_t readCallback(char *buffer, size_t size, size_t nitems) size_t readCallback(char* buffer, size_t size, size_t nitems) {
{ if (readOffset == request.data->length()) return 0;
if (readOffset == request.data->length())
return 0;
auto count = std::min(size * nitems, request.data->length() - readOffset); auto count = std::min(size * nitems, request.data->length() - readOffset);
assert(count); assert(count);
memcpy(buffer, request.data->data() + readOffset, count); memcpy(buffer, request.data->data() + readOffset, count);
@ -238,20 +235,20 @@ struct CurlDownloader : public Downloader
return count; return count;
} }
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) static size_t readCallbackWrapper(char* buffer, size_t size, size_t nitems,
{ void* userp) {
return ((DownloadItem*)userp)->readCallback(buffer, size, nitems); return ((DownloadItem*)userp)->readCallback(buffer, size, nitems);
} }
void init() void init() {
{
if (!req) req = curl_easy_init(); if (!req) req = curl_easy_init();
curl_easy_reset(req); curl_easy_reset(req);
if (verbosity >= lvlVomit) { if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION,
DownloadItem::debugCallback);
} }
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@ -260,7 +257,10 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT, curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); (downloadSettings.userAgentSuffix != ""
? " " + downloadSettings.userAgentSuffix.get()
: ""))
.c_str());
#if LIBCURL_VERSION_NUM >= 0x072b00 #if LIBCURL_VERSION_NUM >= 0x072b00
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
#endif #endif
@ -270,9 +270,11 @@ struct CurlDownloader : public Downloader
else else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
#endif #endif
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEFUNCTION,
DownloadItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEDATA, this); curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERFUNCTION,
DownloadItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@ -281,14 +283,14 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
if (request.head) if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1);
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.data) { if (request.data) {
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
curl_easy_setopt(req, CURLOPT_READDATA, this); curl_easy_setopt(req, CURLOPT_READDATA, this);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE,
(curl_off_t)request.data->length());
} }
if (request.verifyTLS) { if (request.verifyTLS) {
@ -299,14 +301,17 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
} }
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT,
downloadSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME,
downloadSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work /* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */ anyway as if netrc support was disabled. */
curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC_FILE,
settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
if (writtenToSink) if (writtenToSink)
@ -316,17 +321,17 @@ struct CurlDownloader : public Downloader
result.bodySize = 0; result.bodySize = 0;
} }
void finish(CURLcode code) void finish(CURLcode code) {
{
long httpStatus = 0; long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
char* effectiveUriCStr; char* effectiveUriCStr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr) if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr;
result.effectiveUri = effectiveUriCStr;
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", debug(
"finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d "
"bytes",
request.verb(), request.uri, code, httpStatus, result.bodySize); request.verb(), request.uri, code, httpStatus, result.bodySize);
if (decompressionSink) { if (decompressionSink) {
@ -346,8 +351,10 @@ struct CurlDownloader : public Downloader
failEx(writeException); failEx(writeException);
else if (code == CURLE_OK && else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 ||
{ httpStatus == 206 || httpStatus == 304 ||
httpStatus == 226 /* FTP */ ||
httpStatus == 0 /* other protocol */)) {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize); act.progress(result.bodySize, result.bodySize);
done = true; done = true;
@ -358,19 +365,25 @@ struct CurlDownloader : public Downloader
// We treat most errors as transient, but won't retry when hopeless // We treat most errors as transient, but won't retry when hopeless
Error err = Transient; Error err = Transient;
if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { if (httpStatus == 404 || httpStatus == 410 ||
code == CURLE_FILE_COULDNT_READ_FILE) {
// The file is definitely not there // The file is definitely not there
err = NotFound; err = NotFound;
} else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { } else if (httpStatus == 401 || httpStatus == 403 ||
httpStatus == 407) {
// Don't retry on authentication/authorization failures // Don't retry on authentication/authorization failures
err = Forbidden; err = Forbidden;
} else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 &&
// Most 4xx errors are client errors and are probably not worth retrying: httpStatus != 429) {
// Most 4xx errors are client errors and are probably not worth
// retrying:
// * 408 means the server timed out waiting for us, so we try again // * 408 means the server timed out waiting for us, so we try again
// * 429 means too many requests, so we retry (with a delay) // * 429 means too many requests, so we retry (with a delay)
err = Misc; err = Misc;
} else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { } else if (httpStatus == 501 || httpStatus == 505 ||
// Let's treat most 5xx (server) errors as transient, except for a handful: httpStatus == 511) {
// Let's treat most 5xx (server) errors as transient, except for a
// handful:
// * 501 not implemented // * 501 not implemented
// * 505 http version not supported // * 505 http version not supported
// * 511 we're behind a captive portal // * 511 we're behind a captive portal
@ -403,50 +416,57 @@ struct CurlDownloader : public Downloader
auto exc = auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted",
request.verb(), request.uri))
: httpStatus != 0 : httpStatus != 0
? DownloadError(err, ? DownloadError(
fmt("unable to %s '%s': HTTP error %d", err, fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus) request.verb(), request.uri, httpStatus) +
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) (code == CURLE_OK
) ? ""
: DownloadError(err, : fmt(" (curl error: %s)",
fmt("unable to %s '%s': %s (%d)", curl_easy_strerror(code))))
request.verb(), request.uri, curl_easy_strerror(code), code)); : DownloadError(err, fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri,
curl_easy_strerror(code), code));
/* If this is a transient error, then maybe retry the /* If this is a transient error, then maybe retry the
download after a while. If we're writing to a download after a while. If we're writing to a
sink, we can only retry if the server supports sink, we can only retry if the server supports
ranged requests. */ ranged requests. */
if (err == Transient if (err == Transient && attempt < request.tries &&
&& attempt < request.tries (!this->request.dataCallback || writtenToSink == 0 ||
&& (!this->request.dataCallback (acceptRanges && encoding.empty()))) {
|| writtenToSink == 0 int ms = request.baseRetryTimeMs *
|| (acceptRanges && encoding.empty()))) std::pow(2.0f, attempt - 1 +
{ std::uniform_real_distribution<>(
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); 0.0, 0.5)(downloader.mt19937));
if (writtenToSink) if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); warn("%s; retrying from offset %d in %d ms", exc.what(),
writtenToSink, ms);
else else
warn("%s; retrying in %d ms", exc.what(), ms); warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); embargo =
std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this()); downloader.enqueueItem(shared_from_this());
} } else
else
fail(exc); fail(exc);
} }
} }
}; };
struct State struct State {
{
struct EmbargoComparator { struct EmbargoComparator {
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { bool operator()(const std::shared_ptr<DownloadItem>& i1,
const std::shared_ptr<DownloadItem>& i2) {
return i1->embargo > i2->embargo; return i1->embargo > i2->embargo;
} }
}; };
bool quit = false; bool quit = false;
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; std::priority_queue<std::shared_ptr<DownloadItem>,
std::vector<std::shared_ptr<DownloadItem>>,
EmbargoComparator>
incoming;
}; };
Sync<State> state_; Sync<State> state_;
@ -458,9 +478,7 @@ struct CurlDownloader : public Downloader
std::thread workerThread; std::thread workerThread;
CurlDownloader() CurlDownloader() : mt19937(rd()) {
: mt19937(rd())
{
static std::once_flag globalInit; static std::once_flag globalInit;
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
@ -480,8 +498,7 @@ struct CurlDownloader : public Downloader
workerThread = std::thread([&]() { workerThreadEntry(); }); workerThread = std::thread([&]() { workerThreadEntry(); });
} }
~CurlDownloader() ~CurlDownloader() {
{
stopWorkerThread(); stopWorkerThread();
workerThread.join(); workerThread.join();
@ -489,8 +506,7 @@ struct CurlDownloader : public Downloader
if (curlm) curl_multi_cleanup(curlm); if (curlm) curl_multi_cleanup(curlm);
} }
void stopWorkerThread() void stopWorkerThread() {
{
/* Signal the worker thread to exit. */ /* Signal the worker thread to exit. */
{ {
auto state(state_.lock()); auto state(state_.lock());
@ -499,12 +515,9 @@ struct CurlDownloader : public Downloader
writeFull(wakeupPipe.writeSide.get(), " ", false); writeFull(wakeupPipe.writeSide.get(), " ", false);
} }
void workerThreadMain() void workerThreadMain() {
{
/* Cause this thread to be notified on SIGINT. */ /* Cause this thread to be notified on SIGINT. */
auto callback = createInterruptCallback([&]() { auto callback = createInterruptCallback([&]() { stopWorkerThread(); });
stopWorkerThread();
});
std::map<CURL*, std::shared_ptr<DownloadItem>> items; std::map<CURL*, std::shared_ptr<DownloadItem>> items;
@ -519,7 +532,9 @@ struct CurlDownloader : public Downloader
int running; int running;
CURLMcode mc = curl_multi_perform(curlm, &running); CURLMcode mc = curl_multi_perform(curlm, &running);
if (mc != CURLM_OK) if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); throw nix::Error(
format("unexpected error from curl_multi_perform(): %s") %
curl_multi_strerror(mc));
/* Set the promises of any finished requests. */ /* Set the promises of any finished requests. */
CURLMsg* msg; CURLMsg* msg;
@ -544,12 +559,17 @@ struct CurlDownloader : public Downloader
long maxSleepTimeMs = items.empty() ? 10000 : 100; long maxSleepTimeMs = items.empty() ? 10000 : 100;
auto sleepTimeMs = auto sleepTimeMs =
nextWakeup != std::chrono::steady_clock::time_point() nextWakeup != std::chrono::steady_clock::time_point()
? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) ? std::max(
0,
(int)std::chrono::duration_cast<std::chrono::milliseconds>(
nextWakeup - std::chrono::steady_clock::now())
.count())
: maxSleepTimeMs; : maxSleepTimeMs;
vomit("download thread waiting for %d ms", sleepTimeMs); vomit("download thread waiting for %d ms", sleepTimeMs);
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
if (mc != CURLM_OK) if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); throw nix::Error(format("unexpected error from curl_multi_wait(): %s") %
curl_multi_strerror(mc));
nextWakeup = std::chrono::steady_clock::time_point(); nextWakeup = std::chrono::steady_clock::time_point();
@ -574,8 +594,8 @@ struct CurlDownloader : public Downloader
incoming.push_back(item); incoming.push_back(item);
state->incoming.pop(); state->incoming.pop();
} else { } else {
if (nextWakeup == std::chrono::steady_clock::time_point() if (nextWakeup == std::chrono::steady_clock::time_point() ||
|| item->embargo < nextWakeup) item->embargo < nextWakeup)
nextWakeup = item->embargo; nextWakeup = item->embargo;
break; break;
} }
@ -595,8 +615,7 @@ struct CurlDownloader : public Downloader
debug("download thread shutting down"); debug("download thread shutting down");
} }
void workerThreadEntry() void workerThreadEntry() {
{
try { try {
workerThreadMain(); workerThreadMain();
} catch (nix::Interrupted& e) { } catch (nix::Interrupted& e) {
@ -611,30 +630,29 @@ struct CurlDownloader : public Downloader
} }
} }
void enqueueItem(std::shared_ptr<DownloadItem> item) void enqueueItem(std::shared_ptr<DownloadItem> item) {
{ if (item->request.data && !hasPrefix(item->request.uri, "http://") &&
if (item->request.data !hasPrefix(item->request.uri, "https://"))
&& !hasPrefix(item->request.uri, "http://")
&& !hasPrefix(item->request.uri, "https://"))
throw nix::Error("uploading to '%s' is not supported", item->request.uri); throw nix::Error("uploading to '%s' is not supported", item->request.uri);
{ {
auto state(state_.lock()); auto state(state_.lock());
if (state->quit) if (state->quit)
throw nix::Error("cannot enqueue download request because the download thread is shutting down"); throw nix::Error(
"cannot enqueue download request because the download thread is "
"shutting down");
state->incoming.push(item); state->incoming.push(item);
} }
writeFull(wakeupPipe.writeSide.get(), " "); writeFull(wakeupPipe.writeSide.get(), " ");
} }
#ifdef ENABLE_S3 #ifdef ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri) std::tuple<std::string, std::string, Store::Params> parseS3Uri(
{ std::string uri) {
auto [path, params] = splitUriAndParams(uri); auto [path, params] = splitUriAndParams(uri);
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
if (slash == std::string::npos) if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", path);
throw nix::Error("bad S3 URI '%s'", path);
std::string bucketName(path, 5, slash - 5); std::string bucketName(path, 5, slash - 5);
std::string key(path, slash + 1); std::string key(path, slash + 1);
@ -644,8 +662,7 @@ struct CurlDownloader : public Downloader
#endif #endif
void enqueueDownload(const DownloadRequest& request, void enqueueDownload(const DownloadRequest& request,
Callback<DownloadResult> callback) override Callback<DownloadResult> callback) override {
{
/* Ugly hack to support s3:// URIs. */ /* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) { if (hasPrefix(request.uri, "s3://")) {
// FIXME: do this on a worker thread // FIXME: do this on a worker thread
@ -664,36 +681,37 @@ struct CurlDownloader : public Downloader
auto s3Res = s3Helper.getObject(bucketName, key); auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res; DownloadResult res;
if (!s3Res.data) if (!s3Res.data)
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); throw DownloadError(
NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data; res.data = s3Res.data;
callback(std::move(res)); callback(std::move(res));
#else #else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); throw nix::Error(
"cannot download '%s' because Nix is not built with S3 support",
request.uri);
#endif #endif
} catch (...) { callback.rethrow(); } } catch (...) {
callback.rethrow();
}
return; return;
} }
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); enqueueItem(
std::make_shared<DownloadItem>(*this, request, std::move(callback)));
} }
}; };
ref<Downloader> getDownloader() ref<Downloader> getDownloader() {
{
static ref<Downloader> downloader = makeDownloader(); static ref<Downloader> downloader = makeDownloader();
return downloader; return downloader;
} }
ref<Downloader> makeDownloader() ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); }
{
return make_ref<CurlDownloader>();
}
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request) std::future<DownloadResult> Downloader::enqueueDownload(
{ const DownloadRequest& request) {
auto promise = std::make_shared<std::promise<DownloadResult>>(); auto promise = std::make_shared<std::promise<DownloadResult>>();
enqueueDownload(request, enqueueDownload(request, {[promise](std::future<DownloadResult> fut) {
{[promise](std::future<DownloadResult> fut) {
try { try {
promise->set_value(fut.get()); promise->set_value(fut.get());
} catch (...) { } catch (...) {
@ -703,13 +721,11 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
return promise->get_future(); return promise->get_future();
} }
DownloadResult Downloader::download(const DownloadRequest & request) DownloadResult Downloader::download(const DownloadRequest& request) {
{
return enqueueDownload(request).get(); return enqueueDownload(request).get();
} }
void Downloader::download(DownloadRequest && request, Sink & sink) void Downloader::download(DownloadRequest&& request, Sink& sink) {
{
/* Note: we can't call 'sink' via request.dataCallback, because /* Note: we can't call 'sink' via request.dataCallback, because
that would cause the sink to execute on the downloader that would cause the sink to execute on the downloader
thread. If 'sink' is a coroutine, this will fail. Also, if the thread. If 'sink' is a coroutine, this will fail. Also, if the
@ -736,7 +752,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}); });
request.dataCallback = [_state](char* buf, size_t len) { request.dataCallback = [_state](char* buf, size_t len) {
auto state(_state->lock()); auto state(_state->lock());
if (state->quit) return; if (state->quit) return;
@ -757,8 +772,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
state->avail.notify_one(); state->avail.notify_one();
}; };
enqueueDownload(request, enqueueDownload(request, {[_state](std::future<DownloadResult> fut) {
{[_state](std::future<DownloadResult> fut) {
auto state(_state->lock()); auto state(_state->lock());
state->quit = true; state->quit = true;
try { try {
@ -781,7 +795,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
auto state(_state->lock()); auto state(_state->lock());
while (state->data.empty()) { while (state->data.empty()) {
if (state->quit) { if (state->quit) {
if (state->exc) std::rethrow_exception(state->exc); if (state->exc) std::rethrow_exception(state->exc);
return; return;
@ -804,8 +817,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
} }
CachedDownloadResult Downloader::downloadCached( CachedDownloadResult Downloader::downloadCached(
ref<Store> store, const CachedDownloadRequest & request) ref<Store> store, const CachedDownloadRequest& request) {
{
auto url = resolveUri(request.uri); auto url = resolveUri(request.uri);
auto name = request.name; auto name = request.name;
@ -816,7 +828,8 @@ CachedDownloadResult Downloader::downloadCached(
Path expectedStorePath; Path expectedStorePath;
if (request.expectedHash) { if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); expectedStorePath =
store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(expectedStorePath)) { if (store->isValidPath(expectedStorePath)) {
CachedDownloadResult result; CachedDownloadResult result;
result.storePath = expectedStorePath; result.storePath = expectedStorePath;
@ -828,7 +841,8 @@ CachedDownloadResult Downloader::downloadCached(
Path cacheDir = getCacheDir() + "/nix/tarballs"; Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir); createDirs(cacheDir);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); string urlHash = hashString(htSHA256, name + std::string("\0"s) + url)
.to_string(Base32, false);
Path dataFile = cacheDir + "/" + urlHash + ".info"; Path dataFile = cacheDir + "/" + urlHash + ".info";
Path fileLink = cacheDir + "/" + urlHash + "-file"; Path fileLink = cacheDir + "/" + urlHash + "-file";
@ -850,7 +864,8 @@ CachedDownloadResult Downloader::downloadCached(
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) { if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked; time_t lastChecked;
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { if (string2Int(ss[2], lastChecked) &&
(uint64_t)lastChecked + request.ttl >= (uint64_t)time(0)) {
skip = true; skip = true;
result.effectiveUri = request.uri; result.effectiveUri = request.uri;
result.etag = ss[1]; result.etag = ss[1];
@ -864,7 +879,6 @@ CachedDownloadResult Downloader::downloadCached(
} }
if (!skip) { if (!skip) {
try { try {
DownloadRequest request2(url); DownloadRequest request2(url);
request2.expectedETag = expectedETag; request2.expectedETag = expectedETag;
@ -876,7 +890,9 @@ CachedDownloadResult Downloader::downloadCached(
ValidPathInfo info; ValidPathInfo info;
StringSink sink; StringSink sink;
dumpString(*res.data, sink); dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); Hash hash = hashString(
request.expectedHash ? request.expectedHash.type : htSHA256,
*res.data);
info.path = store->makeFixedOutputPath(false, hash, name); info.path = store->makeFixedOutputPath(false, hash, name);
info.narHash = hashString(htSHA256, *sink.s); info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size(); info.narSize = sink.s->size();
@ -888,7 +904,8 @@ CachedDownloadResult Downloader::downloadCached(
assert(!storePath.empty()); assert(!storePath.empty());
replaceSymlink(storePath, fileLink); replaceSymlink(storePath, fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); writeFile(dataFile,
url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError& e) { } catch (DownloadError& e) {
if (storePath.empty()) throw; if (storePath.empty()) throw;
warn("warning: %s; using cached result", e.msg()); warn("warning: %s; using cached result", e.msg());
@ -898,21 +915,24 @@ CachedDownloadResult Downloader::downloadCached(
if (request.unpack) { if (request.unpack) {
Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); PathLocks lock2({unpackedLink},
fmt("waiting for lock on '%1%'...", unpackedLink));
Path unpackedStorePath; Path unpackedStorePath;
if (pathExists(unpackedLink)) { if (pathExists(unpackedLink)) {
unpackedStorePath = readLink(unpackedLink); unpackedStorePath = readLink(unpackedLink);
store->addTempRoot(unpackedStorePath); store->addTempRoot(unpackedStorePath);
if (!store->isValidPath(unpackedStorePath)) if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = "";
unpackedStorePath = "";
} }
if (unpackedStorePath.empty()) { if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url); printInfo(format("unpacking '%1%'...") % url);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true); AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression. // FIXME: this requires GNU tar for decompression.
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); runProgram("tar", true,
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); {"xf", store->toRealPath(storePath), "-C", tmpDir,
"--strip-components", "1"});
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256,
defaultPathFilter, NoRepair);
} }
replaceSymlink(unpackedStorePath, unpackedLink); replaceSymlink(unpackedStorePath, unpackedLink);
storePath = unpackedStorePath; storePath = unpackedStorePath;
@ -920,11 +940,16 @@ CachedDownloadResult Downloader::downloadCached(
if (expectedStorePath != "" && storePath != expectedStorePath) { if (expectedStorePath != "" && storePath != expectedStorePath) {
unsigned int statusCode = 102; unsigned int statusCode = 102;
Hash gotHash = request.unpack Hash gotHash =
? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(storePath))
.first
: hashFile(request.expectedHash.type, store->toRealPath(storePath)); : hashFile(request.expectedHash.type, store->toRealPath(storePath));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", throw nix::Error(statusCode,
url, request.expectedHash.to_string(), gotHash.to_string()); "hash mismatch in file downloaded from '%s':\n wanted: "
"%s\n got: %s",
url, request.expectedHash.to_string(),
gotHash.to_string());
} }
result.storePath = storePath; result.storePath = storePath;
@ -932,15 +957,14 @@ CachedDownloadResult Downloader::downloadCached(
return result; return result;
} }
bool isUri(const string& s) {
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true; if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://"); size_t pos = s.find("://");
if (pos == string::npos) return false; if (pos == string::npos) return false;
string scheme(s, 0, pos); string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; return scheme == "http" || scheme == "https" || scheme == "file" ||
scheme == "channel" || scheme == "git" || scheme == "s3" ||
scheme == "ssh";
} }
} // namespace nix
}

View file

@ -1,40 +1,45 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh"
#include "globals.hh"
#include <string>
#include <future> #include <future>
#include <string>
#include "globals.hh"
#include "hash.hh"
#include "types.hh"
namespace nix { namespace nix {
struct DownloadSettings : Config struct DownloadSettings : Config {
{
Setting<bool> enableHttp2{this, true, "http2", Setting<bool> enableHttp2{this, true, "http2",
"Whether to enable HTTP/2 support."}; "Whether to enable HTTP/2 support."};
Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", Setting<std::string> userAgentSuffix{
this, "", "user-agent-suffix",
"String appended to the user agent in HTTP requests."}; "String appended to the user agent in HTTP requests."};
Setting<size_t> httpConnections{this, 25, "http-connections", Setting<size_t> httpConnections{this,
25,
"http-connections",
"Number of parallel HTTP connections.", "Number of parallel HTTP connections.",
{"binary-caches-parallel-connections"}}; {"binary-caches-parallel-connections"}};
Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", Setting<unsigned long> connectTimeout{
"Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; this, 0, "connect-timeout",
"Timeout for connecting to servers during downloads. 0 means use curl's "
"builtin default."};
Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", Setting<unsigned long> stalledDownloadTimeout{
"Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; this, 300, "stalled-download-timeout",
"Timeout (in seconds) for receiving data from servers during download. "
"Nix cancels idle downloads after this timeout's duration."};
Setting<unsigned int> tries{this, 5, "download-attempts", Setting<unsigned int> tries{
this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."}; "How often Nix will attempt to download a file before giving up."};
}; };
extern DownloadSettings downloadSettings; extern DownloadSettings downloadSettings;
struct DownloadRequest struct DownloadRequest {
{
std::string uri; std::string uri;
std::string expectedETag; std::string expectedETag;
bool verifyTLS = true; bool verifyTLS = true;
@ -50,14 +55,10 @@ struct DownloadRequest
DownloadRequest(const std::string& uri) DownloadRequest(const std::string& uri)
: uri(uri), parentAct(getCurActivity()) {} : uri(uri), parentAct(getCurActivity()) {}
std::string verb() std::string verb() { return data ? "upload" : "download"; }
{
return data ? "upload" : "download";
}
}; };
struct DownloadResult struct DownloadResult {
{
bool cached = false; bool cached = false;
std::string etag; std::string etag;
std::string effectiveUri; std::string effectiveUri;
@ -65,20 +66,17 @@ struct DownloadResult
uint64_t bodySize = 0; uint64_t bodySize = 0;
}; };
struct CachedDownloadRequest struct CachedDownloadRequest {
{
std::string uri; std::string uri;
bool unpack = false; bool unpack = false;
std::string name; std::string name;
Hash expectedHash; Hash expectedHash;
unsigned int ttl = settings.tarballTtl; unsigned int ttl = settings.tarballTtl;
CachedDownloadRequest(const std::string & uri) CachedDownloadRequest(const std::string& uri) : uri(uri) {}
: uri(uri) { }
}; };
struct CachedDownloadResult struct CachedDownloadResult {
{
// Note: 'storePath' may be different from 'path' when using a // Note: 'storePath' may be different from 'path' when using a
// chroot store. // chroot store.
Path storePath; Path storePath;
@ -89,8 +87,7 @@ struct CachedDownloadResult
class Store; class Store;
struct Downloader struct Downloader {
{
virtual ~Downloader() {} virtual ~Downloader() {}
/* Enqueue a download request, returning a future to the result of /* Enqueue a download request, returning a future to the result of
@ -112,7 +109,8 @@ struct Downloader
and is more recent than tarball-ttl seconds. Otherwise, and is more recent than tarball-ttl seconds. Otherwise,
use the recorded ETag to verify if the server has a more use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */ recent version, and if so, download it to the Nix store. */
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); CachedDownloadResult downloadCached(ref<Store> store,
const CachedDownloadRequest& request);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
}; };
@ -124,15 +122,13 @@ ref<Downloader> getDownloader();
/* Return a new Downloader object. */ /* Return a new Downloader object. */
ref<Downloader> makeDownloader(); ref<Downloader> makeDownloader();
class DownloadError : public Error class DownloadError : public Error {
{
public: public:
Downloader::Error error; Downloader::Error error;
DownloadError(Downloader::Error error, const FormatOrString& fs) DownloadError(Downloader::Error error, const FormatOrString& fs)
: Error(fs), error(error) : Error(fs), error(error) {}
{ }
}; };
bool isUri(const string& s); bool isUri(const string& s);
} } // namespace nix

View file

@ -1,31 +1,23 @@
#include "store-api.hh"
#include "archive.hh"
#include "worker-protocol.hh"
#include <algorithm> #include <algorithm>
#include "archive.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
struct HashAndWriteSink : Sink struct HashAndWriteSink : Sink {
{
Sink& writeSink; Sink& writeSink;
HashSink hashSink; HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) HashAndWriteSink(Sink& writeSink)
{ : writeSink(writeSink), hashSink(htSHA256) {}
} virtual void operator()(const unsigned char* data, size_t len) {
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len); writeSink(data, len);
hashSink(data, len); hashSink(data, len);
} }
Hash currentHash() Hash currentHash() { return hashSink.currentHash().first; }
{
return hashSink.currentHash().first;
}
}; };
void Store::exportPaths(const Paths & paths, Sink & sink) void Store::exportPaths(const Paths& paths, Sink& sink) {
{
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
std::reverse(sorted.begin(), sorted.end()); std::reverse(sorted.begin(), sorted.end());
@ -42,8 +34,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
sink << 0; sink << 0;
} }
void Store::exportPath(const Path & path, Sink & sink) void Store::exportPath(const Path& path, Sink& sink) {
{
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink); HashAndWriteSink hashAndWriteSink(sink);
@ -55,19 +46,22 @@ void Store::exportPath(const Path & path, Sink & sink)
Don't complain if the stored hash is zero (unknown). */ Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash(); Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") %
% info->narHash.to_string() % hash.to_string()); path % info->narHash.to_string() % hash.to_string());
hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; hashAndWriteSink << exportMagic << path << info->references << info->deriver
<< 0;
} }
Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) Paths Store::importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
{ CheckSigsFlag checkSigs) {
Paths res; Paths res;
while (true) { while (true) {
auto n = readNum<uint64_t>(source); auto n = readNum<uint64_t>(source);
if (n == 0) break; if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); if (n != 1)
throw Error(
"input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
TeeSink tee(source); TeeSink tee(source);
@ -81,7 +75,8 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
info.path = readStorePath(*this, source); info.path = readStorePath(*this, source);
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); // Activity act(*logger, lvlInfo, format("importing path '%s'") %
// info.path);
info.references = readStorePaths<PathSet>(*this, source); info.references = readStorePaths<PathSet>(*this, source);
@ -92,8 +87,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
info.narSize = tee.source.data->size(); info.narSize = tee.source.data->size();
// Ignore optional legacy signature. // Ignore optional legacy signature.
if (readInt(source) == 1) if (readInt(source) == 1) readString(source);
readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
@ -103,4 +97,4 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
return res; return res;
} }
} } // namespace nix

View file

@ -6,13 +6,11 @@ namespace nix {
/* An abstract class for accessing a filesystem-like structure, such /* An abstract class for accessing a filesystem-like structure, such
as a (possibly remote) Nix store or the contents of a NAR file. */ as a (possibly remote) Nix store or the contents of a NAR file. */
class FSAccessor class FSAccessor {
{
public: public:
enum Type { tMissing, tRegular, tSymlink, tDirectory }; enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat struct Stat {
{
Type type = tMissing; Type type = tMissing;
uint64_t fileSize = 0; // regular files only uint64_t fileSize = 0; // regular files only
bool isExecutable = false; // regular files only bool isExecutable = false; // regular files only
@ -30,4 +28,4 @@ public:
virtual std::string readLink(const Path& path) = 0; virtual std::string readLink(const Path& path) = 0;
}; };
} } // namespace nix

View file

@ -1,42 +1,37 @@
#include "derivations.hh"
#include "globals.hh"
#include "local-store.hh"
#include "finally.hh"
#include <functional>
#include <queue>
#include <algorithm>
#include <regex>
#include <random>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <unistd.h> #include <unistd.h>
#include <algorithm>
#include <climits> #include <climits>
#include <functional>
#include <queue>
#include <random>
#include <regex>
#include "derivations.hh"
#include "finally.hh"
#include "globals.hh"
#include "local-store.hh"
namespace nix { namespace nix {
static string gcLockName = "gc.lock"; static string gcLockName = "gc.lock";
static string gcRootsDir = "gcroots"; static string gcRootsDir = "gcroots";
/* Acquire the global GC lock. This is used to prevent new Nix /* Acquire the global GC lock. This is used to prevent new Nix
processes from starting after the temporary root files have been processes from starting after the temporary root files have been
read. To be precise: when they try to create a new temporary root read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished / file, they will block until the garbage collector has finished /
yielded the GC lock. */ yielded the GC lock. */
AutoCloseFD LocalStore::openGCLock(LockType lockType) AutoCloseFD LocalStore::openGCLock(LockType lockType) {
{ Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str();
Path fnGCLock = (format("%1%/%2%")
% stateDir % gcLockName).str();
debug(format("acquiring global GC lock '%1%'") % fnGCLock); debug(format("acquiring global GC lock '%1%'") % fnGCLock);
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); AutoCloseFD fdGCLock =
open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock) if (!fdGCLock)
throw SysError(format("opening global GC lock '%1%'") % fnGCLock); throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
@ -52,69 +47,60 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
return fdGCLock; return fdGCLock;
} }
static void makeSymlink(const Path& link, const Path& target) {
static void makeSymlink(const Path & link, const Path & target)
{
/* Create directories up to `gcRoot'. */ /* Create directories up to `gcRoot'. */
createDirs(dirOf(link)); createDirs(dirOf(link));
/* Create the new symlink. */ /* Create the new symlink. */
Path tempLink = (format("%1%.tmp-%2%-%3%") Path tempLink =
% link % getpid() % random()).str(); (format("%1%.tmp-%2%-%3%") % link % getpid() % random()).str();
createSymlink(target, tempLink); createSymlink(target, tempLink);
/* Atomically replace the old one. */ /* Atomically replace the old one. */
if (rename(tempLink.c_str(), link.c_str()) == -1) if (rename(tempLink.c_str(), link.c_str()) == -1)
throw SysError(format("cannot rename '%1%' to '%2%'") throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % link);
% tempLink % link);
} }
void LocalStore::syncWithGC() { AutoCloseFD fdGCLock = openGCLock(ltRead); }
void LocalStore::syncWithGC() void LocalStore::addIndirectRoot(const Path& path) {
{
AutoCloseFD fdGCLock = openGCLock(ltRead);
}
void LocalStore::addIndirectRoot(const Path & path)
{
string hash = hashString(htSHA1, path).to_string(Base32, false); string hash = hashString(htSHA1, path).to_string(Base32, false);
Path realRoot = canonPath((format("%1%/%2%/auto/%3%") Path realRoot = canonPath(
% stateDir % gcRootsDir % hash).str()); (format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str());
makeSymlink(realRoot, path); makeSymlink(realRoot, path);
} }
Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot,
Path LocalFSStore::addPermRoot(const Path & _storePath, bool indirect, bool allowOutsideRootsDir) {
const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
{
Path storePath(canonPath(_storePath)); Path storePath(canonPath(_storePath));
Path gcRoot(canonPath(_gcRoot)); Path gcRoot(canonPath(_gcRoot));
assertStorePath(storePath); assertStorePath(storePath);
if (isInStore(gcRoot)) if (isInStore(gcRoot))
throw Error(format( throw Error(format("creating a garbage collector root (%1%) in the Nix "
"creating a garbage collector root (%1%) in the Nix store is forbidden " "store is forbidden "
"(are you running nix-build inside the store?)") % gcRoot); "(are you running nix-build inside the store?)") %
gcRoot);
if (indirect) { if (indirect) {
/* Don't clobber the link if it already exists and doesn't /* Don't clobber the link if it already exists and doesn't
point to the Nix store. */ point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); throw Error(format("cannot create symlink '%1%'; already exists") %
gcRoot);
makeSymlink(gcRoot, storePath); makeSymlink(gcRoot, storePath);
addIndirectRoot(gcRoot); addIndirectRoot(gcRoot);
} }
else { else {
if (!allowOutsideRootsDir) { if (!allowOutsideRootsDir) {
Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); Path rootsDir =
canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(format( throw Error(format("path '%1%' is not a valid garbage collector root; "
"path '%1%' is not a valid garbage collector root; " "it's not in the directory '%2%'") %
"it's not in the directory '%2%'") gcRoot % rootsDir);
% gcRoot % rootsDir);
} }
if (baseNameOf(gcRoot) == baseNameOf(storePath)) if (baseNameOf(gcRoot) == baseNameOf(storePath))
@ -132,10 +118,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
Roots roots = findRoots(false); Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0) if (roots[storePath].count(gcRoot) == 0)
printError( printError(
format( format("warning: '%1%' is not in a directory where the garbage "
"warning: '%1%' is not in a directory where the garbage collector looks for roots; " "collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector") "therefore, '%2%' might be removed by the garbage collector") %
% gcRoot % storePath); gcRoot % storePath);
} }
/* Grab the global GC root, causing us to block while a GC is in /* Grab the global GC root, causing us to block while a GC is in
@ -146,14 +132,11 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
return gcRoot; return gcRoot;
} }
void LocalStore::addTempRoot(const Path& path) {
void LocalStore::addTempRoot(const Path & path)
{
auto state(_state.lock()); auto state(_state.lock());
/* Create the temporary roots file for this process. */ /* Create the temporary roots file for this process. */
if (!state->fdTempRoots) { if (!state->fdTempRoots) {
while (1) { while (1) {
AutoCloseFD fdGCLock = openGCLock(ltRead); AutoCloseFD fdGCLock = openGCLock(ltRead);
@ -180,7 +163,6 @@ void LocalStore::addTempRoot(const Path & path)
get a lock. (It won't delete the file after we get a get a lock. (It won't delete the file after we get a
lock.) Try again. */ lock.) Try again. */
} }
} }
/* Upgrade the lock to a write lock. This will cause us to block /* Upgrade the lock to a write lock. This will cause us to block
@ -196,12 +178,9 @@ void LocalStore::addTempRoot(const Path & path)
lockFile(state->fdTempRoots.get(), ltRead, true); lockFile(state->fdTempRoots.get(), ltRead, true);
} }
static std::string censored = "{censored}"; static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root /* Read the `temproots' directory for per-process temporary root
files. */ files. */
for (auto& i : readDirectory(tempRootsDir)) { for (auto& i : readDirectory(tempRootsDir)) {
@ -255,21 +234,18 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
} }
} }
void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path& path, const Path& target) { auto foundRoot = [&](const Path& path, const Path& target) {
Path storePath = toStorePath(target); Path storePath = toStorePath(target);
if (isStorePath(storePath) && isValidPath(storePath)) if (isStorePath(storePath) && isValidPath(storePath))
roots[storePath].emplace(path); roots[storePath].emplace(path);
else else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); printInfo(format("skipping invalid root from '%1%' to '%2%'") % path %
storePath);
}; };
try { try {
if (type == DT_UNKNOWN) type = getFileType(path);
if (type == DT_UNKNOWN)
type = getFileType(path);
if (type == DT_DIR) { if (type == DT_DIR) {
for (auto& i : readDirectory(path)) for (auto& i : readDirectory(path))
@ -278,15 +254,15 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
else if (type == DT_LNK) { else if (type == DT_LNK) {
Path target = readLink(path); Path target = readLink(path);
if (isInStore(target)) if (isInStore(target)) foundRoot(path, target);
foundRoot(path, target);
/* Handle indirect roots. */ /* Handle indirect roots. */
else { else {
target = absPath(target, dirOf(path)); target = absPath(target, dirOf(path));
if (!pathExists(target)) { if (!pathExists(target)) {
if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); printInfo(format("removing stale link from '%1%' to '%2%'") % path %
target);
unlink(path.c_str()); unlink(path.c_str());
} }
} else { } else {
@ -315,9 +291,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
} }
} }
void LocalStore::findRootsNoTemp(Roots& roots, bool censor) {
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
{
/* Process direct roots in {gcroots,profiles}. */ /* Process direct roots in {gcroots,profiles}. */
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@ -328,9 +302,7 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
findRuntimeRoots(roots, censor); findRuntimeRoots(roots, censor);
} }
Roots LocalStore::findRoots(bool censor) {
Roots LocalStore::findRoots(bool censor)
{
Roots roots; Roots roots;
findRootsNoTemp(roots, censor); findRootsNoTemp(roots, censor);
@ -340,55 +312,49 @@ Roots LocalStore::findRoots(bool censor)
return roots; return roots;
} }
static void readProcLink(const string & file, Roots & roots) static void readProcLink(const string& file, Roots& roots) {
{
/* 64 is the starting buffer size gnu readlink uses... */ /* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64}; auto bufsiz = ssize_t{64};
try_again: try_again:
char buf[bufsiz]; char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz); auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) { if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) if (errno == ENOENT || errno == EACCES || errno == ESRCH) return;
return;
throw SysError("reading symlink"); throw SysError("reading symlink");
} }
if (res == bufsiz) { if (res == bufsiz) {
if (SSIZE_MAX / 2 < bufsiz) if (SSIZE_MAX / 2 < bufsiz) throw Error("stupidly long symlink");
throw Error("stupidly long symlink");
bufsiz *= 2; bufsiz *= 2;
goto try_again; goto try_again;
} }
if (res > 0 && buf[0] == '/') if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)] roots[std::string(static_cast<char*>(buf), res)].emplace(file);
.emplace(file);
} }
static string quoteRegexChars(const string & raw) static string quoteRegexChars(const string& raw) {
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)"); return std::regex_replace(raw, specialRegex, R"(\$&)");
} }
static void readFileRoots(const char * path, Roots & roots) static void readFileRoots(const char* path, Roots& roots) {
{
try { try {
roots[readFile(path)].emplace(path); roots[readFile(path)].emplace(path);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != EACCES) if (e.errNo != ENOENT && e.errNo != EACCES) throw;
throw;
} }
} }
void LocalStore::findRuntimeRoots(Roots & roots, bool censor) void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
{
Roots unchecked; Roots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")}; auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) { if (procDir) {
struct dirent* ent; struct dirent* ent;
auto digitsRegex = std::regex(R"(^\d+$)"); auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); auto mapRegex =
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = std::regex(quoteRegexChars(storeDir) +
R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) { while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt(); checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) { if (std::regex_match(ent->d_name, digitsRegex)) {
@ -398,8 +364,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) { if (!fdDir) {
if (errno == ENOENT || errno == EACCES) if (errno == ENOENT || errno == EACCES) continue;
continue;
throw SysError(format("opening %1%") % fdStr); throw SysError(format("opening %1%") % fdStr);
} }
struct dirent* fd_ent; struct dirent* fd_ent;
@ -408,15 +373,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
} }
if (errno) { if (errno) {
if (errno == ESRCH) if (errno == ESRCH) continue;
continue;
throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
} }
fdDir.reset(); fdDir.reset();
try { try {
auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); auto mapLines = tokenizeString<std::vector<string>>(
readFile(mapFile, true), "\n");
for (const auto& line : mapLines) { for (const auto& line : mapLines) {
auto match = std::smatch{}; auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex)) if (std::regex_match(line, match, mapRegex))
@ -426,28 +391,28 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
auto envFile = fmt("/proc/%s/environ", ent->d_name); auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile, true); auto envString = readFile(envFile, true);
auto env_end = std::sregex_iterator{}; auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) for (auto i = std::sregex_iterator{envString.begin(), envString.end(),
storePathRegex};
i != env_end; ++i)
unchecked[i->str()].emplace(envFile); unchecked[i->str()].emplace(envFile);
} catch (SysError& e) { } catch (SysError& e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue;
continue;
throw; throw;
} }
} }
} }
if (errno) if (errno) throw SysError("iterating /proc");
throw SysError("iterating /proc");
} }
#if !defined(__linux__) #if !defined(__linux__)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test
// See: https://github.com/NixOS/nix/issues/3011 // to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we
// Because of this we disable lsof when running the tests. // disable lsof when running the tests.
if (getEnv("_NIX_TEST_NO_LSOF") == "") { if (getEnv("_NIX_TEST_NO_LSOF") == "") {
try { try {
std::regex lsofRegex(R"(^n(/.*)$)"); std::regex lsofRegex(R"(^n(/.*)$)");
auto lsofLines = auto lsofLines = tokenizeString<std::vector<string>>(
tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n");
for (const auto& line : lsofLines) { for (const auto& line : lsofLines) {
std::smatch match; std::smatch match;
if (std::regex_match(line, match, lsofRegex)) if (std::regex_match(line, match, lsofRegex))
@ -479,12 +444,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
} }
} }
struct GCLimitReached {}; struct GCLimitReached {};
struct LocalStore::GCState {
struct LocalStore::GCState
{
GCOptions options; GCOptions options;
GCResults& results; GCResults& results;
PathSet roots; PathSet roots;
@ -499,25 +461,20 @@ struct LocalStore::GCState
GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {} GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {}
}; };
bool LocalStore::isActiveTempFile(const GCState& state, const Path& path,
bool LocalStore::isActiveTempFile(const GCState & state, const string& suffix) {
const Path & path, const string & suffix) return hasSuffix(path, suffix) &&
{ state.tempRoots.find(string(path, 0, path.size() - suffix.size())) !=
return hasSuffix(path, suffix) state.tempRoots.end();
&& state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
} }
void LocalStore::deleteGarbage(GCState& state, const Path& path) {
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
unsigned long long bytesFreed; unsigned long long bytesFreed;
deletePath(path, bytesFreed); deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed; state.results.bytesFreed += bytesFreed;
} }
void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt(); checkInterrupt();
unsigned long long size = 0; unsigned long long size = 0;
@ -557,26 +514,29 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
throw SysError(format("making '%1%' writable") % realPath); throw SysError(format("making '%1%' writable") % realPath);
Path tmp = trashDir + "/" + baseNameOf(path); Path tmp = trashDir + "/" + baseNameOf(path);
if (rename(realPath.c_str(), tmp.c_str())) if (rename(realPath.c_str(), tmp.c_str()))
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); throw SysError(format("unable to rename '%1%' to '%2%'") % realPath %
tmp);
state.bytesInvalidated += size; state.bytesInvalidated += size;
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOSPC) { if (e.errNo == ENOSPC) {
printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); printInfo(format("note: can't create move '%1%': %2%") % realPath %
e.msg());
deleteGarbage(state, realPath); deleteGarbage(state, realPath);
} }
} }
} else } else
deleteGarbage(state, realPath); deleteGarbage(state, realPath);
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { if (state.results.bytesFreed + state.bytesInvalidated >
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") %
state.options.maxFreed);
throw GCLimitReached(); throw GCLimitReached();
} }
} }
bool LocalStore::canReachRoot(GCState& state, PathSet& visited,
bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) const Path& path) {
{
if (visited.count(path)) return false; if (visited.count(path)) return false;
if (state.alive.count(path)) return true; if (state.alive.count(path)) return true;
@ -611,8 +571,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
are derivers of this path that are not garbage. */ are derivers of this path that are not garbage. */
if (state.gcKeepOutputs) { if (state.gcKeepOutputs) {
PathSet derivers = queryValidDerivers(path); PathSet derivers = queryValidDerivers(path);
for (auto & i : derivers) for (auto& i : derivers) incoming.insert(i);
incoming.insert(i);
} }
for (auto& i : incoming) for (auto& i : incoming)
@ -625,15 +584,14 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
return false; return false;
} }
void LocalStore::tryToDelete(GCState& state, const Path& path) {
void LocalStore::tryToDelete(GCState & state, const Path & path)
{
checkInterrupt(); checkInterrupt();
auto realPath = realStoreDir + "/" + baseNameOf(path); auto realPath = realStoreDir + "/" + baseNameOf(path);
if (realPath == linksDir || realPath == trashDir) return; if (realPath == linksDir || realPath == trashDir) return;
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); // Activity act(*logger, lvlDebug, format("considering whether to delete
// '%1%'") % path);
if (!isStorePath(path) || !isValidPath(path)) { if (!isStorePath(path) || !isValidPath(path)) {
/* A lock file belonging to a path that we're building right /* A lock file belonging to a path that we're building right
@ -660,19 +618,16 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
nix-store --delete doesn't have the unexpected effect of nix-store --delete doesn't have the unexpected effect of
recursing into derivations and outputs. */ recursing into derivations and outputs. */
state.dead.insert(visited.begin(), visited.end()); state.dead.insert(visited.begin(), visited.end());
if (state.shouldDelete) if (state.shouldDelete) deletePathRecursive(state, path);
deletePathRecursive(state, path);
} }
} }
/* Unlink all files in /nix/store/.links that have a link count of 1, /* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases might see a link count of 1 just before optimisePath() increases
the link count. */ the link count. */
void LocalStore::removeUnusedLinks(const GCState & state) void LocalStore::removeUnusedLinks(const GCState& state) {
{
AutoCloseDir dir(opendir(linksDir.c_str())); AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
@ -708,13 +663,11 @@ void LocalStore::removeUnusedLinks(const GCState & state)
throw SysError(format("statting '%1%'") % linksDir); throw SysError(format("statting '%1%'") % linksDir);
long long overhead = st.st_blocks * 512ULL; long long overhead = st.st_blocks * 512ULL;
printInfo(format("note: currently hard linking saves %.2f MiB") printInfo(format("note: currently hard linking saves %.2f MiB") %
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
} }
void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
GCState state(results); GCState state(results);
state.options = options; state.options = options;
state.gcKeepOutputs = settings.gcKeepOutputs; state.gcKeepOutputs = settings.gcKeepOutputs;
@ -729,10 +682,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
state.gcKeepDerivations = false; state.gcKeepDerivations = false;
} }
state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; state.shouldDelete = options.action == GCOptions::gcDeleteDead ||
options.action == GCOptions::gcDeleteSpecific;
if (state.shouldDelete) if (state.shouldDelete) deletePath(reservedPath);
deletePath(reservedPath);
/* Acquire the global GC root. This prevents /* Acquire the global GC root. This prevents
a) New roots from being added. a) New roots from being added.
@ -743,8 +696,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
permanent roots cannot increase now. */ permanent roots cannot increase now. */
printError(format("finding garbage collector roots...")); printError(format("finding garbage collector roots..."));
Roots rootMap; Roots rootMap;
if (!options.ignoreLiveness) if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true);
findRootsNoTemp(rootMap, true);
for (auto& i : rootMap) state.roots.insert(i.first); for (auto& i : rootMap) state.roots.insert(i.first);
@ -754,8 +706,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
FDs fds; FDs fds;
Roots tempRoots; Roots tempRoots;
findTempRoots(fds, tempRoots, true); findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots) for (auto& root : tempRoots) state.tempRoots.insert(root.first);
state.tempRoots.insert(root.first);
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
/* After this point the set of roots or temporary roots cannot /* After this point the set of roots or temporary roots cannot
@ -778,25 +729,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
paths (for gcDeleteSpecific). */ paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific) { if (options.action == GCOptions::gcDeleteSpecific) {
for (auto& i : options.pathsToDelete) { for (auto& i : options.pathsToDelete) {
assertStorePath(i); assertStorePath(i);
tryToDelete(state, i); tryToDelete(state, i);
if (state.dead.find(i) == state.dead.end()) if (state.dead.find(i) == state.dead.end())
throw Error(format("cannot delete path '%1%' since it is still alive") % i); throw Error(format("cannot delete path '%1%' since it is still alive") %
i);
} }
} else if (options.maxFreed > 0) { } else if (options.maxFreed > 0) {
if (state.shouldDelete) if (state.shouldDelete)
printError(format("deleting garbage...")); printError(format("deleting garbage..."));
else else
printError(format("determining live/dead paths...")); printError(format("determining live/dead paths..."));
try { try {
AutoCloseDir dir(opendir(realStoreDir.c_str())); AutoCloseDir dir(opendir(realStoreDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); if (!dir)
throw SysError(format("opening directory '%1%'") % realStoreDir);
/* Read the store and immediately delete all paths that /* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting aren't valid. When using --max-freed etc., deleting
@ -828,8 +778,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
std::mt19937 gen(1); std::mt19937 gen(1);
std::shuffle(entries_.begin(), entries_.end(), gen); std::shuffle(entries_.begin(), entries_.end(), gen);
for (auto & i : entries_) for (auto& i : entries_) tryToDelete(state, i);
tryToDelete(state, i);
} catch (GCLimitReached& e) { } catch (GCLimitReached& e) {
} }
@ -854,7 +803,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
deleteGarbage(state, trashDir); deleteGarbage(state, trashDir);
/* Clean up the links directory. */ /* Clean up the links directory. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { if (options.action == GCOptions::gcDeleteDead ||
options.action == GCOptions::gcDeleteSpecific) {
printError(format("deleting unused links...")); printError(format("deleting unused links..."));
removeUnusedLinks(state); removeUnusedLinks(state);
} }
@ -863,9 +813,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
// if (options.action == GCOptions::gcDeleteDead) vacuumDB(); // if (options.action == GCOptions::gcDeleteDead) vacuumDB();
} }
void LocalStore::autoGC(bool sync) {
void LocalStore::autoGC(bool sync)
{
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
auto getAvail = [this]() -> uint64_t { auto getAvail = [this]() -> uint64_t {
@ -892,7 +840,9 @@ void LocalStore::autoGC(bool sync)
auto now = std::chrono::steady_clock::now(); auto now = std::chrono::steady_clock::now();
if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; if (now < state->lastGCCheck +
std::chrono::seconds(settings.minFreeCheckInterval))
return;
auto avail = getAvail(); auto avail = getAvail();
@ -908,9 +858,7 @@ void LocalStore::autoGC(bool sync)
future = state->gcFuture = promise.get_future().share(); future = state->gcFuture = promise.get_future().share();
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
try { try {
/* Wake up any threads waiting for the auto-GC to finish. */ /* Wake up any threads waiting for the auto-GC to finish. */
Finally wakeup([&]() { Finally wakeup([&]() {
auto state(_state.lock()); auto state(_state.lock());
@ -935,7 +883,6 @@ void LocalStore::autoGC(bool sync)
// future, but we don't really care. // future, but we don't really care.
ignoreException(); ignoreException();
} }
}).detach(); }).detach();
} }
@ -944,5 +891,4 @@ void LocalStore::autoGC(bool sync)
if (sync) future.get(); if (sync) future.get();
} }
} // namespace nix
}

View file

@ -1,17 +1,14 @@
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include <dlfcn.h>
#include "archive.hh"
#include "args.hh"
#include <algorithm> #include <algorithm>
#include <map> #include <map>
#include <thread> #include <thread>
#include <dlfcn.h> #include "archive.hh"
#include "args.hh"
#include "util.hh"
namespace nix { namespace nix {
/* The default location of the daemon socket, relative to nixStateDir. /* The default location of the daemon socket, relative to nixStateDir.
The socket is in a directory to allow you to control access to the The socket is in a directory to allow you to control access to the
Nix daemon by setting the mode/ownership of the directory Nix daemon by setting the mode/ownership of the directory
@ -31,23 +28,25 @@ Settings settings;
static GlobalConfig::Register r1(&settings); static GlobalConfig::Register r1(&settings);
Settings::Settings() Settings::Settings()
: nixPrefix(NIX_PREFIX) : nixPrefix(NIX_PREFIX),
, nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) nixStore(canonPath(
, nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))),
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))),
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))),
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))),
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))),
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))),
, nixManDir(canonPath(NIX_MAN_DIR)) nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))),
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) nixManDir(canonPath(NIX_MAN_DIR)),
{ nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) {
buildUsersGroup = getuid() == 0 ? "nixbld" : ""; buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
if (caFile == "") { if (caFile == "") {
for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) for (auto& fn :
{"/etc/ssl/certs/ca-certificates.crt",
"/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
if (pathExists(fn)) { if (pathExists(fn)) {
caFile = fn; caFile = fn;
break; break;
@ -58,8 +57,7 @@ Settings::Settings()
auto s = getEnv("NIX_REMOTE_SYSTEMS"); auto s = getEnv("NIX_REMOTE_SYSTEMS");
if (s != "") { if (s != "") {
Strings ss; Strings ss;
for (auto & p : tokenizeString<Strings>(s, ":")) for (auto& p : tokenizeString<Strings>(s, ":")) ss.push_back("@" + p);
ss.push_back("@" + p);
builders = concatStringsSep(" ", ss); builders = concatStringsSep(" ", ss);
} }
@ -67,11 +65,11 @@ Settings::Settings()
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
#endif #endif
allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); allowedImpureHostPrefixes =
tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
} }
void loadConfFile() void loadConfFile() {
{
globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
/* We only want to send overrides to the daemon, i.e. stuff from /* We only want to send overrides to the daemon, i.e. stuff from
@ -79,27 +77,25 @@ void loadConfFile()
globalConfig.resetOverriden(); globalConfig.resetOverriden();
auto dirs = getConfigDirs(); auto dirs = getConfigDirs();
// Iterate over them in reverse so that the ones appearing first in the path take priority // Iterate over them in reverse so that the ones appearing first in the path
// take priority
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
} }
} }
unsigned int Settings::getDefaultCores() unsigned int Settings::getDefaultCores() {
{
return std::max(1U, std::thread::hardware_concurrency()); return std::max(1U, std::thread::hardware_concurrency());
} }
StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultSystemFeatures() {
{
/* For backwards compatibility, accept some "features" that are /* For backwards compatibility, accept some "features" that are
used in Nixpkgs to route builds to certain machines but don't used in Nixpkgs to route builds to certain machines but don't
actually require anything special on the machines. */ actually require anything special on the machines. */
StringSet features{"nixos-test", "benchmark", "big-parallel"}; StringSet features{"nixos-test", "benchmark", "big-parallel"};
#if __linux__ #if __linux__
if (access("/dev/kvm", R_OK | W_OK) == 0) if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm");
features.insert("kvm");
#endif #endif
return features; return features;
@ -107,29 +103,38 @@ StringSet Settings::getDefaultSystemFeatures()
const string nixVersion = PACKAGE_VERSION; const string nixVersion = PACKAGE_VERSION;
template<> void BaseSetting<SandboxMode>::set(const std::string & str) template <>
{ void BaseSetting<SandboxMode>::set(const std::string& str) {
if (str == "true") value = smEnabled; if (str == "true")
else if (str == "relaxed") value = smRelaxed; value = smEnabled;
else if (str == "false") value = smDisabled; else if (str == "relaxed")
else throw UsageError("option '%s' has invalid value '%s'", name, str); value = smRelaxed;
else if (str == "false")
value = smDisabled;
else
throw UsageError("option '%s' has invalid value '%s'", name, str);
} }
template<> std::string BaseSetting<SandboxMode>::to_string() template <>
{ std::string BaseSetting<SandboxMode>::to_string() {
if (value == smEnabled) return "true"; if (value == smEnabled)
else if (value == smRelaxed) return "relaxed"; return "true";
else if (value == smDisabled) return "false"; else if (value == smRelaxed)
else abort(); return "relaxed";
else if (value == smDisabled)
return "false";
else
abort();
} }
template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) template <>
{ void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) {
AbstractSetting::toJSON(out); AbstractSetting::toJSON(out);
} }
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) template <>
{ void BaseSetting<SandboxMode>::convertToArg(Args& args,
const std::string& category) {
args.mkFlag() args.mkFlag()
.longName(name) .longName(name)
.description("Enable sandboxing.") .description("Enable sandboxing.")
@ -147,16 +152,15 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
.category(category); .category(category);
} }
void MaxBuildJobsSetting::set(const std::string & str) void MaxBuildJobsSetting::set(const std::string& str) {
{ if (str == "auto")
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); value = std::max(1U, std::thread::hardware_concurrency());
else if (!string2Int(str, value)) else if (!string2Int(str, value))
throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); throw UsageError(
"configuration setting '%s' should be 'auto' or an integer", name);
} }
void initPlugins() {
void initPlugins()
{
for (const auto& pluginFile : settings.pluginFiles.get()) { for (const auto& pluginFile : settings.pluginFiles.get()) {
Paths pluginFiles; Paths pluginFiles;
try { try {
@ -164,17 +168,16 @@ void initPlugins()
for (const auto& ent : ents) for (const auto& ent : ents)
pluginFiles.emplace_back(pluginFile + "/" + ent.name); pluginFiles.emplace_back(pluginFile + "/" + ent.name);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo != ENOTDIR) if (e.errNo != ENOTDIR) throw;
throw;
pluginFiles.emplace_back(pluginFile); pluginFiles.emplace_back(pluginFile);
} }
for (const auto& file : pluginFiles) { for (const auto& file : pluginFiles) {
/* handle is purposefully leaked as there may be state in the /* handle is purposefully leaked as there may be state in the
DSO needed by the action of the plugin. */ DSO needed by the action of the plugin. */
void *handle = void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle) if (!handle)
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); throw Error("could not dynamically open plugin file '%s': %s", file,
dlerror());
} }
} }
@ -184,4 +187,4 @@ void initPlugins()
globalConfig.warnUnknownSettings(); globalConfig.warnUnknownSettings();
} }
} } // namespace nix

View file

@ -1,27 +1,21 @@
#pragma once #pragma once
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include <map>
#include <limits>
#include <sys/types.h> #include <sys/types.h>
#include <limits>
#include <map>
#include "config.hh"
#include "types.hh"
#include "util.hh"
namespace nix { namespace nix {
typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
struct MaxBuildJobsSetting : public BaseSetting<unsigned int> struct MaxBuildJobsSetting : public BaseSetting<unsigned int> {
{ MaxBuildJobsSetting(Config* options, unsigned int def,
MaxBuildJobsSetting(Config * options, const std::string& name, const std::string& description,
unsigned int def,
const std::string & name,
const std::string & description,
const std::set<std::string>& aliases = {}) const std::set<std::string>& aliases = {})
: BaseSetting<unsigned int>(def, name, description, aliases) : BaseSetting<unsigned int>(def, name, description, aliases) {
{
options->addSetting(this); options->addSetting(this);
} }
@ -29,13 +23,11 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
}; };
class Settings : public Config { class Settings : public Config {
unsigned int getDefaultCores(); unsigned int getDefaultCores();
StringSet getDefaultSystemFeatures(); StringSet getDefaultSystemFeatures();
public: public:
Settings(); Settings();
Path nixPrefix; Path nixPrefix;
@ -69,32 +61,45 @@ public:
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
"The default Nix store to use."}; "The default Nix store to use."};
Setting<bool> keepFailed{this, false, "keep-failed", Setting<bool> keepFailed{
this, false, "keep-failed",
"Whether to keep temporary directories of failed builds."}; "Whether to keep temporary directories of failed builds."};
Setting<bool> keepGoing{this, false, "keep-going", Setting<bool> keepGoing{
this, false, "keep-going",
"Whether to keep building derivations when another build fails."}; "Whether to keep building derivations when another build fails."};
Setting<bool> tryFallback{this, false, "fallback", Setting<bool> tryFallback{
this,
false,
"fallback",
"Whether to fall back to building when substitution fails.", "Whether to fall back to building when substitution fails.",
{"build-fallback"}}; {"build-fallback"}};
/* Whether to show build log output in real time. */ /* Whether to show build log output in real time. */
bool verboseBuild = true; bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines", Setting<size_t> logLines{
this, 10, "log-lines",
"If verbose-build is false, the number of lines of the tail of " "If verbose-build is false, the number of lines of the tail of "
"the log to show if a build fails."}; "the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", MaxBuildJobsSetting maxBuildJobs{this,
"Maximum number of parallel build jobs. \"auto\" means use number of cores.", 1,
"max-jobs",
"Maximum number of parallel build jobs. "
"\"auto\" means use number of cores.",
{"build-max-jobs"}}; {"build-max-jobs"}};
Setting<unsigned int> buildCores{this, getDefaultCores(), "cores", Setting<unsigned int> buildCores{
this,
getDefaultCores(),
"cores",
"Number of CPU cores to utilize in parallel within a build, " "Number of CPU cores to utilize in parallel within a build, "
"i.e. by passing this number to Make via '-j'. 0 means that the " "i.e. by passing this number to Make via '-j'. 0 means that the "
"number of actual CPU cores on the local host ought to be " "number of actual CPU cores on the local host ought to be "
"auto-detected.", {"build-cores"}}; "auto-detected.",
{"build-cores"}};
/* Read-only mode. Don't copy stuff to the store, don't change /* Read-only mode. Don't copy stuff to the store, don't change
the database. */ the database. */
@ -103,28 +108,41 @@ public:
Setting<std::string> thisSystem{this, SYSTEM, "system", Setting<std::string> thisSystem{this, SYSTEM, "system",
"The canonical Nix system name."}; "The canonical Nix system name."};
Setting<time_t> maxSilentTime{this, 0, "max-silent-time", Setting<time_t> maxSilentTime{
this,
0,
"max-silent-time",
"The maximum time in seconds that a builer can go without " "The maximum time in seconds that a builer can go without "
"producing any output on stdout/stderr before it is killed. " "producing any output on stdout/stderr before it is killed. "
"0 means infinity.", "0 means infinity.",
{"build-max-silent-time"}}; {"build-max-silent-time"}};
Setting<time_t> buildTimeout{this, 0, "timeout", Setting<time_t> buildTimeout{
this,
0,
"timeout",
"The maximum duration in seconds that a builder can run. " "The maximum duration in seconds that a builder can run. "
"0 means infinity.", {"build-timeout"}}; "0 means infinity.",
{"build-timeout"}};
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote",
"The path of the helper program that executes builds to remote machines."}; "build-hook",
"The path of the helper program that executes builds "
"to remote machines."};
Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", Setting<std::string> builders{this, "@" + nixConfDir + "/machines",
"A semicolon-separated list of build machines, in the format of nix.machines."}; "builders",
"A semicolon-separated list of build machines, "
"in the format of nix.machines."};
Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", Setting<bool> buildersUseSubstitutes{
this, false, "builders-use-substitutes",
"Whether build machines should use their own substitutes for obtaining " "Whether build machines should use their own substitutes for obtaining "
"build dependencies if possible, rather than waiting for this host to " "build dependencies if possible, rather than waiting for this host to "
"upload them."}; "upload them."};
Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", Setting<off_t> reservedSize{
this, 8 * 1024 * 1024, "gc-reserved-space",
"Amount of reserved disk space for the garbage collector."}; "Amount of reserved disk space for the garbage collector."};
Setting<bool> fsyncMetadata{this, true, "fsync-metadata", Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
@ -133,29 +151,43 @@ public:
Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
"Whether SQLite should use WAL mode."}; "Whether SQLite should use WAL mode."};
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", Setting<bool> syncBeforeRegistering{
this, false, "sync-before-registering",
"Whether to call sync() before registering a path as valid."}; "Whether to call sync() before registering a path as valid."};
Setting<bool> useSubstitutes{this, true, "substitute", Setting<bool> useSubstitutes{this,
true,
"substitute",
"Whether to use substitutes.", "Whether to use substitutes.",
{"build-use-substitutes"}}; {"build-use-substitutes"}};
Setting<std::string> buildUsersGroup{this, "", "build-users-group", Setting<std::string> buildUsersGroup{
this, "", "build-users-group",
"The Unix group that contains the build users."}; "The Unix group that contains the build users."};
Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26", Setting<bool> impersonateLinux26{
this,
false,
"impersonate-linux-26",
"Whether to impersonate a Linux 2.6 machine on newer kernels.", "Whether to impersonate a Linux 2.6 machine on newer kernels.",
{"build-impersonate-linux-26"}}; {"build-impersonate-linux-26"}};
Setting<bool> keepLog{this, true, "keep-build-log", Setting<bool> keepLog{this,
true,
"keep-build-log",
"Whether to store build logs.", "Whether to store build logs.",
{"build-keep-log"}}; {"build-keep-log"}};
Setting<bool> compressLog{this, true, "compress-build-log", Setting<bool> compressLog{this,
true,
"compress-build-log",
"Whether to compress logs.", "Whether to compress logs.",
{"build-compress-log"}}; {"build-compress-log"}};
Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size", Setting<unsigned long> maxLogSize{
this,
0,
"max-build-log-size",
"Maximum number of bytes a builder can write to stdout/stderr " "Maximum number of bytes a builder can write to stdout/stderr "
"before being killed (0 means no limit).", "before being killed (0 means no limit).",
{"build-max-log-size"}}; {"build-max-log-size"}};
@ -165,25 +197,37 @@ public:
stderr. Hack to prevent Hydra logs from being polluted. */ stderr. Hack to prevent Hydra logs from being polluted. */
bool printRepeatedBuilds = true; bool printRepeatedBuilds = true;
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", Setting<unsigned int> pollInterval{
this, 5, "build-poll-interval",
"How often (in seconds) to poll for locks."}; "How often (in seconds) to poll for locks."};
Setting<bool> checkRootReachability{this, false, "gc-check-reachability", Setting<bool> checkRootReachability{
this, false, "gc-check-reachability",
"Whether to check if new GC roots can in fact be found by the " "Whether to check if new GC roots can in fact be found by the "
"garbage collector."}; "garbage collector."};
Setting<bool> gcKeepOutputs{this, false, "keep-outputs", Setting<bool> gcKeepOutputs{
this,
false,
"keep-outputs",
"Whether the garbage collector should keep outputs of live derivations.", "Whether the garbage collector should keep outputs of live derivations.",
{"gc-keep-outputs"}}; {"gc-keep-outputs"}};
Setting<bool> gcKeepDerivations{this, true, "keep-derivations", Setting<bool> gcKeepDerivations{
this,
true,
"keep-derivations",
"Whether the garbage collector should keep derivers of live paths.", "Whether the garbage collector should keep derivers of live paths.",
{"gc-keep-derivations"}}; {"gc-keep-derivations"}};
Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
"Whether to automatically replace files with identical contents with hard links."}; "Whether to automatically replace files with "
"identical contents with hard links."};
Setting<bool> envKeepDerivations{this, false, "keep-env-derivations", Setting<bool> envKeepDerivations{
this,
false,
"keep-env-derivations",
"Whether to add derivations as a dependency of user environments " "Whether to add derivations as a dependency of user environments "
"(to prevent them from being GCed).", "(to prevent them from being GCed).",
{"env-keep-derivations"}}; {"env-keep-derivations"}};
@ -192,170 +236,240 @@ public:
bool lockCPU; bool lockCPU;
/* Whether to show a stack trace if Nix evaluation fails. */ /* Whether to show a stack trace if Nix evaluation fails. */
Setting<bool> showTrace{this, false, "show-trace", Setting<bool> showTrace{
this, false, "show-trace",
"Whether to show a stack trace on evaluation errors."}; "Whether to show a stack trace on evaluation errors."};
Setting<SandboxMode> sandboxMode{this, Setting<SandboxMode> sandboxMode {
this,
#if __linux__ #if __linux__
smEnabled smEnabled
#else #else
smDisabled smDisabled
#endif #endif
, "sandbox", ,
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", "sandbox",
{"build-use-chroot", "build-use-sandbox"}}; "Whether to enable sandboxed builds. Can be \"true\", \"false\" or "
"\"relaxed\".",
{
"build-use-chroot", "build-use-sandbox"
}
};
Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths", Setting<PathSet> sandboxPaths{
this,
{},
"sandbox-paths",
"The paths to make available inside the build sandbox.", "The paths to make available inside the build sandbox.",
{"build-chroot-dirs", "build-sandbox-paths"}}; {"build-chroot-dirs", "build-sandbox-paths"}};
Setting<bool> sandboxFallback{this, true, "sandbox-fallback", Setting<bool> sandboxFallback{
this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."}; "Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", Setting<PathSet> extraSandboxPaths{
this,
{},
"extra-sandbox-paths",
"Additional paths to make available inside the build sandbox.", "Additional paths to make available inside the build sandbox.",
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
Setting<size_t> buildRepeat{this, 0, "repeat", Setting<size_t> buildRepeat{
this,
0,
"repeat",
"The number of times to repeat a build in order to verify determinism.", "The number of times to repeat a build in order to verify determinism.",
{"build-repeat"}}; {"build-repeat"}};
#if __linux__ #if __linux__
Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", Setting<std::string> sandboxShmSize{
this, "50%", "sandbox-dev-shm-size",
"The size of /dev/shm in the build sandbox."}; "The size of /dev/shm in the build sandbox."};
Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
"The build directory inside the sandbox."}; "The build directory inside the sandbox."};
#endif #endif
Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", Setting<PathSet> allowedImpureHostPrefixes{
"Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; this,
{},
"allowed-impure-host-deps",
"Which prefixes to allow derivations to ask for access to (primarily for "
"Darwin)."};
#if __APPLE__ #if __APPLE__
Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", Setting<bool> darwinLogSandboxViolations{
this, false, "darwin-log-sandbox-violations",
"Whether to log Darwin sandbox access violations to the system log."}; "Whether to log Darwin sandbox access violations to the system log."};
#endif #endif
Setting<bool> runDiffHook{this, false, "run-diff-hook", Setting<bool> runDiffHook{
this, false, "run-diff-hook",
"Whether to run the program specified by the diff-hook setting " "Whether to run the program specified by the diff-hook setting "
"repeated builds produce a different result. Typically used to " "repeated builds produce a different result. Typically used to "
"plug in diffoscope."}; "plug in diffoscope."};
PathSetting diffHook{this, true, "", "diff-hook", PathSetting diffHook{
this, true, "", "diff-hook",
"A program that prints out the differences between the two paths " "A program that prints out the differences between the two paths "
"specified on its command line."}; "specified on its command line."};
Setting<bool> enforceDeterminism{this, true, "enforce-determinism", Setting<bool> enforceDeterminism{
this, true, "enforce-determinism",
"Whether to fail if repeated builds produce different output."}; "Whether to fail if repeated builds produce different output."};
Setting<Strings> trustedPublicKeys{this, Setting<Strings> trustedPublicKeys{
this,
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
"trusted-public-keys", "trusted-public-keys",
"Trusted public keys for secure substitution.", "Trusted public keys for secure substitution.",
{"binary-cache-public-keys"}}; {"binary-cache-public-keys"}};
Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", Setting<Strings> secretKeyFiles{
this,
{},
"secret-key-files",
"Secret keys with which to sign local builds."}; "Secret keys with which to sign local builds."};
Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", Setting<unsigned int> tarballTtl{
this, 60 * 60, "tarball-ttl",
"How long downloaded files are considered up-to-date."}; "How long downloaded files are considered up-to-date."};
Setting<bool> requireSigs{this, true, "require-sigs", Setting<bool> requireSigs{
this, true, "require-sigs",
"Whether to check that any non-content-addressed path added to the " "Whether to check that any non-content-addressed path added to the "
"Nix store has a valid signature (that is, one signed using a key " "Nix store has a valid signature (that is, one signed using a key "
"listed in 'trusted-public-keys'."}; "listed in 'trusted-public-keys'."};
Setting<StringSet> extraPlatforms{this, Setting<StringSet> extraPlatforms{
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, this,
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"}
: StringSet{},
"extra-platforms", "extra-platforms",
"Additional platforms that can be built on the local system. " "Additional platforms that can be built on the local system. "
"These may be supported natively (e.g. armv7 on some aarch64 CPUs " "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
"or using hacks like qemu-user."}; "or using hacks like qemu-user."};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), Setting<StringSet> systemFeatures{
"system-features", this, getDefaultSystemFeatures(), "system-features",
"Optional features that this system implements (like \"kvm\")."}; "Optional features that this system implements (like \"kvm\")."};
Setting<Strings> substituters{this, Setting<Strings> substituters{
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), this,
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"}
: Strings(),
"substituters", "substituters",
"The URIs of substituters (such as https://cache.nixos.org/).", "The URIs of substituters (such as https://cache.nixos.org/).",
{"binary-caches"}}; {"binary-caches"}};
// FIXME: provide a way to add to option values. // FIXME: provide a way to add to option values.
Setting<Strings> extraSubstituters{this, {}, "extra-substituters", Setting<Strings> extraSubstituters{this,
{},
"extra-substituters",
"Additional URIs of substituters.", "Additional URIs of substituters.",
{"extra-binary-caches"}}; {"extra-binary-caches"}};
Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", Setting<StringSet> trustedSubstituters{
"Disabled substituters that may be enabled via the substituters option by untrusted users.", this,
{},
"trusted-substituters",
"Disabled substituters that may be enabled via the substituters option "
"by untrusted users.",
{"trusted-binary-caches"}}; {"trusted-binary-caches"}};
Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", Setting<Strings> trustedUsers{this,
"Which users or groups are trusted to ask the daemon to do unsafe things."}; {"root"},
"trusted-users",
"Which users or groups are trusted to ask the "
"daemon to do unsafe things."};
Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", Setting<unsigned int> ttlNegativeNarInfoCache{
"The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " this, 3600, "narinfo-cache-negative-ttl",
"The TTL in seconds for negative lookups in the disk cache i.e binary "
"cache lookups that "
"return an invalid path result"}; "return an invalid path result"};
Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", Setting<unsigned int> ttlPositiveNarInfoCache{
"The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
"The TTL in seconds for positive lookups in the disk cache i.e binary "
"cache lookups that "
"return a valid path result."}; "return a valid path result."};
/* ?Who we trust to use the daemon in safe ways */ /* ?Who we trust to use the daemon in safe ways */
Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", Setting<Strings> allowedUsers{
this,
{"*"},
"allowed-users",
"Which users or groups are allowed to connect to the daemon."}; "Which users or groups are allowed to connect to the daemon."};
Setting<bool> printMissing{this, true, "print-missing", Setting<bool> printMissing{
this, true, "print-missing",
"Whether to print what paths need to be built or downloaded."}; "Whether to print what paths need to be built or downloaded."};
Setting<std::string> preBuildHook{this, Setting<std::string> preBuildHook {
this,
#if __APPLE__ #if __APPLE__
nixLibexecDir + "/nix/resolve-system-dependencies", nixLibexecDir + "/nix/resolve-system-dependencies",
#else #else
"", "",
#endif #endif
"pre-build-hook", "pre-build-hook",
"A program to run just before a build to set derivation-specific build settings."}; "A program to run just before a build to set derivation-specific build "
"settings."
};
Setting<std::string> postBuildHook{this, "", "post-build-hook", Setting<std::string> postBuildHook{
this, "", "post-build-hook",
"A program to run just after each successful build."}; "A program to run just after each successful build."};
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"),
"Path to the netrc file used to obtain usernames/passwords for downloads."}; "netrc-file",
"Path to the netrc file used to obtain "
"usernames/passwords for downloads."};
/* Path to the SSL CA file used */ /* Path to the SSL CA file used */
Path caFile; Path caFile;
#if __linux__ #if __linux__
Setting<bool> filterSyscalls{this, true, "filter-syscalls", Setting<bool> filterSyscalls{
this, true, "filter-syscalls",
"Whether to prevent certain dangerous system calls, such as " "Whether to prevent certain dangerous system calls, such as "
"creation of setuid/setgid files or adding ACLs or extended " "creation of setuid/setgid files or adding ACLs or extended "
"attributes. Only disable this if you're aware of the " "attributes. Only disable this if you're aware of the "
"security implications."}; "security implications."};
Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges", Setting<bool> allowNewPrivileges{
this, false, "allow-new-privileges",
"Whether builders can acquire new privileges by calling programs with " "Whether builders can acquire new privileges by calling programs with "
"setuid/setgid bits or with file capabilities."}; "setuid/setgid bits or with file capabilities."};
#endif #endif
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", Setting<Strings> hashedMirrors{
this,
{"http://tarballs.nixos.org/"},
"hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."}; "A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free", Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."}; "Automatically run the garbage collector when free "
"disk space drops below the specified amount."};
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(),
"Stop deleting garbage when free disk space is above the specified amount."}; "max-free",
"Stop deleting garbage when free disk space is "
"above the specified amount."};
Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", Setting<uint64_t> minFreeCheckInterval{
this, 5, "min-free-check-interval",
"Number of seconds between checking free disk space."}; "Number of seconds between checking free disk space."};
Setting<Paths> pluginFiles{this, {}, "plugin-files", Setting<Paths> pluginFiles{
this,
{},
"plugin-files",
"Plugins to dynamically load at nix initialization time."}; "Plugins to dynamically load at nix initialization time."};
}; };
// FIXME: don't use a global variable. // FIXME: don't use a global variable.
extern Settings settings; extern Settings settings;
@ -367,4 +481,4 @@ void loadConfFile();
extern const string nixVersion; extern const string nixVersion;
} } // namespace nix

View file

@ -7,14 +7,11 @@ namespace nix {
MakeError(UploadToHTTP, Error); MakeError(UploadToHTTP, Error);
class HttpBinaryCacheStore : public BinaryCacheStore class HttpBinaryCacheStore : public BinaryCacheStore {
{
private: private:
Path cacheUri; Path cacheUri;
struct State struct State {
{
bool enabled = true; bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil; std::chrono::steady_clock::time_point disabledUntil;
}; };
@ -22,25 +19,16 @@ private:
Sync<State> _state; Sync<State> _state;
public: public:
HttpBinaryCacheStore(const Params& params, const Path& _cacheUri)
HttpBinaryCacheStore( : BinaryCacheStore(params), cacheUri(_cacheUri) {
const Params & params, const Path & _cacheUri) if (cacheUri.back() == '/') cacheUri.pop_back();
: BinaryCacheStore(params)
, cacheUri(_cacheUri)
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
diskCache = getNarInfoDiskCache(); diskCache = getNarInfoDiskCache();
} }
std::string getUri() override std::string getUri() override { return cacheUri; }
{
return cacheUri;
}
void init() override void init() override {
{
// FIXME: do this lazily? // FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
try { try {
@ -53,20 +41,18 @@ public:
} }
protected: protected:
void maybeDisable() {
void maybeDisable()
{
auto state(_state.lock()); auto state(_state.lock());
if (state->enabled && settings.tryFallback) { if (state->enabled && settings.tryFallback) {
int t = 60; int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t); printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false; state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); state->disabledUntil =
std::chrono::steady_clock::now() + std::chrono::seconds(t);
} }
} }
void checkEnabled() void checkEnabled() {
{
auto state(_state.lock()); auto state(_state.lock());
if (state->enabled) return; if (state->enabled) return;
if (std::chrono::steady_clock::now() > state->disabledUntil) { if (std::chrono::steady_clock::now() > state->disabledUntil) {
@ -77,8 +63,7 @@ protected:
throw SubstituterDisabled("substituter '%s' is disabled", getUri()); throw SubstituterDisabled("substituter '%s' is disabled", getUri());
} }
bool fileExists(const std::string & path) override bool fileExists(const std::string& path) override {
{
checkEnabled(); checkEnabled();
try { try {
@ -96,55 +81,55 @@ protected:
} }
} }
void upsertFile(const std::string & path, void upsertFile(const std::string& path, const std::string& data,
const std::string & data, const std::string& mimeType) override {
const std::string & mimeType) override
{
auto req = DownloadRequest(cacheUri + "/" + path); auto req = DownloadRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType; req.mimeType = mimeType;
try { try {
getDownloader()->download(req); getDownloader()->download(req);
} catch (DownloadError& e) { } catch (DownloadError& e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s",
cacheUri, e.msg());
} }
} }
DownloadRequest makeRequest(const std::string & path) DownloadRequest makeRequest(const std::string& path) {
{
DownloadRequest request(cacheUri + "/" + path); DownloadRequest request(cacheUri + "/" + path);
return request; return request;
} }
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string& path, Sink& sink) override {
{
checkEnabled(); checkEnabled();
auto request(makeRequest(path)); auto request(makeRequest(path));
try { try {
getDownloader()->download(std::move(request), sink); getDownloader()->download(std::move(request), sink);
} catch (DownloadError& e) { } catch (DownloadError& e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable(); maybeDisable();
throw; throw;
} }
} }
void getFile(const std::string & path, void getFile(
Callback<std::shared_ptr<std::string>> callback) noexcept override const std::string& path,
{ Callback<std::shared_ptr<std::string>> callback) noexcept override {
checkEnabled(); checkEnabled();
auto request(makeRequest(path)); auto request(makeRequest(path));
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr =
std::make_shared<decltype(callback)>(std::move(callback));
getDownloader()->enqueueDownload(request, getDownloader()->enqueueDownload(
{[callbackPtr, this](std::future<DownloadResult> result) { request, {[callbackPtr, this](std::future<DownloadResult> result) {
try { try {
(*callbackPtr)(result.get().data); (*callbackPtr)(result.get().data);
} catch (DownloadError& e) { } catch (DownloadError& e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) if (e.error == Downloader::NotFound ||
e.error == Downloader::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>()); return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable(); maybeDisable();
callbackPtr->rethrow(); callbackPtr->rethrow();
@ -153,21 +138,19 @@ protected:
} }
}}); }});
} }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{
if (std::string(uri, 0, 7) != "http://" && if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" && std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" ||
) return 0; std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init(); store->init();
return store; return store;
}); });
} } // namespace nix

View file

@ -1,29 +1,34 @@
#include "archive.hh" #include "archive.hh"
#include "derivations.hh"
#include "pool.hh" #include "pool.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
#include "ssh.hh"
#include "store-api.hh" #include "store-api.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
namespace nix { namespace nix {
static std::string uriScheme = "ssh://"; static std::string uriScheme = "ssh://";
struct LegacySSHStore : public Store struct LegacySSHStore : public Store {
{ const Setting<int> maxConnections{
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; this, 1, "max-connections",
"maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; const Setting<bool> compress{this, false, "compress",
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; "whether to compress the connection"};
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; const Setting<Path> remoteProgram{
this, "nix-store", "remote-program",
"path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{
this, "", "remote-store", "URI of the store on the remote system"};
// Hack for getting remote build log output. // Hack for getting remote build log output.
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; const Setting<int> logFD{
this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection struct Connection {
{
std::unique_ptr<SSHMaster::Connection> sshConn; std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to; FdSink to;
FdSource from; FdSource from;
@ -38,29 +43,23 @@ struct LegacySSHStore : public Store
SSHMaster master; SSHMaster master;
LegacySSHStore(const string& host, const Params& params) LegacySSHStore(const string& host, const Params& params)
: Store(params) : Store(params),
, host(host) host(host),
, connections(make_ref<Pool<Connection>>( connections(make_ref<Pool<Connection>>(
std::max(1, (int)maxConnections), std::max(1, (int)maxConnections),
[this]() { return openConnection(); }, [this]() { return openConnection(); },
[](const ref<Connection> & r) { return r->good; } [](const ref<Connection>& r) { return r->good; })),
)) master(host, sshKey,
, master(
host,
sshKey,
// Use SSH master only if using more than 1 connection. // Use SSH master only if using more than 1 connection.
connections->capacity() > 1, connections->capacity() > 1, compress, logFD) {}
compress,
logFD)
{
}
ref<Connection> openConnection() ref<Connection> openConnection() {
{
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand( conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram) fmt("%s --serve --write", remoteProgram) +
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); (remoteStore.get() == ""
? ""
: " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get()); conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get()); conn->from = FdSource(conn->sshConn->out.get());
@ -73,7 +72,8 @@ struct LegacySSHStore : public Store
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
conn->remoteVersion = readInt(conn->from); conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); throw Error("unsupported 'nix-store --serve' protocol version on '%s'",
host);
} catch (EndOfFile& e) { } catch (EndOfFile& e) {
throw Error("cannot connect to '%1%'", host); throw Error("cannot connect to '%1%'", host);
@ -82,14 +82,11 @@ struct LegacySSHStore : public Store
return conn; return conn;
}; };
string getUri() override string getUri() override { return uriScheme + host; }
{
return uriScheme + host;
}
void queryPathInfoUncached(const Path & path, void queryPathInfoUncached(
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override const Path& path,
{ Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override {
try { try {
auto conn(connections->get()); auto conn(connections->get());
@ -120,30 +117,23 @@ struct LegacySSHStore : public Store
assert(s == ""); assert(s == "");
callback(std::move(info)); callback(std::move(info));
} catch (...) { callback.rethrow(); } } catch (...) {
callback.rethrow();
}
} }
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
RepairFlag repair, CheckSigsFlag checkSigs, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override std::shared_ptr<FSAccessor> accessor) override {
{
debug("adding path '%s' to remote host '%s'", info.path, host); debug("adding path '%s' to remote host '%s'", info.path, host);
auto conn(connections->get()); auto conn(connections->get());
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
conn->to << cmdAddToStoreNar << info.path << info.deriver
conn->to << info.narHash.to_string(Base16, false) << info.references
<< cmdAddToStoreNar << info.registrationTime << info.narSize << info.ultimate
<< info.path << info.sigs << info.ca;
<< info.deriver
<< info.narHash.to_string(Base16, false)
<< info.references
<< info.registrationTime
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
try { try {
copyNAR(source, conn->to); copyNAR(source, conn->to);
} catch (...) { } catch (...) {
@ -153,33 +143,24 @@ struct LegacySSHStore : public Store
conn->to.flush(); conn->to.flush();
} else { } else {
conn->to << cmdImportPaths << 1;
conn->to
<< cmdImportPaths
<< 1;
try { try {
copyNAR(source, conn->to); copyNAR(source, conn->to);
} catch (...) { } catch (...) {
conn->good = false; conn->good = false;
throw; throw;
} }
conn->to conn->to << exportMagic << info.path << info.references << info.deriver
<< exportMagic << 0 << 0;
<< info.path
<< info.references
<< info.deriver
<< 0
<< 0;
conn->to.flush(); conn->to.flush();
} }
if (readInt(conn->from) != 1) if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s', info.path, host"); throw Error(
"failed to add path '%s' to remote host '%s', info.path, host");
} }
void narFromPath(const Path & path, Sink & sink) override void narFromPath(const Path& path, Sink& sink) override {
{
auto conn(connections->get()); auto conn(connections->get());
conn->to << cmdDumpStorePath << path; conn->to << cmdDumpStorePath << path;
@ -187,36 +168,31 @@ struct LegacySSHStore : public Store
copyNAR(conn->from, sink); copyNAR(conn->from, sink);
} }
Path queryPathFromHashPart(const string & hashPart) override Path queryPathFromHashPart(const string& hashPart) override {
{ unsupported("queryPathFromHashPart"); } unsupported("queryPathFromHashPart");
}
Path addToStore(const string & name, const Path & srcPath, Path addToStore(const string& name, const Path& srcPath, bool recursive,
bool recursive, HashType hashAlgo, HashType hashAlgo, PathFilter& filter,
PathFilter & filter, RepairFlag repair) override RepairFlag repair) override {
{ unsupported("addToStore"); } unsupported("addToStore");
}
Path addTextToStore(const string& name, const string& s, Path addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair) override const PathSet& references, RepairFlag repair) override {
{ unsupported("addTextToStore"); } unsupported("addTextToStore");
}
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv, BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override BuildMode buildMode) override {
{
auto conn(connections->get()); auto conn(connections->get());
conn->to conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime
<< cmdBuildDerivation
<< drvPath
<< drv
<< settings.maxSilentTime
<< settings.buildTimeout; << settings.buildTimeout;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
conn->to conn->to << settings.maxLogSize;
<< settings.maxLogSize;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->to conn->to << settings.buildRepeat << settings.enforceDeterminism;
<< settings.buildRepeat
<< settings.enforceDeterminism;
conn->to.flush(); conn->to.flush();
@ -225,29 +201,26 @@ struct LegacySSHStore : public Store
conn->from >> status.errorMsg; conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; conn->from >> status.timesBuilt >> status.isNonDeterministic >>
status.startTime >> status.stopTime;
return status; return status;
} }
void ensurePath(const Path & path) override void ensurePath(const Path& path) override { unsupported("ensurePath"); }
{ unsupported("ensurePath"); }
void computeFSClosure(const PathSet & paths, void computeFSClosure(const PathSet& paths, PathSet& out,
PathSet & out, bool flipDirection = false, bool flipDirection = false, bool includeOutputs = false,
bool includeOutputs = false, bool includeDerivers = false) override bool includeDerivers = false) override {
{
if (flipDirection || includeDerivers) { if (flipDirection || includeDerivers) {
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); Store::computeFSClosure(paths, out, flipDirection, includeOutputs,
includeDerivers);
return; return;
} }
auto conn(connections->get()); auto conn(connections->get());
conn->to conn->to << cmdQueryClosure << includeOutputs << paths;
<< cmdQueryClosure
<< includeOutputs
<< paths;
conn->to.flush(); conn->to.flush();
auto res = readStorePaths<PathSet>(*this, conn->from); auto res = readStorePaths<PathSet>(*this, conn->from);
@ -255,39 +228,31 @@ struct LegacySSHStore : public Store
out.insert(res.begin(), res.end()); out.insert(res.begin(), res.end());
} }
PathSet queryValidPaths(const PathSet & paths, PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
SubstituteFlag maybeSubstitute = NoSubstitute) override NoSubstitute) override {
{
auto conn(connections->get()); auto conn(connections->get());
conn->to conn->to << cmdQueryValidPaths << false // lock
<< cmdQueryValidPaths << maybeSubstitute << paths;
<< false // lock
<< maybeSubstitute
<< paths;
conn->to.flush(); conn->to.flush();
return readStorePaths<PathSet>(*this, conn->from); return readStorePaths<PathSet>(*this, conn->from);
} }
void connect() override void connect() override { auto conn(connections->get()); }
{
auto conn(connections->get());
}
unsigned int getProtocol() override unsigned int getProtocol() override {
{
auto conn(connections->get()); auto conn(connections->get());
return conn->remoteVersion; return conn->remoteVersion;
} }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); return std::make_shared<LegacySSHStore>(
std::string(uri, uriScheme.size()), params);
}); });
} } // namespace nix

View file

@ -4,70 +4,54 @@
namespace nix { namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore class LocalBinaryCacheStore : public BinaryCacheStore {
{
private: private:
Path binaryCacheDir; Path binaryCacheDir;
public: public:
LocalBinaryCacheStore(const Params& params, const Path& binaryCacheDir)
LocalBinaryCacheStore( : BinaryCacheStore(params), binaryCacheDir(binaryCacheDir) {}
const Params & params, const Path & binaryCacheDir)
: BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir)
{
}
void init() override; void init() override;
std::string getUri() override std::string getUri() override { return "file://" + binaryCacheDir; }
{
return "file://" + binaryCacheDir;
}
protected: protected:
bool fileExists(const std::string& path) override; bool fileExists(const std::string& path) override;
void upsertFile(const std::string & path, void upsertFile(const std::string& path, const std::string& data,
const std::string & data,
const std::string& mimeType) override; const std::string& mimeType) override;
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string& path, Sink& sink) override {
{
try { try {
readFile(binaryCacheDir + "/" + path, sink); readFile(binaryCacheDir + "/" + path, sink);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOENT) if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache",
path);
} }
} }
PathSet queryAllValidPaths() override PathSet queryAllValidPaths() override {
{
PathSet paths; PathSet paths;
for (auto& entry : readDirectory(binaryCacheDir)) { for (auto& entry : readDirectory(binaryCacheDir)) {
if (entry.name.size() != 40 || if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo"))
!hasSuffix(entry.name, ".narinfo"))
continue; continue;
paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); paths.insert(storeDir + "/" +
entry.name.substr(0, entry.name.size() - 8));
} }
return paths; return paths;
} }
}; };
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init() {
{
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
BinaryCacheStore::init(); BinaryCacheStore::init();
} }
static void atomicWrite(const Path & path, const std::string & s) static void atomicWrite(const Path& path, const std::string& s) {
{
Path tmp = path + ".tmp." + std::to_string(getpid()); Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false); AutoDelete del(tmp, false);
writeFile(tmp, s); writeFile(tmp, s);
@ -76,28 +60,26 @@ static void atomicWrite(const Path & path, const std::string & s)
del.cancel(); del.cancel();
} }
bool LocalBinaryCacheStore::fileExists(const std::string & path) bool LocalBinaryCacheStore::fileExists(const std::string& path) {
{
return pathExists(binaryCacheDir + "/" + path); return pathExists(binaryCacheDir + "/" + path);
} }
void LocalBinaryCacheStore::upsertFile(const std::string& path, void LocalBinaryCacheStore::upsertFile(const std::string& path,
const std::string& data, const std::string& data,
const std::string & mimeType) const std::string& mimeType) {
{
atomicWrite(binaryCacheDir + "/" + path, data); atomicWrite(binaryCacheDir + "/" + path, data);
} }
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
std::string(uri, 0, 7) != "file://") std::string(uri, 0, 7) != "file://")
return 0; return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); auto store =
std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
store->init(); store->init();
return store; return store;
}); });
} } // namespace nix

View file

@ -1,100 +1,84 @@
#include "archive.hh" #include "archive.hh"
#include "fs-accessor.hh"
#include "store-api.hh"
#include "globals.hh"
#include "compression.hh" #include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "store-api.hh"
namespace nix { namespace nix {
LocalFSStore::LocalFSStore(const Params & params) LocalFSStore::LocalFSStore(const Params& params) : Store(params) {}
: Store(params)
{
}
struct LocalStoreAccessor : public FSAccessor struct LocalStoreAccessor : public FSAccessor {
{
ref<LocalFSStore> store; ref<LocalFSStore> store;
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {} LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {}
Path toRealPath(const Path & path) Path toRealPath(const Path& path) {
{
Path storePath = store->toStorePath(path); Path storePath = store->toStorePath(path);
if (!store->isValidPath(storePath)) if (!store->isValidPath(storePath))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); throw InvalidPath(format("path '%1%' is not a valid store path") %
storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size()); return store->getRealStoreDir() + std::string(path, store->storeDir.size());
} }
FSAccessor::Stat stat(const Path & path) override FSAccessor::Stat stat(const Path& path) override {
{
auto realPath = toRealPath(path); auto realPath = toRealPath(path);
struct stat st; struct stat st;
if (lstat(realPath.c_str(), &st)) { if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; if (errno == ENOENT || errno == ENOTDIR)
return {Type::tMissing, 0, false};
throw SysError(format("getting status of '%1%'") % path); throw SysError(format("getting status of '%1%'") % path);
} }
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error(format("file '%1%' has unsupported type") % path); throw Error(format("file '%1%' has unsupported type") % path);
return { return {S_ISREG(st.st_mode)
S_ISREG(st.st_mode) ? Type::tRegular : ? Type::tRegular
S_ISLNK(st.st_mode) ? Type::tSymlink : : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory,
Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0, S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
} }
StringSet readDirectory(const Path & path) override StringSet readDirectory(const Path& path) override {
{
auto realPath = toRealPath(path); auto realPath = toRealPath(path);
auto entries = nix::readDirectory(realPath); auto entries = nix::readDirectory(realPath);
StringSet res; StringSet res;
for (auto & entry : entries) for (auto& entry : entries) res.insert(entry.name);
res.insert(entry.name);
return res; return res;
} }
std::string readFile(const Path & path) override std::string readFile(const Path& path) override {
{
return nix::readFile(toRealPath(path)); return nix::readFile(toRealPath(path));
} }
std::string readLink(const Path & path) override std::string readLink(const Path& path) override {
{
return nix::readLink(toRealPath(path)); return nix::readLink(toRealPath(path));
} }
}; };
ref<FSAccessor> LocalFSStore::getFSAccessor() ref<FSAccessor> LocalFSStore::getFSAccessor() {
{
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
} }
void LocalFSStore::narFromPath(const Path & path, Sink & sink) void LocalFSStore::narFromPath(const Path& path, Sink& sink) {
{ if (!isValidPath(path)) throw Error(format("path '%s' is not valid") % path);
if (!isValidPath(path))
throw Error(format("path '%s' is not valid") % path);
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
} }
const string LocalFSStore::drvsLogDir = "drvs"; const string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) {
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
{
auto path(path_); auto path(path_);
assertStorePath(path); assertStorePath(path);
if (!isDerivation(path)) { if (!isDerivation(path)) {
try { try {
path = queryPathInfo(path)->deriver; path = queryPathInfo(path)->deriver;
@ -107,10 +91,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
string baseName = baseNameOf(path); string baseName = baseNameOf(path);
for (int j = 0; j < 2; j++) { for (int j = 0; j < 2; j++) {
Path logPath = j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir,
Path logPath = string(baseName, 0, 2), string(baseName, 2))
j == 0
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName); : fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2"; Path logBz2Path = logPath + ".bz2";
@ -120,12 +102,12 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
else if (pathExists(logBz2Path)) { else if (pathExists(logBz2Path)) {
try { try {
return decompress("bzip2", readFile(logBz2Path)); return decompress("bzip2", readFile(logBz2Path));
} catch (Error &) { } } catch (Error&) {
}
} }
} }
return nullptr; return nullptr;
} }
} } // namespace nix

File diff suppressed because it is too large Load diff

View file

@ -1,48 +1,37 @@
#pragma once #pragma once
#include "sqlite.hh"
#include "pathlocks.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include "pathlocks.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
namespace nix { namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <= /* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
const int nixSchemaVersion = 10; const int nixSchemaVersion = 10;
struct Derivation; struct Derivation;
struct OptimiseStats {
struct OptimiseStats
{
unsigned long filesLinked = 0; unsigned long filesLinked = 0;
unsigned long long bytesFreed = 0; unsigned long long bytesFreed = 0;
unsigned long long blocksFreed = 0; unsigned long long blocksFreed = 0;
}; };
class LocalStore : public LocalFSStore {
class LocalStore : public LocalFSStore
{
private: private:
/* Lock file used for upgrading. */ /* Lock file used for upgrading. */
AutoCloseFD globalLock; AutoCloseFD globalLock;
struct State struct State {
{
/* The SQLite database object. */ /* The SQLite database object. */
SQLite db; SQLite db;
@ -84,7 +73,6 @@ private:
Sync<State, std::recursive_mutex> _state; Sync<State, std::recursive_mutex> _state;
public: public:
PathSetting realStoreDir_; PathSetting realStoreDir_;
const Path realStoreDir; const Path realStoreDir;
@ -97,15 +85,13 @@ public:
const Path fnTempRoots; const Path fnTempRoots;
private: private:
Setting<bool> requireSigs{
Setting<bool> requireSigs{(Store*) this, (Store*)this, settings.requireSigs, "require-sigs",
settings.requireSigs, "whether store paths should have a trusted signature on import"};
"require-sigs", "whether store paths should have a trusted signature on import"};
const PublicKeys& getPublicKeys(); const PublicKeys& getPublicKeys();
public: public:
// Hack for build-remote.cc. // Hack for build-remote.cc.
PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS")); PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
@ -121,12 +107,13 @@ public:
bool isValidPathUncached(const Path& path) override; bool isValidPathUncached(const Path& path) override;
PathSet queryValidPaths(const PathSet & paths, PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
SubstituteFlag maybeSubstitute = NoSubstitute) override; NoSubstitute) override;
PathSet queryAllValidPaths() override; PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path, void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path& path, PathSet& referrers) override; void queryReferrers(const Path& path, PathSet& referrers) override;
@ -144,20 +131,21 @@ public:
void querySubstitutablePathInfos(const PathSet& paths, void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override; SubstitutablePathInfos& infos) override;
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
RepairFlag repair, CheckSigsFlag checkSigs, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath, Path addToStore(const string& name, const Path& srcPath, bool recursive,
bool recursive, HashType hashAlgo, HashType hashAlgo, PathFilter& filter,
PathFilter & filter, RepairFlag repair) override; RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained /* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive == in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive == true) or simply the contents of a regular file (if recursive ==
false). */ false). */
Path addToStoreFromDump(const string& dump, const string& name, Path addToStoreFromDump(const string& dump, const string& name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); bool recursive = true, HashType hashAlgo = htSHA256,
RepairFlag repair = NoRepair);
Path addTextToStore(const string& name, const string& s, Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override; const PathSet& references, RepairFlag repair) override;
@ -176,14 +164,12 @@ public:
void syncWithGC() override; void syncWithGC() override;
private: private:
typedef std::shared_ptr<AutoCloseFD> FDPtr; typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs; typedef list<FDPtr> FDs;
void findTempRoots(FDs& fds, Roots& roots, bool censor); void findTempRoots(FDs& fds, Roots& roots, bool censor);
public: public:
Roots findRoots(bool censor) override; Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions& options, GCResults& results) override; void collectGarbage(const GCOptions& options, GCResults& results) override;
@ -224,7 +210,6 @@ public:
void autoGC(bool sync = true); void autoGC(bool sync = true);
private: private:
int getSchema(); int getSchema();
void openDB(State& state, bool create); void openDB(State& state, bool create);
@ -233,15 +218,16 @@ private:
uint64_t queryValidPathId(State& state, const Path& path); uint64_t queryValidPathId(State& state, const Path& path);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); uint64_t addValidPath(State& state, const ValidPathInfo& info,
bool checkOutputs = true);
void invalidatePath(State& state, const Path& path); void invalidatePath(State& state, const Path& path);
/* Delete a path from the Nix store. */ /* Delete a path from the Nix store. */
void invalidatePathChecked(const Path& path); void invalidatePathChecked(const Path& path);
void verifyPath(const Path & path, const PathSet & store, void verifyPath(const Path& path, const PathSet& store, PathSet& done,
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); PathSet& validPaths, RepairFlag repair, bool& errors);
void updatePathInfo(State& state, const ValidPathInfo& info); void updatePathInfo(State& state, const ValidPathInfo& info);
@ -260,8 +246,8 @@ private:
void deletePathRecursive(GCState& state, const Path& path); void deletePathRecursive(GCState& state, const Path& path);
bool isActiveTempFile(const GCState & state, bool isActiveTempFile(const GCState& state, const Path& path,
const Path & path, const string & suffix); const string& suffix);
AutoCloseFD openGCLock(LockType lockType); AutoCloseFD openGCLock(LockType lockType);
@ -280,8 +266,10 @@ private:
typedef std::unordered_set<ino_t> InodeHash; typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash(); InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); Strings readDirectoryIgnoringInodes(const Path& path,
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); const InodeHash& inodeHash);
void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path,
InodeHash& inodeHash);
// Internal versions that are not wrapped in retry_sqlite. // Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State& state, const Path& path); bool isValidPath_(State& state, const Path& path);
@ -299,11 +287,9 @@ private:
friend class SubstitutionGoal; friend class SubstitutionGoal;
}; };
typedef std::pair<dev_t, ino_t> Inode; typedef std::pair<dev_t, ino_t> Inode;
typedef set<Inode> InodesSeen; typedef set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path /* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular: after it has been built. In particular:
- the last modification date on each file is set to 1 (i.e., - the last modification date on each file is set to 1 (i.e.,
@ -312,11 +298,12 @@ typedef set<Inode> InodesSeen;
without execute permission; setuid bits etc. are cleared) without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're - the owner and group are set to the Nix user and group, if we're
running as root. */ running as root. */
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); void canonicalisePathMetaData(const Path& path, uid_t fromUid,
InodesSeen& inodesSeen);
void canonicalisePathMetaData(const Path& path, uid_t fromUid); void canonicalisePathMetaData(const Path& path, uid_t fromUid);
void canonicaliseTimestampAndPermissions(const Path& path); void canonicaliseTimestampAndPermissions(const Path& path);
MakeError(PathInUse, Error); MakeError(PathInUse, Error);
} } // namespace nix

View file

@ -1,27 +1,23 @@
#include "machines.hh" #include "machines.hh"
#include "util.hh"
#include "globals.hh"
#include <algorithm> #include <algorithm>
#include "globals.hh"
#include "util.hh"
namespace nix { namespace nix {
Machine::Machine(decltype(storeUri) storeUri, Machine::Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
decltype(speedFactor) speedFactor, decltype(speedFactor) speedFactor,
decltype(supportedFeatures) supportedFeatures, decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures, decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) : decltype(sshPublicHostKey) sshPublicHostKey)
storeUri( : storeUri(
// Backwards compatibility: if the URI is a hostname, // Backwards compatibility: if the URI is a hostname,
// prepend ssh://. // prepend ssh://.
storeUri.find("://") != std::string::npos storeUri.find("://") != std::string::npos ||
|| hasPrefix(storeUri, "local") hasPrefix(storeUri, "local") ||
|| hasPrefix(storeUri, "remote") hasPrefix(storeUri, "remote") ||
|| hasPrefix(storeUri, "auto") hasPrefix(storeUri, "auto") || hasPrefix(storeUri, "/")
|| hasPrefix(storeUri, "/")
? storeUri ? storeUri
: "ssh://" + storeUri), : "ssh://" + storeUri),
systemTypes(systemTypes), systemTypes(systemTypes),
@ -30,8 +26,7 @@ Machine::Machine(decltype(storeUri) storeUri,
speedFactor(std::max(1U, speedFactor)), speedFactor(std::max(1U, speedFactor)),
supportedFeatures(supportedFeatures), supportedFeatures(supportedFeatures),
mandatoryFeatures(mandatoryFeatures), mandatoryFeatures(mandatoryFeatures),
sshPublicHostKey(sshPublicHostKey) sshPublicHostKey(sshPublicHostKey) {}
{}
bool Machine::allSupported(const std::set<string>& features) const { bool Machine::allSupported(const std::set<string>& features) const {
return std::all_of(features.begin(), features.end(), return std::all_of(features.begin(), features.end(),
@ -42,14 +37,12 @@ bool Machine::allSupported(const std::set<string> & features) const {
} }
bool Machine::mandatoryMet(const std::set<string>& features) const { bool Machine::mandatoryMet(const std::set<string>& features) const {
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), return std::all_of(
[&](const string & feature) { mandatoryFeatures.begin(), mandatoryFeatures.end(),
return features.count(feature); [&](const string& feature) { return features.count(feature); });
});
} }
void parseMachines(const std::string & s, Machines & machines) void parseMachines(const std::string& s, Machines& machines) {
{
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
trim(line); trim(line);
line.erase(std::find(line.begin(), line.end(), '#'), line.end()); line.erase(std::find(line.begin(), line.end(), '#'), line.end());
@ -60,8 +53,7 @@ void parseMachines(const std::string & s, Machines & machines)
try { try {
parseMachines(readFile(file), machines); parseMachines(readFile(file), machines);
} catch (const SysError& e) { } catch (const SysError& e) {
if (e.errNo != ENOENT) if (e.errNo != ENOENT) throw;
throw;
debug("cannot find machines file '%s'", file); debug("cannot find machines file '%s'", file);
} }
continue; continue;
@ -69,26 +61,27 @@ void parseMachines(const std::string & s, Machines & machines)
auto tokens = tokenizeString<std::vector<string>>(line); auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size(); auto sz = tokens.size();
if (sz < 1) if (sz < 1) throw FormatError("bad machine specification '%s'", line);
throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) { auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
}; };
machines.emplace_back(tokens[0], machines.emplace_back(
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, tokens[0],
isSet(2) ? tokens[2] : "", isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",")
isSet(3) ? std::stoull(tokens[3]) : 1LL, : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL, isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",")
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",")
: std::set<string>{},
isSet(7) ? tokens[7] : ""); isSet(7) ? tokens[7] : "");
} }
} }
Machines getMachines() Machines getMachines() {
{
static auto machines = [&]() { static auto machines = [&]() {
Machines machines; Machines machines;
parseMachines(settings.builders, machines); parseMachines(settings.builders, machines);
@ -97,4 +90,4 @@ Machines getMachines()
return machines; return machines;
} }
} } // namespace nix

View file

@ -5,7 +5,6 @@
namespace nix { namespace nix {
struct Machine { struct Machine {
const string storeUri; const string storeUri;
const std::vector<string> systemTypes; const std::vector<string> systemTypes;
const string sshKey; const string sshKey;
@ -20,10 +19,8 @@ struct Machine {
bool mandatoryMet(const std::set<string>& features) const; bool mandatoryMet(const std::set<string>& features) const;
Machine(decltype(storeUri) storeUri, Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
decltype(speedFactor) speedFactor, decltype(speedFactor) speedFactor,
decltype(supportedFeatures) supportedFeatures, decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures, decltype(mandatoryFeatures) mandatoryFeatures,
@ -36,4 +33,4 @@ void parseMachines(const std::string & s, Machines & machines);
Machines getMachines(); Machines getMachines();
} } // namespace nix

View file

@ -1,19 +1,16 @@
#include "derivations.hh" #include "derivations.hh"
#include "parsed-derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "local-store.hh" #include "local-store.hh"
#include "parsed-derivations.hh"
#include "store-api.hh" #include "store-api.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
namespace nix { namespace nix {
void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_,
void Store::computeFSClosure(const PathSet & startPaths, bool flipDirection, bool includeOutputs,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) bool includeDerivers) {
{ struct State {
struct State
{
size_t pending; size_t pending;
PathSet& paths; PathSet& paths;
std::exception_ptr exc; std::exception_ptr exc;
@ -34,23 +31,21 @@ void Store::computeFSClosure(const PathSet & startPaths,
state->pending++; state->pending++;
} }
queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) { queryPathInfo(
path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async // FIXME: calls to isValidPath() should be async
try { try {
auto info = fut.get(); auto info = fut.get();
if (flipDirection) { if (flipDirection) {
PathSet referrers; PathSet referrers;
queryReferrers(path, referrers); queryReferrers(path, referrers);
for (auto& ref : referrers) for (auto& ref : referrers)
if (ref != path) if (ref != path) enqueue(ref);
enqueue(ref);
if (includeOutputs) if (includeOutputs)
for (auto & i : queryValidDerivers(path)) for (auto& i : queryValidDerivers(path)) enqueue(i);
enqueue(i);
if (includeDerivers && isDerivation(path)) if (includeDerivers && isDerivation(path))
for (auto& i : queryDerivationOutputs(path)) for (auto& i : queryDerivationOutputs(path))
@ -58,10 +53,8 @@ void Store::computeFSClosure(const PathSet & startPaths,
enqueue(i); enqueue(i);
} else { } else {
for (auto& ref : info->references) for (auto& ref : info->references)
if (ref != path) if (ref != path) enqueue(ref);
enqueue(ref);
if (includeOutputs && isDerivation(path)) if (includeOutputs && isDerivation(path))
for (auto& i : queryDerivationOutputs(path)) for (auto& i : queryDerivationOutputs(path))
@ -69,7 +62,6 @@ void Store::computeFSClosure(const PathSet & startPaths,
if (includeDerivers && isValidPath(info->deriver)) if (includeDerivers && isValidPath(info->deriver))
enqueue(info->deriver); enqueue(info->deriver);
} }
{ {
@ -87,8 +79,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
}}); }});
}; };
for (auto & startPath : startPaths) for (auto& startPath : startPaths) enqueue(startPath);
enqueue(startPath);
{ {
auto state(state_.lock()); auto state(state_.lock());
@ -97,41 +88,40 @@ void Store::computeFSClosure(const PathSet & startPaths,
} }
} }
void Store::computeFSClosure(const Path& startPath, PathSet& paths_,
void Store::computeFSClosure(const Path & startPath, bool flipDirection, bool includeOutputs,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) bool includeDerivers) {
{ computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs,
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); includeDerivers);
} }
void Store::queryMissing(const PathSet& targets, PathSet& willBuild_,
void Store::queryMissing(const PathSet & targets, PathSet& willSubstitute_, PathSet& unknown_,
PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, unsigned long long& downloadSize_,
unsigned long long & downloadSize_, unsigned long long & narSize_) unsigned long long& narSize_) {
{ Activity act(*logger, lvlDebug, actUnknown,
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); "querying info about missing paths");
downloadSize_ = narSize_ = 0; downloadSize_ = narSize_ = 0;
ThreadPool pool; ThreadPool pool;
struct State struct State {
{
PathSet done; PathSet done;
PathSet &unknown, &willSubstitute, &willBuild; PathSet &unknown, &willSubstitute, &willBuild;
unsigned long long& downloadSize; unsigned long long& downloadSize;
unsigned long long& narSize; unsigned long long& narSize;
}; };
struct DrvState struct DrvState {
{
size_t left; size_t left;
bool done = false; bool done = false;
PathSet outPaths; PathSet outPaths;
DrvState(size_t left) : left(left) {} DrvState(size_t left) : left(left) {}
}; };
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_,
downloadSize_, narSize_});
std::function<void(Path)> doPath; std::function<void(Path)> doPath;
@ -142,12 +132,12 @@ void Store::queryMissing(const PathSet & targets,
} }
for (auto& i : drv.inputDrvs) for (auto& i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); pool.enqueue(
std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
}; };
auto checkOutput = [&]( auto checkOutput = [&](const Path& drvPath, ref<Derivation> drv,
const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_) const Path& outPath, ref<Sync<DrvState>> drvState_) {
{
if (drvState_->lock()->done) return; if (drvState_->lock()->done) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
@ -172,7 +162,6 @@ void Store::queryMissing(const PathSet & targets,
}; };
doPath = [&](const Path& path) { doPath = [&](const Path& path) {
{ {
auto state(state_.lock()); auto state(state_.lock());
if (state->done.count(path)) return; if (state->done.count(path)) return;
@ -194,20 +183,19 @@ void Store::queryMissing(const PathSet & targets,
PathSet invalid; PathSet invalid;
for (auto& j : drv.outputs) for (auto& j : drv.outputs)
if (wantOutput(j.first, i2.second) if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path))
&& !isValidPath(j.second.path))
invalid.insert(j.second.path); invalid.insert(j.second.path);
if (invalid.empty()) return; if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto& output : invalid) for (auto& output : invalid)
pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); pool.enqueue(std::bind(checkOutput, i2.first,
make_ref<Derivation>(drv), output, drvState));
} else } else
mustBuildDrv(i2.first, drv); mustBuildDrv(i2.first, drv);
} else { } else {
if (isValidPath(path)) return; if (isValidPath(path)) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
@ -234,15 +222,12 @@ void Store::queryMissing(const PathSet & targets,
} }
}; };
for (auto & path : targets) for (auto& path : targets) pool.enqueue(std::bind(doPath, path));
pool.enqueue(std::bind(doPath, path));
pool.process(); pool.process();
} }
Paths Store::topoSortPaths(const PathSet& paths) {
Paths Store::topoSortPaths(const PathSet & paths)
{
Paths sorted; Paths sorted;
PathSet visited, parents; PathSet visited, parents;
@ -250,7 +235,9 @@ Paths Store::topoSortPaths(const PathSet & paths)
dfsVisit = [&](const Path& path, const Path* parent) { dfsVisit = [&](const Path& path, const Path* parent) {
if (parents.find(path) != parents.end()) if (parents.find(path) != parents.end())
throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); throw BuildError(
format("cycle detected in the references of '%1%' from '%2%'") %
path % *parent);
if (visited.find(path) != visited.end()) return; if (visited.find(path) != visited.end()) return;
visited.insert(path); visited.insert(path);
@ -265,18 +252,15 @@ Paths Store::topoSortPaths(const PathSet & paths)
for (auto& i : references) for (auto& i : references)
/* Don't traverse into paths that don't exist. That can /* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */ happen due to substitutes for non-existent paths. */
if (i != path && paths.find(i) != paths.end()) if (i != path && paths.find(i) != paths.end()) dfsVisit(i, &path);
dfsVisit(i, &path);
sorted.push_front(path); sorted.push_front(path);
parents.erase(path); parents.erase(path);
}; };
for (auto & i : paths) for (auto& i : paths) dfsVisit(i, nullptr);
dfsVisit(i, nullptr);
return sorted; return sorted;
} }
} // namespace nix
}

View file

@ -1,17 +1,14 @@
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include <algorithm>
#include <map>
#include <nlohmann/json.hpp>
#include <stack>
#include "archive.hh" #include "archive.hh"
#include "json.hh" #include "json.hh"
#include <map>
#include <stack>
#include <algorithm>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
struct NarMember struct NarMember {
{
FSAccessor::Type type = FSAccessor::Type::tMissing; FSAccessor::Type type = FSAccessor::Type::tMissing;
bool isExecutable = false; bool isExecutable = false;
@ -26,16 +23,14 @@ struct NarMember
std::map<std::string, NarMember> children; std::map<std::string, NarMember> children;
}; };
struct NarAccessor : public FSAccessor struct NarAccessor : public FSAccessor {
{
std::shared_ptr<const std::string> nar; std::shared_ptr<const std::string> nar;
GetNarBytes getNarBytes; GetNarBytes getNarBytes;
NarMember root; NarMember root;
struct NarIndexer : ParseSink, StringSource struct NarIndexer : ParseSink, StringSource {
{
NarAccessor& acc; NarAccessor& acc;
std::stack<NarMember*> parents; std::stack<NarMember*> parents;
@ -44,8 +39,7 @@ struct NarAccessor : public FSAccessor
bool isExec = false; bool isExec = false;
NarIndexer(NarAccessor& acc, const std::string& nar) NarIndexer(NarAccessor& acc, const std::string& nar)
: StringSource(nar), acc(acc) : StringSource(nar), acc(acc) {}
{ }
void createMember(const Path& path, NarMember member) { void createMember(const Path& path, NarMember member) {
size_t level = std::count(path.begin(), path.end(), '/'); size_t level = std::count(path.begin(), path.end(), '/');
@ -57,36 +51,30 @@ struct NarAccessor : public FSAccessor
} else { } else {
if (parents.top()->type != FSAccessor::Type::tDirectory) if (parents.top()->type != FSAccessor::Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path); throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); auto result = parents.top()->children.emplace(baseNameOf(path),
std::move(member));
parents.push(&result.first->second); parents.push(&result.first->second);
} }
} }
void createDirectory(const Path & path) override void createDirectory(const Path& path) override {
{
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
} }
void createRegularFile(const Path & path) override void createRegularFile(const Path& path) override {
{
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
} }
void isExecutable() override void isExecutable() override { parents.top()->isExecutable = true; }
{
parents.top()->isExecutable = true;
}
void preallocateContents(unsigned long long size) override void preallocateContents(unsigned long long size) override {
{
currentStart = string(s, pos, 16); currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max()); assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size; parents.top()->size = (size_t)size;
parents.top()->start = pos; parents.top()->start = pos;
} }
void receiveContents(unsigned char * data, unsigned int len) override void receiveContents(unsigned char* data, unsigned int len) override {
{
// Sanity check // Sanity check
if (!currentStart.empty()) { if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char*)data, 16)); assert(len < 16 || currentStart == string((char*)data, 16));
@ -94,22 +82,19 @@ struct NarAccessor : public FSAccessor
} }
} }
void createSymlink(const Path & path, const string & target) override void createSymlink(const Path& path, const string& target) override {
{
createMember(path, createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
} }
}; };
NarAccessor(ref<const std::string> nar) : nar(nar) NarAccessor(ref<const std::string> nar) : nar(nar) {
{
NarIndexer indexer(*this, *nar); NarIndexer indexer(*this, *nar);
parseDump(indexer, indexer); parseDump(indexer, indexer);
} }
NarAccessor(const std::string& listing, GetNarBytes getNarBytes) NarAccessor(const std::string& listing, GetNarBytes getNarBytes)
: getNarBytes(getNarBytes) : getNarBytes(getNarBytes) {
{
using json = nlohmann::json; using json = nlohmann::json;
std::function<void(NarMember&, json&)> recurse; std::function<void(NarMember&, json&)> recurse;
@ -131,15 +116,15 @@ struct NarAccessor : public FSAccessor
} else if (type == "symlink") { } else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink; member.type = FSAccessor::Type::tSymlink;
member.target = v.value("target", ""); member.target = v.value("target", "");
} else return; } else
return;
}; };
json v = json::parse(listing); json v = json::parse(listing);
recurse(root, v); recurse(root, v);
} }
NarMember * find(const Path & path) NarMember* find(const Path& path) {
{
Path canon = path == "" ? "" : canonPath(path); Path canon = path == "" ? "" : canonPath(path);
NarMember* current = &root; NarMember* current = &root;
auto end = path.end(); auto end = path.end();
@ -171,33 +156,30 @@ struct NarAccessor : public FSAccessor
return *result; return *result;
} }
Stat stat(const Path & path) override Stat stat(const Path& path) override {
{
auto i = find(path); auto i = find(path);
if (i == nullptr) if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false};
return {FSAccessor::Type::tMissing, 0, false};
return {i->type, i->size, i->isExecutable, i->start}; return {i->type, i->size, i->isExecutable, i->start};
} }
StringSet readDirectory(const Path & path) override StringSet readDirectory(const Path& path) override {
{
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory) if (i.type != FSAccessor::Type::tDirectory)
throw Error(format("path '%1%' inside NAR file is not a directory") % path); throw Error(format("path '%1%' inside NAR file is not a directory") %
path);
StringSet res; StringSet res;
for (auto & child : i.children) for (auto& child : i.children) res.insert(child.first);
res.insert(child.first);
return res; return res;
} }
std::string readFile(const Path & path) override std::string readFile(const Path& path) override {
{
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tRegular) if (i.type != FSAccessor::Type::tRegular)
throw Error(format("path '%1%' inside NAR file is not a regular file") % path); throw Error(format("path '%1%' inside NAR file is not a regular file") %
path);
if (getNarBytes) return getNarBytes(i.start, i.size); if (getNarBytes) return getNarBytes(i.start, i.size);
@ -205,8 +187,7 @@ struct NarAccessor : public FSAccessor
return std::string(*nar, i.start, i.size); return std::string(*nar, i.start, i.size);
} }
std::string readLink(const Path & path) override std::string readLink(const Path& path) override {
{
auto i = get(path); auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink) if (i.type != FSAccessor::Type::tSymlink)
throw Error(format("path '%1%' inside NAR file is not a symlink") % path); throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
@ -214,20 +195,17 @@ struct NarAccessor : public FSAccessor
} }
}; };
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) {
{
return make_ref<NarAccessor>(nar); return make_ref<NarAccessor>(nar);
} }
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing, ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
GetNarBytes getNarBytes) GetNarBytes getNarBytes) {
{
return make_ref<NarAccessor>(listing, getNarBytes); return make_ref<NarAccessor>(listing, getNarBytes);
} }
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
const Path & path, bool recurse) bool recurse) {
{
auto st = accessor->stat(path); auto st = accessor->stat(path);
auto obj = res.object(); auto obj = res.object();
@ -236,10 +214,8 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
case FSAccessor::Type::tRegular: case FSAccessor::Type::tRegular:
obj.attr("type", "regular"); obj.attr("type", "regular");
obj.attr("size", st.fileSize); obj.attr("size", st.fileSize);
if (st.isExecutable) if (st.isExecutable) obj.attr("executable", true);
obj.attr("executable", true); if (st.narOffset) obj.attr("narOffset", st.narOffset);
if (st.narOffset)
obj.attr("narOffset", st.narOffset);
break; break;
case FSAccessor::Type::tDirectory: case FSAccessor::Type::tDirectory:
obj.attr("type", "directory"); obj.attr("type", "directory");
@ -263,4 +239,4 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
} }
} }
} } // namespace nix

View file

@ -1,7 +1,6 @@
#pragma once #pragma once
#include <functional> #include <functional>
#include "fs-accessor.hh" #include "fs-accessor.hh"
namespace nix { namespace nix {
@ -16,15 +15,14 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
inside the NAR. */ inside the NAR. */
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor( ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
const std::string & listing,
GetNarBytes getNarBytes); GetNarBytes getNarBytes);
class JSONPlaceholder; class JSONPlaceholder;
/* Write a JSON representation of the contents of a NAR (except file /* Write a JSON representation of the contents of a NAR (except file
contents). */ contents). */
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
const Path & path, bool recurse); bool recurse);
} } // namespace nix

View file

@ -1,9 +1,8 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"
#include <sqlite3.h> #include <sqlite3.h>
#include "globals.hh"
#include "sqlite.hh"
#include "sync.hh"
namespace nix { namespace nix {
@ -45,32 +44,28 @@ create table if not exists LastPurge (
)sql"; )sql";
class NarInfoDiskCacheImpl : public NarInfoDiskCache class NarInfoDiskCacheImpl : public NarInfoDiskCache {
{
public: public:
/* How often to purge expired entries from the cache. */ /* How often to purge expired entries from the cache. */
const int purgeInterval = 24 * 3600; const int purgeInterval = 24 * 3600;
struct Cache struct Cache {
{
int id; int id;
Path storeDir; Path storeDir;
bool wantMassQuery; bool wantMassQuery;
int priority; int priority;
}; };
struct State struct State {
{
SQLite db; SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR,
purgeCache;
std::map<std::string, Cache> caches; std::map<std::string, Cache> caches;
}; };
Sync<State> _state; Sync<State> _state;
NarInfoDiskCacheImpl() NarInfoDiskCacheImpl() {
{
auto state(_state.lock()); auto state(_state.lock());
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
@ -87,21 +82,33 @@ public:
state->db.exec(schema); state->db.exec(schema);
state->insertCache.create(state->db, state->insertCache.create(
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, "
"wantMassQuery, priority) values (?, ?, ?, ?, ?)");
state->queryCache.create(state->db, state->queryCache.create(state->db,
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); "select id, storeDir, wantMassQuery, priority "
"from BinaryCaches where url = ?");
state->insertNAR.create(state->db, state->insertNAR.create(
"insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " state->db,
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); "insert or replace into NARs(cache, hashPart, namePart, url, "
"compression, fileHash, fileSize, narHash, "
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, "
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
state->insertMissingNAR.create(state->db, state->insertMissingNAR.create(
"insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->db,
"insert or replace into NARs(cache, hashPart, timestamp, present) "
"values (?, ?, ?, 0)");
state->queryNAR.create(state->db, state->queryNAR.create(
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); state->db,
"select present, namePart, url, compression, fileHash, fileSize, "
"narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? "
"and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 "
"and timestamp > ?))");
/* Periodically purge expired entries from the database. */ /* Periodically purge expired entries from the database. */
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
@ -110,46 +117,50 @@ public:
SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
auto queryLastPurge_(queryLastPurge.use()); auto queryLastPurge_(queryLastPurge.use());
if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { if (!queryLastPurge_.next() ||
queryLastPurge_.getInt(0) < now - purgeInterval) {
SQLiteStmt(state->db, SQLiteStmt(state->db,
"delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") "delete from NARs where ((present = 0 and timestamp < ?) or "
.use() "(present = 1 and timestamp < ?))")
(now - settings.ttlNegativeNarInfoCache) .use()(now - settings.ttlNegativeNarInfoCache)(
(now - settings.ttlPositiveNarInfoCache) now - settings.ttlPositiveNarInfoCache)
.exec(); .exec();
debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); debug("deleted %d entries from the NAR info disk cache",
sqlite3_changes(state->db));
SQLiteStmt(state->db, SQLiteStmt(
state->db,
"insert or replace into LastPurge(dummy, value) values ('', ?)") "insert or replace into LastPurge(dummy, value) values ('', ?)")
.use()(now).exec(); .use()(now)
.exec();
} }
}); });
} }
Cache & getCache(State & state, const std::string & uri) Cache& getCache(State& state, const std::string& uri) {
{
auto i = state.caches.find(uri); auto i = state.caches.find(uri);
if (i == state.caches.end()) abort(); if (i == state.caches.end()) abort();
return i->second; return i->second;
} }
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override void createCache(const std::string& uri, const Path& storeDir,
{ bool wantMassQuery, int priority) override {
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
auto state(_state.lock()); auto state(_state.lock());
// FIXME: race // FIXME: race
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)
.exec();
assert(sqlite3_changes(state->db) == 1); assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; state->caches[uri] = Cache{(int)sqlite3_last_insert_rowid(state->db),
storeDir, wantMassQuery, priority};
}); });
} }
bool cacheExists(const std::string & uri, bool cacheExists(const std::string& uri, bool& wantMassQuery,
bool & wantMassQuery, int & priority) override int& priority) override {
{
return retrySQLite<bool>([&]() { return retrySQLite<bool>([&]() {
auto state(_state.lock()); auto state(_state.lock());
@ -157,8 +168,9 @@ public:
if (i == state->caches.end()) { if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri)); auto queryCache(state->queryCache.use()(uri));
if (!queryCache.next()) return false; if (!queryCache.next()) return false;
state->caches.emplace(uri, state->caches.emplace(
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); uri, Cache{(int)queryCache.getInt(0), queryCache.getStr(1),
queryCache.getInt(2) != 0, (int)queryCache.getInt(3)});
} }
auto& cache(getCache(*state, uri)); auto& cache(getCache(*state, uri));
@ -171,8 +183,7 @@ public:
} }
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) override const std::string& uri, const std::string& hashPart) override {
{
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>( return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> { [&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
auto state(_state.lock()); auto state(_state.lock());
@ -181,27 +192,22 @@ public:
auto now = time(0); auto now = time(0);
auto queryNAR(state->queryNAR.use() auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)(
(cache.id) now - settings.ttlNegativeNarInfoCache)(
(hashPart) now - settings.ttlPositiveNarInfoCache));
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache));
if (!queryNAR.next()) if (!queryNAR.next()) return {oUnknown, 0};
return {oUnknown, 0};
if (!queryNAR.getInt(0)) if (!queryNAR.getInt(0)) return {oInvalid, 0};
return {oInvalid, 0};
auto narInfo = make_ref<NarInfo>(); auto narInfo = make_ref<NarInfo>();
auto namePart = queryNAR.getStr(1); auto namePart = queryNAR.getStr(1);
narInfo->path = cache.storeDir + "/" + narInfo->path = cache.storeDir + "/" + hashPart +
hashPart + (namePart.empty() ? "" : "-" + namePart); (namePart.empty() ? "" : "-" + namePart);
narInfo->url = queryNAR.getStr(2); narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3); narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4)) if (!queryNAR.isNull(4)) narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5); narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7); narInfo->narSize = queryNAR.getInt(7);
@ -217,51 +223,43 @@ public:
}); });
} }
void upsertNarInfo( void upsertNarInfo(const std::string& uri, const std::string& hashPart,
const std::string & uri, const std::string & hashPart, std::shared_ptr<ValidPathInfo> info) override {
std::shared_ptr<ValidPathInfo> info) override
{
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
auto state(_state.lock()); auto state(_state.lock());
auto& cache(getCache(*state, uri)); auto& cache(getCache(*state, uri));
if (info) { if (info) {
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
assert(hashPart == storePathToHash(info->path)); assert(hashPart == storePathToHash(info->path));
state->insertNAR.use() state->insertNAR
(cache.id) .use()(cache.id)(hashPart)(storePathToName(info->path))(
(hashPart) narInfo ? narInfo->url : "", narInfo != 0)(
(storePathToName(info->path)) narInfo ? narInfo->compression : "", narInfo != 0)(
(narInfo ? narInfo->url : "", narInfo != 0) narInfo && narInfo->fileHash ? narInfo->fileHash.to_string()
(narInfo ? narInfo->compression : "", narInfo != 0) : "",
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) narInfo && narInfo->fileHash)(
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) narInfo ? narInfo->fileSize : 0,
(info->narHash.to_string()) narInfo != 0 && narInfo->fileSize)(info->narHash.to_string())(
(info->narSize) info->narSize)(concatStringsSep(" ", info->shortRefs()))(
(concatStringsSep(" ", info->shortRefs())) info->deriver != "" ? baseNameOf(info->deriver) : "",
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") info->deriver !=
(concatStringsSep(" ", info->sigs)) "")(concatStringsSep(" ", info->sigs))(info->ca)(time(0))
(info->ca) .exec();
(time(0)).exec();
} else { } else {
state->insertMissingNAR.use() state->insertMissingNAR.use()(cache.id)(hashPart)(time(0)).exec();
(cache.id)
(hashPart)
(time(0)).exec();
} }
}); });
} }
}; };
ref<NarInfoDiskCache> getNarInfoDiskCache() ref<NarInfoDiskCache> getNarInfoDiskCache() {
{
static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>(); static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
return cache; return cache;
} }
} } // namespace nix

View file

@ -1,26 +1,25 @@
#pragma once #pragma once
#include "ref.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "ref.hh"
namespace nix { namespace nix {
class NarInfoDiskCache class NarInfoDiskCache {
{
public: public:
typedef enum { oValid, oInvalid, oUnknown } Outcome; typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual void createCache(const std::string& uri, const Path& storeDir, virtual void createCache(const std::string& uri, const Path& storeDir,
bool wantMassQuery, int priority) = 0; bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri, virtual bool cacheExists(const std::string& uri, bool& wantMassQuery,
bool & wantMassQuery, int & priority) = 0; int& priority) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string& uri, const std::string& hashPart) = 0; const std::string& uri, const std::string& hashPart) = 0;
virtual void upsertNarInfo( virtual void upsertNarInfo(const std::string& uri,
const std::string & uri, const std::string & hashPart, const std::string& hashPart,
std::shared_ptr<ValidPathInfo> info) = 0; std::shared_ptr<ValidPathInfo> info) = 0;
}; };
@ -28,4 +27,4 @@ public:
multiple threads. */ multiple threads. */
ref<NarInfoDiskCache> getNarInfoDiskCache(); ref<NarInfoDiskCache> getNarInfoDiskCache();
} } // namespace nix

View file

@ -1,10 +1,10 @@
#include "globals.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "globals.hh"
namespace nix { namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) NarInfo::NarInfo(const Store& store, const std::string& s,
{ const std::string& whence) {
auto corrupt = [&]() { auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence); throw Error(format("NAR info file '%1%' is corrupt") % whence);
}; };
@ -20,7 +20,6 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
size_t pos = 0; size_t pos = 0;
while (pos < s.size()) { while (pos < s.size()) {
size_t colon = s.find(':', pos); size_t colon = s.find(':', pos);
if (colon == std::string::npos) corrupt(); if (colon == std::string::npos) corrupt();
@ -34,8 +33,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (name == "StorePath") { if (name == "StorePath") {
if (!store.isStorePath(value)) corrupt(); if (!store.isStorePath(value)) corrupt();
path = value; path = value;
} } else if (name == "URL")
else if (name == "URL")
url = value; url = value;
else if (name == "Compression") else if (name == "Compression")
compression = value; compression = value;
@ -43,13 +41,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
fileHash = parseHashField(value); fileHash = parseHashField(value);
else if (name == "FileSize") { else if (name == "FileSize") {
if (!string2Int(value, fileSize)) corrupt(); if (!string2Int(value, fileSize)) corrupt();
} } else if (name == "NarHash")
else if (name == "NarHash")
narHash = parseHashField(value); narHash = parseHashField(value);
else if (name == "NarSize") { else if (name == "NarSize") {
if (!string2Int(value, narSize)) corrupt(); if (!string2Int(value, narSize)) corrupt();
} } else if (name == "References") {
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " "); auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt(); if (!references.empty()) corrupt();
for (auto& r : refs) { for (auto& r : refs) {
@ -57,15 +53,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (!store.isStorePath(r2)) corrupt(); if (!store.isStorePath(r2)) corrupt();
references.insert(r2); references.insert(r2);
} }
} } else if (name == "Deriver") {
else if (name == "Deriver") {
if (value != "unknown-deriver") { if (value != "unknown-deriver") {
auto p = store.storeDir + "/" + value; auto p = store.storeDir + "/" + value;
if (!store.isStorePath(p)) corrupt(); if (!store.isStorePath(p)) corrupt();
deriver = p; deriver = p;
} }
} } else if (name == "System")
else if (name == "System")
system = value; system = value;
else if (name == "Sig") else if (name == "Sig")
sigs.insert(value); sigs.insert(value);
@ -82,8 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
} }
std::string NarInfo::to_string() const std::string NarInfo::to_string() const {
{
std::string res; std::string res;
res += "StorePath: " + path + "\n"; res += "StorePath: " + path + "\n";
res += "URL: " + url + "\n"; res += "URL: " + url + "\n";
@ -98,19 +91,15 @@ std::string NarInfo::to_string() const
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
if (!deriver.empty()) if (!deriver.empty()) res += "Deriver: " + baseNameOf(deriver) + "\n";
res += "Deriver: " + baseNameOf(deriver) + "\n";
if (!system.empty()) if (!system.empty()) res += "System: " + system + "\n";
res += "System: " + system + "\n";
for (auto sig : sigs) for (auto sig : sigs) res += "Sig: " + sig + "\n";
res += "Sig: " + sig + "\n";
if (!ca.empty()) if (!ca.empty()) res += "CA: " + ca + "\n";
res += "CA: " + ca + "\n";
return res; return res;
} }
} } // namespace nix

View file

@ -1,13 +1,12 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "store-api.hh" #include "store-api.hh"
#include "types.hh"
namespace nix { namespace nix {
struct NarInfo : ValidPathInfo struct NarInfo : ValidPathInfo {
{
std::string url; std::string url;
std::string compression; std::string compression;
Hash fileHash; Hash fileHash;
@ -21,4 +20,4 @@ struct NarInfo : ValidPathInfo
std::string to_string() const; std::string to_string() const;
}; };
} } // namespace nix

View file

@ -1,22 +1,18 @@
#include "util.hh"
#include "local-store.hh"
#include "globals.hh"
#include <cstdlib>
#include <cstring>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h> #include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <cstring>
#include <regex> #include <regex>
#include "globals.hh"
#include "local-store.hh"
#include "util.hh"
namespace nix { namespace nix {
static void makeWritable(const Path& path) {
static void makeWritable(const Path & path)
{
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path); throw SysError(format("getting attributes of path '%1%'") % path);
@ -24,13 +20,10 @@ static void makeWritable(const Path & path)
throw SysError(format("changing writability of '%1%'") % path); throw SysError(format("changing writability of '%1%'") % path);
} }
struct MakeReadOnly {
struct MakeReadOnly
{
Path path; Path path;
MakeReadOnly(const Path& path) : path(path) {} MakeReadOnly(const Path& path) : path(path) {}
~MakeReadOnly() ~MakeReadOnly() {
{
try { try {
/* This will make the path read-only. */ /* This will make the path read-only. */
if (path != "") canonicaliseTimestampAndPermissions(path); if (path != "") canonicaliseTimestampAndPermissions(path);
@ -40,9 +33,7 @@ struct MakeReadOnly
} }
}; };
LocalStore::InodeHash LocalStore::loadInodeHash() {
LocalStore::InodeHash LocalStore::loadInodeHash()
{
debug("loading hash inodes in memory"); debug("loading hash inodes in memory");
InodeHash inodeHash; InodeHash inodeHash;
@ -62,9 +53,8 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
return inodeHash; return inodeHash;
} }
Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) const InodeHash& inodeHash) {
{
Strings names; Strings names;
AutoCloseDir dir(opendir(path.c_str())); AutoCloseDir dir(opendir(path.c_str()));
@ -88,10 +78,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
return names; return names;
} }
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats, void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
const Path & path, InodeHash & inodeHash) const Path& path, InodeHash& inodeHash) {
{
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st;
@ -104,8 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
https://github.com/NixOS/nix/issues/1443 for more discussion. */ https://github.com/NixOS/nix/issues/1443 for more discussion. */
if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) {
{
debug(format("'%1%' is not allowed to be linked in macOS") % path); debug(format("'%1%' is not allowed to be linked in macOS") % path);
return; return;
} }
@ -113,8 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash); Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names) for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash);
optimisePath_(act, stats, path + "/" + i, inodeHash);
return; return;
} }
@ -123,7 +109,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
#if CAN_LINK_SYMLINK #if CAN_LINK_SYMLINK
&& !S_ISLNK(st.st_mode) && !S_ISLNK(st.st_mode)
#endif #endif
) return; )
return;
/* Sometimes SNAFUs can cause files in the Nix store to be /* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under modified, in particular when running programs as root under
@ -136,7 +123,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* This can still happen on top-level files. */ /* This can still happen on top-level files. */
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); debug(format("'%1%' is already linked, with %2% other file(s)") % path %
(st.st_nlink - 2));
return; return;
} }
@ -174,7 +162,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
full. When that happens, it's fine to ignore it: we full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this just effectively disable deduplication of this
file. */ file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); printInfo("cannot link '%s' to '%s': %s", linkPath, path,
strerror(errno));
return; return;
default: default:
@ -211,8 +200,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
its timestamp back to 0. */ its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
Path tempLink = (format("%1%/.tmp-link-%2%-%3%") Path tempLink =
% realStoreDir % getpid() % random()).str(); (format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random())
.str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) { if (errno == EMLINK) {
@ -245,13 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks; stats.blocksFreed += st.st_blocks;
if (act) if (act) act->result(resFileLinked, st.st_size, st.st_blocks);
act->result(resFileLinked, st.st_size, st.st_blocks);
} }
void LocalStore::optimiseStore(OptimiseStats& stats) {
void LocalStore::optimiseStore(OptimiseStats & stats)
{
Activity act(*logger, actOptimiseStore); Activity act(*logger, actOptimiseStore);
PathSet paths = queryAllValidPaths(); PathSet paths = queryAllValidPaths();
@ -265,7 +252,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
addTempRoot(i); addTempRoot(i);
if (!isValidPath(i)) continue; /* path was GC'ed, probably */ if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); Activity act(*logger, lvlTalkative, actUnknown,
fmt("optimising path '%s'", i));
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
} }
done++; done++;
@ -273,30 +261,25 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
} }
} }
static string showBytes(unsigned long long bytes) static string showBytes(unsigned long long bytes) {
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
} }
void LocalStore::optimiseStore() void LocalStore::optimiseStore() {
{
OptimiseStats stats; OptimiseStats stats;
optimiseStore(stats); optimiseStore(stats);
printInfo( printInfo(format("%1% freed by hard-linking %2% files") %
format("%1% freed by hard-linking %2% files") showBytes(stats.bytesFreed) % stats.filesLinked);
% showBytes(stats.bytesFreed)
% stats.filesLinked);
} }
void LocalStore::optimisePath(const Path & path) void LocalStore::optimisePath(const Path& path) {
{
OptimiseStats stats; OptimiseStats stats;
InodeHash inodeHash; InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); if (settings.autoOptimiseStore)
optimisePath_(nullptr, stats, path, inodeHash);
} }
} // namespace nix
}

View file

@ -3,28 +3,29 @@
namespace nix { namespace nix {
ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv) ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
: drvPath(drvPath), drv(drv) : drvPath(drvPath), drv(drv) {
{
/* Parse the __json attribute, if any. */ /* Parse the __json attribute, if any. */
auto jsonAttr = drv.env.find("__json"); auto jsonAttr = drv.env.find("__json");
if (jsonAttr != drv.env.end()) { if (jsonAttr != drv.env.end()) {
try { try {
structuredAttrs = nlohmann::json::parse(jsonAttr->second); structuredAttrs = nlohmann::json::parse(jsonAttr->second);
} catch (std::exception& e) { } catch (std::exception& e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); throw Error("cannot process __json attribute of '%s': %s", drvPath,
e.what());
} }
} }
} }
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const std::optional<std::string> ParsedDerivation::getStringAttr(
{ const std::string& name) const {
if (structuredAttrs) { if (structuredAttrs) {
auto i = structuredAttrs->find(name); auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end()) if (i == structuredAttrs->end())
return {}; return {};
else { else {
if (!i->is_string()) if (!i->is_string())
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); throw Error("attribute '%s' of derivation '%s' must be a string", name,
drvPath);
return i->get<std::string>(); return i->get<std::string>();
} }
} else { } else {
@ -36,15 +37,15 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n
} }
} }
bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const {
{
if (structuredAttrs) { if (structuredAttrs) {
auto i = structuredAttrs->find(name); auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end()) if (i == structuredAttrs->end())
return def; return def;
else { else {
if (!i->is_boolean()) if (!i->is_boolean())
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); throw Error("attribute '%s' of derivation '%s' must be a Boolean", name,
drvPath);
return i->get<bool>(); return i->get<bool>();
} }
} else { } else {
@ -56,19 +57,23 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
} }
} }
std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const std::optional<Strings> ParsedDerivation::getStringsAttr(
{ const std::string& name) const {
if (structuredAttrs) { if (structuredAttrs) {
auto i = structuredAttrs->find(name); auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end()) if (i == structuredAttrs->end())
return {}; return {};
else { else {
if (!i->is_array()) if (!i->is_array())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); throw Error(
"attribute '%s' of derivation '%s' must be a list of strings", name,
drvPath);
Strings res; Strings res;
for (auto j = i->begin(); j != i->end(); ++j) { for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string()) if (!j->is_string())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); throw Error(
"attribute '%s' of derivation '%s' must be a list of strings",
name, drvPath);
res.push_back(j->get<std::string>()); res.push_back(j->get<std::string>());
} }
return res; return res;
@ -82,19 +87,16 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
} }
} }
StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet ParsedDerivation::getRequiredSystemFeatures() const {
{
StringSet res; StringSet res;
for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i); res.insert(i);
return res; return res;
} }
bool ParsedDerivation::canBuildLocally() const bool ParsedDerivation::canBuildLocally() const {
{ if (drv.platform != settings.thisSystem.get() &&
if (drv.platform != settings.thisSystem.get() !settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin())
&& !settings.extraPlatforms.get().count(drv.platform)
&& !drv.isBuiltin())
return false; return false;
for (auto& feature : getRequiredSystemFeatures()) for (auto& feature : getRequiredSystemFeatures())
@ -103,14 +105,12 @@ bool ParsedDerivation::canBuildLocally() const
return true; return true;
} }
bool ParsedDerivation::willBuildLocally() const bool ParsedDerivation::willBuildLocally() const {
{
return getBoolAttr("preferLocalBuild") && canBuildLocally(); return getBoolAttr("preferLocalBuild") && canBuildLocally();
} }
bool ParsedDerivation::substitutesAllowed() const bool ParsedDerivation::substitutesAllowed() const {
{
return getBoolAttr("allowSubstitutes", true); return getBoolAttr("allowSubstitutes", true);
} }
} } // namespace nix

View file

@ -1,21 +1,17 @@
#include "derivations.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "derivations.hh"
namespace nix { namespace nix {
class ParsedDerivation class ParsedDerivation {
{
Path drvPath; Path drvPath;
BasicDerivation& drv; BasicDerivation& drv;
std::optional<nlohmann::json> structuredAttrs; std::optional<nlohmann::json> structuredAttrs;
public: public:
ParsedDerivation(const Path& drvPath, BasicDerivation& drv); ParsedDerivation(const Path& drvPath, BasicDerivation& drv);
const std::optional<nlohmann::json> & getStructuredAttrs() const const std::optional<nlohmann::json>& getStructuredAttrs() const {
{
return structuredAttrs; return structuredAttrs;
} }
@ -34,4 +30,4 @@ public:
bool substitutesAllowed() const; bool substitutesAllowed() const;
}; };
} } // namespace nix

View file

@ -1,21 +1,16 @@
#include "pathlocks.hh" #include "pathlocks.hh"
#include "util.hh" #include <fcntl.h>
#include "sync.hh" #include <sys/file.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <cerrno> #include <cerrno>
#include <cstdlib> #include <cstdlib>
#include "sync.hh"
#include <fcntl.h> #include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/file.h>
namespace nix { namespace nix {
AutoCloseFD openLockFile(const Path& path, bool create) {
AutoCloseFD openLockFile(const Path & path, bool create)
{
AutoCloseFD fd; AutoCloseFD fd;
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
@ -25,9 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
return fd; return fd;
} }
void deleteLockFile(const Path& path, int fd) {
void deleteLockFile(const Path & path, int fd)
{
/* Get rid of the lock file. Have to be careful not to introduce /* Get rid of the lock file. Have to be careful not to introduce
races. Write a (meaningless) token to the file to indicate to races. Write a (meaningless) token to the file to indicate to
other processes waiting on this lock that the lock is stale other processes waiting on this lock that the lock is stale
@ -38,14 +31,16 @@ void deleteLockFile(const Path & path, int fd)
file is an optimisation, not a necessity. */ file is an optimisation, not a necessity. */
} }
bool lockFile(int fd, LockType lockType, bool wait) {
bool lockFile(int fd, LockType lockType, bool wait)
{
int type; int type;
if (lockType == ltRead) type = LOCK_SH; if (lockType == ltRead)
else if (lockType == ltWrite) type = LOCK_EX; type = LOCK_SH;
else if (lockType == ltNone) type = LOCK_UN; else if (lockType == ltWrite)
else abort(); type = LOCK_EX;
else if (lockType == ltNone)
type = LOCK_UN;
else
abort();
if (wait) { if (wait) {
while (flock(fd, type) != 0) { while (flock(fd, type) != 0) {
@ -59,31 +54,22 @@ bool lockFile(int fd, LockType lockType, bool wait)
while (flock(fd, type | LOCK_NB) != 0) { while (flock(fd, type | LOCK_NB) != 0) {
checkInterrupt(); checkInterrupt();
if (errno == EWOULDBLOCK) return false; if (errno == EWOULDBLOCK) return false;
if (errno != EINTR) if (errno != EINTR) throw SysError(format("acquiring/releasing lock"));
throw SysError(format("acquiring/releasing lock"));
} }
} }
return true; return true;
} }
PathLocks::PathLocks() : deletePaths(false) {}
PathLocks::PathLocks()
: deletePaths(false)
{
}
PathLocks::PathLocks(const PathSet& paths, const string& waitMsg) PathLocks::PathLocks(const PathSet& paths, const string& waitMsg)
: deletePaths(false) : deletePaths(false) {
{
lockPaths(paths, waitMsg); lockPaths(paths, waitMsg);
} }
bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
bool PathLocks::lockPaths(const PathSet & paths, bool wait) {
const string & waitMsg, bool wait)
{
assert(fds.empty()); assert(fds.empty());
/* Note that `fds' is built incrementally so that the destructor /* Note that `fds' is built incrementally so that the destructor
@ -101,7 +87,6 @@ bool PathLocks::lockPaths(const PathSet & paths,
AutoCloseFD fd; AutoCloseFD fd;
while (1) { while (1) {
/* Open/create the lock file. */ /* Open/create the lock file. */
fd = openLockFile(lockPath, true); fd = openLockFile(lockPath, true);
@ -142,9 +127,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
return true; return true;
} }
PathLocks::~PathLocks() {
PathLocks::~PathLocks()
{
try { try {
unlock(); unlock();
} catch (...) { } catch (...) {
@ -152,15 +135,13 @@ PathLocks::~PathLocks()
} }
} }
void PathLocks::unlock() {
void PathLocks::unlock()
{
for (auto& i : fds) { for (auto& i : fds) {
if (deletePaths) deleteLockFile(i.second, i.first); if (deletePaths) deleteLockFile(i.second, i.first);
if (close(i.first) == -1) if (close(i.first) == -1)
printError( printError(format("error (ignored): cannot close lock file on '%1%'") %
format("error (ignored): cannot close lock file on '%1%'") % i.second); i.second);
debug(format("lock released on '%1%'") % i.second); debug(format("lock released on '%1%'") % i.second);
} }
@ -168,11 +149,8 @@ void PathLocks::unlock()
fds.clear(); fds.clear();
} }
void PathLocks::setDeletion(bool deletePaths) {
void PathLocks::setDeletion(bool deletePaths)
{
this->deletePaths = deletePaths; this->deletePaths = deletePaths;
} }
} // namespace nix
}

View file

@ -16,8 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(int fd, LockType lockType, bool wait); bool lockFile(int fd, LockType lockType, bool wait);
class PathLocks class PathLocks {
{
private: private:
typedef std::pair<int, Path> FDPair; typedef std::pair<int, Path> FDPair;
list<FDPair> fds; list<FDPair> fds;
@ -25,14 +24,12 @@ private:
public: public:
PathLocks(); PathLocks();
PathLocks(const PathSet & paths, PathLocks(const PathSet& paths, const string& waitMsg = "");
const string & waitMsg = ""); bool lockPaths(const PathSet& _paths, const string& waitMsg = "",
bool lockPaths(const PathSet & _paths,
const string & waitMsg = "",
bool wait = true); bool wait = true);
~PathLocks(); ~PathLocks();
void unlock(); void unlock();
void setDeletion(bool deletePaths); void setDeletion(bool deletePaths);
}; };
} } // namespace nix

View file

@ -1,27 +1,21 @@
#include "profiles.hh" #include "profiles.hh"
#include <errno.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
namespace nix { namespace nix {
static bool cmpGensByNumber(const Generation& a, const Generation& b) {
static bool cmpGensByNumber(const Generation & a, const Generation & b)
{
return a.number < b.number; return a.number < b.number;
} }
/* Parse a generation name of the format /* Parse a generation name of the format
`<profilename>-<number>-link'. */ `<profilename>-<number>-link'. */
static int parseName(const string & profileName, const string & name) static int parseName(const string& profileName, const string& name) {
{
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
string s = string(name, profileName.size() + 1); string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link"); string::size_type p = s.find("-link");
@ -33,10 +27,7 @@ static int parseName(const string & profileName, const string & name)
return -1; return -1;
} }
Generations findGenerations(Path profile, int& curGen) {
Generations findGenerations(Path profile, int & curGen)
{
Generations gens; Generations gens;
Path profileDir = dirOf(profile); Path profileDir = dirOf(profile);
@ -58,24 +49,17 @@ Generations findGenerations(Path profile, int & curGen)
gens.sort(cmpGensByNumber); gens.sort(cmpGensByNumber);
curGen = pathExists(profile) curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1;
? parseName(profileName, readLink(profile))
: -1;
return gens; return gens;
} }
static void makeName(const Path& profile, unsigned int num, Path& outLink) {
static void makeName(const Path & profile, unsigned int num,
Path & outLink)
{
Path prefix = (format("%1%-%2%") % profile % num).str(); Path prefix = (format("%1%-%2%") % profile % num).str();
outLink = prefix + "-link"; outLink = prefix + "-link";
} }
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) {
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
{
/* The new generation number should be higher than old the /* The new generation number should be higher than old the
previous ones. */ previous ones. */
int dummy; int dummy;
@ -113,24 +97,19 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
return generation; return generation;
} }
static void removeFile(const Path& path) {
static void removeFile(const Path & path)
{
if (remove(path.c_str()) == -1) if (remove(path.c_str()) == -1)
throw SysError(format("cannot unlink '%1%'") % path); throw SysError(format("cannot unlink '%1%'") % path);
} }
void deleteGeneration(const Path& profile, unsigned int gen) {
void deleteGeneration(const Path & profile, unsigned int gen)
{
Path generation; Path generation;
makeName(profile, gen, generation); makeName(profile, gen, generation);
removeFile(generation); removeFile(generation);
} }
static void deleteGeneration2(const Path& profile, unsigned int gen,
static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) bool dryRun) {
{
if (dryRun) if (dryRun)
printInfo(format("would remove generation %1%") % gen); printInfo(format("would remove generation %1%") % gen);
else { else {
@ -139,9 +118,9 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
} }
} }
void deleteGenerations(const Path& profile,
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun) const std::set<unsigned int>& gensToDelete,
{ bool dryRun) {
PathLocks lock; PathLocks lock;
lockProfile(lock, profile); lockProfile(lock, profile);
@ -149,7 +128,8 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
Generations gens = findGenerations(profile, curGen); Generations gens = findGenerations(profile, curGen);
if (gensToDelete.find(curGen) != gensToDelete.end()) if (gensToDelete.find(curGen) != gensToDelete.end())
throw Error(format("cannot delete current generation of profile %1%'") % profile); throw Error(format("cannot delete current generation of profile %1%'") %
profile);
for (auto& i : gens) { for (auto& i : gens) {
if (gensToDelete.find(i.number) == gensToDelete.end()) continue; if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@ -157,8 +137,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
} }
} }
void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) {
{
PathLocks lock; PathLocks lock;
lockProfile(lock, profile); lockProfile(lock, profile);
@ -181,8 +160,7 @@ void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
} }
} }
void deleteOldGenerations(const Path & profile, bool dryRun) void deleteOldGenerations(const Path& profile, bool dryRun) {
{
PathLocks lock; PathLocks lock;
lockProfile(lock, profile); lockProfile(lock, profile);
@ -190,13 +168,10 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
Generations gens = findGenerations(profile, curGen); Generations gens = findGenerations(profile, curGen);
for (auto& i : gens) for (auto& i : gens)
if (i.number != curGen) if (i.number != curGen) deleteGeneration2(profile, i.number, dryRun);
deleteGeneration2(profile, i.number, dryRun);
} }
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) {
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
{
PathLocks lock; PathLocks lock;
lockProfile(lock, profile); lockProfile(lock, profile);
@ -207,8 +182,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
for (auto i = gens.rbegin(); i != gens.rend(); ++i) for (auto i = gens.rbegin(); i != gens.rend(); ++i)
if (canDelete) { if (canDelete) {
assert(i->creationTime < t); assert(i->creationTime < t);
if (i->number != curGen) if (i->number != curGen) deleteGeneration2(profile, i->number, dryRun);
deleteGeneration2(profile, i->number, dryRun);
} else if (i->creationTime < t) { } else if (i->creationTime < t) {
/* We may now start deleting generations, but we don't /* We may now start deleting generations, but we don't
delete this generation yet, because this generation was delete this generation yet, because this generation was
@ -218,9 +192,8 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
} }
} }
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) bool dryRun) {
{
time_t curTime = time(0); time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1); string strDays = string(timeSpec, 0, timeSpec.size() - 1);
int days; int days;
@ -233,27 +206,21 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
deleteGenerationsOlderThan(profile, oldTime, dryRun); deleteGenerationsOlderThan(profile, oldTime, dryRun);
} }
void switchLink(Path link, Path target) {
void switchLink(Path link, Path target)
{
/* Hacky. */ /* Hacky. */
if (dirOf(target) == dirOf(link)) target = baseNameOf(target); if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
replaceSymlink(target, link); replaceSymlink(target, link);
} }
void lockProfile(PathLocks& lock, const Path& profile) {
void lockProfile(PathLocks & lock, const Path & profile) lock.lockPaths({profile},
{ (format("waiting for lock on profile '%1%'") % profile).str());
lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());
lock.setDeletion(true); lock.setDeletion(true);
} }
string optimisticLockProfile(const Path& profile) {
string optimisticLockProfile(const Path & profile)
{
return pathExists(profile) ? readLink(profile) : ""; return pathExists(profile) ? readLink(profile) : "";
} }
} // namespace nix
}

View file

@ -1,32 +1,21 @@
#pragma once #pragma once
#include "types.hh"
#include "pathlocks.hh"
#include <time.h> #include <time.h>
#include "pathlocks.hh"
#include "types.hh"
namespace nix { namespace nix {
struct Generation {
struct Generation
{
int number; int number;
Path path; Path path;
time_t creationTime; time_t creationTime;
Generation() Generation() { number = -1; }
{ operator bool() const { return number != -1; }
number = -1;
}
operator bool() const
{
return number != -1;
}
}; };
typedef list<Generation> Generations; typedef list<Generation> Generations;
/* Returns the list of currently present generations for the specified /* Returns the list of currently present generations for the specified
profile, sorted by generation number. */ profile, sorted by generation number. */
Generations findGenerations(Path profile, int& curGen); Generations findGenerations(Path profile, int& curGen);
@ -37,15 +26,18 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
void deleteGeneration(const Path& profile, unsigned int gen); void deleteGeneration(const Path& profile, unsigned int gen);
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun); void deleteGenerations(const Path& profile,
const std::set<unsigned int>& gensToDelete, bool dryRun);
void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); void deleteGenerationsGreaterThan(const Path& profile, const int max,
bool dryRun);
void deleteOldGenerations(const Path& profile, bool dryRun); void deleteOldGenerations(const Path& profile, bool dryRun);
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun); void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun); void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
bool dryRun);
void switchLink(Path link, Path target); void switchLink(Path link, Path target);
@ -64,4 +56,4 @@ void lockProfile(PathLocks & lock, const Path & profile);
rebuilt. */ rebuilt. */
string optimisticLockProfile(const Path& profile); string optimisticLockProfile(const Path& profile);
} } // namespace nix

View file

@ -1,21 +1,16 @@
#include "references.hh" #include "references.hh"
#include <cstdlib>
#include <map>
#include "archive.hh"
#include "hash.hh" #include "hash.hh"
#include "util.hh" #include "util.hh"
#include "archive.hh"
#include <map>
#include <cstdlib>
namespace nix { namespace nix {
static unsigned int refLength = 32; /* characters */ static unsigned int refLength = 32; /* characters */
static void search(const unsigned char* s, size_t len, StringSet& hashes,
static void search(const unsigned char * s, size_t len, StringSet& seen) {
StringSet & hashes, StringSet & seen)
{
static bool initialised = false; static bool initialised = false;
static bool isBase32[256]; static bool isBase32[256];
if (!initialised) { if (!initialised) {
@ -37,8 +32,7 @@ static void search(const unsigned char * s, size_t len,
if (!match) continue; if (!match) continue;
string ref((const char*)s + i, refLength); string ref((const char*)s + i, refLength);
if (hashes.find(ref) != hashes.end()) { if (hashes.find(ref) != hashes.end()) {
debug(format("found reference to '%1%' at offset '%2%'") debug(format("found reference to '%1%' at offset '%2%'") % ref % i);
% ref % i);
seen.insert(ref); seen.insert(ref);
hashes.erase(ref); hashes.erase(ref);
} }
@ -46,9 +40,7 @@ static void search(const unsigned char * s, size_t len,
} }
} }
struct RefScanSink : Sink {
struct RefScanSink : Sink
{
HashSink hashSink; HashSink hashSink;
StringSet hashes; StringSet hashes;
StringSet seen; StringSet seen;
@ -60,29 +52,27 @@ struct RefScanSink : Sink
void operator()(const unsigned char* data, size_t len); void operator()(const unsigned char* data, size_t len);
}; };
void RefScanSink::operator()(const unsigned char* data, size_t len) {
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
hashSink(data, len); hashSink(data, len);
/* It's possible that a reference spans the previous and current /* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */ previous fragment and the start of the current fragment. */
string s = tail + string((const char *) data, len > refLength ? refLength : len); string s =
tail + string((const char*)data, len > refLength ? refLength : len);
search((const unsigned char*)s.data(), s.size(), hashes, seen); search((const unsigned char*)s.data(), s.size(), hashes, seen);
search(data, len, hashes, seen); search(data, len, hashes, seen);
size_t tailLen = len <= refLength ? len : refLength; size_t tailLen = len <= refLength ? len : refLength;
tail = tail = string(tail, tail.size() < refLength - tailLen
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + ? 0
: tail.size() - (refLength - tailLen)) +
string((const char*)data + len - tailLen, tailLen); string((const char*)data + len - tailLen, tailLen);
} }
PathSet scanForReferences(const string& path, const PathSet& refs,
PathSet scanForReferences(const string & path, HashResult& hash) {
const PathSet & refs, HashResult & hash)
{
RefScanSink sink; RefScanSink sink;
std::map<string, Path> backMap; std::map<string, Path> backMap;
@ -92,8 +82,7 @@ PathSet scanForReferences(const string & path,
for (auto& i : refs) { for (auto& i : refs) {
string baseName = baseNameOf(i); string baseName = baseNameOf(i);
string::size_type pos = baseName.find('-'); string::size_type pos = baseName.find('-');
if (pos == string::npos) if (pos == string::npos) throw Error(format("bad reference '%1%'") % i);
throw Error(format("bad reference '%1%'") % i);
string s = string(baseName, 0, pos); string s = string(baseName, 0, pos);
assert(s.size() == refLength); assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end()); assert(backMap.find(s) == backMap.end());
@ -118,5 +107,4 @@ PathSet scanForReferences(const string & path,
return found; return found;
} }
} // namespace nix
}

View file

@ -1,7 +1,7 @@
#pragma once #pragma once
#include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "types.hh"
namespace nix { namespace nix {

View file

@ -1,30 +1,25 @@
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "nar-accessor.hh"
#include "json.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "json.hh"
#include "nar-accessor.hh"
namespace nix { namespace nix {
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir) RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir)
: store(store) : store(store), cacheDir(cacheDir) {
, cacheDir(cacheDir) if (cacheDir != "") createDirs(cacheDir);
{
if (cacheDir != "")
createDirs(cacheDir);
} }
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) Path RemoteFSAccessor::makeCacheFile(const Path& storePath,
{ const std::string& ext) {
assert(cacheDir != ""); assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
} }
void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar, void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar,
ref<FSAccessor> narAccessor) ref<FSAccessor> narAccessor) {
{
nars.emplace(storePath, narAccessor); nars.emplace(storePath, narAccessor);
if (cacheDir != "") { if (cacheDir != "") {
@ -43,15 +38,15 @@ void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & na
} }
} }
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) {
{
auto path = canonPath(path_); auto path = canonPath(path_);
auto storePath = store->toStorePath(path); auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size()); std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath)) if (!store->isValidPath(storePath))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); throw InvalidPath(format("path '%1%' is not a valid store path") %
storePath);
auto i = nars.find(storePath); auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath}; if (i != nars.end()) return {i->second, restPath};
@ -60,17 +55,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
std::string listing; std::string listing;
Path cacheFile; Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { if (cacheDir != "" &&
pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
try { try {
listing = nix::readFile(makeCacheFile(storePath, "ls")); listing = nix::readFile(makeCacheFile(storePath, "ls"));
auto narAccessor = makeLazyNarAccessor(listing, auto narAccessor = makeLazyNarAccessor(
[cacheFile](uint64_t offset, uint64_t length) { listing, [cacheFile](uint64_t offset, uint64_t length) {
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile);
throw SysError("opening NAR cache file '%s'", cacheFile);
if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset) if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset)
throw SysError("seeking in '%s'", cacheFile); throw SysError("seeking in '%s'", cacheFile);
@ -84,7 +77,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
nars.emplace(storePath, narAccessor); nars.emplace(storePath, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError&) {
}
try { try {
*sink.s = nix::readFile(cacheFile); *sink.s = nix::readFile(cacheFile);
@ -93,7 +87,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
nars.emplace(storePath, narAccessor); nars.emplace(storePath, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError&) {
}
} }
store->narFromPath(storePath, sink); store->narFromPath(storePath, sink);
@ -102,28 +97,24 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return {narAccessor, restPath}; return {narAccessor, restPath};
} }
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) {
{
auto res = fetch(path); auto res = fetch(path);
return res.first->stat(res.second); return res.first->stat(res.second);
} }
StringSet RemoteFSAccessor::readDirectory(const Path & path) StringSet RemoteFSAccessor::readDirectory(const Path& path) {
{
auto res = fetch(path); auto res = fetch(path);
return res.first->readDirectory(res.second); return res.first->readDirectory(res.second);
} }
std::string RemoteFSAccessor::readFile(const Path & path) std::string RemoteFSAccessor::readFile(const Path& path) {
{
auto res = fetch(path); auto res = fetch(path);
return res.first->readFile(res.second); return res.first->readFile(res.second);
} }
std::string RemoteFSAccessor::readLink(const Path & path) std::string RemoteFSAccessor::readLink(const Path& path) {
{
auto res = fetch(path); auto res = fetch(path);
return res.first->readLink(res.second); return res.first->readLink(res.second);
} }
} } // namespace nix

View file

@ -6,8 +6,7 @@
namespace nix { namespace nix {
class RemoteFSAccessor : public FSAccessor class RemoteFSAccessor : public FSAccessor {
{
ref<Store> store; ref<Store> store;
std::map<Path, ref<FSAccessor>> nars; std::map<Path, ref<FSAccessor>> nars;
@ -24,7 +23,6 @@ class RemoteFSAccessor : public FSAccessor
ref<FSAccessor> narAccessor); ref<FSAccessor> narAccessor);
public: public:
RemoteFSAccessor(ref<Store> store, RemoteFSAccessor(ref<Store> store,
const /* FIXME: use std::optional */ Path& cacheDir = ""); const /* FIXME: use std::optional */ Path& cacheDir = "");
@ -37,4 +35,4 @@ public:
std::string readLink(const Path& path) override; std::string readLink(const Path& path) override;
}; };
} } // namespace nix

View file

@ -1,37 +1,32 @@
#include "serialise.hh"
#include "util.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "worker-protocol.hh"
#include "archive.hh"
#include "affinity.hh"
#include "globals.hh"
#include "derivations.hh"
#include "pool.hh"
#include "finally.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h> #include <unistd.h>
#include <cstring> #include <cstring>
#include "affinity.hh"
#include "archive.hh"
#include "derivations.hh"
#include "finally.hh"
#include "globals.hh"
#include "pool.hh"
#include "serialise.hh"
#include "util.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
Path readStorePath(Store& store, Source& from) {
Path readStorePath(Store & store, Source & from)
{
Path path = readString(from); Path path = readString(from);
store.assertStorePath(path); store.assertStorePath(path);
return path; return path;
} }
template <class T>
template<class T> T readStorePaths(Store & store, Source & from) T readStorePaths(Store& store, Source& from) {
{
T paths = readStrings<T>(from); T paths = readStrings<T>(from);
for (auto& i : paths) store.assertStorePath(i); for (auto& i : paths) store.assertStorePath(i);
return paths; return paths;
@ -40,28 +35,24 @@ template<class T> T readStorePaths(Store & store, Source & from)
template PathSet readStorePaths(Store& store, Source& from); template PathSet readStorePaths(Store& store, Source& from);
template Paths readStorePaths(Store& store, Source& from); template Paths readStorePaths(Store& store, Source& from);
/* TODO: Separate these store impls into different files, give them better names */ /* TODO: Separate these store impls into different files, give them better names
*/
RemoteStore::RemoteStore(const Params& params) RemoteStore::RemoteStore(const Params& params)
: Store(params) : Store(params),
, connections(make_ref<Pool<Connection>>( connections(make_ref<Pool<Connection>>(
std::max(1, (int)maxConnections), std::max(1, (int)maxConnections),
[this]() { return openConnectionWrapper(); }, [this]() { return openConnectionWrapper(); },
[this](const ref<Connection>& r) { [this](const ref<Connection>& r) {
return return r->to.good() && r->from.good() &&
r->to.good() std::chrono::duration_cast<std::chrono::seconds>(
&& r->from.good() std::chrono::steady_clock::now() - r->startTime)
&& std::chrono::duration_cast<std::chrono::seconds>( .count() < maxConnectionAge;
std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; })) {}
}
))
{
}
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() {
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
{
if (failed) if (failed)
throw Error("opening a connection to remote store '%s' previously failed", getUri()); throw Error("opening a connection to remote store '%s' previously failed",
getUri());
try { try {
return openConnection(); return openConnection();
} catch (...) { } catch (...) {
@ -70,26 +61,16 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
} }
} }
UDSRemoteStore::UDSRemoteStore(const Params& params) UDSRemoteStore::UDSRemoteStore(const Params& params)
: Store(params) : Store(params), LocalFSStore(params), RemoteStore(params) {}
, LocalFSStore(params)
, RemoteStore(params)
{
}
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params) UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params)
: Store(params) : Store(params),
, LocalFSStore(params) LocalFSStore(params),
, RemoteStore(params) RemoteStore(params),
, path(socket_path) path(socket_path) {}
{
}
std::string UDSRemoteStore::getUri() {
std::string UDSRemoteStore::getUri()
{
if (path) { if (path) {
return std::string("unix://") + *path; return std::string("unix://") + *path;
} else { } else {
@ -97,19 +78,18 @@ std::string UDSRemoteStore::getUri()
} }
} }
ref<RemoteStore::Connection> UDSRemoteStore::openConnection() {
ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
{
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
/* Connect to a daemon that does the privileged work for us. */ /* Connect to a daemon that does the privileged work for us. */
conn->fd = socket(PF_UNIX, SOCK_STREAM conn->fd = socket(PF_UNIX,
SOCK_STREAM
#ifdef SOCK_CLOEXEC #ifdef SOCK_CLOEXEC
| SOCK_CLOEXEC | SOCK_CLOEXEC
#endif #endif
, 0); ,
if (!conn->fd) 0);
throw SysError("cannot create Unix domain socket"); if (!conn->fd) throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get()); closeOnExec(conn->fd.get());
string socketPath = path ? *path : settings.nixDaemonSocketFile; string socketPath = path ? *path : settings.nixDaemonSocketFile;
@ -133,9 +113,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
return conn; return conn;
} }
void RemoteStore::initConnection(Connection& conn) {
void RemoteStore::initConnection(Connection & conn)
{
/* Send the magic greeting, check for the reply. */ /* Send the magic greeting, check for the reply. */
try { try {
conn.to << WORKER_MAGIC_1; conn.to << WORKER_MAGIC_1;
@ -144,7 +122,8 @@ void RemoteStore::initConnection(Connection & conn)
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
conn.from >> conn.daemonVersion; conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) if (GET_PROTOCOL_MAJOR(conn.daemonVersion) !=
GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported"); throw Error("Nix daemon protocol version not supported");
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
throw Error("the Nix daemon version is too old"); throw Error("the Nix daemon version is too old");
@ -158,35 +137,26 @@ void RemoteStore::initConnection(Connection & conn)
conn.to << 0; conn.to << 0;
} }
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false;
conn.to << false;
auto ex = conn.processStderr(); auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex); if (ex) std::rethrow_exception(ex);
} } catch (Error& e) {
catch (Error & e) { throw Error("cannot open connection to remote store '%s': %s", getUri(),
throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); e.what());
} }
setOptions(conn); setOptions(conn);
} }
void RemoteStore::setOptions(Connection& conn) {
void RemoteStore::setOptions(Connection & conn) conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing
{ << settings.tryFallback << verbosity << settings.maxBuildJobs
conn.to << wopSetOptions << settings.maxSilentTime << true
<< settings.keepFailed
<< settings.keepGoing
<< settings.tryFallback
<< verbosity
<< settings.maxBuildJobs
<< settings.maxSilentTime
<< true
<< (settings.verboseBuild ? lvlError : lvlVomit) << (settings.verboseBuild ? lvlError : lvlVomit)
<< 0 // obsolete log type << 0 // obsolete log type
<< 0 /* obsolete print build trace */ << 0 /* obsolete print build trace */
<< settings.buildCores << settings.buildCores << settings.useSubstitutes;
<< settings.useSubstitutes;
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
std::map<std::string, Config::SettingInfo> overrides; std::map<std::string, Config::SettingInfo> overrides;
@ -200,35 +170,28 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.useSubstitutes.name); overrides.erase(settings.useSubstitutes.name);
overrides.erase(settings.showTrace.name); overrides.erase(settings.showTrace.name);
conn.to << overrides.size(); conn.to << overrides.size();
for (auto & i : overrides) for (auto& i : overrides) conn.to << i.first << i.second.value;
conn.to << i.first << i.second.value;
} }
auto ex = conn.processStderr(); auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex); if (ex) std::rethrow_exception(ex);
} }
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks /* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
the connection as bad (causing it to be closed) if a non-daemon the connection as bad (causing it to be closed) if a non-daemon
exception is thrown before the handle is closed. Such an exception exception is thrown before the handle is closed. Such an exception
causes a deviation from the expected protocol and therefore a causes a deviation from the expected protocol and therefore a
desynchronization between the client and daemon. */ desynchronization between the client and daemon. */
struct ConnectionHandle struct ConnectionHandle {
{
Pool<RemoteStore::Connection>::Handle handle; Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false; bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle) ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle)
: handle(std::move(handle)) : handle(std::move(handle)) {}
{ }
ConnectionHandle(ConnectionHandle && h) ConnectionHandle(ConnectionHandle&& h) : handle(std::move(h.handle)) {}
: handle(std::move(h.handle))
{ }
~ConnectionHandle() ~ConnectionHandle() {
{
if (!daemonException && std::uncaught_exceptions()) { if (!daemonException && std::uncaught_exceptions()) {
handle.markBad(); handle.markBad();
debug("closing daemon connection because of an exception"); debug("closing daemon connection because of an exception");
@ -237,8 +200,7 @@ struct ConnectionHandle
RemoteStore::Connection* operator->() { return &*handle; } RemoteStore::Connection* operator->() { return &*handle; }
void processStderr(Sink * sink = 0, Source * source = 0) void processStderr(Sink* sink = 0, Source* source = 0) {
{
auto ex = handle->processStderr(sink, source); auto ex = handle->processStderr(sink, source);
if (ex) { if (ex) {
daemonException = true; daemonException = true;
@ -247,24 +209,19 @@ struct ConnectionHandle
} }
}; };
ConnectionHandle RemoteStore::getConnection() {
ConnectionHandle RemoteStore::getConnection()
{
return ConnectionHandle(connections->get()); return ConnectionHandle(connections->get());
} }
bool RemoteStore::isValidPathUncached(const Path& path) {
bool RemoteStore::isValidPathUncached(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopIsValidPath << path; conn->to << wopIsValidPath << path;
conn.processStderr(); conn.processStderr();
return readInt(conn->from); return readInt(conn->from);
} }
PathSet RemoteStore::queryValidPaths(const PathSet& paths,
PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) SubstituteFlag maybeSubstitute) {
{
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res; PathSet res;
@ -278,18 +235,14 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
} }
} }
PathSet RemoteStore::queryAllValidPaths() {
PathSet RemoteStore::queryAllValidPaths()
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryAllValidPaths; conn->to << wopQueryAllValidPaths;
conn.processStderr(); conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from); return readStorePaths<PathSet>(*this, conn->from);
} }
PathSet RemoteStore::querySubstitutablePaths(const PathSet& paths) {
PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
{
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res; PathSet res;
@ -306,16 +259,13 @@ PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
} }
} }
void RemoteStore::querySubstitutablePathInfos(const PathSet& paths, void RemoteStore::querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos & infos) SubstitutablePathInfos& infos) {
{
if (paths.empty()) return; if (paths.empty()) return;
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
for (auto& i : paths) { for (auto& i : paths) {
SubstitutablePathInfo info; SubstitutablePathInfo info;
conn->to << wopQuerySubstitutablePathInfo << i; conn->to << wopQuerySubstitutablePathInfo << i;
@ -331,7 +281,6 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
} }
} else { } else {
conn->to << wopQuerySubstitutablePathInfos << paths; conn->to << wopQuerySubstitutablePathInfos << paths;
conn.processStderr(); conn.processStderr();
size_t count = readNum<size_t>(conn->from); size_t count = readNum<size_t>(conn->from);
@ -344,14 +293,12 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
info.downloadSize = readLongLong(conn->from); info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from);
} }
} }
} }
void RemoteStore::queryPathInfoUncached(
void RemoteStore::queryPathInfoUncached(const Path & path, const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
{
try { try {
std::shared_ptr<ValidPathInfo> info; std::shared_ptr<ValidPathInfo> info;
{ {
@ -366,7 +313,8 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
throw; throw;
} }
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid; conn->from >> valid; bool valid;
conn->from >> valid;
if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
} }
info = std::make_shared<ValidPathInfo>(); info = std::make_shared<ValidPathInfo>();
@ -383,13 +331,12 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
} }
} }
callback(std::move(info)); callback(std::move(info));
} catch (...) { callback.rethrow(); } } catch (...) {
callback.rethrow();
}
} }
void RemoteStore::queryReferrers(const Path& path, PathSet& referrers) {
void RemoteStore::queryReferrers(const Path & path,
PathSet & referrers)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryReferrers << path; conn->to << wopQueryReferrers << path;
conn.processStderr(); conn.processStderr();
@ -397,36 +344,28 @@ void RemoteStore::queryReferrers(const Path & path,
referrers.insert(referrers2.begin(), referrers2.end()); referrers.insert(referrers2.begin(), referrers2.end());
} }
PathSet RemoteStore::queryValidDerivers(const Path& path) {
PathSet RemoteStore::queryValidDerivers(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryValidDerivers << path; conn->to << wopQueryValidDerivers << path;
conn.processStderr(); conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from); return readStorePaths<PathSet>(*this, conn->from);
} }
PathSet RemoteStore::queryDerivationOutputs(const Path& path) {
PathSet RemoteStore::queryDerivationOutputs(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << path; conn->to << wopQueryDerivationOutputs << path;
conn.processStderr(); conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from); return readStorePaths<PathSet>(*this, conn->from);
} }
PathSet RemoteStore::queryDerivationOutputNames(const Path& path) {
PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryDerivationOutputNames << path; conn->to << wopQueryDerivationOutputNames << path;
conn.processStderr(); conn.processStderr();
return readStrings<PathSet>(conn->from); return readStrings<PathSet>(conn->from);
} }
Path RemoteStore::queryPathFromHashPart(const string& hashPart) {
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopQueryPathFromHashPart << hashPart; conn->to << wopQueryPathFromHashPart << hashPart;
conn.processStderr(); conn.processStderr();
@ -435,10 +374,9 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
return path; return path;
} }
void RemoteStore::addToStore(const ValidPathInfo& info, Source& source, void RemoteStore::addToStore(const ValidPathInfo& info, Source& source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) RepairFlag repair, CheckSigsFlag checkSigs,
{ std::shared_ptr<FSAccessor> accessor) {
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
@ -448,11 +386,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
sink << 1 // == path follows sink << 1 // == path follows
; ;
copyNAR(source, sink); copyNAR(source, sink);
sink sink << exportMagic << info.path << info.references << info.deriver
<< exportMagic
<< info.path
<< info.references
<< info.deriver
<< 0 // == no legacy signature << 0 // == no legacy signature
<< 0 // == no path follows << 0 // == no path follows
; ;
@ -465,31 +399,32 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
} }
else { else {
conn->to << wopAddToStoreNar conn->to << wopAddToStoreNar << info.path << info.deriver
<< info.path << info.deriver << info.narHash.to_string(Base16, false) << info.narHash.to_string(Base16, false) << info.references
<< info.references << info.registrationTime << info.narSize << info.registrationTime << info.narSize << info.ultimate
<< info.ultimate << info.sigs << info.ca << info.sigs << info.ca << repair << !checkSigs;
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to); if (!tunnel) copyNAR(source, conn->to);
conn.processStderr(0, tunnel ? &source : nullptr); conn.processStderr(0, tunnel ? &source : nullptr);
} }
} }
Path RemoteStore::addToStore(const string& name, const Path& _srcPath, Path RemoteStore::addToStore(const string& name, const Path& _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) bool recursive, HashType hashAlgo,
{ PathFilter& filter, RepairFlag repair) {
if (repair) throw Error("repairing is not supported when building through the Nix daemon"); if (repair)
throw Error(
"repairing is not supported when building through the Nix daemon");
auto conn(getConnection()); auto conn(getConnection());
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
conn->to << wopAddToStore << name conn->to << wopAddToStore << name
<< ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ << ((hashAlgo == htSHA256 && recursive)
<< (recursive ? 1 : 0) ? 0
<< printHashType(hashAlgo); : 1) /* backwards compatibility hack */
<< (recursive ? 1 : 0) << printHashType(hashAlgo);
try { try {
conn->to.written = 0; conn->to.written = 0;
@ -504,21 +439,21 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
} catch (SysError& e) { } catch (SysError& e) {
/* Daemon closed while we were sending the path. Probably OOM /* Daemon closed while we were sending the path. Probably OOM
or I/O error. */ or I/O error. */
if (e.errNo == EPIPE) if (e.errNo == EPIPE) try {
try {
conn.processStderr(); conn.processStderr();
} catch (EndOfFile & e) { } } catch (EndOfFile& e) {
}
throw; throw;
} }
return readStorePath(*this, conn->from); return readStorePath(*this, conn->from);
} }
Path RemoteStore::addTextToStore(const string& name, const string& s, Path RemoteStore::addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair) const PathSet& references, RepairFlag repair) {
{ if (repair)
if (repair) throw Error("repairing is not supported when building through the Nix daemon"); throw Error(
"repairing is not supported when building through the Nix daemon");
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopAddTextToStore << name << s << references; conn->to << wopAddTextToStore << name << s << references;
@ -527,9 +462,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
return readStorePath(*this, conn->from); return readStorePath(*this, conn->from);
} }
void RemoteStore::buildPaths(const PathSet& drvPaths, BuildMode buildMode) {
void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopBuildPaths; conn->to << wopBuildPaths;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
@ -540,23 +473,23 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
/* Old daemons did not take a 'buildMode' parameter, so we /* Old daemons did not take a 'buildMode' parameter, so we
need to validate it here on the client side. */ need to validate it here on the client side. */
if (buildMode != bmNormal) if (buildMode != bmNormal)
throw Error("repairing or checking is not supported when building through the Nix daemon"); throw Error(
"repairing or checking is not supported when building through the "
"Nix daemon");
} else { } else {
/* For backwards compatibility with old daemons, strip output /* For backwards compatibility with old daemons, strip output
identifiers. */ identifiers. */
PathSet drvPaths2; PathSet drvPaths2;
for (auto & i : drvPaths) for (auto& i : drvPaths) drvPaths2.insert(string(i, 0, i.find('!')));
drvPaths2.insert(string(i, 0, i.find('!')));
conn->to << drvPaths2; conn->to << drvPaths2;
} }
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
BuildResult RemoteStore::buildDerivation(const Path& drvPath,
BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, const BasicDerivation& drv,
BuildMode buildMode) BuildMode buildMode) {
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopBuildDerivation << drvPath << drv << buildMode; conn->to << wopBuildDerivation << drvPath << drv << buildMode;
conn.processStderr(); conn.processStderr();
@ -567,45 +500,35 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
return res; return res;
} }
void RemoteStore::ensurePath(const Path& path) {
void RemoteStore::ensurePath(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopEnsurePath << path; conn->to << wopEnsurePath << path;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
void RemoteStore::addTempRoot(const Path& path) {
void RemoteStore::addTempRoot(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopAddTempRoot << path; conn->to << wopAddTempRoot << path;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
void RemoteStore::addIndirectRoot(const Path& path) {
void RemoteStore::addIndirectRoot(const Path & path)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopAddIndirectRoot << path; conn->to << wopAddIndirectRoot << path;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
void RemoteStore::syncWithGC() {
void RemoteStore::syncWithGC()
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopSyncWithGC; conn->to << wopSyncWithGC;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
Roots RemoteStore::findRoots(bool censor) {
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopFindRoots; conn->to << wopFindRoots;
conn.processStderr(); conn.processStderr();
@ -619,13 +542,11 @@ Roots RemoteStore::findRoots(bool censor)
return result; return result;
} }
void RemoteStore::collectGarbage(const GCOptions& options, GCResults& results) {
void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to conn->to << wopCollectGarbage << options.action << options.pathsToDelete
<< wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness << options.ignoreLiveness
<< options.maxFreed << options.maxFreed
/* removed options */ /* removed options */
<< 0 << 0 << 0; << 0 << 0 << 0;
@ -642,38 +563,31 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
} }
} }
void RemoteStore::optimiseStore() {
void RemoteStore::optimiseStore()
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopOptimiseStore; conn->to << wopOptimiseStore;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) {
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopVerifyStore << checkContents << repair; conn->to << wopVerifyStore << checkContents << repair;
conn.processStderr(); conn.processStderr();
return readInt(conn->from); return readInt(conn->from);
} }
void RemoteStore::addSignatures(const Path& storePath, const StringSet& sigs) {
void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
{
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopAddSignatures << storePath << sigs; conn->to << wopAddSignatures << storePath << sigs;
conn.processStderr(); conn.processStderr();
readInt(conn->from); readInt(conn->from);
} }
void RemoteStore::queryMissing(const PathSet& targets, PathSet& willBuild,
void RemoteStore::queryMissing(const PathSet & targets, PathSet& willSubstitute, PathSet& unknown,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long& downloadSize,
unsigned long long & downloadSize, unsigned long long & narSize) unsigned long long& narSize) {
{
{ {
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
@ -690,32 +604,20 @@ void RemoteStore::queryMissing(const PathSet & targets,
} }
fallback: fallback:
return Store::queryMissing(targets, willBuild, willSubstitute, return Store::queryMissing(targets, willBuild, willSubstitute, unknown,
unknown, downloadSize, narSize); downloadSize, narSize);
} }
void RemoteStore::connect() { auto conn(getConnection()); }
void RemoteStore::connect() unsigned int RemoteStore::getProtocol() {
{
auto conn(getConnection());
}
unsigned int RemoteStore::getProtocol()
{
auto conn(connections->get()); auto conn(connections->get());
return conn->daemonVersion; return conn->daemonVersion;
} }
void RemoteStore::flushBadConnections() { connections->flushBad(); }
void RemoteStore::flushBadConnections() RemoteStore::Connection::~Connection() {
{
connections->flushBad();
}
RemoteStore::Connection::~Connection()
{
try { try {
to.flush(); to.flush();
} catch (...) { } catch (...) {
@ -723,9 +625,7 @@ RemoteStore::Connection::~Connection()
} }
} }
static Logger::Fields readFields(Source& from) {
static Logger::Fields readFields(Source & from)
{
Logger::Fields fields; Logger::Fields fields;
size_t size = readInt(from); size_t size = readInt(from);
for (size_t n = 0; n < size; n++) { for (size_t n = 0; n < size; n++) {
@ -740,13 +640,11 @@ static Logger::Fields readFields(Source & from)
return fields; return fields;
} }
std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink,
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) Source* source) {
{
to.flush(); to.flush();
while (true) { while (true) {
auto msg = readNum<uint64_t>(from); auto msg = readNum<uint64_t>(from);
if (msg == STDERR_WRITE) { if (msg == STDERR_WRITE) {
@ -806,12 +704,12 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
static std::string uriScheme = "unix://"; static std::string uriScheme = "unix://";
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); return std::make_shared<UDSRemoteStore>(
std::string(uri, uriScheme.size()), params);
}); });
} } // namespace nix

View file

@ -2,31 +2,28 @@
#include <limits> #include <limits>
#include <string> #include <string>
#include "store-api.hh" #include "store-api.hh"
namespace nix { namespace nix {
class Pipe; class Pipe;
class Pid; class Pid;
struct FdSink; struct FdSink;
struct FdSource; struct FdSource;
template<typename T> class Pool; template <typename T>
class Pool;
struct ConnectionHandle; struct ConnectionHandle;
/* FIXME: RemoteStore is a misnomer - should be something like /* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */ DaemonStore. */
class RemoteStore : public virtual Store class RemoteStore : public virtual Store {
{
public: public:
const Setting<int> maxConnections{
(Store*)this, 1, "max-connections",
"maximum number of concurrent connections to the Nix daemon"};
const Setting<int> maxConnections{(Store*) this, 1, const Setting<unsigned int> maxConnectionAge{
"max-connections", "maximum number of concurrent connections to the Nix daemon"}; (Store*)this, std::numeric_limits<unsigned int>::max(),
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"}; "max-connection-age", "number of seconds to reuse a connection"};
virtual bool sameMachine() = 0; virtual bool sameMachine() = 0;
@ -37,12 +34,13 @@ public:
bool isValidPathUncached(const Path& path) override; bool isValidPathUncached(const Path& path) override;
PathSet queryValidPaths(const PathSet & paths, PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
SubstituteFlag maybeSubstitute = NoSubstitute) override; NoSubstitute) override;
PathSet queryAllValidPaths() override; PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path, void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path& path, PathSet& referrers) override; void queryReferrers(const Path& path, PathSet& referrers) override;
@ -60,13 +58,14 @@ public:
void querySubstitutablePathInfos(const PathSet& paths, void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override; SubstitutablePathInfos& infos) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo& info, Source& nar, RepairFlag repair,
RepairFlag repair, CheckSigsFlag checkSigs, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string& name, const Path& srcPath, Path addToStore(const string& name, const Path& srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; PathFilter& filter = defaultPathFilter,
RepairFlag repair = NoRepair) override;
Path addTextToStore(const string& name, const string& s, Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override; const PathSet& references, RepairFlag repair) override;
@ -94,9 +93,10 @@ public:
void addSignatures(const Path& storePath, const StringSet& sigs) override; void addSignatures(const Path& storePath, const StringSet& sigs) override;
void queryMissing(const PathSet & targets, void queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, PathSet& willSubstitute, PathSet& unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override; unsigned long long& downloadSize,
unsigned long long& narSize) override;
void connect() override; void connect() override;
@ -105,9 +105,7 @@ public:
void flushBadConnections(); void flushBadConnections();
protected: protected:
struct Connection {
struct Connection
{
AutoCloseFD fd; AutoCloseFD fd;
FdSink to; FdSink to;
FdSource from; FdSource from;
@ -134,28 +132,21 @@ protected:
friend struct ConnectionHandle; friend struct ConnectionHandle;
private: private:
std::atomic_bool failed{false}; std::atomic_bool failed{false};
}; };
class UDSRemoteStore : public LocalFSStore, public RemoteStore class UDSRemoteStore : public LocalFSStore, public RemoteStore {
{
public: public:
UDSRemoteStore(const Params& params); UDSRemoteStore(const Params& params);
UDSRemoteStore(std::string path, const Params& params); UDSRemoteStore(std::string path, const Params& params);
std::string getUri() override; std::string getUri() override;
bool sameMachine() bool sameMachine() { return true; }
{ return true; }
private: private:
ref<RemoteStore::Connection> openConnection() override; ref<RemoteStore::Connection> openConnection() override;
std::optional<std::string> path; std::optional<std::string> path;
}; };
} // namespace nix
}

View file

@ -1,14 +1,6 @@
#if ENABLE_S3 #if ENABLE_S3
#include "s3.hh"
#include "s3-binary-cache-store.hh" #include "s3-binary-cache-store.hh"
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h> #include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h> #include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h> #include <aws/core/auth/AWSCredentialsProvider.h>
@ -24,13 +16,19 @@
#include <aws/s3/model/ListObjectsRequest.h> #include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h> #include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h> #include <aws/transfer/TransferManager.h>
#include "compression.hh"
#include "download.hh"
#include "globals.hh"
#include "istringstream_nocopy.hh"
#include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include "s3.hh"
using namespace Aws::Transfer; using namespace Aws::Transfer;
namespace nix { namespace nix {
struct S3Error : public Error struct S3Error : public Error {
{
Aws::S3::S3Errors err; Aws::S3::S3Errors err;
S3Error(Aws::S3::S3Errors err, const FormatOrString& fs) S3Error(Aws::S3::S3Errors err, const FormatOrString& fs)
: Error(fs), err(err){}; : Error(fs), err(err){};
@ -39,27 +37,22 @@ struct S3Error : public Error
/* Helper: given an Outcome<R, E>, return R in case of success, or /* Helper: given an Outcome<R, E>, return R in case of success, or
throw an exception in case of an error. */ throw an exception in case of an error. */
template <typename R, typename E> template <typename R, typename E>
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) {
{
if (!outcome.IsSuccess()) if (!outcome.IsSuccess())
throw S3Error( throw S3Error(outcome.GetError().GetErrorType(),
outcome.GetError().GetErrorType(),
fs.s + ": " + outcome.GetError().GetMessage()); fs.s + ": " + outcome.GetError().GetMessage());
return outcome.GetResultWithOwnership(); return outcome.GetResultWithOwnership();
} }
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem {
{
using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
void ProcessFormattedStatement(Aws::String && statement) override void ProcessFormattedStatement(Aws::String&& statement) override {
{
debug("AWS: %s", chomp(statement)); debug("AWS: %s", chomp(statement));
} }
}; };
static void initAWS() static void initAWS() {
{
static std::once_flag flag; static std::once_flag flag;
std::call_once(flag, []() { std::call_once(flag, []() {
Aws::SDKOptions options; Aws::SDKOptions options;
@ -70,8 +63,7 @@ static void initAWS()
if (verbosity >= lvlDebug) { if (verbosity >= lvlDebug) {
options.loggingOptions.logLevel = options.loggingOptions.logLevel =
verbosity == lvlDebug verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug
? Aws::Utils::Logging::LogLevel::Debug
: Aws::Utils::Logging::LogLevel::Trace; : Aws::Utils::Logging::LogLevel::Trace;
options.loggingOptions.logger_create_fn = [options]() { options.loggingOptions.logger_create_fn = [options]() {
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
@ -82,14 +74,18 @@ static void initAWS()
}); });
} }
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) S3Helper::S3Helper(const string& profile, const string& region,
: config(makeConfig(region, scheme, endpoint)) const string& scheme, const string& endpoint)
, client(make_ref<Aws::S3::S3Client>( : config(makeConfig(region, scheme, endpoint)),
client(make_ref<Aws::S3::S3Client>(
profile == "" profile == ""
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>()) std::make_shared<
Aws::Auth::DefaultAWSCredentialsProviderChain>())
: std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())), std::make_shared<
Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(
profile.c_str())),
*config, *config,
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 #if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
@ -97,25 +93,25 @@ S3Helper::S3Helper(const string & profile, const string & region, const string &
#else #else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif #endif
endpoint.empty())) endpoint.empty())) {
{
} }
/* Log AWS retries. */ /* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy class RetryStrategy : public Aws::Client::DefaultRetryStrategy {
{ bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override long attemptedRetries) const override {
{ auto retry =
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
if (retry) if (retry)
printError("AWS error '%s' (%s), will retry in %d ms", printError("AWS error '%s' (%s), will retry in %d ms",
error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); error.GetExceptionName(), error.GetMessage(),
CalculateDelayBeforeNextRetry(error, attemptedRetries));
return retry; return retry;
} }
}; };
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
{ const string& region, const string& scheme, const string& endpoint) {
initAWS(); initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>(); auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = region; res->region = region;
@ -132,30 +128,26 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
return res; return res;
} }
S3Helper::DownloadResult S3Helper::getObject( S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName,
const std::string & bucketName, const std::string & key) const std::string& key) {
{
debug("fetching 's3://%s/%s'...", bucketName, key); debug("fetching 's3://%s/%s'...", bucketName, key);
auto request = auto request =
Aws::S3::Model::GetObjectRequest() Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key);
.WithBucket(bucketName)
.WithKey(key);
request.SetResponseStreamFactory([&]() { request.SetResponseStreamFactory(
return Aws::New<std::stringstream>("STRINGSTREAM"); [&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); });
});
DownloadResult res; DownloadResult res;
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
try { try {
auto result = checkAws(fmt("AWS error fetching '%s'", key), auto result = checkAws(fmt("AWS error fetching '%s'", key),
client->GetObject(request)); client->GetObject(request));
res.data = decompress(result.GetContentEncoding(), res.data =
decompress(result.GetContentEncoding(),
dynamic_cast<std::stringstream&>(result.GetBody()).str()); dynamic_cast<std::stringstream&>(result.GetBody()).str());
} catch (S3Error& e) { } catch (S3Error& e) {
@ -164,24 +156,35 @@ S3Helper::DownloadResult S3Helper::getObject(
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); res.durationMs =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
return res; return res;
} }
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore {
{ const Setting<std::string> profile{
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; const Setting<std::string> region{
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> scheme{
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; this, "", "scheme",
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; "The scheme to use for S3 requests, https by default."};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; const Setting<std::string> endpoint{
const Setting<bool> multipartUpload{ this, "", "endpoint",
this, false, "multipart-upload", "whether to use multi-part uploads"}; "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression",
"compression method for .ls files"};
const Setting<std::string> logCompression{
this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{this, false, "multipart-upload",
"whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{ const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; this, 5 * 1024 * 1024, "buffer-size",
"size (in bytes) of each part in multi-part uploads"};
std::string bucketName; std::string bucketName;
@ -189,41 +192,30 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
S3Helper s3Helper; S3Helper s3Helper;
S3BinaryCacheStoreImpl( S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName)
const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params),
: S3BinaryCacheStore(params) bucketName(bucketName),
, bucketName(bucketName) s3Helper(profile, region, scheme, endpoint) {
, s3Helper(profile, region, scheme, endpoint)
{
diskCache = getNarInfoDiskCache(); diskCache = getNarInfoDiskCache();
} }
std::string getUri() override std::string getUri() override { return "s3://" + bucketName; }
{
return "s3://" + bucketName;
}
void init() override void init() override {
{
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
BinaryCacheStore::init(); BinaryCacheStore::init();
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
} }
} }
const Stats & getS3Stats() override const Stats& getS3Stats() override { return stats; }
{
return stats;
}
/* This is a specialisation of isValidPath() that optimistically /* This is a specialisation of isValidPath() that optimistically
fetches the .narinfo file, rather than first checking for its fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing existence via a HEAD request. Since .narinfos are small, doing
a GET is unlikely to be slower than HEAD. */ a GET is unlikely to be slower than HEAD. */
bool isValidPathUncached(const Path & storePath) override bool isValidPathUncached(const Path& storePath) override {
{
try { try {
queryPathInfo(storePath); queryPathInfo(storePath);
return true; return true;
@ -232,23 +224,22 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
} }
} }
bool fileExists(const std::string & path) override bool fileExists(const std::string& path) override {
{
stats.head++; stats.head++;
auto res = s3Helper.client->HeadObject( auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest()
Aws::S3::Model::HeadObjectRequest()
.WithBucket(bucketName) .WithBucket(bucketName)
.WithKey(path)); .WithKey(path));
if (!res.IsSuccess()) { if (!res.IsSuccess()) {
auto& error = res.GetError(); auto& error = res.GetError();
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND ||
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
// If bucket listing is disabled, 404s turn into 403s // If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
return false; return false;
throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); throw Error(format("AWS error fetching '%s': %s") % path %
error.GetMessage());
} }
return true; return true;
@ -259,17 +250,17 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
void uploadFile(const std::string& path, const std::string& data, void uploadFile(const std::string& path, const std::string& data,
const std::string& mimeType, const std::string& mimeType,
const std::string & contentEncoding) const std::string& contentEncoding) {
{
auto stream = std::make_shared<istringstream_nocopy>(data); auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency(); auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); executor =
std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(
maxThreads);
std::call_once(transferManagerCreated, [&]() std::call_once(transferManagerCreated, [&]() {
{
if (multipartUpload) { if (multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get()); TransferManagerConfiguration transferConfig(executor.get());
@ -278,9 +269,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
transferConfig.uploadProgressCallback = transferConfig.uploadProgressCallback =
[](const TransferManager* transferManager, [](const TransferManager* transferManager,
const std::shared_ptr<const TransferHandle> const std::shared_ptr<const TransferHandle>& transferHandle) {
&transferHandle)
{
// FIXME: find a way to properly abort the multipart upload. // FIXME: find a way to properly abort the multipart upload.
// checkInterrupt(); // checkInterrupt();
debug("upload progress ('%s'): '%d' of '%d' bytes", debug("upload progress ('%s'): '%d' of '%d' bytes",
@ -296,37 +285,35 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
if (transferManager) { if (transferManager) {
if (contentEncoding != "") if (contentEncoding != "")
throw Error("setting a content encoding is not supported with S3 multi-part uploads"); throw Error(
"setting a content encoding is not supported with S3 multi-part "
"uploads");
std::shared_ptr<TransferHandle> transferHandle = std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile( transferManager->UploadFile(stream, bucketName, path, mimeType,
stream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(), Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */); nullptr /*, contentEncoding */);
transferHandle->WaitUntilFinished(); transferHandle->WaitUntilFinished();
if (transferHandle->GetStatus() == TransferStatus::FAILED) if (transferHandle->GetStatus() == TransferStatus::FAILED)
throw Error("AWS error: failed to upload 's3://%s/%s': %s", throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName,
bucketName, path, transferHandle->GetLastError().GetMessage()); path, transferHandle->GetLastError().GetMessage());
if (transferHandle->GetStatus() != TransferStatus::COMPLETED) if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", throw Error(
"AWS error: transfer status of 's3://%s/%s' in unexpected state",
bucketName, path); bucketName, path);
} else { } else {
auto request = Aws::S3::Model::PutObjectRequest()
auto request =
Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName) .WithBucket(bucketName)
.WithKey(path); .WithKey(path);
request.SetContentType(mimeType); request.SetContentType(mimeType);
if (contentEncoding != "") if (contentEncoding != "") request.SetContentEncoding(contentEncoding);
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data); auto stream = std::make_shared<istringstream_nocopy>(data);
@ -351,20 +338,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
} }
void upsertFile(const std::string& path, const std::string& data, void upsertFile(const std::string& path, const std::string& data,
const std::string & mimeType) override const std::string& mimeType) override {
{
if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); uploadFile(path, *compress(narinfoCompression, data), mimeType,
narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls")) else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/")) else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression); uploadFile(path, *compress(logCompression, data), mimeType,
logCompression);
else else
uploadFile(path, data, mimeType, ""); uploadFile(path, data, mimeType, "");
} }
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string& path, Sink& sink) override {
{
stats.get++; stats.get++;
// FIXME: stream output to sink. // FIXME: stream output to sink.
@ -374,33 +361,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
stats.getTimeMs += res.durationMs; stats.getTimeMs += res.durationMs;
if (res.data) { if (res.data) {
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName,
bucketName, path, res.data->size(), res.durationMs); path, res.data->size(), res.durationMs);
sink((unsigned char*)res.data->data(), res.data->size()); sink((unsigned char*)res.data->data(), res.data->size());
} else } else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri());
} }
PathSet queryAllValidPaths() override PathSet queryAllValidPaths() override {
{
PathSet paths; PathSet paths;
std::string marker; std::string marker;
do { do {
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName %
marker);
auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, auto res = checkAws(
s3Helper.client->ListObjects( format("AWS error listing bucket '%s'") % bucketName,
Aws::S3::Model::ListObjectsRequest() s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest()
.WithBucket(bucketName) .WithBucket(bucketName)
.WithDelimiter("/") .WithDelimiter("/")
.WithMarker(marker))); .WithMarker(marker)));
auto& contents = res.GetContents(); auto& contents = res.GetContents();
debug(format("got %d keys, next marker '%s'") debug(format("got %d keys, next marker '%s'") % contents.size() %
% contents.size() % res.GetNextMarker()); res.GetNextMarker());
for (auto object : contents) { for (auto object : contents) {
auto& key = object.GetKey(); auto& key = object.GetKey();
@ -413,19 +401,18 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return paths; return paths;
} }
}; };
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore(
const std::string & uri, const Store::Params & params) [](const std::string& uri,
-> std::shared_ptr<Store> const Store::Params& params) -> std::shared_ptr<Store> {
{
if (std::string(uri, 0, 5) != "s3://") return 0; if (std::string(uri, 0, 5) != "s3://") return 0;
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); auto store =
std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
store->init(); store->init();
return store; return store;
}); });
} } // namespace nix
#endif #endif

View file

@ -1,23 +1,16 @@
#pragma once #pragma once
#include "binary-cache-store.hh"
#include <atomic> #include <atomic>
#include "binary-cache-store.hh"
namespace nix { namespace nix {
class S3BinaryCacheStore : public BinaryCacheStore class S3BinaryCacheStore : public BinaryCacheStore {
{
protected: protected:
S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {}
S3BinaryCacheStore(const Params & params)
: BinaryCacheStore(params)
{ }
public: public:
struct Stats {
struct Stats
{
std::atomic<uint64_t> put{0}; std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0}; std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0}; std::atomic<uint64_t> putTimeMs{0};
@ -30,4 +23,4 @@ public:
virtual const Stats& getS3Stats() = 0; virtual const Stats& getS3Stats() = 0;
}; };
} } // namespace nix

View file

@ -4,30 +4,39 @@
#include "ref.hh" #include "ref.hh"
namespace Aws { namespace Client { class ClientConfiguration; } } namespace Aws {
namespace Aws { namespace S3 { class S3Client; } } namespace Client {
class ClientConfiguration;
}
} // namespace Aws
namespace Aws {
namespace S3 {
class S3Client;
}
} // namespace Aws
namespace nix { namespace nix {
struct S3Helper struct S3Helper {
{
ref<Aws::Client::ClientConfiguration> config; ref<Aws::Client::ClientConfiguration> config;
ref<Aws::S3::S3Client> client; ref<Aws::S3::S3Client> client;
S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); S3Helper(const std::string& profile, const std::string& region,
const std::string& scheme, const std::string& endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region,
const std::string& scheme,
const std::string& endpoint);
struct DownloadResult struct DownloadResult {
{
std::shared_ptr<std::string> data; std::shared_ptr<std::string> data;
unsigned int durationMs; unsigned int durationMs;
}; };
DownloadResult getObject( DownloadResult getObject(const std::string& bucketName,
const std::string & bucketName, const std::string & key); const std::string& key);
}; };
} } // namespace nix
#endif #endif

View file

@ -21,4 +21,4 @@ typedef enum {
cmdAddToStoreNar = 9, cmdAddToStoreNar = 9,
} ServeCommand; } ServeCommand;
} } // namespace nix

View file

@ -1,14 +1,11 @@
#include "sqlite.hh" #include "sqlite.hh"
#include "util.hh"
#include <sqlite3.h> #include <sqlite3.h>
#include <atomic> #include <atomic>
#include "util.hh"
namespace nix { namespace nix {
[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) [[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) {
{
int err = sqlite3_errcode(db); int err = sqlite3_errcode(db);
int exterr = sqlite3_extended_errcode(db); int exterr = sqlite3_extended_errcode(db);
@ -20,20 +17,18 @@ namespace nix {
err == SQLITE_PROTOCOL err == SQLITE_PROTOCOL
? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
: fmt("SQLite database '%s' is busy", path)); : fmt("SQLite database '%s' is busy", path));
} } else
else
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
} }
SQLite::SQLite(const Path & path) SQLite::SQLite(const Path& path) {
{
if (sqlite3_open_v2(path.c_str(), &db, if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path); throw Error(format("cannot open SQLite database '%s'") % path);
} }
SQLite::~SQLite() SQLite::~SQLite() {
{
try { try {
if (db && sqlite3_close(db) != SQLITE_OK) if (db && sqlite3_close(db) != SQLITE_OK)
throwSQLiteError(db, "closing database"); throwSQLiteError(db, "closing database");
@ -42,16 +37,14 @@ SQLite::~SQLite()
} }
} }
void SQLite::exec(const std::string & stmt) void SQLite::exec(const std::string& stmt) {
{
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
}); });
} }
void SQLiteStmt::create(sqlite3 * db, const string & sql) void SQLiteStmt::create(sqlite3* db, const string& sql) {
{
checkInterrupt(); checkInterrupt();
assert(!stmt); assert(!stmt);
if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
@ -60,8 +53,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & sql)
this->sql = sql; this->sql = sql;
} }
SQLiteStmt::~SQLiteStmt() SQLiteStmt::~SQLiteStmt() {
{
try { try {
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
@ -70,32 +62,27 @@ SQLiteStmt::~SQLiteStmt()
} }
} }
SQLiteStmt::Use::Use(SQLiteStmt & stmt) SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) {
: stmt(stmt)
{
assert(stmt.stmt); assert(stmt.stmt);
/* Note: sqlite3_reset() returns the error code for the most /* Note: sqlite3_reset() returns the error code for the most
recent call to sqlite3_step(). So ignore it. */ recent call to sqlite3_step(). So ignore it. */
sqlite3_reset(stmt); sqlite3_reset(stmt);
} }
SQLiteStmt::Use::~Use() SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); }
{
sqlite3_reset(stmt);
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value,
{ bool notNull) {
if (notNull) { if (notNull) {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1,
SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument"); throwSQLiteError(stmt.db, "binding argument");
} else } else
bind(); bind();
return *this; return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) {
{
if (notNull) { if (notNull) {
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument"); throwSQLiteError(stmt.db, "binding argument");
@ -104,69 +91,57 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
return *this; return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::bind() SQLiteStmt::Use& SQLiteStmt::Use::bind() {
{
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument"); throwSQLiteError(stmt.db, "binding argument");
return *this; return *this;
} }
int SQLiteStmt::Use::step() int SQLiteStmt::Use::step() { return sqlite3_step(stmt); }
{
return sqlite3_step(stmt);
}
void SQLiteStmt::Use::exec() void SQLiteStmt::Use::exec() {
{
int r = step(); int r = step();
assert(r != SQLITE_ROW); assert(r != SQLITE_ROW);
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
} }
bool SQLiteStmt::Use::next() bool SQLiteStmt::Use::next() {
{
int r = step(); int r = step();
if (r != SQLITE_DONE && r != SQLITE_ROW) if (r != SQLITE_DONE && r != SQLITE_ROW)
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
return r == SQLITE_ROW; return r == SQLITE_ROW;
} }
std::string SQLiteStmt::Use::getStr(int col) std::string SQLiteStmt::Use::getStr(int col) {
{
auto s = (const char*)sqlite3_column_text(stmt, col); auto s = (const char*)sqlite3_column_text(stmt, col);
assert(s); assert(s);
return s; return s;
} }
int64_t SQLiteStmt::Use::getInt(int col) int64_t SQLiteStmt::Use::getInt(int col) {
{
// FIXME: detect nulls? // FIXME: detect nulls?
return sqlite3_column_int64(stmt, col); return sqlite3_column_int64(stmt, col);
} }
bool SQLiteStmt::Use::isNull(int col) bool SQLiteStmt::Use::isNull(int col) {
{
return sqlite3_column_type(stmt, col) == SQLITE_NULL; return sqlite3_column_type(stmt, col) == SQLITE_NULL;
} }
SQLiteTxn::SQLiteTxn(sqlite3 * db) SQLiteTxn::SQLiteTxn(sqlite3* db) {
{
this->db = db; this->db = db;
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "starting transaction"); throwSQLiteError(db, "starting transaction");
active = true; active = true;
} }
void SQLiteTxn::commit() void SQLiteTxn::commit() {
{
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "committing transaction"); throwSQLiteError(db, "committing transaction");
active = false; active = false;
} }
SQLiteTxn::~SQLiteTxn() SQLiteTxn::~SQLiteTxn() {
{
try { try {
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "aborting transaction"); throwSQLiteError(db, "aborting transaction");
@ -175,8 +150,7 @@ SQLiteTxn::~SQLiteTxn()
} }
} }
void handleSQLiteBusy(const SQLiteBusy & e) void handleSQLiteBusy(const SQLiteBusy& e) {
{
static std::atomic<time_t> lastWarned{0}; static std::atomic<time_t> lastWarned{0};
time_t now = time(0); time_t now = time(0);
@ -195,4 +169,4 @@ void handleSQLiteBusy(const SQLiteBusy & e)
nanosleep(&t, 0); nanosleep(&t, 0);
} }
} } // namespace nix

View file

@ -2,7 +2,6 @@
#include <functional> #include <functional>
#include <string> #include <string>
#include "types.hh" #include "types.hh"
class sqlite3; class sqlite3;
@ -11,14 +10,17 @@ class sqlite3_stmt;
namespace nix { namespace nix {
/* RAII wrapper to close a SQLite database automatically. */ /* RAII wrapper to close a SQLite database automatically. */
struct SQLite struct SQLite {
{
sqlite3* db = 0; sqlite3* db = 0;
SQLite() {} SQLite() {}
SQLite(const Path& path); SQLite(const Path& path);
SQLite(const SQLite& from) = delete; SQLite(const SQLite& from) = delete;
SQLite& operator=(const SQLite& from) = delete; SQLite& operator=(const SQLite& from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } SQLite& operator=(SQLite&& from) {
db = from.db;
from.db = 0;
return *this;
}
~SQLite(); ~SQLite();
operator sqlite3*() { return db; } operator sqlite3*() { return db; }
@ -26,8 +28,7 @@ struct SQLite
}; };
/* RAII wrapper to create and destroy SQLite prepared statements. */ /* RAII wrapper to create and destroy SQLite prepared statements. */
struct SQLiteStmt struct SQLiteStmt {
{
sqlite3* db = 0; sqlite3* db = 0;
sqlite3_stmt* stmt = 0; sqlite3_stmt* stmt = 0;
std::string sql; std::string sql;
@ -38,16 +39,15 @@ struct SQLiteStmt
operator sqlite3_stmt*() { return stmt; } operator sqlite3_stmt*() { return stmt; }
/* Helper for binding / executing statements. */ /* Helper for binding / executing statements. */
class Use class Use {
{
friend struct SQLiteStmt; friend struct SQLiteStmt;
private: private:
SQLiteStmt& stmt; SQLiteStmt& stmt;
unsigned int curArg = 1; unsigned int curArg = 1;
Use(SQLiteStmt& stmt); Use(SQLiteStmt& stmt);
public: public:
~Use(); ~Use();
/* Bind the next parameter. */ /* Bind the next parameter. */
@ -69,16 +69,12 @@ struct SQLiteStmt
bool isNull(int col); bool isNull(int col);
}; };
Use use() Use use() { return Use(*this); }
{
return Use(*this);
}
}; };
/* RAII helper that ensures transactions are aborted unless explicitly /* RAII helper that ensures transactions are aborted unless explicitly
committed. */ committed. */
struct SQLiteTxn struct SQLiteTxn {
{
bool active = false; bool active = false;
sqlite3* db; sqlite3* db;
@ -89,7 +85,6 @@ struct SQLiteTxn
~SQLiteTxn(); ~SQLiteTxn();
}; };
MakeError(SQLiteError, Error); MakeError(SQLiteError, Error);
MakeError(SQLiteBusy, SQLiteError); MakeError(SQLiteBusy, SQLiteError);
@ -100,8 +95,7 @@ void handleSQLiteBusy(const SQLiteBusy & e);
/* Convenience function for retrying a SQLite transaction when the /* Convenience function for retrying a SQLite transaction when the
database is busy. */ database is busy. */
template <typename T> template <typename T>
T retrySQLite(std::function<T()> fun) T retrySQLite(std::function<T()> fun) {
{
while (true) { while (true) {
try { try {
return fun(); return fun();
@ -111,4 +105,4 @@ T retrySQLite(std::function<T()> fun)
} }
} }
} } // namespace nix

View file

@ -1,51 +1,40 @@
#include "store-api.hh"
#include "remote-store.hh"
#include "remote-fs-accessor.hh"
#include "archive.hh" #include "archive.hh"
#include "worker-protocol.hh"
#include "pool.hh" #include "pool.hh"
#include "remote-fs-accessor.hh"
#include "remote-store.hh"
#include "ssh.hh" #include "ssh.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix { namespace nix {
static std::string uriScheme = "ssh-ng://"; static std::string uriScheme = "ssh-ng://";
class SSHStore : public RemoteStore class SSHStore : public RemoteStore {
{
public: public:
const Setting<Path> sshKey{(Store*)this, "", "ssh-key",
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; "path to an SSH private key"};
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; const Setting<bool> compress{(Store*)this, false, "compress",
"whether to compress the connection"};
SSHStore(const std::string& host, const Params& params) SSHStore(const std::string& host, const Params& params)
: Store(params) : Store(params),
, RemoteStore(params) RemoteStore(params),
, host(host) host(host),
, master( master(host, sshKey,
host,
sshKey,
// Use SSH master only if using more than 1 connection. // Use SSH master only if using more than 1 connection.
connections->capacity() > 1, connections->capacity() > 1, compress) {}
compress)
{
}
std::string getUri() override std::string getUri() override { return uriScheme + host; }
{
return uriScheme + host;
}
bool sameMachine() bool sameMachine() { return false; }
{ return false; }
void narFromPath(const Path& path, Sink& sink) override; void narFromPath(const Path& path, Sink& sink) override;
ref<FSAccessor> getFSAccessor() override; ref<FSAccessor> getFSAccessor() override;
private: private:
struct Connection : RemoteStore::Connection {
struct Connection : RemoteStore::Connection
{
std::unique_ptr<SSHMaster::Connection> sshConn; std::unique_ptr<SSHMaster::Connection> sshConn;
}; };
@ -55,8 +44,7 @@ private:
SSHMaster master; SSHMaster master;
void setOptions(RemoteStore::Connection & conn) override void setOptions(RemoteStore::Connection& conn) override{
{
/* TODO Add a way to explicitly ask for some options to be /* TODO Add a way to explicitly ask for some options to be
forwarded. One option: A way to query the daemon for its forwarded. One option: A way to query the daemon for its
settings, and then a series of params to SSHStore like settings, and then a series of params to SSHStore like
@ -66,21 +54,18 @@ private:
}; };
}; };
void SSHStore::narFromPath(const Path & path, Sink & sink) void SSHStore::narFromPath(const Path& path, Sink& sink) {
{
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopNarFromPath << path; conn->to << wopNarFromPath << path;
conn->processStderr(); conn->processStderr();
copyNAR(conn->from, sink); copyNAR(conn->from, sink);
} }
ref<FSAccessor> SSHStore::getFSAccessor() ref<FSAccessor> SSHStore::getFSAccessor() {
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
} }
ref<RemoteStore::Connection> SSHStore::openConnection() ref<RemoteStore::Connection> SSHStore::openConnection() {
{
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand("nix-daemon --stdio"); conn->sshConn = master.startCommand("nix-daemon --stdio");
conn->to = FdSink(conn->sshConn->in.get()); conn->to = FdSink(conn->sshConn->in.get());
@ -89,12 +74,11 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
return conn; return conn;
} }
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore([](const std::string& uri,
const std::string & uri, const Store::Params & params) const Store::Params& params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store> {
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
}); });
} } // namespace nix

View file

@ -2,30 +2,27 @@
namespace nix { namespace nix {
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) SSHMaster::SSHMaster(const std::string& host, const std::string& keyFile,
: host(host) bool useMaster, bool compress, int logFD)
, fakeSSH(host == "localhost") : host(host),
, keyFile(keyFile) fakeSSH(host == "localhost"),
, useMaster(useMaster && !fakeSSH) keyFile(keyFile),
, compress(compress) useMaster(useMaster && !fakeSSH),
, logFD(logFD) compress(compress),
{ logFD(logFD) {
if (host == "" || hasPrefix(host, "-")) if (host == "" || hasPrefix(host, "-"))
throw Error("invalid SSH host name '%s'", host); throw Error("invalid SSH host name '%s'", host);
} }
void SSHMaster::addCommonSSHOpts(Strings & args) void SSHMaster::addCommonSSHOpts(Strings& args) {
{
for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS"))) for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
args.push_back(i); args.push_back(i);
if (!keyFile.empty()) if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile});
args.insert(args.end(), {"-i", keyFile}); if (compress) args.push_back("-C");
if (compress)
args.push_back("-C");
} }
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
{ const std::string& command) {
Path socketPath = startMaster(); Path socketPath = startMaster();
Pipe in, out; Pipe in, out;
@ -36,7 +33,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
ProcessOptions options; ProcessOptions options;
options.dieWithParent = false; options.dieWithParent = false;
conn->sshPid = startProcess([&]() { conn->sshPid = startProcess(
[&]() {
restoreSignals(); restoreSignals();
close(in.writeSide.get()); close(in.writeSide.get());
@ -56,10 +54,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
} else { } else {
args = {"ssh", host.c_str(), "-x", "-a"}; args = {"ssh", host.c_str(), "-x", "-a"};
addCommonSSHOpts(args); addCommonSSHOpts(args);
if (socketPath != "") if (socketPath != "") args.insert(args.end(), {"-S", socketPath});
args.insert(args.end(), {"-S", socketPath}); if (verbosity >= lvlChatty) args.push_back("-v");
if (verbosity >= lvlChatty)
args.push_back("-v");
} }
args.push_back(command); args.push_back(command);
@ -67,8 +63,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
// could not exec ssh/bash // could not exec ssh/bash
throw SysError("unable to execute '%s'", args.front()); throw SysError("unable to execute '%s'", args.front());
}, options); },
options);
in.readSide = -1; in.readSide = -1;
out.writeSide = -1; out.writeSide = -1;
@ -79,15 +75,15 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
return conn; return conn;
} }
Path SSHMaster::startMaster() Path SSHMaster::startMaster() {
{
if (!useMaster) return ""; if (!useMaster) return "";
auto state(state_.lock()); auto state(state_.lock());
if (state->sshMaster != -1) return state->socketPath; if (state->sshMaster != -1) return state->socketPath;
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700)); state->tmpDir =
std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->socketPath = (Path)*state->tmpDir + "/ssh.sock"; state->socketPath = (Path)*state->tmpDir + "/ssh.sock";
@ -97,7 +93,8 @@ Path SSHMaster::startMaster()
ProcessOptions options; ProcessOptions options;
options.dieWithParent = false; options.dieWithParent = false;
state->sshMaster = startProcess([&]() { state->sshMaster = startProcess(
[&]() {
restoreSignals(); restoreSignals();
close(out.readSide.get()); close(out.readSide.get());
@ -105,25 +102,26 @@ Path SSHMaster::startMaster()
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("duping over stdout"); throw SysError("duping over stdout");
Strings args = Strings args = {"ssh", host.c_str(),
{ "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath "-M", "-N",
, "-o", "LocalCommand=echo started" "-S", state->socketPath,
, "-o", "PermitLocalCommand=yes" "-o", "LocalCommand=echo started",
}; "-o", "PermitLocalCommand=yes"};
if (verbosity >= lvlChatty) if (verbosity >= lvlChatty) args.push_back("-v");
args.push_back("-v");
addCommonSSHOpts(args); addCommonSSHOpts(args);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
throw SysError("unable to execute '%s'", args.front()); throw SysError("unable to execute '%s'", args.front());
}, options); },
options);
out.writeSide = -1; out.writeSide = -1;
std::string reply; std::string reply;
try { try {
reply = readLine(out.readSide.get()); reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { } } catch (EndOfFile& e) {
}
if (reply != "started") if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host); throw Error("failed to start SSH master connection to '%s'", host);
@ -131,4 +129,4 @@ Path SSHMaster::startMaster()
return state->socketPath; return state->socketPath;
} }
} } // namespace nix

View file

@ -1,14 +1,12 @@
#pragma once #pragma once
#include "util.hh"
#include "sync.hh" #include "sync.hh"
#include "util.hh"
namespace nix { namespace nix {
class SSHMaster class SSHMaster {
{
private: private:
const std::string host; const std::string host;
bool fakeSSH; bool fakeSSH;
const std::string keyFile; const std::string keyFile;
@ -16,8 +14,7 @@ private:
const bool compress; const bool compress;
const int logFD; const int logFD;
struct State struct State {
{
Pid sshMaster; Pid sshMaster;
std::unique_ptr<AutoDelete> tmpDir; std::unique_ptr<AutoDelete> tmpDir;
Path socketPath; Path socketPath;
@ -28,11 +25,10 @@ private:
void addCommonSSHOpts(Strings& args); void addCommonSSHOpts(Strings& args);
public: public:
SSHMaster(const std::string& host, const std::string& keyFile, bool useMaster,
bool compress, int logFD = -1);
SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); struct Connection {
struct Connection
{
Pid sshPid; Pid sshPid;
AutoCloseFD out, in; AutoCloseFD out, in;
}; };
@ -42,4 +38,4 @@ public:
Path startMaster(); Path startMaster();
}; };
} } // namespace nix

View file

@ -1,41 +1,31 @@
#include "crypto.hh"
#include "globals.hh"
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include <future>
#include "crypto.hh"
#include "derivations.hh"
#include "globals.hh"
#include "json.hh"
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh" #include "util.hh"
#include "derivations.hh"
#include <future>
namespace nix { namespace nix {
bool Store::isInStore(const Path& path) const {
bool Store::isInStore(const Path & path) const
{
return isInDir(path, storeDir); return isInDir(path, storeDir);
} }
bool Store::isStorePath(const Path& path) const {
bool Store::isStorePath(const Path & path) const return isInStore(path) &&
{ path.size() >= storeDir.size() + 1 + storePathHashLen &&
return isInStore(path) path.find('/', storeDir.size() + 1) == Path::npos;
&& path.size() >= storeDir.size() + 1 + storePathHashLen
&& path.find('/', storeDir.size() + 1) == Path::npos;
} }
void Store::assertStorePath(const Path& path) const {
void Store::assertStorePath(const Path & path) const
{
if (!isStorePath(path)) if (!isStorePath(path))
throw Error(format("path '%1%' is not in the Nix store") % path); throw Error(format("path '%1%' is not in the Nix store") % path);
} }
Path Store::toStorePath(const Path& path) const {
Path Store::toStorePath(const Path & path) const
{
if (!isInStore(path)) if (!isInStore(path))
throw Error(format("path '%1%' is not in the Nix store") % path); throw Error(format("path '%1%' is not in the Nix store") % path);
Path::size_type slash = path.find('/', storeDir.size() + 1); Path::size_type slash = path.find('/', storeDir.size() + 1);
@ -45,9 +35,7 @@ Path Store::toStorePath(const Path & path) const
return Path(path, 0, slash); return Path(path, 0, slash);
} }
Path Store::followLinksToStore(const Path& _path) const {
Path Store::followLinksToStore(const Path & _path) const
{
Path path = absPath(_path); Path path = absPath(_path);
while (!isInStore(path)) { while (!isInStore(path)) {
if (!isLink(path)) break; if (!isLink(path)) break;
@ -59,39 +47,36 @@ Path Store::followLinksToStore(const Path & _path) const
return path; return path;
} }
Path Store::followLinksToStorePath(const Path& path) const {
Path Store::followLinksToStorePath(const Path & path) const
{
return toStorePath(followLinksToStore(path)); return toStorePath(followLinksToStore(path));
} }
string storePathToName(const Path& path) {
string storePathToName(const Path & path)
{
auto base = baseNameOf(path); auto base = baseNameOf(path);
assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); assert(base.size() == storePathHashLen ||
return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
return base.size() == storePathHashLen ? ""
: string(base, storePathHashLen + 1);
} }
string storePathToHash(const Path& path) {
string storePathToHash(const Path & path)
{
auto base = baseNameOf(path); auto base = baseNameOf(path);
assert(base.size() >= storePathHashLen); assert(base.size() >= storePathHashLen);
return string(base, 0, storePathHashLen); return string(base, 0, storePathHashLen);
} }
void checkStoreName(const string& name) {
void checkStoreName(const string & name)
{
string validChars = "+-._?="; string validChars = "+-._?=";
auto baseError = format("The path name '%2%' is invalid: %3%. " auto baseError =
format(
"The path name '%2%' is invalid: %3%. "
"Path names are alphanumeric and can include the symbols %1% " "Path names are alphanumeric and can include the symbols %1% "
"and must not begin with a period. " "and must not begin with a period. "
"Note: If '%2%' is a source file and you cannot rename it on " "Note: If '%2%' is a source file and you cannot rename it on "
"disk, builtins.path { name = ... } can be used to give it an " "disk, builtins.path { name = ... } can be used to give it an "
"alternative name.") % validChars % name; "alternative name.") %
validChars % name;
/* Disallow names starting with a dot for possible security /* Disallow names starting with a dot for possible security
reasons (e.g., "." and ".."). */ reasons (e.g., "." and ".."). */
@ -102,16 +87,12 @@ void checkStoreName(const string & name)
if (name.length() > 211) if (name.length() > 211)
throw Error(baseError % "name must be less than 212 characters"); throw Error(baseError % "name must be less than 212 characters");
for (auto& i : name) for (auto& i : name)
if (!((i >= 'A' && i <= 'Z') || if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') ||
(i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) {
(i >= '0' && i <= '9') ||
validChars.find(i) != string::npos))
{
throw Error(baseError % (format("the '%1%' character is invalid") % i)); throw Error(baseError % (format("the '%1%' character is invalid") % i));
} }
} }
/* Store paths have the following form: /* Store paths have the following form:
<store>/<h>-<name> <store>/<h>-<name>
@ -182,44 +163,38 @@ void checkStoreName(const string & name)
"source:". "source:".
*/ */
Path Store::makeStorePath(const string& type, const Hash& hash,
Path Store::makeStorePath(const string & type, const string& name) const {
const Hash & hash, const string & name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
checkStoreName(name); checkStoreName(name);
return storeDir + "/" return storeDir + "/" +
+ compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) +
+ "-" + name; "-" + name;
} }
Path Store::makeOutputPath(const string& id, const Hash& hash,
Path Store::makeOutputPath(const string & id, const string& name) const {
const Hash & hash, const string & name) const
{
return makeStorePath("output:" + id, hash, return makeStorePath("output:" + id, hash,
name + (id == "out" ? "" : "-" + id)); name + (id == "out" ? "" : "-" + id));
} }
Path Store::makeFixedOutputPath(bool recursive, const Hash& hash,
Path Store::makeFixedOutputPath(bool recursive, const string& name) const {
const Hash & hash, const string & name) const
{
return hash.type == htSHA256 && recursive return hash.type == htSHA256 && recursive
? makeStorePath("source", hash, name) ? makeStorePath("source", hash, name)
: makeStorePath("output:out", hashString(htSHA256, : makeStorePath(
"output:out",
hashString(htSHA256,
"fixed:out:" + (recursive ? (string) "r:" : "") + "fixed:out:" + (recursive ? (string) "r:" : "") +
hash.to_string(Base16) + ":"), hash.to_string(Base16) + ":"),
name); name);
} }
Path Store::makeTextPath(const string& name, const Hash& hash, Path Store::makeTextPath(const string& name, const Hash& hash,
const PathSet & references) const const PathSet& references) const {
{
assert(hash.type == htSHA256); assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit /* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be hacky, but we can't put them in `s' since that would be
@ -232,38 +207,28 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
return makeStorePath(type, hash, name); return makeStorePath(type, hash, name);
} }
std::pair<Path, Hash> Store::computeStorePathForPath(const string& name, std::pair<Path, Hash> Store::computeStorePathForPath(const string& name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const const Path& srcPath,
{ bool recursive,
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); HashType hashAlgo,
PathFilter& filter) const {
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
Path dstPath = makeFixedOutputPath(recursive, h, name); Path dstPath = makeFixedOutputPath(recursive, h, name);
return std::pair<Path, Hash>(dstPath, h); return std::pair<Path, Hash>(dstPath, h);
} }
Path Store::computeStorePathForText(const string& name, const string& s, Path Store::computeStorePathForText(const string& name, const string& s,
const PathSet & references) const const PathSet& references) const {
{
return makeTextPath(name, hashString(htSHA256, s), references); return makeTextPath(name, hashString(htSHA256, s), references);
} }
Store::Store(const Params& params) Store::Store(const Params& params)
: Config(params) : Config(params), state({(size_t)pathInfoCacheSize}) {}
, state({(size_t) pathInfoCacheSize})
{
}
std::string Store::getUri() { return ""; }
std::string Store::getUri() bool Store::isValidPath(const Path& storePath) {
{
return "";
}
bool Store::isValidPath(const Path & storePath)
{
assertStorePath(storePath); assertStorePath(storePath);
auto hashPart = storePathToHash(storePath); auto hashPart = storePathToHash(storePath);
@ -282,8 +247,8 @@ bool Store::isValidPath(const Path & storePath)
if (res.first != NarInfoDiskCache::oUnknown) { if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++; stats.narInfoReadAverted++;
auto state_(state.lock()); auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, state_->pathInfoCache.upsert(
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
return res.first == NarInfoDiskCache::oValid; return res.first == NarInfoDiskCache::oValid;
} }
} }
@ -297,11 +262,9 @@ bool Store::isValidPath(const Path & storePath)
return valid; return valid;
} }
/* Default implementation for stores that only implement /* Default implementation for stores that only implement
queryPathInfoUncached(). */ queryPathInfoUncached(). */
bool Store::isValidPathUncached(const Path & path) bool Store::isValidPathUncached(const Path& path) {
{
try { try {
queryPathInfo(path); queryPathInfo(path);
return true; return true;
@ -310,13 +273,10 @@ bool Store::isValidPathUncached(const Path & path)
} }
} }
ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) {
ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
{
std::promise<ref<ValidPathInfo>> promise; std::promise<ref<ValidPathInfo>> promise;
queryPathInfo(storePath, queryPathInfo(storePath, {[&](std::future<ref<ValidPathInfo>> result) {
{[&](std::future<ref<ValidPathInfo>> result) {
try { try {
promise.set_value(result.get()); promise.set_value(result.get());
} catch (...) { } catch (...) {
@ -327,10 +287,8 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
return promise.get_future().get(); return promise.get_future().get();
} }
void Store::queryPathInfo(const Path& storePath, void Store::queryPathInfo(const Path& storePath,
Callback<ref<ValidPathInfo>> callback) noexcept Callback<ref<ValidPathInfo>> callback) noexcept {
{
std::string hashPart; std::string hashPart;
try { try {
@ -354,51 +312,53 @@ void Store::queryPathInfo(const Path & storePath,
stats.narInfoReadAverted++; stats.narInfoReadAverted++;
{ {
auto state_(state.lock()); auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, state_->pathInfoCache.upsert(
hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
if (res.first == NarInfoDiskCache::oInvalid || if (res.first == NarInfoDiskCache::oInvalid ||
(res.second->path != storePath && storePathToName(storePath) != "")) (res.second->path != storePath &&
storePathToName(storePath) != ""))
throw InvalidPath(format("path '%s' is not valid") % storePath); throw InvalidPath(format("path '%s' is not valid") % storePath);
} }
return callback(ref<ValidPathInfo>(res.second)); return callback(ref<ValidPathInfo>(res.second));
} }
} }
} catch (...) { return callback.rethrow(); } } catch (...) {
return callback.rethrow();
}
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath, queryPathInfoUncached(
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { storePath, {[this, storePath, hashPart, callbackPtr](
std::future<std::shared_ptr<ValidPathInfo>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
if (diskCache) if (diskCache) diskCache->upsertNarInfo(getUri(), hashPart, info);
diskCache->upsertNarInfo(getUri(), hashPart, info);
{ {
auto state_(state.lock()); auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, info); state_->pathInfoCache.upsert(hashPart, info);
} }
if (!info if (!info ||
|| (info->path != storePath && storePathToName(storePath) != "")) (info->path != storePath && storePathToName(storePath) != "")) {
{
stats.narInfoMissing++; stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath); throw InvalidPath("path '%s' is not valid", storePath);
} }
(*callbackPtr)(ref<ValidPathInfo>(info)); (*callbackPtr)(ref<ValidPathInfo>(info));
} catch (...) { callbackPtr->rethrow(); } } catch (...) {
callbackPtr->rethrow();
}
}}); }});
} }
PathSet Store::queryValidPaths(const PathSet& paths,
PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) SubstituteFlag maybeSubstitute) {
{ struct State {
struct State
{
size_t left; size_t left;
PathSet valid; PathSet valid;
std::exception_ptr exc; std::exception_ptr exc;
@ -411,7 +371,8 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
auto doQuery = [&](const Path& path) { auto doQuery = [&](const Path& path) {
checkInterrupt(); checkInterrupt();
queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { queryPathInfo(
path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
auto state(state_.lock()); auto state(state_.lock());
try { try {
auto info = fut.get(); auto info = fut.get();
@ -421,13 +382,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
state->exc = std::current_exception(); state->exc = std::current_exception();
} }
assert(state->left); assert(state->left);
if (!--state->left) if (!--state->left) wakeup.notify_one();
wakeup.notify_one();
}}); }});
}; };
for (auto & path : paths) for (auto& path : paths) pool.enqueue(std::bind(doQuery, path));
pool.enqueue(std::bind(doQuery, path));
pool.process(); pool.process();
@ -441,13 +400,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
} }
} }
/* Return a string accepted by decodeValidPathInfo() that /* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */ responsibility of the caller to provide a closure. */
string Store::makeValidityRegistration(const PathSet & paths, string Store::makeValidityRegistration(const PathSet& paths, bool showDerivers,
bool showDerivers, bool showHash) bool showHash) {
{
string s = ""; string s = "";
for (auto& i : paths) { for (auto& i : paths) {
@ -465,17 +422,15 @@ string Store::makeValidityRegistration(const PathSet & paths,
s += (format("%1%\n") % info->references.size()).str(); s += (format("%1%\n") % info->references.size()).str();
for (auto & j : info->references) for (auto& j : info->references) s += j + "\n";
s += j + "\n";
} }
return s; return s;
} }
void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths,
bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) bool includeImpureInfo, bool showClosureSize,
{ AllowInvalidFlag allowInvalid) {
auto jsonList = jsonOut.list(); auto jsonList = jsonOut.list();
for (auto storePath : storePaths) { for (auto storePath : storePaths) {
@ -486,18 +441,15 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
auto info = queryPathInfo(storePath); auto info = queryPathInfo(storePath);
storePath = info->path; storePath = info->path;
jsonPath jsonPath.attr("narHash", info->narHash.to_string())
.attr("narHash", info->narHash.to_string())
.attr("narSize", info->narSize); .attr("narSize", info->narSize);
{ {
auto jsonRefs = jsonPath.list("references"); auto jsonRefs = jsonPath.list("references");
for (auto & ref : info->references) for (auto& ref : info->references) jsonRefs.elem(ref);
jsonRefs.elem(ref);
} }
if (info->ca != "") if (info->ca != "") jsonPath.attr("ca", info->ca);
jsonPath.attr("ca", info->ca);
std::pair<uint64_t, uint64_t> closureSizes; std::pair<uint64_t, uint64_t> closureSizes;
@ -507,28 +459,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
} }
if (includeImpureInfo) { if (includeImpureInfo) {
if (info->deriver != "") jsonPath.attr("deriver", info->deriver);
if (info->deriver != "")
jsonPath.attr("deriver", info->deriver);
if (info->registrationTime) if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime); jsonPath.attr("registrationTime", info->registrationTime);
if (info->ultimate) if (info->ultimate) jsonPath.attr("ultimate", info->ultimate);
jsonPath.attr("ultimate", info->ultimate);
if (!info->sigs.empty()) { if (!info->sigs.empty()) {
auto jsonSigs = jsonPath.list("signatures"); auto jsonSigs = jsonPath.list("signatures");
for (auto & sig : info->sigs) for (auto& sig : info->sigs) jsonSigs.elem(sig);
jsonSigs.elem(sig);
} }
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
if (narInfo) { if (narInfo) {
if (!narInfo->url.empty()) if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url);
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash) if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
if (narInfo->fileSize) if (narInfo->fileSize)
@ -544,9 +491,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
} }
} }
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) {
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
{
uint64_t totalNarSize = 0, totalDownloadSize = 0; uint64_t totalNarSize = 0, totalDownloadSize = 0;
PathSet closure; PathSet closure;
computeFSClosure(storePath, closure, false, false); computeFSClosure(storePath, closure, false, false);
@ -555,15 +500,12 @@ std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
totalNarSize += info->narSize; totalNarSize += info->narSize;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
if (narInfo) if (narInfo) totalDownloadSize += narInfo->fileSize;
totalDownloadSize += narInfo->fileSize;
} }
return {totalNarSize, totalDownloadSize}; return {totalNarSize, totalDownloadSize};
} }
const Store::Stats& Store::getStats() {
const Store::Stats & Store::getStats()
{
{ {
auto state_(state.lock()); auto state_(state.lock());
stats.pathInfoCacheSize = state_->pathInfoCache.size(); stats.pathInfoCacheSize = state_->pathInfoCache.size();
@ -571,21 +513,16 @@ const Store::Stats & Store::getStats()
return stats; return stats;
} }
void Store::buildPaths(const PathSet& paths, BuildMode buildMode) {
void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
{
for (auto& path : paths) for (auto& path : paths)
if (isDerivation(path)) if (isDerivation(path)) unsupported("buildPaths");
unsupported("buildPaths");
if (queryValidPaths(paths).size() != paths.size()) if (queryValidPaths(paths).size() != paths.size()) unsupported("buildPaths");
unsupported("buildPaths");
} }
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) const Path& storePath, RepairFlag repair,
{ CheckSigsFlag checkSigs) {
auto srcUri = srcStore->getUri(); auto srcUri = srcStore->getUri();
auto dstUri = dstStore->getUri(); auto dstUri = dstStore->getUri();
@ -594,7 +531,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
? fmt("copying path '%s' to '%s'", storePath, dstUri) ? fmt("copying path '%s' to '%s'", storePath, dstUri)
: dstUri == "local" || dstUri == "daemon" : dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", storePath, srcUri) ? fmt("copying path '%s' from '%s'", storePath, srcUri)
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), : fmt("copying path '%s' from '%s' to '%s'", storePath,
srcUri, dstUri),
{storePath, srcUri, dstUri}); {storePath, srcUri, dstUri});
PushActivity pact(act.id); PushActivity pact(act.id);
@ -622,24 +560,26 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
info = info2; info = info2;
} }
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource(
[&](Sink& sink) {
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
sink(data, len); sink(data, len);
total += len; total += len;
act.progress(total, info->narSize); act.progress(total, info->narSize);
}); });
srcStore->narFromPath({storePath}, wrapperSink); srcStore->narFromPath({storePath}, wrapperSink);
}, [&]() { },
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); [&]() {
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete",
storePath, srcStore->getUri());
}); });
dstStore->addToStore(*info, *source, repair, checkSigs); dstStore->addToStore(*info, *source, repair, checkSigs);
} }
void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, const PathSet& storePaths, RepairFlag repair,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) CheckSigsFlag checkSigs, SubstituteFlag substitute) {
{
PathSet valid = dstStore->queryValidPaths(storePaths, substitute); PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
PathSet missing; PathSet missing;
@ -648,7 +588,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
if (missing.empty()) return; if (missing.empty()) return;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); Activity act(*logger, lvlInfo, actCopyPaths,
fmt("copying %d paths", missing.size()));
std::atomic<size_t> nrDone{0}; std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0}; std::atomic<size_t> nrFailed{0};
@ -661,8 +602,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
ThreadPool pool; ThreadPool pool;
processGraph<Path>(pool, processGraph<Path>(
PathSet(missing.begin(), missing.end()), pool, PathSet(missing.begin(), missing.end()),
[&](const Path& storePath) { [&](const Path& storePath) {
if (dstStore->isValidPath(storePath)) { if (dstStore->isValidPath(storePath)) {
@ -689,9 +630,9 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
} catch (Error& e) { } catch (Error& e) {
nrFailed++; nrFailed++;
if (!settings.keepGoing) if (!settings.keepGoing) throw e;
throw e; logger->log(lvlError,
logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); format("could not copy %s: %s") % storePath % e.what());
showProgress(); showProgress();
return; return;
} }
@ -702,22 +643,21 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
}); });
} }
void copyClosure(ref<Store> srcStore, ref<Store> dstStore, void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, const PathSet& storePaths, RepairFlag repair,
SubstituteFlag substitute) CheckSigsFlag checkSigs, SubstituteFlag substitute) {
{
PathSet closure; PathSet closure;
srcStore->computeFSClosure({storePaths}, closure); srcStore->computeFSClosure({storePaths}, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
} }
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) {
ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
{
ValidPathInfo info; ValidPathInfo info;
getline(str, info.path); getline(str, info.path);
if (str.eof()) { info.path = ""; return info; } if (str.eof()) {
info.path = "";
return info;
}
if (hashGiven) { if (hashGiven) {
string s; string s;
getline(str, s); getline(str, s);
@ -726,7 +666,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
if (!string2Int(s, info.narSize)) throw Error("number expected"); if (!string2Int(s, info.narSize)) throw Error("number expected");
} }
getline(str, info.deriver); getline(str, info.deriver);
string s; int n; string s;
int n;
getline(str, s); getline(str, s);
if (!string2Int(s, n)) throw Error("number expected"); if (!string2Int(s, n)) throw Error("number expected");
while (n--) { while (n--) {
@ -737,9 +678,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
return info; return info;
} }
string showPaths(const PathSet& paths) {
string showPaths(const PathSet & paths)
{
string s; string s;
for (auto& i : paths) { for (auto& i : paths) {
if (s.size() != 0) s += ", "; if (s.size() != 0) s += ", ";
@ -748,30 +687,24 @@ string showPaths(const PathSet & paths)
return s; return s;
} }
std::string ValidPathInfo::fingerprint() const {
std::string ValidPathInfo::fingerprint() const
{
if (narSize == 0 || !narHash) if (narSize == 0 || !narHash)
throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") throw Error(format("cannot calculate fingerprint of path '%s' because its "
% path); "size/hash is not known") %
return path);
"1;" + path + ";" return "1;" + path + ";" + narHash.to_string(Base32) + ";" +
+ narHash.to_string(Base32) + ";" std::to_string(narSize) + ";" + concatStringsSep(",", references);
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", references);
} }
void ValidPathInfo::sign(const SecretKey& secretKey) {
void ValidPathInfo::sign(const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint())); sigs.insert(secretKey.signDetached(fingerprint()));
} }
bool ValidPathInfo::isContentAddressed(const Store& store) const {
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto warn = [&]() { auto warn = [&]() {
printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); printError(
format("warning: path '%s' claims to be content-addressed but isn't") %
path);
}; };
if (hasPrefix(ca, "text:")) { if (hasPrefix(ca, "text:")) {
@ -786,7 +719,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
bool recursive = ca.compare(6, 2, "r:") == 0; bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6)); Hash hash(std::string(ca, recursive ? 8 : 6));
if (references.empty() && if (references.empty() &&
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) store.makeFixedOutputPath(recursive, hash, storePathToName(path)) ==
path)
return true; return true;
else else
warn(); warn();
@ -795,70 +729,58 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
return false; return false;
} }
size_t ValidPathInfo::checkSignatures(const Store& store,
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const const PublicKeys& publicKeys) const {
{
if (isContentAddressed(store)) return maxSigs; if (isContentAddressed(store)) return maxSigs;
size_t good = 0; size_t good = 0;
for (auto& sig : sigs) for (auto& sig : sigs)
if (checkSignature(publicKeys, sig)) if (checkSignature(publicKeys, sig)) good++;
good++;
return good; return good;
} }
bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys,
bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const const std::string& sig) const {
{
return verifyDetached(fingerprint(), sig, publicKeys); return verifyDetached(fingerprint(), sig, publicKeys);
} }
Strings ValidPathInfo::shortRefs() const {
Strings ValidPathInfo::shortRefs() const
{
Strings refs; Strings refs;
for (auto & r : references) for (auto& r : references) refs.push_back(baseNameOf(r));
refs.push_back(baseNameOf(r));
return refs; return refs;
} }
std::string makeFixedOutputCA(bool recursive, const Hash& hash) {
std::string makeFixedOutputCA(bool recursive, const Hash & hash)
{
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
} }
void Store::addToStore(const ValidPathInfo& info, Source& narSource, void Store::addToStore(const ValidPathInfo& info, Source& narSource,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) std::shared_ptr<FSAccessor> accessor) {
{ addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs,
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); accessor);
} }
void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar, void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) std::shared_ptr<FSAccessor> accessor) {
{
StringSource source(*nar); StringSource source(*nar);
addToStore(info, source, repair, checkSigs, accessor); addToStore(info, source, repair, checkSigs, accessor);
} }
} } // namespace nix
#include "local-store.hh" #include "local-store.hh"
#include "remote-store.hh" #include "remote-store.hh"
namespace nix { namespace nix {
RegisterStoreImplementation::Implementations*
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; RegisterStoreImplementation::implementations = 0;
/* Split URI into protocol+hierarchy part and its parameter set. */ /* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) std::pair<std::string, Store::Params> splitUriAndParams(
{ const std::string& uri_) {
auto uri(uri_); auto uri(uri_);
Store::Params params; Store::Params params;
auto q = uri.find('?'); auto q = uri.find('?');
@ -890,8 +812,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
} }
ref<Store> openStore(const std::string& uri_, ref<Store> openStore(const std::string& uri_,
const Store::Params & extraParams) const Store::Params& extraParams) {
{
auto [uri, uriParams] = splitUriAndParams(uri_); auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams; auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end()); params.insert(uriParams.begin(), uriParams.end());
@ -907,9 +828,7 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri); throw Error("don't know how to open Nix store '%s'", uri);
} }
StoreType getStoreType(const std::string& uri, const std::string& stateDir) {
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
if (uri == "daemon") { if (uri == "daemon") {
return tDaemon; return tDaemon;
} else if (uri == "local" || hasPrefix(uri, "/")) { } else if (uri == "local" || hasPrefix(uri, "/")) {
@ -926,18 +845,15 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
} }
} }
static RegisterStoreImplementation regStore([](const std::string& uri,
static RegisterStoreImplementation regStore([]( const Store::Params& params)
const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> {
-> std::shared_ptr<Store>
{
switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
case tDaemon: case tDaemon:
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: { case tLocal: {
Store::Params params2 = params; Store::Params params2 = params;
if (hasPrefix(uri, "/")) if (hasPrefix(uri, "/")) params2["root"] = uri;
params2["root"] = uri;
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
} }
default: default:
@ -945,9 +861,7 @@ static RegisterStoreImplementation regStore([](
} }
}); });
std::list<ref<Store>> getDefaultSubstituters() {
std::list<ref<Store>> getDefaultSubstituters()
{
static auto stores([]() { static auto stores([]() {
std::list<ref<Store>> stores; std::list<ref<Store>> stores;
@ -963,11 +877,9 @@ std::list<ref<Store>> getDefaultSubstituters()
} }
}; };
for (auto uri : settings.substituters.get()) for (auto uri : settings.substituters.get()) addStore(uri);
addStore(uri);
for (auto uri : settings.extraSubstituters.get()) for (auto uri : settings.extraSubstituters.get()) addStore(uri);
addStore(uri);
stores.sort([](ref<Store>& a, ref<Store>& b) { stores.sort([](ref<Store>& a, ref<Store>& b) {
return a->getPriority() < b->getPriority(); return a->getPriority() < b->getPriority();
@ -979,5 +891,4 @@ std::list<ref<Store>> getDefaultSubstituters()
return stores; return stores;
} }
} // namespace nix
}

View file

@ -1,32 +1,26 @@
#pragma once #pragma once
#include "hash.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
#include <atomic> #include <atomic>
#include <limits> #include <limits>
#include <map> #include <map>
#include <unordered_map>
#include <unordered_set>
#include <memory> #include <memory>
#include <string> #include <string>
#include <unordered_map>
#include <unordered_set>
#include "config.hh"
#include "crypto.hh"
#include "globals.hh"
#include "hash.hh"
#include "lru-cache.hh"
#include "serialise.hh"
#include "sync.hh"
namespace nix { namespace nix {
MakeError(SubstError, Error) MakeError(SubstError, Error)
MakeError(BuildError, Error) /* denotes a permanent build failure */ MakeError(BuildError, Error) /* denotes a permanent build failure */
MakeError(InvalidPath, Error) MakeError(InvalidPath, Error) MakeError(Unsupported, Error)
MakeError(Unsupported, Error) MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error)
MakeError(SubstituteGone, Error)
MakeError(SubstituterDisabled, Error)
struct BasicDerivation; struct BasicDerivation;
struct Derivation; struct Derivation;
@ -35,25 +29,20 @@ class NarInfoDiskCache;
class Store; class Store;
class JSONPlaceholder; class JSONPlaceholder;
enum RepairFlag : bool { NoRepair = false, Repair = true }; enum RepairFlag : bool { NoRepair = false, Repair = true };
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Size of the hash part of store paths, in base-32 characters. */ /* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits const size_t storePathHashLen = 32; // i.e. 160 bits
/* Magic header of exportPath() output (obsolete). */ /* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e; const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
struct GCOptions {
struct GCOptions
{
/* Garbage collector operation: /* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from - `gcReturnLive': return the set of paths reachable from
@ -89,9 +78,7 @@ struct GCOptions
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
}; };
struct GCResults {
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would /* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */ be or have been deleted. */
PathSet paths; PathSet paths;
@ -101,9 +88,7 @@ struct GCResults
unsigned long long bytesFreed = 0; unsigned long long bytesFreed = 0;
}; };
struct SubstitutablePathInfo {
struct SubstitutablePathInfo
{
Path deriver; Path deriver;
PathSet references; PathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */ unsigned long long downloadSize; /* 0 = unknown or inapplicable */
@ -112,9 +97,7 @@ struct SubstitutablePathInfo
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo {
struct ValidPathInfo
{
Path path; Path path;
Path deriver; Path deriver;
Hash narHash; Hash narHash;
@ -157,12 +140,8 @@ struct ValidPathInfo
*/ */
std::string ca; std::string ca;
bool operator == (const ValidPathInfo & i) const bool operator==(const ValidPathInfo& i) const {
{ return path == i.path && narHash == i.narHash && references == i.references;
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
} }
/* Return a fingerprint of the store path to be used in binary /* Return a fingerprint of the store path to be used in binary
@ -183,10 +162,12 @@ struct ValidPathInfo
/* Return the number of signatures on this .narinfo that were /* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path produced by one of the specified keys, or maxSigs if the path
is content-addressed. */ is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; size_t checkSignatures(const Store& store,
const PublicKeys& publicKeys) const;
/* Verify a single signature. */ /* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; bool checkSignature(const PublicKeys& publicKeys,
const std::string& sig) const;
Strings shortRefs() const; Strings shortRefs() const;
@ -195,12 +176,9 @@ struct ValidPathInfo
typedef list<ValidPathInfo> ValidPathInfos; typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck }; enum BuildMode { bmNormal, bmRepair, bmCheck };
struct BuildResult {
struct BuildResult
{
/* Note: don't remove status codes, and only add new status codes /* Note: don't remove status codes, and only add new status codes
at the end of the list, to prevent client/server at the end of the list, to prevent client/server
incompatibilities in the nix-store --serve protocol. */ incompatibilities in the nix-store --serve protocol. */
@ -239,25 +217,25 @@ struct BuildResult
} }
}; };
class Store : public std::enable_shared_from_this<Store>, public Config {
class Store : public std::enable_shared_from_this<Store>, public Config
{
public: public:
typedef std::map<std::string, std::string> Params; typedef std::map<std::string, std::string> Params;
const PathSetting storeDir_{this, false, settings.nixStore, const PathSetting storeDir_{this, false, settings.nixStore, "store",
"store", "path to the Nix store"}; "path to the Nix store"};
const Path storeDir = storeDir_; const Path storeDir = storeDir_;
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; const Setting<int> pathInfoCacheSize{
this, 65536, "path-info-cache-size",
"size of the in-memory store path information cache"};
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; const Setting<bool> isTrusted{
this, false, "trusted",
"whether paths from this store can be used as substitutes even when they "
"lack trusted signatures"};
protected: protected:
struct State {
struct State
{
LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache; LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
}; };
@ -268,7 +246,6 @@ protected:
Store(const Params& params); Store(const Params& params);
public: public:
virtual ~Store() {} virtual ~Store() {}
virtual std::string getUri() = 0; virtual std::string getUri() = 0;
@ -296,14 +273,14 @@ public:
Path followLinksToStorePath(const Path& path) const; Path followLinksToStorePath(const Path& path) const;
/* Constructs a unique store path name. */ /* Constructs a unique store path name. */
Path makeStorePath(const string & type, Path makeStorePath(const string& type, const Hash& hash,
const Hash & hash, const string & name) const; const string& name) const;
Path makeOutputPath(const string & id, Path makeOutputPath(const string& id, const Hash& hash,
const Hash & hash, const string & name) const; const string& name) const;
Path makeFixedOutputPath(bool recursive, Path makeFixedOutputPath(bool recursive, const Hash& hash,
const Hash & hash, const string & name) const; const string& name) const;
Path makeTextPath(const string& name, const Hash& hash, Path makeTextPath(const string& name, const Hash& hash,
const PathSet& references) const; const PathSet& references) const;
@ -311,9 +288,10 @@ public:
/* This is the preparatory part of addToStore(); it computes the /* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */ path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const string & name, std::pair<Path, Hash> computeStorePathForPath(
const Path & srcPath, bool recursive = true, const string& name, const Path& srcPath, bool recursive = true,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; HashType hashAlgo = htSHA256,
PathFilter& filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore(). /* Preparatory part of addTextToStore().
@ -336,23 +314,20 @@ public:
bool isValidPath(const Path& path); bool isValidPath(const Path& path);
protected: protected:
virtual bool isValidPathUncached(const Path& path); virtual bool isValidPathUncached(const Path& path);
public: public:
/* Query which of the given paths is valid. Optionally, try to /* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */ substitute missing paths. */
virtual PathSet queryValidPaths(const PathSet & paths, virtual PathSet queryValidPaths(
SubstituteFlag maybeSubstitute = NoSubstitute); const PathSet& paths, SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store /* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash> rather than (i.e. you'll get /nix/store/<hash> rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */ full store path. */
virtual PathSet queryAllValidPaths() virtual PathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); }
{ unsupported("queryAllValidPaths"); }
/* Query information about a valid path. It is permitted to omit /* Query information about a valid path. It is permitted to omit
the name part of the store path. */ the name part of the store path. */
@ -363,16 +338,16 @@ public:
Callback<ref<ValidPathInfo>> callback) noexcept; Callback<ref<ValidPathInfo>> callback) noexcept;
protected: protected:
virtual void queryPathInfoUncached(
virtual void queryPathInfoUncached(const Path & path, const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
public: public:
/* Queries the set of incoming FS references for a store path. /* Queries the set of incoming FS references for a store path.
The result is not cleared. */ The result is not cleared. */
virtual void queryReferrers(const Path & path, PathSet & referrers) virtual void queryReferrers(const Path& path, PathSet& referrers) {
{ unsupported("queryReferrers"); } unsupported("queryReferrers");
}
/* Return all currently valid derivations that have `path' as an /* Return all currently valid derivations that have `path' as an
output. (Note that the result of `queryDeriver()' is the output. (Note that the result of `queryDeriver()' is the
@ -381,12 +356,14 @@ public:
virtual PathSet queryValidDerivers(const Path& path) { return {}; }; virtual PathSet queryValidDerivers(const Path& path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */ /* Query the outputs of the derivation denoted by `path'. */
virtual PathSet queryDerivationOutputs(const Path & path) virtual PathSet queryDerivationOutputs(const Path& path) {
{ unsupported("queryDerivationOutputs"); } unsupported("queryDerivationOutputs");
}
/* Query the output names of the derivation denoted by `path'. */ /* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path) virtual StringSet queryDerivationOutputNames(const Path& path) {
{ unsupported("queryDerivationOutputNames"); } unsupported("queryDerivationOutputNames");
}
/* Query the full store path given the hash part of a valid store /* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */ path, or "" if the path doesn't exist. */
@ -399,18 +376,23 @@ public:
sizes) of a set of paths. If a path does not have substitute sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting infos map. */ info, it's omitted from the resulting infos map. */
virtual void querySubstitutablePathInfos(const PathSet& paths, virtual void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos & infos) { return; }; SubstitutablePathInfos& infos) {
return;
};
virtual bool wantMassQuery() { return false; } virtual bool wantMassQuery() { return false; }
/* Import a path into the store. */ /* Import a path into the store. */
virtual void addToStore(const ValidPathInfo& info, Source& narSource, virtual void addToStore(const ValidPathInfo& info, Source& narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0); std::shared_ptr<FSAccessor> accessor = 0);
// FIXME: remove // FIXME: remove
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, virtual void addToStore(const ValidPathInfo& info,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, const ref<std::string>& nar,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0); std::shared_ptr<FSAccessor> accessor = 0);
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
@ -419,12 +401,14 @@ public:
libutil/archive.hh). */ libutil/archive.hh). */
virtual Path addToStore(const string& name, const Path& srcPath, virtual Path addToStore(const string& name, const Path& srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; PathFilter& filter = defaultPathFilter,
RepairFlag repair = NoRepair) = 0;
/* Like addToStore, but the contents written to the output path is /* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */ a regular file containing the given string. */
virtual Path addTextToStore(const string& name, const string& s, virtual Path addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair = NoRepair) = 0; const PathSet& references,
RepairFlag repair = NoRepair) = 0;
/* Write a NAR dump of a store path. */ /* Write a NAR dump of a store path. */
virtual void narFromPath(const Path& path, Sink& sink) = 0; virtual void narFromPath(const Path& path, Sink& sink) = 0;
@ -442,7 +426,8 @@ public:
/* Build a single non-materialized derivation (i.e. not from an /* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). Note that drvPath is only used for on-disk .drv file). Note that drvPath is only used for
informational purposes. */ informational purposes. */
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, virtual BuildResult buildDerivation(const Path& drvPath,
const BasicDerivation& drv,
BuildMode buildMode = bmNormal) = 0; BuildMode buildMode = bmNormal) = 0;
/* Ensure that a path is valid. If it is not currently valid, it /* Ensure that a path is valid. If it is not currently valid, it
@ -452,16 +437,16 @@ public:
/* Add a store path as a temporary root of the garbage collector. /* Add a store path as a temporary root of the garbage collector.
The root disappears as soon as we exit. */ The root disappears as soon as we exit. */
virtual void addTempRoot(const Path & path) virtual void addTempRoot(const Path& path) { unsupported("addTempRoot"); }
{ unsupported("addTempRoot"); }
/* Add an indirect root, which is merely a symlink to `path' from /* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that automatically remove the indirect root when it finds that
`path' has disappeared. */ `path' has disappeared. */
virtual void addIndirectRoot(const Path & path) virtual void addIndirectRoot(const Path& path) {
{ unsupported("addIndirectRoot"); } unsupported("addIndirectRoot");
}
/* Acquire the global GC lock, then immediately release it. This /* Acquire the global GC lock, then immediately release it. This
function must be called after registering a new permanent root, function must be called after registering a new permanent root,
@ -488,18 +473,18 @@ public:
outside of the Nix store that point to `storePath'. If outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots 'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */ found in /proc is censored. */
virtual Roots findRoots(bool censor) virtual Roots findRoots(bool censor) { unsupported("findRoots"); }
{ unsupported("findRoots"); }
/* Perform a garbage collection. */ /* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results) virtual void collectGarbage(const GCOptions& options, GCResults& results) {
{ unsupported("collectGarbage"); } unsupported("collectGarbage");
}
/* Return a string representing information about the path that /* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */ `nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths, string makeValidityRegistration(const PathSet& paths, bool showDerivers,
bool showDerivers, bool showHash); bool showHash);
/* Write a JSON representation of store path metadata, such as the /* Write a JSON representation of store path metadata, such as the
hash and the references. If includeImpureInfo is true, hash and the references. If includeImpureInfo is true,
@ -521,16 +506,18 @@ public:
/* Check the integrity of the Nix store. Returns true if errors /* Check the integrity of the Nix store. Returns true if errors
remain. */ remain. */
virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) {
return false;
};
/* Return an object to access files in the Nix store. */ /* Return an object to access files in the Nix store. */
virtual ref<FSAccessor> getFSAccessor() virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); }
{ unsupported("getFSAccessor"); }
/* Add signatures to the specified store path. The signatures are /* Add signatures to the specified store path. The signatures are
not verified. */ not verified. */
virtual void addSignatures(const Path & storePath, const StringSet & sigs) virtual void addSignatures(const Path& storePath, const StringSet& sigs) {
{ unsupported("addSignatures"); } unsupported("addSignatures");
}
/* Utility functions. */ /* Utility functions. */
@ -545,20 +532,22 @@ public:
`storePath' is returned; that is, the closures under the `storePath' is returned; that is, the closures under the
`referrers' relation instead of the `references' relation is `referrers' relation instead of the `references' relation is
returned. */ returned. */
virtual void computeFSClosure(const PathSet & paths, virtual void computeFSClosure(const PathSet& paths, PathSet& out,
PathSet & out, bool flipDirection = false, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false); bool includeOutputs = false,
bool includeDerivers = false);
void computeFSClosure(const Path & path, void computeFSClosure(const Path& path, PathSet& out,
PathSet & out, bool flipDirection = false, bool flipDirection = false, bool includeOutputs = false,
bool includeOutputs = false, bool includeDerivers = false); bool includeDerivers = false);
/* Given a set of paths that are to be built, return the set of /* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths derivations that will be built, and the set of output paths
that will be substituted. */ that will be substituted. */
virtual void queryMissing(const PathSet & targets, virtual void queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, PathSet& willSubstitute, PathSet& unknown,
unsigned long long & downloadSize, unsigned long long & narSize); unsigned long long& downloadSize,
unsigned long long& narSize);
/* Sort a set of paths topologically under the references /* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */ relation. If p refers to q, then p precedes q in this list. */
@ -577,8 +566,7 @@ public:
Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor, Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs); CheckSigsFlag checkSigs = CheckSigs);
struct Stats struct Stats {
{
std::atomic<uint64_t> narInfoRead{0}; std::atomic<uint64_t> narInfoRead{0};
std::atomic<uint64_t> narInfoReadAverted{0}; std::atomic<uint64_t> narInfoReadAverted{0};
std::atomic<uint64_t> narInfoMissing{0}; std::atomic<uint64_t> narInfoMissing{0};
@ -598,67 +586,55 @@ public:
/* Return the build log of the specified store path, if available, /* Return the build log of the specified store path, if available,
or null otherwise. */ or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const Path & path) virtual std::shared_ptr<std::string> getBuildLog(const Path& path) {
{ return nullptr; } return nullptr;
}
/* Hack to allow long-running processes like hydra-queue-runner to /* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */ occasionally flush their path info cache. */
void clearPathInfoCache() void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); }
{
state.lock()->pathInfoCache.clear();
}
/* Establish a connection to the store, for store types that have /* Establish a connection to the store, for store types that have
a notion of connection. Otherwise this is a no-op. */ a notion of connection. Otherwise this is a no-op. */
virtual void connect(){}; virtual void connect(){};
/* Get the protocol version of this store or it's connection. */ /* Get the protocol version of this store or it's connection. */
virtual unsigned int getProtocol() virtual unsigned int getProtocol() { return 0; };
{
return 0;
};
/* Get the priority of the store, used to order substituters. In /* Get the priority of the store, used to order substituters. In
particular, binary caches can specify a priority field in their particular, binary caches can specify a priority field in their
"nix-cache-info" file. Lower value means higher priority. */ "nix-cache-info" file. Lower value means higher priority. */
virtual int getPriority() { return 0; } virtual int getPriority() { return 0; }
virtual Path toRealPath(const Path & storePath) virtual Path toRealPath(const Path& storePath) { return storePath; }
{
return storePath;
}
virtual void createUser(const std::string & userName, uid_t userId) virtual void createUser(const std::string& userName, uid_t userId) {}
{ }
protected: protected:
Stats stats; Stats stats;
/* Unsupported methods. */ /* Unsupported methods. */
[[noreturn]] void unsupported(const std::string & op) [[noreturn]] void unsupported(const std::string& op) {
{ throw Unsupported("operation '%s' is not supported by store '%s'", op,
throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); getUri());
} }
}; };
class LocalFSStore : public virtual Store {
class LocalFSStore : public virtual Store
{
public: public:
// FIXME: the (Store*) cast works around a bug in gcc that causes // FIXME: the (Store*) cast works around a bug in gcc that causes
// it to emit the call to the Option constructor. Clang works fine // it to emit the call to the Option constructor. Clang works fine
// either way. // either way.
const PathSetting rootDir{(Store*) this, true, "", const PathSetting rootDir{(Store*)this, true, "", "root",
"root", "directory prefixed to all other paths"}; "directory prefixed to all other paths"};
const PathSetting stateDir{(Store*) this, false, const PathSetting stateDir{
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, (Store*)this, false,
"state", "directory where Nix will store state"}; rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state",
const PathSetting logDir{(Store*) this, false, "directory where Nix will store state"};
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, const PathSetting logDir{
"log", "directory where Nix will store state"}; (Store*)this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log",
"directory where Nix will store state"};
const static string drvsLogDir; const static string drvsLogDir;
@ -668,21 +644,20 @@ public:
ref<FSAccessor> getFSAccessor() override; ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */ /* Register a permanent GC root. */
Path addPermRoot(const Path & storePath, Path addPermRoot(const Path& storePath, const Path& gcRoot, bool indirect,
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); bool allowOutsideRootsDir = false);
virtual Path getRealStoreDir() { return storeDir; } virtual Path getRealStoreDir() { return storeDir; }
Path toRealPath(const Path & storePath) override Path toRealPath(const Path& storePath) override {
{
assert(isInStore(storePath)); assert(isInStore(storePath));
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); return getRealStoreDir() + "/" +
std::string(storePath, storeDir.size() + 1);
} }
std::shared_ptr<std::string> getBuildLog(const Path& path) override; std::shared_ptr<std::string> getBuildLog(const Path& path) override;
}; };
/* Extract the name part of the given store path. */ /* Extract the name part of the given store path. */
string storePathToName(const Path& path); string storePathToName(const Path& path);
@ -694,37 +669,32 @@ string storePathToHash(const Path & path);
a dot. */ a dot. */
void checkStoreName(const string& name); void checkStoreName(const string& name);
/* Copy a path from one store to another. */ /* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); const Path& storePath, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
/* Copy store paths from one store to another. The paths may be copied /* Copy store paths from one store to another. The paths may be copied
in parallel. They are copied in a topologically sorted order in parallel. They are copied in a topologically sorted order
(i.e. if A is a reference of B, then A is copied before B), but (i.e. if A is a reference of B, then A is copied before B), but
the set of store paths is not automatically closed; use the set of store paths is not automatically closed; use
copyClosure() for that. */ copyClosure() for that. */
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
RepairFlag repair = NoRepair, const PathSet& storePaths, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute); SubstituteFlag substitute = NoSubstitute);
/* Copy the closure of the specified paths from one store to another. */ /* Copy the closure of the specified paths from one store to another. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore, void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths, const PathSet& storePaths, RepairFlag repair = NoRepair,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs, CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute); SubstituteFlag substitute = NoSubstitute);
/* Remove the temporary roots file for this process. Any temporary /* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered root becomes garbage after this point unless it has been registered
as a (permanent) root. */ as a (permanent) root. */
void removeTempRoots(); void removeTempRoots();
/* Return a Store object to access the Nix store denoted by /* Return a Store object to access the Nix store denoted by
uri (slight misnomer...). Supported values are: uri (slight misnomer...). Supported values are:
@ -757,13 +727,7 @@ void removeTempRoots();
ref<Store> openStore(const std::string& uri = settings.storeUri.get(), ref<Store> openStore(const std::string& uri = settings.storeUri.get(),
const Store::Params& extraParams = Store::Params()); const Store::Params& extraParams = Store::Params());
enum StoreType { tDaemon, tLocal, tOther };
enum StoreType {
tDaemon,
tLocal,
tOther
};
StoreType getStoreType(const std::string& uri = settings.storeUri.get(), StoreType getStoreType(const std::string& uri = settings.storeUri.get(),
const std::string& stateDir = settings.nixStateDir); const std::string& stateDir = settings.nixStateDir);
@ -772,40 +736,32 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
substituters option and various legacy options. */ substituters option and various legacy options. */
std::list<ref<Store>> getDefaultSubstituters(); std::list<ref<Store>> getDefaultSubstituters();
/* Store implementation registration. */ /* Store implementation registration. */
typedef std::function<std::shared_ptr<Store>( typedef std::function<std::shared_ptr<Store>(const std::string& uri,
const std::string & uri, const Store::Params & params)> OpenStore; const Store::Params& params)>
OpenStore;
struct RegisterStoreImplementation struct RegisterStoreImplementation {
{
typedef std::vector<OpenStore> Implementations; typedef std::vector<OpenStore> Implementations;
static Implementations* implementations; static Implementations* implementations;
RegisterStoreImplementation(OpenStore fun) RegisterStoreImplementation(OpenStore fun) {
{
if (!implementations) implementations = new Implementations; if (!implementations) implementations = new Implementations;
implementations->push_back(fun); implementations->push_back(fun);
} }
}; };
/* Display a set of paths in human-readable form (i.e., between quotes /* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */ and separated by commas). */
string showPaths(const PathSet& paths); string showPaths(const PathSet& paths);
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven = false);
ValidPathInfo decodeValidPathInfo(std::istream & str,
bool hashGiven = false);
/* Compute the content-addressability assertion (ValidPathInfo::ca) /* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */ for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash& hash); std::string makeFixedOutputCA(bool recursive, const Hash& hash);
/* Split URI into protocol+hierarchy part and its parameter set. */ /* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri); std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri);
} } // namespace nix

View file

@ -2,7 +2,6 @@
namespace nix { namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
@ -10,7 +9,6 @@ namespace nix {
#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00) #define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff) #define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
typedef enum { typedef enum {
wopIsValidPath = 1, wopIsValidPath = 1,
wopHasSubstitutes = 3, wopHasSubstitutes = 3,
@ -51,7 +49,6 @@ typedef enum {
wopQueryMissing = 40, wopQueryMissing = 40,
} WorkerOp; } WorkerOp;
#define STDERR_NEXT 0x6f6c6d67 #define STDERR_NEXT 0x6f6c6d67
#define STDERR_READ 0x64617461 // data needed from source #define STDERR_READ 0x64617461 // data needed from source
#define STDERR_WRITE 0x64617416 // data for sink #define STDERR_WRITE 0x64617416 // data for sink
@ -61,9 +58,8 @@ typedef enum {
#define STDERR_STOP_ACTIVITY 0x53544f50 #define STDERR_STOP_ACTIVITY 0x53544f50
#define STDERR_RESULT 0x52534c54 #define STDERR_RESULT 0x52534c54
Path readStorePath(Store& store, Source& from); Path readStorePath(Store& store, Source& from);
template<class T> T readStorePaths(Store & store, Source & from); template <class T>
T readStorePaths(Store& store, Source& from);
} // namespace nix
}

View file

@ -1,6 +1,6 @@
#include "affinity.hh"
#include "types.hh" #include "types.hh"
#include "util.hh" #include "util.hh"
#include "affinity.hh"
#if __linux__ #if __linux__
#include <sched.h> #include <sched.h>
@ -8,15 +8,12 @@
namespace nix { namespace nix {
#if __linux__ #if __linux__
static bool didSaveAffinity = false; static bool didSaveAffinity = false;
static cpu_set_t savedAffinity; static cpu_set_t savedAffinity;
#endif #endif
void setAffinityTo(int cpu) {
void setAffinityTo(int cpu)
{
#if __linux__ #if __linux__
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true; didSaveAffinity = true;
@ -29,9 +26,7 @@ void setAffinityTo(int cpu)
#endif #endif
} }
int lockToCurrentCPU() {
int lockToCurrentCPU()
{
#if __linux__ #if __linux__
int cpu = sched_getcpu(); int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu); if (cpu != -1) setAffinityTo(cpu);
@ -41,9 +36,7 @@ int lockToCurrentCPU()
#endif #endif
} }
void restoreAffinity() {
void restoreAffinity()
{
#if __linux__ #if __linux__
if (!didSaveAffinity) return; if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
@ -51,5 +44,4 @@ void restoreAffinity()
#endif #endif
} }
} // namespace nix
}

View file

@ -6,4 +6,4 @@ void setAffinityTo(int cpu);
int lockToCurrentCPU(); int lockToCurrentCPU();
void restoreAffinity(); void restoreAffinity();
} } // namespace nix

View file

@ -1,32 +1,31 @@
#include <cerrno> #include "archive.hh"
#include <algorithm>
#include <vector>
#include <map>
#include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h> #include <dirent.h>
#include <fcntl.h> #include <fcntl.h>
#include <strings.h> // for strcasecmp
#include "archive.hh" #include <sys/stat.h>
#include "util.hh" #include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <cerrno>
#include <map>
#include <vector>
#include "config.hh" #include "config.hh"
#include "util.hh"
namespace nix { namespace nix {
struct ArchiveSettings : Config struct ArchiveSettings : Config {
{ Setting<bool> useCaseHack {
Setting<bool> useCaseHack{this, this,
#if __APPLE__ #if __APPLE__
true, true,
#else #else
false, false,
#endif #endif
"use-case-hack", "use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."}; "Whether to enable a Darwin-specific hack for dealing with file name "
"collisions."
};
}; };
static ArchiveSettings archiveSettings; static ArchiveSettings archiveSettings;
@ -39,10 +38,7 @@ static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path&) { return true; }; PathFilter defaultPathFilter = [](const Path&) { return true; };
static void dumpContents(const Path& path, size_t size, Sink& sink) {
static void dumpContents(const Path & path, size_t size,
Sink & sink)
{
sink << "contents" << size; sink << "contents" << size;
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
@ -61,9 +57,7 @@ static void dumpContents(const Path & path, size_t size,
writePadding(size, sink); writePadding(size, sink);
} }
static void dump(const Path& path, Sink& sink, PathFilter& filter) {
static void dump(const Path & path, Sink & sink, PathFilter & filter)
{
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st;
@ -73,14 +67,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "("; sink << "(";
if (S_ISREG(st.st_mode)) { if (S_ISREG(st.st_mode)) {
sink << "type" << "regular"; sink << "type"
<< "regular";
if (st.st_mode & S_IXUSR) if (st.st_mode & S_IXUSR)
sink << "executable" << ""; sink << "executable"
<< "";
dumpContents(path, (size_t)st.st_size, sink); dumpContents(path, (size_t)st.st_size, sink);
} }
else if (S_ISDIR(st.st_mode)) { else if (S_ISDIR(st.st_mode)) {
sink << "type" << "directory"; sink << "type"
<< "directory";
/* If we're on a case-insensitive system like macOS, undo /* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */ the case hack applied by restorePath(). */
@ -90,52 +87,54 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
string name(i.name); string name(i.name);
size_t pos = i.name.find(caseHackSuffix); size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) { if (pos != string::npos) {
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); debug(format("removing case hack suffix from '%1%'") %
(path + "/" + i.name));
name.erase(pos); name.erase(pos);
} }
if (unhacked.find(name) != unhacked.end()) if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'") throw Error(format("file name collision in between '%1%' and '%2%'") %
% (path + "/" + unhacked[name]) % (path + "/" + i.name)); (path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name; unhacked[name] = i.name;
} else } else
unhacked[i.name] = i.name; unhacked[i.name] = i.name;
for (auto& i : unhacked) for (auto& i : unhacked)
if (filter(path + "/" + i.first)) { if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node"; sink << "entry"
<< "("
<< "name" << i.first << "node";
dump(path + "/" + i.second, sink, filter); dump(path + "/" + i.second, sink, filter);
sink << ")"; sink << ")";
} }
} }
else if (S_ISLNK(st.st_mode)) else if (S_ISLNK(st.st_mode))
sink << "type" << "symlink" << "target" << readLink(path); sink << "type"
<< "symlink"
<< "target" << readLink(path);
else throw Error(format("file '%1%' has an unsupported type") % path); else
throw Error(format("file '%1%' has an unsupported type") % path);
sink << ")"; sink << ")";
} }
void dumpPath(const Path& path, Sink& sink, PathFilter& filter) {
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
sink << narVersionMagic1; sink << narVersionMagic1;
dump(path, sink, filter); dump(path, sink, filter);
} }
void dumpString(const std::string& s, Sink& sink) {
void dumpString(const std::string & s, Sink & sink) sink << narVersionMagic1 << "("
{ << "type"
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; << "regular"
<< "contents" << s << ")";
} }
static SerialisationError badArchive(string s) {
static SerialisationError badArchive(string s)
{
return SerialisationError("bad archive: " + s); return SerialisationError("bad archive: " + s);
} }
#if 0 #if 0
static void skipGeneric(Source & source) static void skipGeneric(Source & source)
{ {
@ -146,9 +145,7 @@ static void skipGeneric(Source & source)
} }
#endif #endif
static void parseContents(ParseSink& sink, Source& source, const Path& path) {
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
unsigned long long size = readLongLong(source); unsigned long long size = readLongLong(source);
sink.preallocateContents(size); sink.preallocateContents(size);
@ -168,18 +165,13 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
readPadding(size, source); readPadding(size, source);
} }
struct CaseInsensitiveCompare {
struct CaseInsensitiveCompare bool operator()(const string& a, const string& b) const {
{
bool operator() (const string & a, const string & b) const
{
return strcasecmp(a.c_str(), b.c_str()) < 0; return strcasecmp(a.c_str(), b.c_str()) < 0;
} }
}; };
static void parse(ParseSink& sink, Source& source, const Path& path) {
static void parse(ParseSink & sink, Source & source, const Path & path)
{
string s; string s;
s = readString(source); s = readString(source);
@ -199,8 +191,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
} }
else if (s == "type") { else if (s == "type") {
if (type != tpUnknown) if (type != tpUnknown) throw badArchive("multiple type fields");
throw badArchive("multiple type fields");
string t = readString(source); string t = readString(source);
if (t == "regular") { if (t == "regular") {
@ -217,7 +208,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
type = tpSymlink; type = tpSymlink;
} }
else throw badArchive("unknown file type " + t); else
throw badArchive("unknown file type " + t);
} }
@ -246,15 +238,17 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
break; break;
} else if (s == "name") { } else if (s == "name") {
name = readString(source); name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) if (name.empty() || name == "." || name == ".." ||
name.find('/') != string::npos ||
name.find((char)0) != string::npos)
throw Error(format("NAR contains invalid file name '%1%'") % name); throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName) if (name <= prevName) throw Error("NAR directory is not sorted");
throw Error("NAR directory is not sorted");
prevName = name; prevName = name;
if (archiveSettings.useCaseHack) { if (archiveSettings.useCaseHack) {
auto i = names.find(name); auto i = names.find(name);
if (i != names.end()) { if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") % i->first % name); debug(format("case collision between '%1%' and '%2%'") %
i->first % name);
name += caseHackSuffix; name += caseHackSuffix;
name += std::to_string(++i->second); name += std::to_string(++i->second);
} else } else
@ -278,9 +272,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
} }
} }
void parseDump(ParseSink& sink, Source& source) {
void parseDump(ParseSink & sink, Source & source)
{
string version; string version;
try { try {
version = readString(source, narVersionMagic1.size()); version = readString(source, narVersionMagic1.size());
@ -293,37 +285,30 @@ void parseDump(ParseSink & sink, Source & source)
parse(sink, source, ""); parse(sink, source, "");
} }
struct RestoreSink : ParseSink {
struct RestoreSink : ParseSink
{
Path dstPath; Path dstPath;
AutoCloseFD fd; AutoCloseFD fd;
void createDirectory(const Path & path) void createDirectory(const Path& path) {
{
Path p = dstPath + path; Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1) if (mkdir(p.c_str(), 0777) == -1)
throw SysError(format("creating directory '%1%'") % p); throw SysError(format("creating directory '%1%'") % p);
}; };
void createRegularFile(const Path & path) void createRegularFile(const Path& path) {
{
Path p = dstPath + path; Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError(format("creating file '%1%'") % p); if (!fd) throw SysError(format("creating file '%1%'") % p);
} }
void isExecutable() void isExecutable() {
{
struct stat st; struct stat st;
if (fstat(fd.get(), &st) == -1) if (fstat(fd.get(), &st) == -1) throw SysError("fstat");
throw SysError("fstat");
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
throw SysError("fchmod"); throw SysError("fchmod");
} }
void preallocateContents(unsigned long long len) void preallocateContents(unsigned long long len) {
{
#if HAVE_POSIX_FALLOCATE #if HAVE_POSIX_FALLOCATE
if (len) { if (len) {
errno = posix_fallocate(fd.get(), 0, len); errno = posix_fallocate(fd.get(), 0, len);
@ -337,29 +322,23 @@ struct RestoreSink : ParseSink
#endif #endif
} }
void receiveContents(unsigned char * data, unsigned int len) void receiveContents(unsigned char* data, unsigned int len) {
{
writeFull(fd.get(), data, len); writeFull(fd.get(), data, len);
} }
void createSymlink(const Path & path, const string & target) void createSymlink(const Path& path, const string& target) {
{
Path p = dstPath + path; Path p = dstPath + path;
nix::createSymlink(target, p); nix::createSymlink(target, p);
} }
}; };
void restorePath(const Path& path, Source& source) {
void restorePath(const Path & path, Source & source)
{
RestoreSink sink; RestoreSink sink;
sink.dstPath = path; sink.dstPath = path;
parseDump(sink, source); parseDump(sink, source);
} }
void copyNAR(Source& source, Sink& sink) {
void copyNAR(Source & source, Sink & sink)
{
// FIXME: if 'source' is the output of dumpPath() followed by EOF, // FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing. // we should just forward all data directly without parsing.
@ -374,5 +353,4 @@ void copyNAR(Source & source, Sink & sink)
parseDump(parseSink, wrapper); parseDump(parseSink, wrapper);
} }
} // namespace nix
}

View file

@ -1,12 +1,10 @@
#pragma once #pragma once
#include "types.hh"
#include "serialise.hh" #include "serialise.hh"
#include "types.hh"
namespace nix { namespace nix {
/* dumpPath creates a Nix archive of the specified path. The format /* dumpPath creates a Nix archive of the specified path. The format
is as follows: is as follows:
@ -44,15 +42,13 @@ namespace nix {
`+' denotes string concatenation. */ `+' denotes string concatenation. */
void dumpPath(const Path& path, Sink& sink, void dumpPath(const Path& path, Sink& sink,
PathFilter& filter = defaultPathFilter); PathFilter& filter = defaultPathFilter);
void dumpString(const std::string& s, Sink& sink); void dumpString(const std::string& s, Sink& sink);
/* FIXME: fix this API, it sucks. */ /* FIXME: fix this API, it sucks. */
struct ParseSink struct ParseSink {
{
virtual void createDirectory(const Path& path){}; virtual void createDirectory(const Path& path){};
virtual void createRegularFile(const Path& path){}; virtual void createRegularFile(const Path& path){};
@ -63,8 +59,7 @@ struct ParseSink
virtual void createSymlink(const Path& path, const string& target){}; virtual void createSymlink(const Path& path, const string& target){};
}; };
struct TeeSink : ParseSink struct TeeSink : ParseSink {
{
TeeSource source; TeeSource source;
TeeSink(Source& source) : source(source) {} TeeSink(Source& source) : source(source) {}
@ -77,8 +72,6 @@ void restorePath(const Path & path, Source & source);
/* Read a NAR from 'source' and write it to 'sink'. */ /* Read a NAR from 'source' and write it to 'sink'. */
void copyNAR(Source& source, Sink& sink); void copyNAR(Source& source, Sink& sink);
extern const std::string narVersionMagic1; extern const std::string narVersionMagic1;
} // namespace nix
}

View file

@ -3,34 +3,30 @@
namespace nix { namespace nix {
Args::FlagMaker Args::mkFlag() Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); }
{
return FlagMaker(*this);
}
Args::FlagMaker::~FlagMaker() Args::FlagMaker::~FlagMaker() {
{
assert(flag->longName != ""); assert(flag->longName != "");
args.longFlags[flag->longName] = flag; args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag; if (flag->shortName) args.shortFlags[flag->shortName] = flag;
} }
void Args::parseCmdline(const Strings & _cmdline) void Args::parseCmdline(const Strings& _cmdline) {
{
Strings pendingArgs; Strings pendingArgs;
bool dashDash = false; bool dashDash = false;
Strings cmdline(_cmdline); Strings cmdline(_cmdline);
for (auto pos = cmdline.begin(); pos != cmdline.end();) { for (auto pos = cmdline.begin(); pos != cmdline.end();) {
auto arg = *pos; auto arg = *pos;
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */ `-j3` -> `-j 3`). */
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' &&
isalpha(arg[1])) {
*pos = (string) "-" + arg[1]; *pos = (string) "-" + arg[1];
auto next = pos; ++next; auto next = pos;
++next;
for (unsigned int j = 2; j < arg.length(); j++) for (unsigned int j = 2; j < arg.length(); j++)
if (isalpha(arg[j])) if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]); cmdline.insert(next, (string) "-" + arg[j]);
@ -44,23 +40,19 @@ void Args::parseCmdline(const Strings & _cmdline)
if (!dashDash && arg == "--") { if (!dashDash && arg == "--") {
dashDash = true; dashDash = true;
++pos; ++pos;
} } else if (!dashDash && std::string(arg, 0, 1) == "-") {
else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end())) if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg); throw UsageError(format("unrecognised flag '%1%'") % arg);
} } else {
else {
pendingArgs.push_back(*pos++); pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false)) if (processArgs(pendingArgs, false)) pendingArgs.clear();
pendingArgs.clear();
} }
} }
processArgs(pendingArgs, true); processArgs(pendingArgs, true);
} }
void Args::printHelp(const string & programName, std::ostream & out) void Args::printHelp(const string& programName, std::ostream& out) {
{
std::cout << "Usage: " << programName << " <FLAGS>..."; std::cout << "Usage: " << programName << " <FLAGS>...";
for (auto& exp : expectedArgs) { for (auto& exp : expectedArgs) {
std::cout << renderLabels({exp.label}); std::cout << renderLabels({exp.label});
@ -71,8 +63,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
std::cout << "\n"; std::cout << "\n";
auto s = description(); auto s = description();
if (s != "") if (s != "") std::cout << "\nSummary: " << s << ".\n";
std::cout << "\nSummary: " << s << ".\n";
if (longFlags.size()) { if (longFlags.size()) {
std::cout << "\n"; std::cout << "\n";
@ -81,21 +72,21 @@ void Args::printHelp(const string & programName, std::ostream & out)
} }
} }
void Args::printFlags(std::ostream & out) void Args::printFlags(std::ostream& out) {
{
Table2 table; Table2 table;
for (auto& flag : longFlags) { for (auto& flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue; if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair( table.push_back(std::make_pair(
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") (flag.second->shortName
+ "--" + flag.first + renderLabels(flag.second->labels), ? std::string("-") + flag.second->shortName + ", "
: " ") +
"--" + flag.first + renderLabels(flag.second->labels),
flag.second->description)); flag.second->description));
} }
printTable(out, table); printTable(out, table);
} }
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) {
{
assert(pos != end); assert(pos != end);
auto process = [&](const std::string& name, const Flag& flag) -> bool { auto process = [&](const std::string& name, const Flag& flag) -> bool {
@ -104,8 +95,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
for (size_t n = 0; n < flag.arity; ++n) { for (size_t n = 0; n < flag.arity; ++n) {
if (pos == end) { if (pos == end) {
if (flag.arity == ArityAny) break; if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)") throw UsageError(format("flag '%1%' requires %2% argument(s)") % name %
% name % flag.arity); flag.arity);
} }
args.push_back(*pos++); args.push_back(*pos++);
} }
@ -129,8 +120,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
return false; return false;
} }
bool Args::processArgs(const Strings & args, bool finish) bool Args::processArgs(const Strings& args, bool finish) {
{
if (expectedArgs.empty()) { if (expectedArgs.empty()) {
if (!args.empty()) if (!args.empty())
throw UsageError(format("unexpected argument '%1%'") % args.front()); throw UsageError(format("unexpected argument '%1%'") % args.front());
@ -142,8 +132,7 @@ bool Args::processArgs(const Strings & args, bool finish)
bool res = false; bool res = false;
if ((exp.arity == 0 && finish) || if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity)) (exp.arity > 0 && args.size() == exp.arity)) {
{
std::vector<std::string> ss; std::vector<std::string> ss;
for (auto& s : args) ss.push_back(s); for (auto& s : args) ss.push_back(s);
exp.handler(std::move(ss)); exp.handler(std::move(ss));
@ -157,29 +146,26 @@ bool Args::processArgs(const Strings & args, bool finish)
return res; return res;
} }
Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) {
{
arity(1); arity(1);
label("type"); label("type");
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
handler([ht](std::string s) { handler([ht](std::string s) {
*ht = parseHashType(s); *ht = parseHashType(s);
if (*ht == htUnknown) if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s);
throw UsageError("unknown hash type '%1%'", s);
}); });
return *this; return *this;
} }
Strings argvToStrings(int argc, char * * argv) Strings argvToStrings(int argc, char** argv) {
{
Strings args; Strings args;
argc--; argv++; argc--;
argv++;
while (argc--) args.push_back(*argv++); while (argc--) args.push_back(*argv++);
return args; return args;
} }
std::string renderLabels(const Strings & labels) std::string renderLabels(const Strings& labels) {
{
std::string res; std::string res;
for (auto label : labels) { for (auto label : labels) {
for (auto& c : label) c = std::toupper(c); for (auto& c : label) c = std::toupper(c);
@ -188,16 +174,13 @@ std::string renderLabels(const Strings & labels)
return res; return res;
} }
void printTable(std::ostream & out, const Table2 & table) void printTable(std::ostream& out, const Table2& table) {
{
size_t max = 0; size_t max = 0;
for (auto & row : table) for (auto& row : table) max = std::max(max, row.first.size());
max = std::max(max, row.first.size());
for (auto& row : table) { for (auto& row : table) {
out << " " << row.first out << " " << row.first << std::string(max - row.first.size() + 2, ' ')
<< std::string(max - row.first.size() + 2, ' ')
<< row.second << "\n"; << row.second << "\n";
} }
} }
} } // namespace nix

View file

@ -3,7 +3,6 @@
#include <iostream> #include <iostream>
#include <map> #include <map>
#include <memory> #include <memory>
#include "util.hh" #include "util.hh"
namespace nix { namespace nix {
@ -12,10 +11,8 @@ MakeError(UsageError, Error);
enum HashType : char; enum HashType : char;
class Args class Args {
{
public: public:
/* Parse the command line, throwing a UsageError if something goes /* Parse the command line, throwing a UsageError if something goes
wrong. */ wrong. */
void parseCmdline(const Strings& cmdline); void parseCmdline(const Strings& cmdline);
@ -25,12 +22,10 @@ public:
virtual std::string description() { return ""; } virtual std::string description() { return ""; }
protected: protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max(); static const size_t ArityAny = std::numeric_limits<size_t>::max();
/* Flags. */ /* Flags. */
struct Flag struct Flag {
{
typedef std::shared_ptr<Flag> ptr; typedef std::shared_ptr<Flag> ptr;
std::string longName; std::string longName;
char shortName = 0; char shortName = 0;
@ -49,8 +44,7 @@ protected:
virtual void printFlags(std::ostream& out); virtual void printFlags(std::ostream& out);
/* Positional arguments. */ /* Positional arguments. */
struct ExpectedArg struct ExpectedArg {
{
std::string label; std::string label;
size_t arity; // 0 = any size_t arity; // 0 = any
bool optional; bool optional;
@ -64,41 +58,69 @@ protected:
std::set<std::string> hiddenCategories; std::set<std::string> hiddenCategories;
public: public:
class FlagMaker {
class FlagMaker
{
Args& args; Args& args;
Flag::ptr flag; Flag::ptr flag;
friend class Args; friend class Args;
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){}; FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
public: public:
~FlagMaker(); ~FlagMaker();
FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; FlagMaker& longName(const std::string& s) {
FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; flag->longName = s;
FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; return *this;
FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; };
FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; FlagMaker& shortName(char s) {
FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; flag->shortName = s;
FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }; return *this;
FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }; };
FlagMaker & handler(std::function<void(std::string)> handler) { FlagMaker& description(const std::string& s) {
flag->arity = 1; flag->description = s;
flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); }; return *this;
};
FlagMaker& label(const std::string& l) {
flag->arity = 1;
flag->labels = {l};
return *this;
};
FlagMaker& labels(const Strings& ls) {
flag->arity = ls.size();
flag->labels = ls;
return *this;
};
FlagMaker& arity(size_t arity) {
flag->arity = arity;
return *this;
};
FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) {
flag->handler = handler;
return *this;
};
FlagMaker& handler(std::function<void()> handler) {
flag->handler = [handler](std::vector<std::string>) { handler(); };
return *this;
};
FlagMaker& handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) {
handler(std::move(ss[0]));
};
return *this;
};
FlagMaker& category(const std::string& s) {
flag->category = s;
return *this; return *this;
}; };
FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
template <class T> template <class T>
FlagMaker & dest(T * dest) FlagMaker& dest(T* dest) {
{
flag->arity = 1; flag->arity = 1;
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this; return *this;
}; };
template <class T> template <class T>
FlagMaker & set(T * dest, const T & val) FlagMaker& set(T* dest, const T& val) {
{
flag->arity = 0; flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
return *this; return *this;
@ -114,8 +136,7 @@ public:
void mkFlag1(char shortName, const std::string& longName, void mkFlag1(char shortName, const std::string& longName,
const std::string& label, const std::string& description, const std::string& label, const std::string& description,
std::function<void(std::string)> fun) std::function<void(std::string)> fun) {
{
mkFlag() mkFlag()
.shortName(shortName) .shortName(shortName)
.longName(longName) .longName(longName)
@ -126,15 +147,13 @@ public:
} }
void mkFlag(char shortName, const std::string& name, void mkFlag(char shortName, const std::string& name,
const std::string & description, bool * dest) const std::string& description, bool* dest) {
{
mkFlag(shortName, name, description, dest, true); mkFlag(shortName, name, description, dest, true);
} }
template <class T> template <class T>
void mkFlag(char shortName, const std::string & longName, const std::string & description, void mkFlag(char shortName, const std::string& longName,
T * dest, const T & value) const std::string& description, T* dest, const T& value) {
{
mkFlag() mkFlag()
.shortName(shortName) .shortName(shortName)
.longName(longName) .longName(longName)
@ -144,17 +163,13 @@ public:
template <class I> template <class I>
void mkIntFlag(char shortName, const std::string& longName, void mkIntFlag(char shortName, const std::string& longName,
const std::string & description, I * dest) const std::string& description, I* dest) {
{ mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; });
mkFlag<I>(shortName, longName, description, [=](I n) {
*dest = n;
});
} }
template <class I> template <class I>
void mkFlag(char shortName, const std::string& longName, void mkFlag(char shortName, const std::string& longName,
const std::string & description, std::function<void(I)> fun) const std::string& description, std::function<void(I)> fun) {
{
mkFlag() mkFlag()
.shortName(shortName) .shortName(shortName)
.longName(longName) .longName(longName)
@ -164,25 +179,25 @@ public:
.handler([=](std::vector<std::string> ss) { .handler([=](std::vector<std::string> ss) {
I n; I n;
if (!string2Int(ss[0], n)) if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument", longName); throw UsageError("flag '--%s' requires a integer argument",
longName);
fun(n); fun(n);
}); });
} }
/* Expect a string argument. */ /* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false) void expectArg(const std::string& label, string* dest,
{ bool optional = false) {
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { expectedArgs.push_back(
*dest = ss[0]; ExpectedArg{label, 1, optional,
}}); [=](std::vector<std::string> ss) { *dest = ss[0]; }});
} }
/* Expect 0 or more arguments. */ /* Expect 0 or more arguments. */
void expectArgs(const std::string & label, std::vector<std::string> * dest) void expectArgs(const std::string& label, std::vector<std::string>* dest) {
{ expectedArgs.push_back(ExpectedArg{
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { label, 0, false,
*dest = std::move(ss); [=](std::vector<std::string> ss) { *dest = std::move(ss); }});
}});
} }
friend class MultiCommand; friend class MultiCommand;
@ -198,4 +213,4 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream& out, const Table2& table); void printTable(std::ostream& out, const Table2& table);
} } // namespace nix

Some files were not shown because too many files have changed in this diff Show more