maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

style(3p/nix): Reformat project in Google C++ style

Browse source 
Reformatted with:

    fd . -e hh -e cc | xargs clang-format -i
This commit is contained in:
Vincent Ambo 2020-05-17 16:31:57 +01:00
parent 65a1aae98c
commit 0f2cf531f7
175 changed files with 32126 additions and 34689 deletions
Download patch file Download diff file Expand all files Collapse all files

103
third_party/nix/src/build-remote/build-remote.cc vendored
View file

@ -1,41 +1,39 @@
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include <set>
#include <memory>
#include <tuple>
#include <iomanip>
#include <memory>
#include <set>
#include <tuple>
#if __APPLE__
#include <sys/time.h>
#endif
#include "machines.hh"
#include "shared.hh"
#include "pathlocks.hh"
#include "globals.hh"
#include "serialise.hh"
#include "store-api.hh"
#include "derivations.hh"
#include "local-store.hh"
#include "globals.hh"
#include "legacy.hh"
#include "local-store.hh"
#include "machines.hh"
#include "pathlocks.hh"
#include "serialise.hh"
#include "shared.hh"
#include "store-api.hh"
using namespace nix;
using std::cin;
static void handleAlarm(int sig) {
}
static void handleAlarm(int sig) {}
std::string escapeUri(std::string uri)
{
std::string escapeUri(std::string uri) {
std::replace(uri.begin(), uri.end(), '/', '_');
return uri;
}
static string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) {
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot),
true);
}
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
@ -44,8 +42,7 @@ static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
return true;
}
static int _main(int argc, char * * argv)
{
static int _main(int argc, char** argv) {
{
logger = makeJSONLogger(*logger);
@ -53,8 +50,7 @@ static int _main(int argc, char * * argv)
unsetenv("DISPLAY");
unsetenv("SSH_ASKPASS");
if (argc != 2)
throw UsageError("called without required arguments");
if (argc != 2) throw UsageError("called without required arguments");
verbosity = (Verbosity)std::stoll(argv[1]);
@ -92,21 +88,23 @@ static int _main(int argc, char * * argv)
string storeUri;
while (true) {
try {
auto s = readString(source);
if (s != "try") return 0;
} catch (EndOfFile &) { return 0; }
} catch (EndOfFile&) {
return 0;
}
auto amWilling = readInt(source);
auto neededSystem = readString(source);
source >> drvPath;
auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
&& allSupportedLocally(requiredFeatures);
auto canBuildLocally =
amWilling &&
(neededSystem == settings.thisSystem ||
settings.extraPlatforms.get().count(neededSystem) > 0) &&
allSupportedLocally(requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
@ -123,8 +121,8 @@ static int _main(int argc, char * * argv)
for (auto& m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
if (m.enabled && std::find(m.systemTypes.begin(),
m.systemTypes.end(),
if (m.enabled &&
std::find(m.systemTypes.begin(), m.systemTypes.end(),
neededSystem) != m.systemTypes.end() &&
m.allSupported(requiredFeatures) &&
m.mandatoryMet(requiredFeatures)) {
@ -147,9 +145,11 @@ static int _main(int argc, char * * argv)
bool best = false;
if (!bestSlotLock) {
best = true;
} else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) {
} else if (load / m.speedFactor <
bestLoad / bestMachine->speedFactor) {
best = true;
} else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) {
} else if (load / m.speedFactor ==
bestLoad / bestMachine->speedFactor) {
if (m.speedFactor > bestMachine->speedFactor) {
best = true;
} else if (m.speedFactor == bestMachine->speedFactor) {
@ -183,8 +183,8 @@ static int _main(int argc, char * * argv)
lock = -1;
try {
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
Activity act(*logger, lvlTalkative, actUnknown,
fmt("connecting to '%s'", bestMachine->storeUri));
Store::Params storeParams;
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
@ -200,9 +200,8 @@ static int _main(int argc, char * * argv)
} catch (std::exception& e) {
auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg));
printError("cannot build on '%s': %s%s", bestMachine->storeUri,
e.what(), (msg.empty() ? "" : ": " + msg));
bestMachine->enabled = false;
continue;
}
@ -219,44 +218,54 @@ connected:
auto inputs = readStrings<PathSet>(source);
auto outputs = readStrings<PathSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
AutoCloseFD uploadLock = openLockFile(
currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown,
fmt("waiting for the upload lock to '%s'", storeUri));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
if (!lockFile(uploadLock.get(), ltWrite, true))
printError("somebody is hogging the upload lock for '%s', continuing...");
printError(
"somebody is hogging the upload lock for '%s', continuing...");
alarm(0);
signal(SIGALRM, old);
}
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
auto substitute =
settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
Activity act(*logger, lvlTalkative, actUnknown,
fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs,
substitute);
}
uploadLock = -1;
BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
BasicDerivation drv(
readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
drv.inputSrcs = inputs;
auto result = sshStore->buildDerivation(drvPath, drv);
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri,
result.errorMsg);
PathSet missing;
for (auto& path : outputs)
if (!store->isValidPath(path)) missing.insert(path);
if (!missing.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown,
fmt("copying outputs from '%s'", storeUri));
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs,
NoSubstitute);
}
return 0;

45
third_party/nix/src/libexpr/attr-path.cc vendored
View file

@ -2,12 +2,9 @@
#include "eval-inline.hh"
#include "util.hh"
namespace nix {
static Strings parseAttrPath(const string & s)
{
static Strings parseAttrPath(const string& s) {
Strings res;
string cur;
string::const_iterator i = s.begin();
@ -19,7 +16,8 @@ static Strings parseAttrPath(const string & s)
++i;
while (1) {
if (i == s.end())
throw Error(format("missing closing quote in selection path '%1%'") % s);
throw Error(format("missing closing quote in selection path '%1%'") %
s);
if (*i == '"') break;
cur.push_back(*i++);
}
@ -31,19 +29,17 @@ static Strings parseAttrPath(const string & s)
return res;
}
Value* findAlongAttrPath(EvalState& state, const string& attrPath,
Bindings & autoArgs, Value & vIn)
{
Bindings& autoArgs, Value& vIn) {
Strings tokens = parseAttrPath(attrPath);
Error attrError =
Error(format("attribute selection path '%1%' does not match expression") % attrPath);
Error(format("attribute selection path '%1%' does not match expression") %
attrPath);
Value* v = &vIn;
for (auto& attr : tokens) {
/* Is i an index (integer) or a normal attribute name? */
enum { apAttr, apIndex } apType = apAttr;
unsigned int attrIndex;
@ -59,38 +55,39 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
according to what is specified in the attrPath. */
if (apType == apAttr) {
if (v->type != tAttrs)
throw TypeError(
format("the expression selected by the selection path '%1%' should be a set but is %2%")
% attrPath % showType(*v));
throw TypeError(format("the expression selected by the selection path "
"'%1%' should be a set but is %2%") %
attrPath % showType(*v));
if (attr.empty())
throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
throw Error(format("empty attribute name in selection path '%1%'") %
attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end())
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
throw Error(
format("attribute '%1%' in selection path '%2%' not found") % attr %
attrPath);
v = &*a->value;
}
else if (apType == apIndex) {
if (!v->isList())
throw TypeError(
format("the expression selected by the selection path '%1%' should be a list but is %2%")
% attrPath % showType(*v));
throw TypeError(format("the expression selected by the selection path "
"'%1%' should be a list but is %2%") %
attrPath % showType(*v));
if (attrIndex >= v->listSize())
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
throw Error(
format("list index %1% in selection path '%2%' is out of range") %
attrIndex % attrPath);
v = v->listElems()[attrIndex];
}
}
return v;
}
}
} // namespace nix

5
third_party/nix/src/libexpr/attr-path.hh vendored
View file

@ -1,9 +1,8 @@
#pragma once
#include "eval.hh"
#include <string>
#include <map>
#include <string>
#include "eval.hh"
namespace nix {

28
third_party/nix/src/libexpr/attr-set.cc vendored
View file

@ -1,25 +1,20 @@
#include "attr-set.hh"
#include <algorithm>
#include "eval-inline.hh"
#include <algorithm>
namespace nix {
/* Allocate a new array of attributes for an attribute set with a specific
capacity. The space is implicitly reserved after the Bindings
structure. */
Bindings * EvalState::allocBindings(size_t capacity)
{
Bindings* EvalState::allocBindings(size_t capacity) {
if (capacity > std::numeric_limits<Bindings::size_t>::max())
throw Error("attribute set of size %d is too big", capacity);
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity))
Bindings((Bindings::size_t)capacity);
}
void EvalState::mkAttrs(Value & v, size_t capacity)
{
void EvalState::mkAttrs(Value& v, size_t capacity) {
if (capacity == 0) {
v = vEmptySet;
return;
@ -31,22 +26,15 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
nrAttrsInAttrsets += capacity;
}
/* Create a new attribute named 'name' on an existing attribute set stored
in 'vAttrs' and return the newly allocated Value which is associated with
this attribute. */
Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
{
Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) {
Value* v = allocValue();
vAttrs.attrs->push_back(Attr(name, v));
return v;
}
void Bindings::sort() { std::sort(begin(), end()); }
void Bindings::sort()
{
std::sort(begin(), end());
}
}
} // namespace nix

35
third_party/nix/src/libexpr/attr-set.hh vendored
View file

@ -1,37 +1,30 @@
#pragma once
#include <algorithm>
#include "nixexpr.hh"
#include "symbol-table.hh"
#include <algorithm>
namespace nix {
class EvalState;
struct Value;
/* Map one attribute name to its value. */
struct Attr
{
struct Attr {
Symbol name;
Value* value;
Pos* pos;
Attr(Symbol name, Value* value, Pos* pos = &noPos)
: name(name), value(value), pos(pos){};
Attr() : pos(&noPos){};
bool operator < (const Attr & a) const
{
return name < a.name;
}
bool operator<(const Attr& a) const { return name < a.name; }
};
/* Bindings contains all the attributes of an attribute set. It is defined
by its size and its capacity, the capacity being the number of Attr
elements allocated after this structure, while the size corresponds to
the number of elements already inserted in this structure. */
class Bindings
{
class Bindings {
public:
typedef uint32_t size_t;
@ -49,14 +42,12 @@ public:
typedef Attr* iterator;
void push_back(const Attr & attr)
{
void push_back(const Attr& attr) {
assert(size_ < capacity_);
attrs[size_++] = attr;
}
iterator find(const Symbol & name)
{
iterator find(const Symbol& name) {
Attr key(name, 0);
iterator i = std::lower_bound(begin(), end(), key);
if (i != end() && i->name == name) return i;
@ -66,22 +57,17 @@ public:
iterator begin() { return &attrs[0]; }
iterator end() { return &attrs[size_]; }
Attr & operator[](size_t pos)
{
return attrs[pos];
}
Attr& operator[](size_t pos) { return attrs[pos]; }
void sort();
size_t capacity() { return capacity_; }
/* Returns the attributes in lexicographically sorted order. */
std::vector<const Attr *> lexicographicOrder() const
{
std::vector<const Attr*> lexicographicOrder() const {
std::vector<const Attr*> res;
res.reserve(size_);
for (size_t n = 0; n < size_; n++)
res.emplace_back(&attrs[n]);
for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]);
std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) {
return (const string&)a->name < (const string&)b->name;
});
@ -91,5 +77,4 @@ public:
friend class EvalState;
};
}
} // namespace nix

28
third_party/nix/src/libexpr/common-eval-args.cc vendored
View file

@ -1,40 +1,43 @@
#include "common-eval-args.hh"
#include "shared.hh"
#include "download.hh"
#include "util.hh"
#include "eval.hh"
#include "shared.hh"
#include "util.hh"
namespace nix {
MixEvalArgs::MixEvalArgs()
{
MixEvalArgs::MixEvalArgs() {
mkFlag()
.longName("arg")
.description("argument to be passed to Nix functions")
.labels({"name", "expr"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
.handler(
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
mkFlag()
.longName("argstr")
.description("string-valued argument to be passed to Nix functions")
.labels({"name", "string"})
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
.handler(
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
mkFlag()
.shortName('I')
.longName("include")
.description("add a path to the list of locations used to look up <...> file names")
.description(
"add a path to the list of locations used to look up <...> file "
"names")
.label("path")
.handler([&](std::string s) { searchPath.push_back(s); });
}
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
{
Bindings* MixEvalArgs::getAutoArgs(EvalState& state) {
Bindings* res = state.allocBindings(autoArgs.size());
for (auto& i : autoArgs) {
Value* v = state.allocValue();
if (i.second[0] == 'E')
state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
state.mkThunk_(
*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
else
mkString(*v, string(i.second, 1));
res->push_back(Attr(state.symbols.create(i.first), v));
@ -43,8 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
return res;
}
Path lookupFileArg(EvalState & state, string s)
{
Path lookupFileArg(EvalState& state, string s) {
if (isUri(s)) {
CachedDownloadRequest request(s);
request.unpack = true;
@ -56,4 +58,4 @@ Path lookupFileArg(EvalState & state, string s)
return absPath(s);
}
}
} // namespace nix

6
third_party/nix/src/libexpr/common-eval-args.hh vendored
View file

@ -8,8 +8,7 @@ class Store;
class EvalState;
class Bindings;
struct MixEvalArgs : virtual Args
{
struct MixEvalArgs : virtual Args {
MixEvalArgs();
Bindings* getAutoArgs(EvalState& state);
@ -17,10 +16,9 @@ struct MixEvalArgs : virtual Args
Strings searchPath;
private:
std::map<std::string, std::string> autoArgs;
};
Path lookupFileArg(EvalState& state, string s);
}
} // namespace nix

51
third_party/nix/src/libexpr/eval-inline.hh vendored
View file

@ -2,30 +2,29 @@
#include "eval.hh"
#define LocalNoInline(f) static f __attribute__((noinline)); f
#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f
#define LocalNoInline(f) \
static f __attribute__((noinline)); \
f
#define LocalNoInlineNoReturn(f) \
static f __attribute__((noinline, noreturn)); \
f
namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
{
LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) {
throw EvalError(format(s) % pos);
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
{
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) {
throw TypeError(format(s) % showType(v));
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
{
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v,
const Pos& pos)) {
throw TypeError(format(s) % showType(v) % pos);
}
void EvalState::forceValue(Value & v, const Pos & pos)
{
void EvalState::forceValue(Value& v, const Pos& pos) {
if (v.type == tThunk) {
Env* env = v.thunk.env;
Expr* expr = v.thunk.expr;
@ -39,48 +38,37 @@ void EvalState::forceValue(Value & v, const Pos & pos)
v.thunk.expr = expr;
throw;
}
}
else if (v.type == tApp)
} else if (v.type == tApp)
callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole)
throwEvalError("infinite recursion encountered, at %1%", pos);
}
inline void EvalState::forceAttrs(Value & v)
{
inline void EvalState::forceAttrs(Value& v) {
forceValue(v);
if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected", v);
}
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
inline void EvalState::forceAttrs(Value& v, const Pos& pos) {
forceValue(v);
if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
}
inline void EvalState::forceList(Value & v)
{
inline void EvalState::forceList(Value& v) {
forceValue(v);
if (!v.isList())
throwTypeError("value is %1% while a list was expected", v);
if (!v.isList()) throwTypeError("value is %1% while a list was expected", v);
}
inline void EvalState::forceList(Value & v, const Pos & pos)
{
inline void EvalState::forceList(Value& v, const Pos& pos) {
forceValue(v);
if (!v.isList())
throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
}
/* Note: Various places expect the allocated memory to be zeroed. */
inline void * allocBytes(size_t n)
{
inline void* allocBytes(size_t n) {
void* p;
#if HAVE_BOEHMGC
p = GC_MALLOC(n);
@ -91,5 +79,4 @@ inline void * allocBytes(size_t n)
return p;
}
}
} // namespace nix

873
third_party/nix/src/libexpr/eval.cc vendored
View file

File diff suppressed because it is too large Load diff

116
third_party/nix/src/libexpr/eval.hh vendored
View file

@ -1,30 +1,25 @@
#pragma once
#include "attr-set.hh"
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "hash.hh"
#include "config.hh"
#include <map>
#include <optional>
#include <unordered_map>
#include "attr-set.hh"
#include "config.hh"
#include "hash.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "value.hh"
namespace nix {
class Store;
class EvalState;
enum RepairFlag : bool;
typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args,
Value& v);
typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v);
struct PrimOp
{
struct PrimOp {
PrimOpFun fun;
size_t arity;
Symbol name;
@ -32,9 +27,7 @@ struct PrimOp
: fun(fun), arity(arity), name(name) {}
};
struct Env
{
struct Env {
Env* up;
unsigned short size; // used by ‘valueSize’
unsigned short prevWith : 14; // nr of levels up to next `with' environment
@ -42,37 +35,29 @@ struct Env
Value* values[0];
};
Value& mkString(Value& v, const string& s, const PathSet& context = PathSet());
void copyContext(const Value& v, PathSet& context);
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
typedef std::map<Path, Path> SrcToStore;
std::ostream& operator<<(std::ostream& str, const Value& v);
typedef std::pair<std::string, std::string> SearchPathElem;
typedef std::list<SearchPathElem> SearchPath;
/* Initialise the Boehm GC, if applicable. */
void initGC();
class EvalState
{
class EvalState {
public:
SymbolTable symbols;
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem,
sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn,
sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode;
Symbol sDerivationNix;
@ -93,7 +78,9 @@ private:
/* A cache from path names to parse trees. */
#if HAVE_BOEHMGC
typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
typedef std::map<Path, Expr*, std::less<Path>,
traceable_allocator<std::pair<const Path, Expr*>>>
FileParseCache;
#else
typedef std::map<Path, Expr*> FileParseCache;
#endif
@ -101,7 +88,9 @@ private:
/* A cache from path names to values. */
#if HAVE_BOEHMGC
typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
typedef std::map<Path, Value, std::less<Path>,
traceable_allocator<std::pair<const Path, Value>>>
FileEvalCache;
#else
typedef std::map<Path, Value> FileEvalCache;
#endif
@ -115,7 +104,6 @@ private:
std::unordered_map<Path, Path> resolvedPaths;
public:
EvalState(const Strings& _searchPath, ref<Store> store);
~EvalState();
@ -141,7 +129,8 @@ public:
Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv);
/* Parse a Nix expression from the specified string. */
Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
Expr* parseExprFromString(const string& s, const Path& basePath,
StaticEnv& staticEnv);
Expr* parseExprFromString(const string& s, const Path& basePath);
Expr* parseStdin();
@ -154,10 +143,12 @@ public:
/* Look up a file in the search path. */
Path findFile(const string& path);
Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos);
Path findFile(SearchPath& searchPath, const string& path,
const Pos& pos = noPos);
/* If the specified search path element is a URI, download it. */
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
std::pair<bool, std::string> resolveSearchPathElem(
const SearchPathElem& elem);
/* Evaluate an expression to normal form, storing the result in
value `v'. */
@ -197,7 +188,9 @@ public:
bool isDerivation(Value& v);
std::optional<string> tryAttrsToString(const Pos& pos, Value& v,
PathSet & context, bool coerceMore = false, bool copyToStore = true);
PathSet& context,
bool coerceMore = false,
bool copyToStore = true);
/* String coercion. Converts strings, paths and derivations to a
string. If `coerceMore' is set, also converts nulls, integers,
@ -214,7 +207,6 @@ public:
Path coerceToPath(const Pos& pos, Value& v, PathSet& context);
public:
/* The base environment, containing the builtin functions and
values. */
Env& baseEnv;
@ -223,33 +215,28 @@ public:
StaticEnv staticBaseEnv; // !!! should be private
private:
unsigned int baseEnvDispl = 0;
void createBaseEnv();
Value* addConstant(const string& name, Value& v);
Value * addPrimOp(const string & name,
size_t arity, PrimOpFun primOp);
Value* addPrimOp(const string& name, size_t arity, PrimOpFun primOp);
public:
Value& getBuiltin(const string& name);
private:
inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval);
friend struct ExprVar;
friend struct ExprAttrs;
friend struct ExprLet;
Expr * parse(const char * text, const Path & path,
const Path & basePath, StaticEnv & staticEnv);
Expr* parse(const char* text, const Path& path, const Path& basePath,
StaticEnv& staticEnv);
public:
/* Do a deep equality test between two values. That is, list
elements and attributes are compared recursively. */
bool eqValues(Value& v1, Value& v2);
@ -284,7 +271,6 @@ public:
void realiseContext(const PathSet& context);
private:
unsigned long nrEnvs = 0;
unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0;
@ -313,10 +299,10 @@ private:
friend struct ExprOpUpdate;
friend struct ExprOpConcatLists;
friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args,
Value& v);
};
/* Return a string representing the type of the value `v'. */
string showType(const Value& v);
@ -327,8 +313,7 @@ std::pair<string, string> decodeContext(const string & s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);
struct InvalidPathError : EvalError
{
struct InvalidPathError : EvalError {
Path path;
InvalidPathError(const Path& path);
#ifdef EXCEPTION_NEEDS_THROW_SPEC
@ -336,28 +321,37 @@ struct InvalidPathError : EvalError
#endif
};
struct EvalSettings : Config
{
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
struct EvalSettings : Config {
Setting<bool> enableNativeCode{this, false,
"allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow "
"executing native code should be enabled."};
Setting<bool> restrictEval{this, false, "restrict-eval",
Setting<bool> restrictEval{
this, false, "restrict-eval",
"Whether to restrict file system access to paths in $NIX_PATH, "
"and network access to the URI prefixes listed in 'allowed-uris'."};
Setting<bool> pureEval{this, false, "pure-eval",
"Whether to restrict file system and network access to files specified by cryptographic hash."};
"Whether to restrict file system and network access "
"to files specified by cryptographic hash."};
Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
"Whether the evaluator allows importing the result of a derivation."};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
Setting<Strings> allowedUris{
this,
{},
"allowed-uris",
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit "
"are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
"Emit log messages for each function entry "
"and exit at the 'vomit' log level (-vvvv)"};
};
extern EvalSettings evalSettings;
}
} // namespace nix

2
third_party/nix/src/libexpr/function-trace.cc vendored
View file

@ -14,4 +14,4 @@ FunctionCallTrace::~FunctionCallTrace() {
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
}
}
} // namespace nix

8
third_party/nix/src/libexpr/function-trace.hh vendored
View file

@ -1,15 +1,13 @@
#pragma once
#include "eval.hh"
#include <chrono>
#include "eval.hh"
namespace nix {
struct FunctionCallTrace
{
struct FunctionCallTrace {
const Pos& pos;
FunctionCallTrace(const Pos& pos);
~FunctionCallTrace();
};
}
} // namespace nix

199
third_party/nix/src/libexpr/get-drvs.cc vendored
View file

@ -1,24 +1,18 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include <cstring>
#include <regex>
#include "derivations.hh"
#include "eval-inline.hh"
#include "util.hh"
namespace nix {
DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs)
: state(&state), attrs(attrs), attrPath(attrPath)
{
}
: state(&state), attrs(attrs), attrPath(attrPath) {}
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("")
{
DrvInfo::DrvInfo(EvalState& state, ref<Store> store,
const std::string& drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("") {
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
drvPath = spec.first;
@ -28,23 +22,22 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
name = storePathToName(drvPath);
if (spec.second.size() > 1)
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
throw Error(
"building more than one derivation output is not supported, in '%s'",
drvPathWithOutputs);
outputName =
spec.second.empty()
? get(drv.env, "outputName", "out")
outputName = spec.second.empty() ? get(drv.env, "outputName", "out")
: *spec.second.begin();
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
throw Error("derivation '%s' does not have output '%s'", drvPath,
outputName);
outPath = i->second.path;
}
string DrvInfo::queryName() const
{
string DrvInfo::queryName() const {
if (name == "" && attrs) {
auto i = attrs->find(state->sName);
if (i == attrs->end()) throw TypeError("derivation name missing");
@ -53,41 +46,38 @@ string DrvInfo::queryName() const
return name;
}
string DrvInfo::querySystem() const
{
string DrvInfo::querySystem() const {
if (system == "" && attrs) {
auto i = attrs->find(state->sSystem);
system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
system = i == attrs->end() ? "unknown"
: state->forceStringNoCtx(*i->value, *i->pos);
}
return system;
}
string DrvInfo::queryDrvPath() const
{
string DrvInfo::queryDrvPath() const {
if (drvPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sDrvPath);
PathSet context;
drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
drvPath = i != attrs->end()
? state->coerceToPath(*i->pos, *i->value, context)
: "";
}
return drvPath;
}
string DrvInfo::queryOutPath() const
{
string DrvInfo::queryOutPath() const {
if (outPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
outPath = i != attrs->end()
? state->coerceToPath(*i->pos, *i->value, context)
: "";
}
return outPath;
}
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
{
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) {
if (outputs.empty()) {
/* Get the ‘outputs’ list. */
Bindings::iterator i;
@ -97,22 +87,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
/* For each output... */
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
/* Evaluate the corresponding set. */
string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
string name =
state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
Bindings::iterator out = attrs->find(state->symbols.create(name));
if (out == attrs->end()) continue; // FIXME: throw error?
state->forceAttrs(*out->value);
/* And evaluate its ‘outPath’ attribute. */
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
if (outPath == out->value->attrs->end())
continue; // FIXME: throw error?
PathSet context;
outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
outputs[name] =
state->coerceToPath(*outPath->pos, *outPath->value, context);
}
} else
outputs["out"] = queryOutPath();
}
if (!onlyOutputsToInstall || !attrs)
return outputs;
if (!onlyOutputsToInstall || !attrs) return outputs;
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
const Value* outTI = queryMeta("outputsToInstall");
@ -121,7 +113,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
/* ^ this shows during `nix-env -i` right under the bad derivation */
if (!outTI->isList()) throw errMsg;
Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize();
++i) {
if ((*i)->type != tString) throw errMsg;
auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg;
@ -130,9 +123,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
return result;
}
string DrvInfo::queryOutputName() const
{
string DrvInfo::queryOutputName() const {
if (outputName == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutputName);
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
@ -140,9 +131,7 @@ string DrvInfo::queryOutputName() const
return outputName;
}
Bindings * DrvInfo::getMeta()
{
Bindings* DrvInfo::getMeta() {
if (meta) return meta;
if (!attrs) return 0;
Bindings::iterator a = attrs->find(state->sMeta);
@ -152,56 +141,44 @@ Bindings * DrvInfo::getMeta()
return meta;
}
StringSet DrvInfo::queryMetaNames()
{
StringSet DrvInfo::queryMetaNames() {
StringSet res;
if (!getMeta()) return res;
for (auto & i : *meta)
res.insert(i.name);
for (auto& i : *meta) res.insert(i.name);
return res;
}
bool DrvInfo::checkMeta(Value & v)
{
bool DrvInfo::checkMeta(Value& v) {
state->forceValue(v);
if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n)
if (!checkMeta(*v.listElems()[n])) return false;
return true;
}
else if (v.type == tAttrs) {
} else if (v.type == tAttrs) {
Bindings::iterator i = v.attrs->find(state->sOutPath);
if (i != v.attrs->end()) return false;
for (auto& i : *v.attrs)
if (!checkMeta(*i.value)) return false;
return true;
}
else return v.type == tInt || v.type == tBool || v.type == tString ||
} else
return v.type == tInt || v.type == tBool || v.type == tString ||
v.type == tFloat;
}
Value * DrvInfo::queryMeta(const string & name)
{
Value* DrvInfo::queryMeta(const string& name) {
if (!getMeta()) return 0;
Bindings::iterator a = meta->find(state->symbols.create(name));
if (a == meta->end() || !checkMeta(*a->value)) return 0;
return a->value;
}
string DrvInfo::queryMetaString(const string & name)
{
string DrvInfo::queryMetaString(const string& name) {
Value* v = queryMeta(name);
if (!v || v->type != tString) return "";
return v->string.s;
}
NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
{
NixInt DrvInfo::queryMetaInt(const string& name, NixInt def) {
Value* v = queryMeta(name);
if (!v) return def;
if (v->type == tInt) return v->integer;
@ -214,8 +191,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
return def;
}
NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
{
NixFloat DrvInfo::queryMetaFloat(const string& name, NixFloat def) {
Value* v = queryMeta(name);
if (!v) return def;
if (v->type == tFloat) return v->fpoint;
@ -228,9 +204,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
return def;
}
bool DrvInfo::queryMetaBool(const string & name, bool def)
{
bool DrvInfo::queryMetaBool(const string& name, bool def) {
Value* v = queryMeta(name);
if (!v) return def;
if (v->type == tBool) return v->boolean;
@ -243,34 +217,28 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
return def;
}
void DrvInfo::setMeta(const string & name, Value * v)
{
void DrvInfo::setMeta(const string& name, Value* v) {
getMeta();
Bindings* old = meta;
meta = state->allocBindings(1 + (old ? old->size() : 0));
Symbol sym = state->symbols.create(name);
if (old)
for (auto i : *old)
if (i.name != sym)
meta->push_back(i);
if (i.name != sym) meta->push_back(i);
if (v) meta->push_back(Attr(sym, v));
meta->sort();
}
/* Cache for already considered attrsets. */
typedef set<Bindings*> Done;
/* Evaluate value `v'. If it evaluates to a set of type `derivation',
then put information about it in `drvs' (unless it's already in `done').
The result boolean indicates whether it makes sense
for the caller to recursively search for derivations in `v'. */
static bool getDerivation(EvalState & state, Value & v,
const string & attrPath, DrvInfos & drvs, Done & done,
bool ignoreAssertionFailures)
{
static bool getDerivation(EvalState& state, Value& v, const string& attrPath,
DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures) {
try {
state.forceValue(v);
if (!state.isDerivation(v)) return true;
@ -294,10 +262,8 @@ static bool getDerivation(EvalState & state, Value & v,
}
}
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
bool ignoreAssertionFailures)
{
bool ignoreAssertionFailures) {
Done done;
DrvInfos drvs;
getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
@ -305,32 +271,29 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
return std::move(drvs.front());
}
static string addToPath(const string & s1, const string & s2)
{
static string addToPath(const string& s1, const string& s2) {
return s1.empty() ? s2 : s1 + "." + s2;
}
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
static void getDerivations(EvalState& state, Value& vIn,
const string& pathPrefix, Bindings& autoArgs,
DrvInfos& drvs, Done& done,
bool ignoreAssertionFailures)
{
bool ignoreAssertionFailures) {
Value v;
state.autoCallFunction(autoArgs, vIn, v);
/* Process the expression. */
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures))
;
else if (v.type == tAttrs) {
/* !!! undocumented hackery to support combining channels in
nix-env.cc. */
bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
bool combineChannels =
v.attrs->find(state.symbols.create("_combineChannels")) !=
v.attrs->end();
/* Consider the attributes in sorted order to get more
deterministic behaviour in nix-env operations (e.g. when
@ -339,19 +302,23 @@ static void getDerivations(EvalState & state, Value & vIn,
precedence). */
for (auto& i : v.attrs->lexicographicOrder()) {
debug("evaluating attribute '%1%'", i->name);
if (!std::regex_match(std::string(i->name), attrRegex))
continue;
if (!std::regex_match(std::string(i->name), attrRegex)) continue;
string pathPrefix2 = addToPath(pathPrefix, i->name);
if (combineChannels)
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
ignoreAssertionFailures);
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done,
ignoreAssertionFailures)) {
/* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type == tAttrs) {
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
Bindings::iterator j = i->value->attrs->find(
state.symbols.create("recurseForDerivations"));
if (j != i->value->attrs->end() &&
state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
ignoreAssertionFailures);
}
}
}
@ -360,21 +327,25 @@ static void getDerivations(EvalState & state, Value & vIn,
else if (v.isList()) {
for (unsigned int n = 0; n < v.listSize(); ++n) {
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done,
ignoreAssertionFailures))
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs,
done, ignoreAssertionFailures);
}
}
else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
else
throw TypeError(
"expression does not evaluate to a derivation (or a set or list of "
"those)");
}
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
{
Bindings& autoArgs, DrvInfos& drvs,
bool ignoreAssertionFailures) {
Done done;
getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures);
getDerivations(state, v, pathPrefix, autoArgs, drvs, done,
ignoreAssertionFailures);
}
}
} // namespace nix

25
third_party/nix/src/libexpr/get-drvs.hh vendored
View file

@ -1,16 +1,12 @@
#pragma once
#include "eval.hh"
#include <string>
#include <map>
#include <string>
#include "eval.hh"
namespace nix {
struct DrvInfo
{
struct DrvInfo {
public:
typedef std::map<string, Path> Outputs;
@ -37,14 +33,16 @@ public:
DrvInfo(EvalState& state) : state(&state){};
DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs);
DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
DrvInfo(EvalState& state, ref<Store> store,
const std::string& drvPathWithOutputs);
string queryName() const;
string querySystem() const;
string queryDrvPath() const;
string queryOutPath() const;
string queryOutputName() const;
/** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
/** Return the list of outputs. The "outputs to install" are determined by
* `meta.outputsToInstall`. */
Outputs queryOutputs(bool onlyOutputsToInstall = false);
StringSet queryMetaNames();
@ -68,22 +66,19 @@ public:
bool hasFailed() { return failed; };
};
#if HAVE_BOEHMGC
typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
#else
typedef list<DrvInfo> DrvInfos;
#endif
/* If value `v' denotes a derivation, return a DrvInfo object
describing it. Otherwise return nothing. */
std::optional<DrvInfo> getDerivation(EvalState & state,
Value & v, bool ignoreAssertionFailures);
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
bool ignoreAssertionFailures);
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
Bindings& autoArgs, DrvInfos& drvs,
bool ignoreAssertionFailures);
}
} // namespace nix

76
third_party/nix/src/libexpr/json-to-value.cc vendored
View file

@ -1,35 +1,42 @@
#include "json-to-value.hh"
#include <cstring>
namespace nix {
static void skipWhitespace(const char * & s)
{
static void skipWhitespace(const char*& s) {
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
}
static string parseJSONString(const char * & s)
{
static string parseJSONString(const char*& s) {
string res;
if (*s++ != '"') throw JSONParseError("expected JSON string");
while (*s != '"') {
if (!*s) throw JSONParseError("got end-of-string in JSON string");
if (*s == '\\') {
s++;
if (*s == '"') res += '"';
else if (*s == '\\') res += '\\';
else if (*s == '/') res += '/';
else if (*s == '/') res += '/';
else if (*s == 'b') res += '\b';
else if (*s == 'f') res += '\f';
else if (*s == 'n') res += '\n';
else if (*s == 'r') res += '\r';
else if (*s == 't') res += '\t';
else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
else throw JSONParseError("invalid escaped character in JSON string");
if (*s == '"')
res += '"';
else if (*s == '\\')
res += '\\';
else if (*s == '/')
res += '/';
else if (*s == '/')
res += '/';
else if (*s == 'b')
res += '\b';
else if (*s == 'f')
res += '\f';
else if (*s == 'n')
res += '\n';
else if (*s == 'r')
res += '\r';
else if (*s == 't')
res += '\t';
else if (*s == 'u')
throw JSONParseError(
"\\u characters in JSON strings are currently not supported");
else
throw JSONParseError("invalid escaped character in JSON string");
s++;
} else
res += *s++;
@ -38,9 +45,7 @@ static string parseJSONString(const char * & s)
return res;
}
static void parseJSON(EvalState & state, const char * & s, Value & v)
{
static void parseJSON(EvalState& state, const char*& s, Value& v) {
skipWhitespace(s);
if (!*s) throw JSONParseError("expected JSON value");
@ -57,13 +62,13 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
values.push_back(v2);
skipWhitespace(s);
if (*s == ']') break;
if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
if (*s != ',')
throw JSONParseError("expected ',' or ']' after JSON array element");
s++;
}
s++;
state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n)
v.listElems()[n] = values[n];
for (size_t n = 0; n < values.size(); ++n) v.listElems()[n] = values[n];
}
else if (*s == '{') {
@ -81,12 +86,12 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
attrs[state.symbols.create(name)] = v2;
skipWhitespace(s);
if (*s == '}') break;
if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
if (*s != ',')
throw JSONParseError("expected ',' or '}' after JSON member");
s++;
}
state.mkAttrs(v, attrs.size());
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second));
for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second));
v.attrs->sort();
s++;
}
@ -101,8 +106,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
ValueType number_type = tInt;
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
if (*s == '.' || *s == 'e' || *s == 'E')
number_type = tFloat;
if (*s == '.' || *s == 'e' || *s == 'E') number_type = tFloat;
tmp_number += *s++;
}
@ -133,17 +137,17 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
mkNull(v);
}
else throw JSONParseError("unrecognised JSON value");
else
throw JSONParseError("unrecognised JSON value");
}
void parseJSON(EvalState & state, const string & s_, Value & v)
{
void parseJSON(EvalState& state, const string& s_, Value& v) {
const char* s = s_.c_str();
parseJSON(state, s, v);
skipWhitespace(s);
if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
if (*s)
throw JSONParseError(
format("expected end-of-string while parsing JSON value: %1%") % s);
}
}
} // namespace nix

3
third_party/nix/src/libexpr/json-to-value.hh vendored
View file

@ -1,8 +1,7 @@
#pragma once
#include "eval.hh"
#include <string>
#include "eval.hh"
namespace nix {

71
third_party/nix/src/libexpr/names.cc vendored
View file

@ -1,23 +1,16 @@
#include "names.hh"
#include "util.hh"
namespace nix {
DrvName::DrvName()
{
name = "";
}
DrvName::DrvName() { name = ""; }
/* Parse a derivation name. The `name' part of a derivation name is
everything up to but not including the first dash *not* followed by
a letter. The `version' part is the rest (excluding the separating
dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
'2.0.48'). */
DrvName::DrvName(const string & s) : hits(0)
{
DrvName::DrvName(const string& s) : hits(0) {
name = fullName = s;
for (unsigned int i = 0; i < s.size(); ++i) {
/* !!! isalpha/isdigit are affected by the locale. */
@ -29,21 +22,19 @@ DrvName::DrvName(const string & s) : hits(0)
}
}
bool DrvName::matches(DrvName & n)
{
bool DrvName::matches(DrvName& n) {
if (name != "*") {
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
if (!regex)
regex = std::unique_ptr<std::regex>(
new std::regex(name, std::regex::extended));
if (!std::regex_match(n.name, *regex)) return false;
}
if (version != "" && version != n.version) return false;
return true;
}
string nextComponent(string::const_iterator& p,
const string::const_iterator end)
{
const string::const_iterator end) {
/* Skip any dots and dashes (component separators). */
while (p != end && (*p == '.' || *p == '-')) ++p;
@ -56,52 +47,52 @@ string nextComponent(string::const_iterator & p,
if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++;
else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
s += *p++;
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) s += *p++;
return s;
}
static bool componentsLT(const string & c1, const string & c2)
{
static bool componentsLT(const string& c1, const string& c2) {
int n1, n2;
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
if (c1Num && c2Num) return n1 < n2;
else if (c1 == "" && c2Num) return true;
else if (c1 == "pre" && c2 != "pre") return true;
else if (c2 == "pre") return false;
if (c1Num && c2Num)
return n1 < n2;
else if (c1 == "" && c2Num)
return true;
else if (c1 == "pre" && c2 != "pre")
return true;
else if (c2 == "pre")
return false;
/* Assume that `2.3a' < `2.3.1'. */
else if (c2Num) return true;
else if (c1Num) return false;
else return c1 < c2;
else if (c2Num)
return true;
else if (c1Num)
return false;
else
return c1 < c2;
}
int compareVersions(const string & v1, const string & v2)
{
int compareVersions(const string& v1, const string& v2) {
string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin();
while (p1 != v1.end() || p2 != v2.end()) {
string c1 = nextComponent(p1, v1.end());
string c2 = nextComponent(p2, v2.end());
if (componentsLT(c1, c2)) return -1;
else if (componentsLT(c2, c1)) return 1;
if (componentsLT(c1, c2))
return -1;
else if (componentsLT(c2, c1))
return 1;
}
return 0;
}
DrvNames drvNamesFromArgs(const Strings & opArgs)
{
DrvNames drvNamesFromArgs(const Strings& opArgs) {
DrvNames result;
for (auto & i : opArgs)
result.push_back(DrvName(i));
for (auto& i : opArgs) result.push_back(DrvName(i));
return result;
}
}
} // namespace nix

8
third_party/nix/src/libexpr/names.hh vendored
View file

@ -1,14 +1,12 @@
#pragma once
#include <memory>
#include "types.hh"
#include <regex>
#include "types.hh"
namespace nix {
struct DrvName
{
struct DrvName {
string fullName;
string name;
string version;
@ -29,4 +27,4 @@ string nextComponent(string::const_iterator & p,
int compareVersions(const string& v1, const string& v2);
DrvNames drvNamesFromArgs(const Strings& opArgs);
}
} // namespace nix

267
third_party/nix/src/libexpr/nixexpr.cc vendored
View file

@ -1,35 +1,34 @@
#include "nixexpr.hh"
#include <cstdlib>
#include "derivations.hh"
#include "util.hh"
#include <cstdlib>
namespace nix {
/* Displaying abstract syntax trees. */
std::ostream & operator << (std::ostream & str, const Expr & e)
{
std::ostream& operator<<(std::ostream& str, const Expr& e) {
e.show(str);
return str;
}
static void showString(std::ostream & str, const string & s)
{
static void showString(std::ostream& str, const string& s) {
str << '"';
for (auto c : (string)s)
if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
else if (c == '\n') str << "\\n";
else if (c == '\r') str << "\\r";
else if (c == '\t') str << "\\t";
else str << c;
if (c == '"' || c == '\\' || c == '$')
str << "\\" << c;
else if (c == '\n')
str << "\\n";
else if (c == '\r')
str << "\\r";
else if (c == '\t')
str << "\\t";
else
str << c;
str << '"';
}
static void showId(std::ostream & str, const string & s)
{
static void showId(std::ostream& str, const string& s) {
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
@ -41,10 +40,8 @@ static void showId(std::ostream & str, const string & s)
return;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) {
showString(str, s);
return;
}
@ -52,60 +49,39 @@ static void showId(std::ostream & str, const string & s)
}
}
std::ostream & operator << (std::ostream & str, const Symbol & sym)
{
std::ostream& operator<<(std::ostream& str, const Symbol& sym) {
showId(str, *sym.s);
return str;
}
void Expr::show(std::ostream & str) const
{
abort();
}
void Expr::show(std::ostream& str) const { abort(); }
void ExprInt::show(std::ostream & str) const
{
str << n;
}
void ExprInt::show(std::ostream& str) const { str << n; }
void ExprFloat::show(std::ostream & str) const
{
str << nf;
}
void ExprFloat::show(std::ostream& str) const { str << nf; }
void ExprString::show(std::ostream & str) const
{
showString(str, s);
}
void ExprString::show(std::ostream& str) const { showString(str, s); }
void ExprPath::show(std::ostream & str) const
{
str << s;
}
void ExprPath::show(std::ostream& str) const { str << s; }
void ExprVar::show(std::ostream & str) const
{
str << name;
}
void ExprVar::show(std::ostream& str) const { str << name; }
void ExprSelect::show(std::ostream & str) const
{
void ExprSelect::show(std::ostream& str) const {
str << "(" << *e << ")." << showAttrPath(attrPath);
if (def) str << " or (" << *def << ")";
}
void ExprOpHasAttr::show(std::ostream & str) const
{
void ExprOpHasAttr::show(std::ostream& str) const {
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
}
void ExprAttrs::show(std::ostream & str) const
{
void ExprAttrs::show(std::ostream& str) const {
if (recursive) str << "rec ";
str << "{ ";
for (auto& i : attrs)
if (i.second.inherited)
str << "inherit " << i.first << " " << "; ";
str << "inherit " << i.first << " "
<< "; ";
else
str << i.first << " = " << *i.second.e << "; ";
for (auto& i : dynamicAttrs)
@ -113,22 +89,22 @@ void ExprAttrs::show(std::ostream & str) const
str << "}";
}
void ExprList::show(std::ostream & str) const
{
void ExprList::show(std::ostream& str) const {
str << "[ ";
for (auto & i : elems)
str << "(" << *i << ") ";
for (auto& i : elems) str << "(" << *i << ") ";
str << "]";
}
void ExprLambda::show(std::ostream & str) const
{
void ExprLambda::show(std::ostream& str) const {
str << "(";
if (matchAttrs) {
str << "{ ";
bool first = true;
for (auto& i : formals->formals) {
if (first) first = false; else str << ", ";
if (first)
first = false;
else
str << ", ";
str << i.name;
if (i.def) str << " ? " << *i.def;
}
@ -143,71 +119,63 @@ void ExprLambda::show(std::ostream & str) const
str << ": " << *body << ")";
}
void ExprLet::show(std::ostream & str) const
{
void ExprLet::show(std::ostream& str) const {
str << "(let ";
for (auto& i : attrs->attrs)
if (i.second.inherited) {
str << "inherit " << i.first << "; ";
}
else
} else
str << i.first << " = " << *i.second.e << "; ";
str << "in " << *body << ")";
}
void ExprWith::show(std::ostream & str) const
{
void ExprWith::show(std::ostream& str) const {
str << "(with " << *attrs << "; " << *body << ")";
}
void ExprIf::show(std::ostream & str) const
{
void ExprIf::show(std::ostream& str) const {
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
}
void ExprAssert::show(std::ostream & str) const
{
void ExprAssert::show(std::ostream& str) const {
str << "assert " << *cond << "; " << *body;
}
void ExprOpNot::show(std::ostream & str) const
{
str << "(! " << *e << ")";
}
void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; }
void ExprConcatStrings::show(std::ostream & str) const
{
void ExprConcatStrings::show(std::ostream& str) const {
bool first = true;
str << "(";
for (auto& i : *es) {
if (first) first = false; else str << " + ";
if (first)
first = false;
else
str << " + ";
str << *i;
}
str << ")";
}
void ExprPos::show(std::ostream & str) const
{
str << "__curPos";
}
void ExprPos::show(std::ostream& str) const { str << "__curPos"; }
std::ostream & operator << (std::ostream & str, const Pos & pos)
{
std::ostream& operator<<(std::ostream& str, const Pos& pos) {
if (!pos)
str << "undefined position";
else
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string)pos.file %
pos.line % pos.column)
.str();
return str;
}
string showAttrPath(const AttrPath & attrPath)
{
string showAttrPath(const AttrPath& attrPath) {
std::ostringstream out;
bool first = true;
for (auto& i : attrPath) {
if (!first) out << '.'; else first = false;
if (!first)
out << '.';
else
first = false;
if (i.symbol.set())
out << i.symbol;
else
@ -216,35 +184,21 @@ string showAttrPath(const AttrPath & attrPath)
return out.str();
}
Pos noPos;
/* Computing levels/displacements for variables. */
void Expr::bindVars(const StaticEnv & env)
{
abort();
}
void Expr::bindVars(const StaticEnv& env) { abort(); }
void ExprInt::bindVars(const StaticEnv & env)
{
}
void ExprInt::bindVars(const StaticEnv& env) {}
void ExprFloat::bindVars(const StaticEnv & env)
{
}
void ExprFloat::bindVars(const StaticEnv& env) {}
void ExprString::bindVars(const StaticEnv & env)
{
}
void ExprString::bindVars(const StaticEnv& env) {}
void ExprPath::bindVars(const StaticEnv & env)
{
}
void ExprPath::bindVars(const StaticEnv& env) {}
void ExprVar::bindVars(const StaticEnv & env)
{
void ExprVar::bindVars(const StaticEnv& env) {
/* Check whether the variable appears in the environment. If so,
set its level and displacement. */
const StaticEnv* curEnv;
@ -267,31 +221,28 @@ void ExprVar::bindVars(const StaticEnv & env)
/* Otherwise, the variable must be obtained from the nearest
enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
if (withLevel == -1)
throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name %
pos);
fromWith = true;
this->level = withLevel;
}
void ExprSelect::bindVars(const StaticEnv & env)
{
void ExprSelect::bindVars(const StaticEnv& env) {
e->bindVars(env);
if (def) def->bindVars(env);
for (auto& i : attrPath)
if (!i.symbol.set())
i.expr->bindVars(env);
if (!i.symbol.set()) i.expr->bindVars(env);
}
void ExprOpHasAttr::bindVars(const StaticEnv & env)
{
void ExprOpHasAttr::bindVars(const StaticEnv& env) {
e->bindVars(env);
for (auto& i : attrPath)
if (!i.symbol.set())
i.expr->bindVars(env);
if (!i.symbol.set()) i.expr->bindVars(env);
}
void ExprAttrs::bindVars(const StaticEnv & env)
{
void ExprAttrs::bindVars(const StaticEnv& env) {
const StaticEnv* dynamicEnv = &env;
StaticEnv newEnv(false, &env);
@ -299,16 +250,14 @@ void ExprAttrs::bindVars(const StaticEnv & env)
dynamicEnv = &newEnv;
unsigned int displ = 0;
for (auto & i : attrs)
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs) newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
}
else
for (auto & i : attrs)
i.second.e->bindVars(env);
for (auto& i : attrs) i.second.e->bindVars(env);
for (auto& i : dynamicAttrs) {
i.nameExpr->bindVars(*dynamicEnv);
@ -316,14 +265,11 @@ void ExprAttrs::bindVars(const StaticEnv & env)
}
}
void ExprList::bindVars(const StaticEnv & env)
{
for (auto & i : elems)
i->bindVars(env);
void ExprList::bindVars(const StaticEnv& env) {
for (auto& i : elems) i->bindVars(env);
}
void ExprLambda::bindVars(const StaticEnv & env)
{
void ExprLambda::bindVars(const StaticEnv& env) {
StaticEnv newEnv(false, &env);
unsigned int displ = 0;
@ -331,8 +277,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
if (!arg.empty()) newEnv.vars[arg] = displ++;
if (matchAttrs) {
for (auto & i : formals->formals)
newEnv.vars[i.name] = displ++;
for (auto& i : formals->formals) newEnv.vars[i.name] = displ++;
for (auto& i : formals->formals)
if (i.def) i.def->bindVars(newEnv);
@ -341,13 +286,11 @@ void ExprLambda::bindVars(const StaticEnv & env)
body->bindVars(newEnv);
}
void ExprLet::bindVars(const StaticEnv & env)
{
void ExprLet::bindVars(const StaticEnv& env) {
StaticEnv newEnv(false, &env);
unsigned int displ = 0;
for (auto & i : attrs->attrs)
newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs->attrs) newEnv.vars[i.first] = i.second.displ = displ++;
for (auto& i : attrs->attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
@ -355,8 +298,7 @@ void ExprLet::bindVars(const StaticEnv & env)
body->bindVars(newEnv);
}
void ExprWith::bindVars(const StaticEnv & env)
{
void ExprWith::bindVars(const StaticEnv& env) {
/* Does this `with' have an enclosing `with'? If so, record its
level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */
@ -374,65 +316,46 @@ void ExprWith::bindVars(const StaticEnv & env)
body->bindVars(newEnv);
}
void ExprIf::bindVars(const StaticEnv & env)
{
void ExprIf::bindVars(const StaticEnv& env) {
cond->bindVars(env);
then->bindVars(env);
else_->bindVars(env);
}
void ExprAssert::bindVars(const StaticEnv & env)
{
void ExprAssert::bindVars(const StaticEnv& env) {
cond->bindVars(env);
body->bindVars(env);
}
void ExprOpNot::bindVars(const StaticEnv & env)
{
e->bindVars(env);
}
void ExprConcatStrings::bindVars(const StaticEnv & env)
{
for (auto & i : *es)
i->bindVars(env);
}
void ExprPos::bindVars(const StaticEnv & env)
{
void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); }
void ExprConcatStrings::bindVars(const StaticEnv& env) {
for (auto& i : *es) i->bindVars(env);
}
void ExprPos::bindVars(const StaticEnv& env) {}
/* Storing function names. */
void Expr::setName(Symbol & name)
{
}
void Expr::setName(Symbol& name) {}
void ExprLambda::setName(Symbol & name)
{
void ExprLambda::setName(Symbol& name) {
this->name = name;
body->setName(name);
}
string ExprLambda::showNamePos() const
{
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
string ExprLambda::showNamePos() const {
return (format("%1% at %2%") %
(name.set() ? "'" + (string)name + "'" : "anonymous function") % pos)
.str();
}
/* Symbol table. */
size_t SymbolTable::totalSize() const
{
size_t SymbolTable::totalSize() const {
size_t n = 0;
for (auto & i : symbols)
n += i.size();
for (auto& i : symbols) n += i.size();
return n;
}
}
} // namespace nix

161
third_party/nix/src/libexpr/nixexpr.hh vendored
View file

@ -1,39 +1,27 @@
#pragma once
#include "value.hh"
#include "symbol-table.hh"
#include <map>
#include "symbol-table.hh"
#include "value.hh"
namespace nix {
MakeError(EvalError, Error)
MakeError(ParseError, Error)
MakeError(AssertionError, EvalError)
MakeError(ThrownError, AssertionError)
MakeError(Abort, EvalError)
MakeError(TypeError, EvalError)
MakeError(EvalError, Error) MakeError(ParseError, Error)
MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError)
MakeError(Abort, EvalError) MakeError(TypeError, EvalError)
MakeError(UndefinedVarError, Error)
MakeError(RestrictedPathError, Error)
/* Position objects. */
struct Pos
{
struct Pos {
Symbol file;
unsigned int line, column;
Pos() : line(0), column(0){};
Pos(const Symbol& file, unsigned int line, unsigned int column)
: file(file), line(line), column(column){};
operator bool() const
{
return line != 0;
}
bool operator < (const Pos & p2) const
{
operator bool() const { return line != 0; }
bool operator<(const Pos& p2) const {
if (!line) return p2.line;
if (!p2.line) return false;
int d = ((string)file).compare((string)p2.file);
@ -49,16 +37,13 @@ extern Pos noPos;
std::ostream& operator<<(std::ostream& str, const Pos& pos);
struct Env;
struct Value;
class EvalState;
struct StaticEnv;
/* An attribute path is a sequence of attribute names. */
struct AttrName
{
struct AttrName {
Symbol symbol;
Expr* expr;
AttrName(const Symbol& s) : symbol(s){};
@ -69,11 +54,9 @@ typedef std::vector<AttrName> AttrPath;
string showAttrPath(const AttrPath& attrPath);
/* Abstract syntax of Nix expressions. */
struct Expr
{
struct Expr {
virtual ~Expr(){};
virtual void show(std::ostream& str) const;
virtual void bindVars(const StaticEnv& env);
@ -89,8 +72,7 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
void eval(EvalState& state, Env& env, Value& v); \
void bindVars(const StaticEnv& env);
struct ExprInt : Expr
{
struct ExprInt : Expr {
NixInt n;
Value v;
ExprInt(NixInt n) : n(n) { mkInt(v, n); };
@ -98,8 +80,7 @@ struct ExprInt : Expr
Value* maybeThunk(EvalState& state, Env& env);
};
struct ExprFloat : Expr
{
struct ExprFloat : Expr {
NixFloat nf;
Value v;
ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
@ -107,8 +88,7 @@ struct ExprFloat : Expr
Value* maybeThunk(EvalState& state, Env& env);
};
struct ExprString : Expr
{
struct ExprString : Expr {
Symbol s;
Value v;
ExprString(const Symbol& s) : s(s) { mkString(v, s); };
@ -117,14 +97,12 @@ struct ExprString : Expr
};
/* Temporary class used during parsing of indented strings. */
struct ExprIndStr : Expr
{
struct ExprIndStr : Expr {
string s;
ExprIndStr(const string& s) : s(s){};
};
struct ExprPath : Expr
{
struct ExprPath : Expr {
string s;
Value v;
ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
@ -132,8 +110,7 @@ struct ExprPath : Expr
Value* maybeThunk(EvalState& state, Env& env);
};
struct ExprVar : Expr
{
struct ExprVar : Expr {
Pos pos;
Symbol name;
@ -156,26 +133,27 @@ struct ExprVar : Expr
Value* maybeThunk(EvalState& state, Env& env);
};
struct ExprSelect : Expr
{
struct ExprSelect : Expr {
Pos pos;
Expr *e, *def;
AttrPath attrPath;
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def)
: pos(pos), e(e), def(def), attrPath(attrPath){};
ExprSelect(const Pos& pos, Expr* e, const Symbol& name)
: pos(pos), e(e), def(0) {
attrPath.push_back(AttrName(name));
};
COMMON_METHODS
};
struct ExprOpHasAttr : Expr
{
struct ExprOpHasAttr : Expr {
Expr* e;
AttrPath attrPath;
ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){};
COMMON_METHODS
};
struct ExprAttrs : Expr
{
struct ExprAttrs : Expr {
bool recursive;
struct AttrDef {
bool inherited;
@ -200,116 +178,109 @@ struct ExprAttrs : Expr
COMMON_METHODS
};
struct ExprList : Expr
{
struct ExprList : Expr {
std::vector<Expr*> elems;
ExprList(){};
COMMON_METHODS
};
struct Formal
{
struct Formal {
Symbol name;
Expr* def;
Formal(const Symbol& name, Expr* def) : name(name), def(def){};
};
struct Formals
{
struct Formals {
typedef std::list<Formal> Formals_;
Formals_ formals;
std::set<Symbol> argNames; // used during parsing
bool ellipsis;
};
struct ExprLambda : Expr
{
struct ExprLambda : Expr {
Pos pos;
Symbol name;
Symbol arg;
bool matchAttrs;
Formals* formals;
Expr* body;
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(format("duplicate formal function argument '%1%' at %2%")
% arg % pos);
ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs,
Formals* formals, Expr* body)
: pos(pos),
arg(arg),
matchAttrs(matchAttrs),
formals(formals),
body(body) {
if (!arg.empty() && formals &&
formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(
format("duplicate formal function argument '%1%' at %2%") % arg %
pos);
};
void setName(Symbol& name);
string showNamePos() const;
COMMON_METHODS
};
struct ExprLet : Expr
{
struct ExprLet : Expr {
ExprAttrs* attrs;
Expr* body;
ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){};
COMMON_METHODS
};
struct ExprWith : Expr
{
struct ExprWith : Expr {
Pos pos;
Expr *attrs, *body;
size_t prevWith;
ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
ExprWith(const Pos& pos, Expr* attrs, Expr* body)
: pos(pos), attrs(attrs), body(body){};
COMMON_METHODS
};
struct ExprIf : Expr
{
struct ExprIf : Expr {
Expr *cond, *then, *else_;
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
ExprIf(Expr* cond, Expr* then, Expr* else_)
: cond(cond), then(then), else_(else_){};
COMMON_METHODS
};
struct ExprAssert : Expr
{
struct ExprAssert : Expr {
Pos pos;
Expr *cond, *body;
ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
ExprAssert(const Pos& pos, Expr* cond, Expr* body)
: pos(pos), cond(cond), body(body){};
COMMON_METHODS
};
struct ExprOpNot : Expr
{
struct ExprOpNot : Expr {
Expr* e;
ExprOpNot(Expr* e) : e(e){};
COMMON_METHODS
};
#define MakeBinOp(name, s) \
struct name : Expr \
{ \
struct name : Expr { \
Pos pos; \
Expr *e1, *e2; \
name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \
name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \
void show(std::ostream & str) const \
{ \
void show(std::ostream& str) const { \
str << "(" << *e1 << " " s " " << *e2 << ")"; \
} \
void bindVars(const StaticEnv & env) \
{ \
e1->bindVars(env); e2->bindVars(env); \
void bindVars(const StaticEnv& env) { \
e1->bindVars(env); \
e2->bindVars(env); \
} \
void eval(EvalState& state, Env& env, Value& v); \
};
MakeBinOp(ExprApp, "")
MakeBinOp(ExprOpEq, "==")
MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpAnd, "&&")
MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpImpl, "->")
MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||")
MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//")
MakeBinOp(ExprOpConcatLists, "++")
struct ExprConcatStrings : Expr
{
struct ExprConcatStrings : Expr {
Pos pos;
bool forceString;
vector<Expr*>* es;
@ -318,19 +289,16 @@ struct ExprConcatStrings : Expr
COMMON_METHODS
};
struct ExprPos : Expr
{
struct ExprPos : Expr {
Pos pos;
ExprPos(const Pos& pos) : pos(pos){};
COMMON_METHODS
};
/* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at
runtime. */
struct StaticEnv
{
struct StaticEnv {
bool isWith;
const StaticEnv* up;
typedef std::map<Symbol, unsigned int> Vars;
@ -338,5 +306,4 @@ struct StaticEnv
StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){};
};
}
} // namespace nix

980
third_party/nix/src/libexpr/primops.cc vendored
View file

File diff suppressed because it is too large Load diff

11
third_party/nix/src/libexpr/primops.hh vendored
View file

@ -1,12 +1,10 @@
#include "eval.hh"
#include <tuple>
#include <vector>
#include "eval.hh"
namespace nix {
struct RegisterPrimOp
{
struct RegisterPrimOp {
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
static PrimOps* primOps;
/* You can register a constant by passing an arity of 0. fun
@ -19,8 +17,9 @@ struct RegisterPrimOp
may wish to use them in limited contexts without globally enabling
them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
void prim_importNative(EvalState& state, const Pos& pos, Value** args,
Value& v);
/* Execute a program and parse its output */
void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v);
}
} // namespace nix

69
third_party/nix/src/libexpr/primops/context.cc vendored
View file

@ -1,21 +1,21 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix {
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos,
Value** args, Value& v) {
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
mkString(v, s, PathSet());
}
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1,
prim_unsafeDiscardStringContext);
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_hasContext(EvalState& state, const Pos& pos, Value** args,
Value& v) {
PathSet context;
state.forceString(*args[0], context, pos);
mkBool(v, !context.empty());
@ -23,27 +23,25 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos,
Value** args, Value& v) {
PathSet context;
string s = state.coerceToString(pos, *args[0], context);
PathSet context2;
for (auto & p : context)
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
for (auto& p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p);
mkString(v, s, context2);
}
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1,
prim_unsafeDiscardOutputDependency);
/* Extract the context of a string as a structured Nix value.
@ -64,8 +62,8 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar
Note that for a given path any combination of the above attributes
may be present.
*/
static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_getContext(EvalState& state, const Pos& pos, Value** args,
Value& v) {
struct ContextInfo {
bool path = false;
bool allOutputs = false;
@ -92,7 +90,10 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
auto iter = contextInfos.find(*path);
if (iter == contextInfos.end()) {
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
contextInfos.emplace(
*path,
ContextInfo{isPath, isAllOutputs,
output.empty() ? Strings{} : Strings{std::move(output)}});
} else {
if (isPath)
iter->second.path = true;
@ -110,8 +111,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
for (const auto& info : contextInfos) {
auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first));
state.mkAttrs(infoVal, 3);
if (info.second.path)
mkBool(*state.allocAttr(infoVal, sPath), true);
if (info.second.path) mkBool(*state.allocAttr(infoVal, sPath), true);
if (info.second.allOutputs)
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
if (!info.second.outputs.empty()) {
@ -129,14 +129,13 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
static RegisterPrimOp r4("__getContext", 1, prim_getContext);
/* Append the given context to a given string.
See the commentary above unsafeGetContext for details of the
context representation.
*/
static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_appendContext(EvalState& state, const Pos& pos, Value** args,
Value& v) {
PathSet context;
auto orig = state.forceString(*args[0], context, pos);
@ -146,21 +145,23 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
auto sAllOutputs = state.symbols.create("allOutputs");
for (auto& i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
if (!settings.readOnlyMode)
state.store->ensurePath(i.name);
throw EvalError("Context key '%s' is not a store path, at %s", i.name,
i.pos);
if (!settings.readOnlyMode) state.store->ensurePath(i.name);
state.forceAttrs(*i.value, *i.pos);
auto iter = i.value->attrs->find(sPath);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos))
context.insert(i.name);
if (state.forceBool(*iter->value, *iter->pos)) context.insert(i.name);
}
iter = i.value->attrs->find(sAllOutputs);
if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
throw EvalError(
"Tried to add all-outputs context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
}
context.insert("=" + string(i.name));
}
@ -170,10 +171,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
throw EvalError(
"Tried to add derivation output context of %s, which is not a "
"derivation, to a string, at %s",
i.name, i.pos);
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
auto name =
state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
context.insert("!" + name + "!" + string(i.name));
}
}
@ -184,4 +189,4 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
}
} // namespace nix

76
third_party/nix/src/libexpr/primops/fetchGit.cc vendored
View file

@ -1,22 +1,18 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"
#include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "hash.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals;
namespace nix {
struct GitInfo
{
struct GitInfo {
Path storePath;
std::string rev;
std::string shortRev;
@ -27,24 +23,22 @@ std::regex revRegex("^[0-9a-fA-F]{40}$");
GitInfo exportGit(ref<Store> store, const std::string& uri,
std::optional<std::string> ref, std::string rev,
const std::string & name)
{
const std::string& name) {
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
bool clean = true;
try {
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
runProgram("git", true,
{"-C", uri, "diff-index", "--quiet", "HEAD", "--"});
} catch (ExecError& e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
clean = false;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
@ -70,7 +64,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
return files.count(file);
};
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
gitInfo.storePath =
store->addToStore("source", uri, true, htSHA256, filter);
return gitInfo;
}
@ -87,7 +82,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
Path cacheDir = getCacheDir() + "/nix/gitv2/" +
hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
@ -122,13 +118,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
runProgram("git", true,
{"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri,
fmt("%s:%s", *ref, *ref)});
struct timeval times[2];
times[0].tv_sec = now;
@ -146,9 +144,12 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev)
.to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...",
storeLink)); // FIXME: broken
try {
auto json = nlohmann::json::parse(readFile(storeLink));
@ -177,7 +178,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
gitInfo.storePath = store->addToStore(name, tmpDir);
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
gitInfo.revCount = std::stoull(runProgram(
"git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev}));
nlohmann::json json;
json["storePath"] = gitInfo.storePath;
@ -191,8 +193,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
return gitInfo;
}
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args,
Value& v) {
std::string url;
std::optional<std::string> ref;
std::string rev;
@ -202,13 +204,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto& attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev")
@ -216,7 +218,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s",
attr.name, *attr.pos);
}
if (url.empty())
@ -232,10 +235,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
auto gitInfo = exportGit(state.store, url, ref, rev, name);
state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath,
PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
gitInfo.revCount);
v.attrs->sort();
if (state.allowedPaths)
@ -244,4 +250,4 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
}
} // namespace nix

98
third_party/nix/src/libexpr/primops/fetchMercurial.cc vendored
View file

@ -1,21 +1,17 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp>
#include <regex>
#include "download.hh"
#include "eval-inline.hh"
#include "pathlocks.hh"
#include "primops.hh"
#include "store-api.hh"
using namespace std::string_literals;
namespace nix {
struct HgInfo
{
struct HgInfo {
Path storePath;
std::string branch;
std::string rev;
@ -25,17 +21,18 @@ struct HgInfo
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string& uri,
std::string rev, const std::string & name)
{
std::string rev, const std::string& name) {
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
throw Error(
"in pure evaluation mode, 'fetchMercurial' requires a Mercurial "
"revision");
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
bool clean = runProgram("hg", true,
{"status", "-R", uri, "--modified", "--added",
"--removed"}) == "";
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
@ -46,7 +43,10 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri}));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
runProgram("hg", true,
{"status", "-R", uri, "--clean", "--modified", "--added",
"--no-status", "--print0"}),
"\0"s);
PathFilter filter = [&](const Path& p) -> bool {
assert(hasPrefix(p, uri));
@ -63,7 +63,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
return files.count(file);
};
hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
hgInfo.storePath =
store->addToStore("source", uri, true, htSHA256, filter);
return hgInfo;
}
@ -71,39 +72,40 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
if (rev == "") rev = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(),
hashString(htSHA256, uri).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir,
hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
do so now. */
time_t now = time(0);
struct stat st;
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
{
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now) {
/* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex)
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) &&
runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev,
"--template", "1"})
.killStderr(true))
.second == "1")) {
Activity act(*logger, lvlTalkative, actUnknown,
fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
}
catch (ExecError & e) {
} catch (ExecError& e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, {"recover", "-R", cacheDir});
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
throw ExecError(e.status,
fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
@ -116,7 +118,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
runProgram("hg", true,
{"log", "-R", cacheDir, "-r", rev, "--template",
"{node} {rev} {branch}"}));
assert(tokens.size() == 3);
HgInfo hgInfo;
@ -124,7 +128,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
std::string storeLinkName =
hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev)
.to_string(Base32, false);
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
try {
@ -135,7 +141,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
hgInfo.storePath = json["storePath"];
if (store->isValidPath(hgInfo.storePath)) {
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
printTalkative("using cached Mercurial store path '%s'",
hgInfo.storePath);
return hgInfo;
}
@ -165,8 +172,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
return hgInfo;
}
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args,
Value& v) {
std::string url;
std::string rev;
std::string name = "source";
@ -175,19 +182,20 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto& attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
url =
state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s",
attr.name, *attr.pos);
}
if (url.empty())
@ -203,10 +211,12 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
auto hgInfo = exportMercurial(state.store, url, rev, name);
state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath,
PathSet({hgInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
std::string(hgInfo.rev, 0, 12));
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
v.attrs->sort();
@ -216,4 +226,4 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
}
} // namespace nix

24
third_party/nix/src/libexpr/primops/fromTOML.cc vendored
View file

@ -1,12 +1,11 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "cpptoml/cpptoml.h"
#include "eval-inline.hh"
#include "primops.hh"
namespace nix {
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args,
Value& v) {
using namespace cpptoml;
auto toml = state.forceStringNoCtx(*args[0], pos);
@ -16,11 +15,12 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
std::function<void(Value&, std::shared_ptr<base>)> visit;
visit = [&](Value& v, std::shared_ptr<base> t) {
if (auto t2 = t->as_table()) {
size_t size = 0;
for (auto & i : *t2) { (void) i; size++; }
for (auto& i : *t2) {
(void)i;
size++;
}
state.mkAttrs(v, size);
@ -32,8 +32,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
state.mkList(v2, size2);
for (size_t j = 0; j < size2; ++j)
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
}
else
} else
visit(v2, i.second);
}
@ -75,7 +74,8 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
throw EvalError("unsupported value type in TOML");
}
else abort();
else
abort();
};
try {
@ -87,4 +87,4 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
}
} // namespace nix

53
third_party/nix/src/libexpr/symbol-table.hh vendored
View file

@ -2,7 +2,6 @@
#include <map>
#include <unordered_set>
#include "types.hh"
namespace nix {
@ -13,8 +12,7 @@ namespace nix {
they can be compared efficiently (using a pointer equality test),
because the symbol table stores only one copy of each string. */
class Symbol
{
class Symbol {
private:
const string* s; // pointer into SymbolTable
Symbol(const string* s) : s(s){};
@ -23,65 +21,40 @@ private:
public:
Symbol() : s(0){};
bool operator == (const Symbol & s2) const
{
return s == s2.s;
}
bool operator==(const Symbol& s2) const { return s == s2.s; }
bool operator != (const Symbol & s2) const
{
return s != s2.s;
}
bool operator!=(const Symbol& s2) const { return s != s2.s; }
bool operator < (const Symbol & s2) const
{
return s < s2.s;
}
bool operator<(const Symbol& s2) const { return s < s2.s; }
operator const string & () const
{
return *s;
}
operator const string&() const { return *s; }
bool set() const
{
return s;
}
bool set() const { return s; }
bool empty() const
{
return s->empty();
}
bool empty() const { return s->empty(); }
friend std::ostream& operator<<(std::ostream& str, const Symbol& sym);
};
class SymbolTable
{
class SymbolTable {
private:
typedef std::unordered_set<string> Symbols;
Symbols symbols;
public:
Symbol create(const string & s)
{
Symbol create(const string& s) {
std::pair<Symbols::iterator, bool> res = symbols.insert(s);
return Symbol(&*res.first);
}
size_t size() const
{
return symbols.size();
}
size_t size() const { return symbols.size(); }
size_t totalSize() const;
template <typename T>
void dump(T callback)
{
for (auto & s : symbols)
callback(s);
void dump(T callback) {
for (auto& s : symbols) callback(s);
}
};
}
} // namespace nix

39
third_party/nix/src/libexpr/value-to-json.cc vendored
View file

@ -1,23 +1,19 @@
#include "value-to-json.hh"
#include "json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include <cstdlib>
#include <iomanip>
#include "eval-inline.hh"
#include "json.hh"
#include "util.hh"
namespace nix {
void printValueAsJSON(EvalState & state, bool strict,
Value & v, JSONPlaceholder & out, PathSet & context)
{
void printValueAsJSON(EvalState& state, bool strict, Value& v,
JSONPlaceholder& out, PathSet& context) {
checkInterrupt();
if (strict) state.forceValue(v);
switch (v.type) {
case tInt:
out.write(v.integer);
break;
@ -40,7 +36,8 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
case tAttrs: {
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
auto maybeString =
state.tryAttrsToString(noPos, v, context, false, false);
if (maybeString) {
out.write(*maybeString);
break;
@ -49,8 +46,7 @@ void printValueAsJSON(EvalState & state, bool strict,
if (i == v.attrs->end()) {
auto obj(out.object());
StringSet names;
for (auto & j : *v.attrs)
names.insert(j.name);
for (auto& j : *v.attrs) names.insert(j.name);
for (auto& j : names) {
Attr& a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j));
@ -61,11 +57,14 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
}
case tList1: case tList2: case tListN: {
case tList1:
case tList2:
case tListN: {
auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context);
printValueAsJSON(state, strict, *v.listElems()[n], placeholder,
context);
}
break;
}
@ -83,18 +82,16 @@ void printValueAsJSON(EvalState & state, bool strict,
}
}
void printValueAsJSON(EvalState & state, bool strict,
Value & v, std::ostream & str, PathSet & context)
{
void printValueAsJSON(EvalState& state, bool strict, Value& v,
std::ostream& str, PathSet& context) {
JSONPlaceholder out(str);
printValueAsJSON(state, strict, v, out, context);
}
void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict,
JSONPlaceholder & out, PathSet & context) const
{
JSONPlaceholder& out,
PathSet& context) const {
throw TypeError(format("cannot convert %1% to JSON") % showType());
}
}
} // namespace nix

17
third_party/nix/src/libexpr/value-to-json.hh vendored
View file

@ -1,19 +1,18 @@
#pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix {
class JSONPlaceholder;
void printValueAsJSON(EvalState & state, bool strict,
Value & v, JSONPlaceholder & out, PathSet & context);
void printValueAsJSON(EvalState& state, bool strict, Value& v,
JSONPlaceholder& out, PathSet& context);
void printValueAsJSON(EvalState & state, bool strict,
Value & v, std::ostream & str, PathSet & context);
void printValueAsJSON(EvalState& state, bool strict, Value& v,
std::ostream& str, PathSet& context);
}
} // namespace nix

75
third_party/nix/src/libexpr/value-to-xml.cc vendored
View file

@ -1,41 +1,33 @@
#include "value-to-xml.hh"
#include "xml-writer.hh"
#include <cstdlib>
#include "eval-inline.hh"
#include "util.hh"
#include <cstdlib>
#include "xml-writer.hh"
namespace nix {
static XMLAttrs singletonAttrs(const string & name, const string & value)
{
static XMLAttrs singletonAttrs(const string& name, const string& value) {
XMLAttrs attrs;
attrs[name] = value;
return attrs;
}
static void printValueAsXML(EvalState& state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
Value& v, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen);
static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
{
static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) {
xmlAttrs["path"] = pos.file;
xmlAttrs["line"] = (format("%1%") % pos.line).str();
xmlAttrs["column"] = (format("%1%") % pos.column).str();
}
static void showAttrs(EvalState& state, bool strict, bool location,
Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
{
Bindings& attrs, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) {
StringSet names;
for (auto & i : attrs)
names.insert(i.name);
for (auto& i : attrs) names.insert(i.name);
for (auto& i : names) {
Attr& a(*attrs.find(state.symbols.create(i)));
@ -45,27 +37,26 @@ static void showAttrs(EvalState & state, bool strict, bool location,
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
XMLOpenElement _(doc, "attr", xmlAttrs);
printValueAsXML(state, strict, location,
*a.value, doc, context, drvsSeen);
printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen);
}
}
static void printValueAsXML(EvalState& state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
{
Value& v, XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) {
checkInterrupt();
if (strict) state.forceValue(v);
switch (v.type) {
case tInt:
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
doc.writeEmptyElement(
"int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break;
case tBool:
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
doc.writeEmptyElement(
"bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break;
case tString:
@ -86,7 +77,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (state.isDerivation(v)) {
XMLAttrs xmlAttrs;
Bindings::iterator a = v.attrs->find(state.symbols.create("derivation"));
Bindings::iterator a =
v.attrs->find(state.symbols.create("derivation"));
Path drvPath;
a = v.attrs->find(state.sDrvPath);
@ -119,10 +111,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break;
case tList1: case tList2: case tListN: {
case tList1:
case tList2:
case tListN: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
printValueAsXML(state, strict, location, *v.listElems()[n], doc,
context, drvsSeen);
break;
}
@ -139,17 +134,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
for (auto& i : v.lambda.fun->formals->formals)
doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
} else
doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
doc.writeEmptyElement("varpat",
singletonAttrs("name", v.lambda.fun->arg));
break;
}
case tExternal:
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
v.external->printValueAsXML(state, strict, location, doc, context,
drvsSeen);
break;
case tFloat:
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
doc.writeEmptyElement(
"float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
break;
default:
@ -157,22 +155,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
}
}
void ExternalValueBase::printValueAsXML(EvalState& state, bool strict,
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
{
bool location, XMLWriter& doc,
PathSet& context,
PathSet& drvsSeen) const {
doc.writeEmptyElement("unevaluated");
}
void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context)
{
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
std::ostream& out, PathSet& context) {
XMLWriter doc(true, out);
XMLOpenElement root(doc, "expr");
PathSet drvsSeen;
printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
}
}
} // namespace nix

11
third_party/nix/src/libexpr/value-to-xml.hh vendored
View file

@ -1,14 +1,13 @@
#pragma once
#include "nixexpr.hh"
#include "eval.hh"
#include <string>
#include <map>
#include <string>
#include "eval.hh"
#include "nixexpr.hh"
namespace nix {
void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context);
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
std::ostream& out, PathSet& context);
}

92
third_party/nix/src/libexpr/value.hh vendored
View file

@ -8,7 +8,6 @@
namespace nix {
typedef enum {
tInt = 1,
tBool,
@ -29,7 +28,6 @@ typedef enum {
tFloat
} ValueType;
class Bindings;
struct Env;
struct Expr;
@ -42,16 +40,16 @@ class EvalState;
class XMLWriter;
class JSONPlaceholder;
typedef int64_t NixInt;
typedef double NixFloat;
/* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented
*/
class ExternalValueBase
{
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
class ExternalValueBase {
friend std::ostream& operator<<(std::ostream& str,
const ExternalValueBase& v);
protected:
/* Print out the value */
virtual std::ostream& print(std::ostream& str) const = 0;
@ -69,7 +67,8 @@ class ExternalValueBase
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* error
*/
virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
virtual string coerceToString(const Pos& pos, PathSet& context, bool copyMore,
bool copyToStore) const;
/* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false.
@ -82,21 +81,17 @@ class ExternalValueBase
/* Print the value as XML. Defaults to unevaluated */
virtual void printValueAsXML(EvalState& state, bool strict, bool location,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const;
XMLWriter& doc, PathSet& context,
PathSet& drvsSeen) const;
virtual ~ExternalValueBase()
{
};
virtual ~ExternalValueBase(){};
};
std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v);
struct Value
{
struct Value {
ValueType type;
union
{
union {
NixInt integer;
bool boolean;
@ -151,124 +146,95 @@ struct Value
NixFloat fpoint;
};
bool isList() const
{
bool isList() const {
return type == tList1 || type == tList2 || type == tListN;
}
Value * * listElems()
{
Value** listElems() {
return type == tList1 || type == tList2 ? smallList : bigList.elems;
}
const Value * const * listElems() const
{
const Value* const* listElems() const {
return type == tList1 || type == tList2 ? smallList : bigList.elems;
}
size_t listSize() const
{
size_t listSize() const {
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
}
};
/* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */
static inline void clearValue(Value & v)
{
v.app.left = v.app.right = 0;
}
static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; }
static inline void mkInt(Value & v, NixInt n)
{
static inline void mkInt(Value& v, NixInt n) {
clearValue(v);
v.type = tInt;
v.integer = n;
}
static inline void mkFloat(Value & v, NixFloat n)
{
static inline void mkFloat(Value& v, NixFloat n) {
clearValue(v);
v.type = tFloat;
v.fpoint = n;
}
static inline void mkBool(Value & v, bool b)
{
static inline void mkBool(Value& v, bool b) {
clearValue(v);
v.type = tBool;
v.boolean = b;
}
static inline void mkNull(Value & v)
{
static inline void mkNull(Value& v) {
clearValue(v);
v.type = tNull;
}
static inline void mkApp(Value & v, Value & left, Value & right)
{
static inline void mkApp(Value& v, Value& left, Value& right) {
v.type = tApp;
v.app.left = &left;
v.app.right = &right;
}
static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
{
static inline void mkPrimOpApp(Value& v, Value& left, Value& right) {
v.type = tPrimOpApp;
v.app.left = &left;
v.app.right = &right;
}
static inline void mkStringNoCopy(Value & v, const char * s)
{
static inline void mkStringNoCopy(Value& v, const char* s) {
v.type = tString;
v.string.s = s;
v.string.context = 0;
}
static inline void mkString(Value & v, const Symbol & s)
{
static inline void mkString(Value& v, const Symbol& s) {
mkStringNoCopy(v, ((const string&)s).c_str());
}
void mkString(Value& v, const char* s);
static inline void mkPathNoCopy(Value & v, const char * s)
{
static inline void mkPathNoCopy(Value& v, const char* s) {
clearValue(v);
v.type = tPath;
v.path = s;
}
void mkPath(Value& v, const char* s);
/* Compute the size in bytes of the given value, including all values
and environments reachable from it. Static expressions (Exprs) are
not included. */
size_t valueSize(Value& v);
#if HAVE_BOEHMGC
typedef std::vector<Value*, gc_allocator<Value*> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
typedef std::map<Symbol, Value*, std::less<Symbol>,
gc_allocator<std::pair<const Symbol, Value*> > >
ValueMap;
#else
typedef std::vector<Value*> ValueVector;
typedef std::map<Symbol, Value*> ValueMap;
#endif
}
} // namespace nix

21
third_party/nix/src/libmain/common-args.cc vendored
View file

@ -4,8 +4,7 @@
namespace nix {
MixCommonArgs::MixCommonArgs(const string& programName)
: programName(programName)
{
: programName(programName) {
mkFlag()
.longName("verbose")
.shortName('v')
@ -15,12 +14,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
mkFlag()
.longName("quiet")
.description("decrease verbosity level")
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
.handler([]() {
verbosity =
verbosity > lvlError ? (Verbosity)(verbosity - 1) : lvlError;
});
mkFlag()
.longName("debug")
.description("enable debug output")
.handler([]() { verbosity = lvlDebug; });
mkFlag().longName("debug").description("enable debug output").handler([]() {
verbosity = lvlDebug;
});
mkFlag()
.longName("option")
@ -40,9 +41,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
.shortName('j')
.label("jobs")
.description("maximum number of parallel builds")
.handler([=](std::string s) {
settings.set("max-jobs", s);
});
.handler([=](std::string s) { settings.set("max-jobs", s); });
std::string cat = "config";
globalConfig.convertToArgs(*this, cat);
@ -53,4 +52,4 @@ MixCommonArgs::MixCommonArgs(const string & programName)
hiddenCategories.insert(cat);
}
}
} // namespace nix

22
third_party/nix/src/libmain/common-args.hh vendored
View file

@ -4,30 +4,24 @@
namespace nix {
struct MixCommonArgs : virtual Args
{
struct MixCommonArgs : virtual Args {
string programName;
MixCommonArgs(const string& programName);
};
struct MixDryRun : virtual Args
{
struct MixDryRun : virtual Args {
bool dryRun = false;
MixDryRun()
{
mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
MixDryRun() {
mkFlag(0, "dry-run", "show what this command would do without doing it",
&dryRun);
}
};
struct MixJSON : virtual Args
{
struct MixJSON : virtual Args {
bool json = false;
MixJSON()
{
mkFlag(0, "json", "produce JSON output", &json);
}
MixJSON() { mkFlag(0, "json", "produce JSON output", &json); }
};
}
} // namespace nix

175
third_party/nix/src/libmain/shared.cc vendored
View file

@ -1,30 +1,24 @@
#include "globals.hh"
#include "shared.hh"
#include "store-api.hh"
#include "util.hh"
#include <openssl/crypto.h>
#include <signal.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <exception>
#include <iostream>
#include <mutex>
#include <cstdlib>
#include <sys/time.h>
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#include <openssl/crypto.h>
#include "globals.hh"
#include "store-api.hh"
#include "util.hh"
namespace nix {
static bool gcWarning = true;
void printGCWarning()
{
void printGCWarning() {
if (!gcWarning) return;
static bool haveWarned = false;
warnOnce(haveWarned,
@ -32,62 +26,58 @@ void printGCWarning()
"the result might be removed by the garbage collector");
}
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
{
void printMissing(ref<Store> store, const PathSet& paths, Verbosity lvl) {
unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize,
narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize,
lvl);
}
void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
{
unsigned long long downloadSize, unsigned long long narSize,
Verbosity lvl) {
if (!willBuild.empty()) {
printMsg(lvl, "these derivations will be built:");
Paths sorted = store->topoSortPaths(willBuild);
reverse(sorted.begin(), sorted.end());
for (auto & i : sorted)
printMsg(lvl, fmt(" %s", i));
for (auto& i : sorted) printMsg(lvl, fmt(" %s", i));
}
if (!willSubstitute.empty()) {
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f "
"MiB unpacked):",
downloadSize / (1024.0 * 1024.0),
narSize / (1024.0 * 1024.0)));
for (auto & i : willSubstitute)
printMsg(lvl, fmt(" %s", i));
for (auto& i : willSubstitute) printMsg(lvl, fmt(" %s", i));
}
if (!unknown.empty()) {
printMsg(lvl, fmt("don't know how to build these paths%s:",
(settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
for (auto & i : unknown)
printMsg(lvl, fmt(" %s", i));
(settings.readOnlyMode
? " (may be caused by read-only store access)"
: "")));
for (auto& i : unknown) printMsg(lvl, fmt(" %s", i));
}
}
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end)
{
string getArg(const string& opt, Strings::iterator& i,
const Strings::iterator& end) {
++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
return *i;
}
#if OPENSSL_VERSION_NUMBER < 0x10101000L
/* OpenSSL is not thread-safe by default - it will randomly crash
unless the user supplies a mutex locking function. So let's do
that. */
static std::vector<std::mutex> opensslLocks;
static void opensslLockCallback(int mode, int type, const char * file, int line)
{
static void opensslLockCallback(int mode, int type, const char* file,
int line) {
if (mode & CRYPTO_LOCK)
opensslLocks[type].lock();
else
@ -95,12 +85,9 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
}
#endif
static void sigHandler(int signo) {}
void initNix()
{
void initNix() {
/* Turn on buffering for cerr. */
#if HAVE_PUBSETBUF
static char buf[1024];
@ -122,8 +109,7 @@ void initNix()
sigemptyset(&act.sa_mask);
act.sa_handler = SIG_DFL;
act.sa_flags = 0;
if (sigaction(SIGCHLD, &act, 0))
throw SysError("resetting SIGCHLD");
if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD");
/* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
act.sa_handler = sigHandler;
@ -160,11 +146,11 @@ void initNix()
#endif
}
LegacyArgs::LegacyArgs(const std::string & programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
: MixCommonArgs(programName), parseArg(parseArg)
{
LegacyArgs::LegacyArgs(
const std::string& programName,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg)
: MixCommonArgs(programName), parseArg(parseArg) {
mkFlag()
.longName("no-build-output")
.shortName('Q')
@ -189,15 +175,20 @@ LegacyArgs::LegacyArgs(const std::string & programName,
.set(&(bool&)settings.tryFallback, true);
auto intSettingAlias = [&](char shortName, const std::string& longName,
const std::string & description, const std::string & dest) {
const std::string& description,
const std::string& dest) {
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
settings.set(dest, std::to_string(n));
});
};
intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
intSettingAlias(0, "cores",
"maximum number of CPU cores to use inside a build", "cores");
intSettingAlias(0, "max-silent-time",
"number of seconds of silence before a build is killed",
"max-silent-time");
intSettingAlias(0, "timeout", "number of seconds before a build is killed",
"timeout");
mkFlag(0, "readonly-mode", "do not write to the Nix store",
&settings.readOnlyMode);
@ -212,18 +203,14 @@ LegacyArgs::LegacyArgs(const std::string & programName,
.dest(&(std::string&)settings.storeUri);
}
bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end)
{
bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) {
if (MixCommonArgs::processFlag(pos, end)) return true;
bool res = parseArg(pos, end);
if (res) ++pos;
return res;
}
bool LegacyArgs::processArgs(const Strings & args, bool finish)
{
bool LegacyArgs::processArgs(const Strings& args, bool finish) {
if (args.empty()) return true;
assert(args.size() == 1);
Strings ss(args);
@ -233,23 +220,21 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
return true;
}
void parseCmdLine(int argc, char * * argv,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
{
void parseCmdLine(
int argc, char** argv,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg) {
parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
}
void parseCmdLine(const string & programName, const Strings & args,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
{
void parseCmdLine(
const string& programName, const Strings& args,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg) {
LegacyArgs(programName, parseArg).parseCmdline(args);
}
void printVersion(const string & programName)
{
void printVersion(const string& programName) {
std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
if (verbosity > lvlInfo) {
Strings cfg;
@ -260,25 +245,22 @@ void printVersion(const string & programName)
cfg.push_back("signed-caches");
#endif
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf"
<< "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
}
throw Exit();
}
void showManPage(const string & name)
{
void showManPage(const string& name) {
restoreSignals();
setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr);
throw SysError(format("command 'man %1%' failed") % name.c_str());
}
int handleExceptions(const string & programName, std::function<void()> fun)
{
int handleExceptions(const string& programName, std::function<void()> fun) {
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
string error = ANSI_RED "error:" ANSI_NORMAL " ";
@ -296,12 +278,12 @@ int handleExceptions(const string & programName, std::function<void()> fun)
} catch (Exit& e) {
return e.status;
} catch (UsageError& e) {
printError(
format(error + "%1%\nTry '%2% --help' for more information.")
% e.what() % programName);
printError(format(error + "%1%\nTry '%2% --help' for more information.") %
e.what() % programName);
return 1;
} catch (BaseError& e) {
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
printError(format(error + "%1%%2%") %
(settings.showTrace ? e.prefix() : "") % e.msg());
if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)");
return e.status;
@ -316,9 +298,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
return 0;
}
RunPager::RunPager()
{
RunPager::RunPager() {
if (!isatty(STDOUT_FILENO)) return;
char* pager = getenv("NIX_PAGER");
if (!pager) pager = getenv("PAGER");
@ -330,11 +310,9 @@ RunPager::RunPager()
pid = startProcess([&]() {
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (!getenv("LESS"))
setenv("LESS", "FRSXMK", 1);
if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1);
restoreSignals();
if (pager)
execl("/bin/sh", "sh", "-c", pager, nullptr);
if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr);
execlp("more", "more", nullptr);
@ -347,9 +325,7 @@ RunPager::RunPager()
throw SysError("dupping stdout");
}
RunPager::~RunPager()
{
RunPager::~RunPager() {
try {
if (pid != -1) {
std::cout.flush();
@ -361,21 +337,16 @@ RunPager::~RunPager()
}
}
string showBytes(unsigned long long bytes)
{
string showBytes(unsigned long long bytes) {
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
PrintFreed::~PrintFreed()
{
PrintFreed::~PrintFreed() {
if (show)
std::cout << format("%1% store paths deleted, %2% freed\n")
% results.paths.size()
% showBytes(results.bytesFreed);
std::cout << format("%1% store paths deleted, %2% freed\n") %
results.paths.size() % showBytes(results.bytesFreed);
}
Exit::~Exit() {}
}
} // namespace nix

80
third_party/nix/src/libmain/shared.hh vendored
View file

@ -1,18 +1,14 @@
#pragma once
#include "util.hh"
#include <signal.h>
#include <locale>
#include "args.hh"
#include "common-args.hh"
#include <signal.h>
#include <locale>
#include "util.hh"
namespace nix {
class Exit : public std::exception
{
class Exit : public std::exception {
public:
int status;
Exit() : status(0) {}
@ -25,11 +21,15 @@ int handleExceptions(const string & programName, std::function<void()> fun);
/* Don't forget to call initPlugins() after settings are initialized! */
void initNix();
void parseCmdLine(int argc, char * * argv,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
void parseCmdLine(
int argc, char** argv,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void parseCmdLine(const string & programName, const Strings & args,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
void parseCmdLine(
const string& programName, const Strings& args,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
void printVersion(const string& programName);
@ -38,18 +38,20 @@ void printGCWarning();
class Store;
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const PathSet& paths,
Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const PathSet& willBuild,
const PathSet& willSubstitute, const PathSet& unknown,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
unsigned long long downloadSize, unsigned long long narSize,
Verbosity lvl = lvlInfo);
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end);
string getArg(const string& opt, Strings::iterator& i,
const Strings::iterator& end);
template<class N> N getIntArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
{
template <class N>
N getIntArg(const string& opt, Strings::iterator& i,
const Strings::iterator& end, bool allowUnit) {
++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
string s = *i;
@ -57,11 +59,16 @@ template<class N> N getIntArg(const string & opt,
if (allowUnit && !s.empty()) {
char u = std::toupper(*s.rbegin());
if (std::isalpha(u)) {
if (u == 'K') multiplier = 1ULL << 10;
else if (u == 'M') multiplier = 1ULL << 20;
else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError(format("invalid unit specifier '%1%'") % u);
if (u == 'K')
multiplier = 1ULL << 10;
else if (u == 'M')
multiplier = 1ULL << 20;
else if (u == 'G')
multiplier = 1ULL << 30;
else if (u == 'T')
multiplier = 1ULL << 40;
else
throw UsageError(format("invalid unit specifier '%1%'") % u);
s.resize(s.size() - 1);
}
}
@ -71,27 +78,26 @@ template<class N> N getIntArg(const string & opt,
return n * multiplier;
}
struct LegacyArgs : public MixCommonArgs {
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg;
struct LegacyArgs : public MixCommonArgs
{
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;
LegacyArgs(const std::string & programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
LegacyArgs(
const std::string& programName,
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
parseArg);
bool processFlag(Strings::iterator& pos, Strings::iterator end) override;
bool processArgs(const Strings& args, bool finish) override;
};
/* Show the manual page for the specified program. */
void showManPage(const string& name);
/* The constructor of this class starts a pager if stdout is a
terminal and $PAGER is set. Stdout is redirected to the pager. */
class RunPager
{
class RunPager {
public:
RunPager();
~RunPager();
@ -102,15 +108,13 @@ private:
extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
string showBytes(unsigned long long bytes);
struct GCResults;
struct PrintFreed
{
struct PrintFreed {
bool show;
const GCResults& results;
PrintFreed(bool show, const GCResults& results)
@ -118,9 +122,7 @@ struct PrintFreed
~PrintFreed();
};
/* Install a SIGSEGV handler to detect stack overflows. */
void detectStackOverflow();
}
} // namespace nix

27
third_party/nix/src/libmain/stack.cc vendored
View file

@ -1,17 +1,13 @@
#include "types.hh"
#include <cstring>
#include <signal.h>
#include <unistd.h>
#include <cstddef>
#include <cstdlib>
#include <unistd.h>
#include <signal.h>
#include <cstring>
#include "types.hh"
namespace nix {
static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
{
static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) {
/* Detect stack overflows by comparing the faulting address with
the stack pointer. Unfortunately, getting the stack pointer is
not portable. */
@ -43,9 +39,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
if (sigaction(SIGSEGV, &act, 0)) abort();
}
void detectStackOverflow()
{
void detectStackOverflow() {
#if defined(SA_SIGINFO) && defined(SA_ONSTACK)
/* Install a SIGSEGV handler to detect stack overflows. This
requires an alternative stack, otherwise the signal cannot be
@ -56,16 +50,15 @@ void detectStackOverflow()
stack.ss_sp = stackBuf->data();
if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
stack.ss_flags = 0;
if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");
if (sigaltstack(&stack, 0) == -1)
throw SysError("cannot set alternative stack");
struct sigaction act;
sigfillset(&act.sa_mask);
act.sa_sigaction = sigsegvHandler;
act.sa_flags = SA_SIGINFO | SA_ONSTACK;
if (sigaction(SIGSEGV, &act, 0))
throw SysError("resetting SIGSEGV");
if (sigaction(SIGSEGV, &act, 0)) throw SysError("resetting SIGSEGV");
#endif
}
}
} // namespace nix

173
third_party/nix/src/libstore/binary-cache-store.cc vendored
View file

@ -1,40 +1,37 @@
#include "archive.hh"
#include "binary-cache-store.hh"
#include <chrono>
#include <future>
#include "archive.hh"
#include "compression.hh"
#include "derivations.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "nar-info.hh"
#include "sync.hh"
#include "remote-fs-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "nar-accessor.hh"
#include "json.hh"
#include <chrono>
#include <future>
#include "nar-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include "remote-fs-accessor.hh"
#include "sync.hh"
namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
{
BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) {
if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
secretKey =
std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
StringSink sink;
sink << narVersionMagic1;
narMagic = *sink.s;
}
void BinaryCacheStore::init()
{
void BinaryCacheStore::init() {
std::string cacheInfoFile = "nix-cache-info";
auto cacheInfo = getFile(cacheInfoFile);
if (!cacheInfo) {
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n",
"text/x-nix-cache-info");
} else {
for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
size_t colon = line.find(':');
@ -43,8 +40,9 @@ void BinaryCacheStore::init()
auto value = trim(line.substr(colon + 1, std::string::npos));
if (name == "StoreDir") {
if (value != storeDir)
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
% getUri() % value % storeDir);
throw Error(format("binary cache '%s' is for Nix stores with prefix "
"'%s', not '%s'") %
getUri() % value % storeDir);
} else if (name == "WantMassQuery") {
wantMassQuery_ = value == "1";
} else if (name == "Priority") {
@ -54,19 +52,19 @@ void BinaryCacheStore::init()
}
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
{
void BinaryCacheStore::getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept {
try {
callback(getFile(path));
} catch (...) { callback.rethrow(); }
} catch (...) {
callback.rethrow();
}
}
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
void BinaryCacheStore::getFile(const std::string& path, Sink& sink) {
std::promise<std::shared_ptr<std::string>> promise;
getFile(path,
{[&](std::future<std::shared_ptr<std::string>> result) {
getFile(path, {[&](std::future<std::shared_ptr<std::string>> result) {
try {
promise.set_value(result.get());
} catch (...) {
@ -77,8 +75,8 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
sink((unsigned char*)data->data(), data->size());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
{
std::shared_ptr<std::string> BinaryCacheStore::getFile(
const std::string& path) {
StringSink sink;
try {
getFile(path, sink);
@ -88,14 +86,12 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
return sink.s;
}
Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
{
Path BinaryCacheStore::narInfoFileFor(const Path& storePath) {
assertStorePath(storePath);
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
{
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) {
auto narInfoFile = narInfoFileFor(narInfo->path);
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
@ -108,23 +104,24 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
diskCache->upsertNarInfo(getUri(), hashPart,
std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{
void BinaryCacheStore::addToStore(const ValidPathInfo& info,
const ref<std::string>& nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) {
if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
for (auto & ref : info.references)
try {
if (ref != info.path)
queryPathInfo(ref);
for (auto& ref : info.references) try {
if (ref != info.path) queryPathInfo(ref);
} catch (InvalidPath&) {
throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
% info.path % ref);
throw Error(format("cannot add '%s' to the binary cache because the "
"reference '%s' is not valid") %
info.path % ref);
}
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
@ -135,7 +132,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
throw Error(
format("refusing to copy corrupted path '%1%' to binary cache") %
info.path);
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
@ -150,8 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
if (accessor_) accessor_->addToCache(info.path, *nar, narAccessor);
{
auto res = jsonRoot.placeholder("root");
@ -159,12 +157,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
}
}
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(),
"application/json");
}
else {
if (accessor_)
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
if (accessor_) accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
}
/* Compress the NAR. */
@ -175,18 +173,22 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% narInfo->path % narInfo->narSize
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration);
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printMsg(lvlTalkative,
format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% "
"ms) to binary cache") %
narInfo->path % narInfo->narSize %
((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) %
duration);
/* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" +
(compression == "xz" ? ".xz"
: compression == "bzip2"
? ".bz2"
: compression == "br" ? ".br" : "");
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
@ -205,16 +207,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
stats.narInfoWrite++;
}
bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
{
bool BinaryCacheStore::isValidPathUncached(const Path& storePath) {
// FIXME: this only checks whether a .narinfo with a matching hash
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
// though they shouldn't. Not easily fixed.
return fileExists(narInfoFileFor(storePath));
}
void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
{
void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) {
auto info = queryPathInfo(storePath).cast<const NarInfo>();
uint64_t narSize = 0;
@ -239,20 +239,22 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
stats.narReadBytes += narSize;
}
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
{
void BinaryCacheStore::queryPathInfoUncached(
const Path& storePath,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
auto uri = getUri();
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
auto act = std::make_shared<Activity>(
*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri),
Logger::Fields{storePath, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile,
{[=](std::future<std::shared_ptr<std::string>> fut) {
getFile(
narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) {
try {
auto data = fut.get();
@ -260,10 +262,12 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
stats.narInfoRead++;
(*callbackPtr)((std::shared_ptr<ValidPathInfo>)
std::make_shared<NarInfo>(*this, *data, narInfoFile));
(*callbackPtr)(
(std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>(
*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
(void)
act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callbackPtr->rethrow();
}
@ -271,8 +275,8 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
}
Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
bool recursive, HashType hashAlgo,
PathFilter& filter, RepairFlag repair) {
// FIXME: some cut&paste from LocalStore::addToStore().
/* Read the whole path into memory. This is not a very scalable
@ -298,8 +302,8 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
}
Path BinaryCacheStore::addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair)
{
const PathSet& references,
RepairFlag repair) {
ValidPathInfo info;
info.path = computeStorePathForText(name, s, references);
info.references = references;
@ -313,13 +317,13 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
return info.path;
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
ref<FSAccessor> BinaryCacheStore::getFSAccessor() {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()),
localNarCache);
}
void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
{
void BinaryCacheStore::addSignatures(const Path& storePath,
const StringSet& sigs) {
/* Note: this is inherently racy since there is no locking on
binary caches. In particular, with S3 this unreliable, even
when addSignatures() is called sequentially on a path, because
@ -334,8 +338,7 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
{
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) {
Path drvPath;
if (isDerivation(path))
@ -358,4 +361,4 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
return getFile(logPath);
}
}
} // namespace nix

68
third_party/nix/src/libstore/binary-cache-store.hh vendored
View file

@ -1,41 +1,41 @@
#pragma once
#include "crypto.hh"
#include "store-api.hh"
#include "pool.hh"
#include <atomic>
#include "crypto.hh"
#include "pool.hh"
#include "store-api.hh"
namespace nix {
struct NarInfo;
class BinaryCacheStore : public Store
{
class BinaryCacheStore : public Store {
public:
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
const Setting<std::string> compression{
this, "xz", "compression",
"NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{
this, false, "write-nar-listing",
"whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{
this, "", "secret-key",
"path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"path to a local cache of NARs"};
const Setting<bool> parallelCompression{
this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
private:
std::unique_ptr<SecretKey> secretKey;
protected:
BinaryCacheStore(const Params& params);
public:
virtual bool fileExists(const std::string& path) = 0;
virtual void upsertFile(const std::string & path,
const std::string & data,
virtual void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) = 0;
/* Note: subclasses must implement at least one of the two
@ -46,22 +46,20 @@ public:
/* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */
virtual void getFile(const std::string & path,
virtual void getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept;
std::shared_ptr<std::string> getFile(const std::string& path);
protected:
bool wantMassQuery_ = false;
int priority = 50;
public:
virtual void init();
private:
std::string narMagic;
std::string narInfoFileFor(const Path& storePath);
@ -69,14 +67,15 @@ private:
void writeNarInfo(ref<NarInfo> narInfo);
public:
bool isValidPathUncached(const Path& path) override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
Path queryPathFromHashPart(const string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
Path queryPathFromHashPart(const string& hashPart) override {
unsupported("queryPathFromHashPart");
}
bool wantMassQuery() override { return wantMassQuery_; }
@ -84,9 +83,9 @@ public:
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
Path addToStore(const string& name, const Path& srcPath, bool recursive,
HashType hashAlgo, PathFilter& filter,
RepairFlag repair) override;
Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override;
@ -94,11 +93,11 @@ public:
void narFromPath(const Path& path, Sink& sink) override;
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override
{ unsupported("buildDerivation"); }
BuildMode buildMode) override {
unsupported("buildDerivation");
}
void ensurePath(const Path & path) override
{ unsupported("ensurePath"); }
void ensurePath(const Path& path) override { unsupported("ensurePath"); }
ref<FSAccessor> getFSAccessor() override;
@ -107,9 +106,8 @@ public:
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
int getPriority() override { return priority; }
};
MakeError(NoSuchBinaryCacheFile, Error);
}
} // namespace nix

1651
third_party/nix/src/libstore/build.cc vendored
View file

File diff suppressed because it is too large Load diff

2
third_party/nix/src/libstore/builtins.hh vendored
View file

@ -8,4 +8,4 @@ namespace nix {
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData);
void builtinBuildenv(const BasicDerivation& drv);
}
} // namespace nix

70
third_party/nix/src/libstore/builtins/buildenv.cc vendored
View file

@ -1,9 +1,8 @@
#include "builtins.hh"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <algorithm>
#include "builtins.hh"
namespace nix {
@ -16,23 +15,24 @@ static Priorities priorities;
static unsigned long symlinks;
/* For each activated package, create symlinks */
static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
{
static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
DirEntries srcFiles;
try {
srcFiles = readDirectory(srcDir);
} catch (SysError& e) {
if (e.errNo == ENOTDIR) {
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
printError(
"warning: not including '%s' in the user environment because it's "
"not a directory",
srcDir);
return;
}
throw;
}
for (const auto& ent : srcFiles) {
if (ent.name[0] == '.')
/* not matched by glob */
if (ent.name[0] == '.') /* not matched by glob */
continue;
auto srcFile = srcDir + "/" + ent.name;
auto dstFile = dstDir + "/" + ent.name;
@ -58,8 +58,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
if (hasSuffix(srcFile, "/propagated-build-inputs") ||
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") ||
hasSuffix(srcFile, "/log"))
hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/log"))
continue;
else if (S_ISDIR(srcSt.st_mode)) {
@ -72,7 +71,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
} else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
throw Error("collision between '%1%' and non-directory '%2%'",
srcFile, target);
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1)
@ -98,12 +98,13 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
"to change the priority of one of the conflicting packages"
" (0 being the highest priority)",
srcFile, readLink(dstFile), priority);
if (prevPriority < priority)
continue;
if (prevPriority < priority) continue;
if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile);
} else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
throw Error(
"collision between non-directory '%1%' and directory '%2%'",
srcFile, dstFile);
} else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile);
}
@ -121,17 +122,16 @@ static FileProp postponed = FileProp{};
static Path out;
static void addPkg(const Path & pkgDir, int priority)
{
static void addPkg(const Path& pkgDir, int priority) {
if (done.count(pkgDir)) return;
done.insert(pkgDir);
createLinks(pkgDir, out, priority);
try {
for (const auto& p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
readFile(pkgDir + "/nix-support/propagated-user-env-packages"),
" \n"))
if (!done.count(p)) postponed.insert(p);
} catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
}
@ -141,13 +141,13 @@ struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
Package(Path path, bool active, int priority)
: path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;
void builtinBuildenv(const BasicDerivation & drv)
{
void builtinBuildenv(const BasicDerivation& drv) {
auto getAttr = [&](const string& name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
@ -162,12 +162,17 @@ void builtinBuildenv(const BasicDerivation & drv)
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
/* !!! We're trusting the caller to structure derivations env var correctly
*/
auto active = derivations.front();
derivations.pop_front();
auto priority = stoi(derivations.front());
derivations.pop_front();
auto outputs = stoi(derivations.front());
derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
auto path = derivations.front();
derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
}
}
@ -177,11 +182,11 @@ void builtinBuildenv(const BasicDerivation & drv)
* symlink/unlink steps.
*/
std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) {
return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
return a.priority < b.priority ||
(a.priority == b.priority && a.path < b.path);
});
for (const auto& pkg : pkgs)
if (pkg.active)
addPkg(pkg.path, pkg.priority);
if (pkg.active) addPkg(pkg.path, pkg.priority);
/* Symlink to the packages that have been "propagated" by packages
* installed by the user (i.e., package X declares that it wants Y
@ -192,8 +197,7 @@ void builtinBuildenv(const BasicDerivation & drv)
while (!postponed.empty()) {
auto pkgDirs = postponed;
postponed = FileProp{};
for (const auto & pkgDir : pkgDirs)
addPkg(pkgDir, priorityCounter++);
for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++);
}
printError("created %d symlinks in user environment", symlinks);
@ -201,4 +205,4 @@ void builtinBuildenv(const BasicDerivation & drv)
createSymlink(getAttr("manifest"), out + "/manifest.nix");
}
}
} // namespace nix

20
third_party/nix/src/libstore/builtins/fetchurl.cc vendored
View file

@ -1,13 +1,12 @@
#include "archive.hh"
#include "builtins.hh"
#include "compression.hh"
#include "download.hh"
#include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
namespace nix {
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
{
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) {
/* Make the host's netrc data available. Too bad curl requires
this to be stored in a file. It would be nice if we could just
pass a pointer to the data. */
@ -18,7 +17,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto getAttr = [&](const string& name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
if (i == drv.env.end())
throw Error(format("attribute '%s' missing") % name);
return i->second;
};
@ -31,9 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto downloader = makeDownloader();
auto fetch = [&](const std::string& url) {
auto source = sinkToSource([&](Sink& sink) {
/* No need to do TLS verification, because we check the hash of
the result anyway. */
DownloadRequest request(url);
@ -60,12 +58,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get())
try {
for (auto hashedMirror : settings.hashedMirrors.get()) try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashType(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
fetch(hashedMirror + printHashType(h.type) + "/" +
h.to_string(Base16, false));
return;
} catch (Error& e) {
debug(e.what());
@ -75,4 +73,4 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
fetch(mainUrl);
}
}
} // namespace nix

52
third_party/nix/src/libstore/crypto.cc vendored
View file

@ -1,6 +1,6 @@
#include "crypto.hh"
#include "util.hh"
#include "globals.hh"
#include "util.hh"
#if HAVE_SODIUM
#include <sodium.h>
@ -8,30 +8,24 @@
namespace nix {
static std::pair<std::string, std::string> split(const string & s)
{
static std::pair<std::string, std::string> split(const string& s) {
size_t colon = s.find(':');
if (colon == std::string::npos || colon == 0)
return {"", ""};
if (colon == std::string::npos || colon == 0) return {"", ""};
return {std::string(s, 0, colon), std::string(s, colon + 1)};
}
Key::Key(const string & s)
{
Key::Key(const string& s) {
auto ss = split(s);
name = ss.first;
key = ss.second;
if (name == "" || key == "")
throw Error("secret key is corrupt");
if (name == "" || key == "") throw Error("secret key is corrupt");
key = base64Decode(key);
}
SecretKey::SecretKey(const string & s)
: Key(s)
{
SecretKey::SecretKey(const string& s) : Key(s) {
#if HAVE_SODIUM
if (key.size() != crypto_sign_SECRETKEYBYTES)
throw Error("secret key is not valid");
@ -39,14 +33,14 @@ SecretKey::SecretKey(const string & s)
}
#if !HAVE_SODIUM
[[noreturn]] static void noSodium()
{
throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
[[noreturn]] static void noSodium() {
throw Error(
"Nix was not compiled with libsodium, required for signed binary cache "
"support");
}
#endif
std::string SecretKey::signDetached(const std::string & data) const
{
std::string SecretKey::signDetached(const std::string& data) const {
#if HAVE_SODIUM
unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen;
@ -58,8 +52,7 @@ std::string SecretKey::signDetached(const std::string & data) const
#endif
}
PublicKey SecretKey::toPublicKey() const
{
PublicKey SecretKey::toPublicKey() const {
#if HAVE_SODIUM
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data());
@ -69,9 +62,7 @@ PublicKey SecretKey::toPublicKey() const
#endif
}
PublicKey::PublicKey(const string & s)
: Key(s)
{
PublicKey::PublicKey(const string& s) : Key(s) {
#if HAVE_SODIUM
if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid");
@ -79,8 +70,7 @@ PublicKey::PublicKey(const string & s)
}
bool verifyDetached(const std::string& data, const std::string& sig,
const PublicKeys & publicKeys)
{
const PublicKeys& publicKeys) {
#if HAVE_SODIUM
auto ss = split(sig);
@ -88,19 +78,17 @@ bool verifyDetached(const std::string & data, const std::string & sig,
if (key == publicKeys.end()) return false;
auto sig2 = base64Decode(ss.second);
if (sig2.size() != crypto_sign_BYTES)
throw Error("signature is not valid");
if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid");
return crypto_sign_verify_detached((unsigned char *) sig2.data(),
(unsigned char *) data.data(), data.size(),
(unsigned char *) key->second.key.data()) == 0;
return crypto_sign_verify_detached(
(unsigned char*)sig2.data(), (unsigned char*)data.data(),
data.size(), (unsigned char*)key->second.key.data()) == 0;
#else
noSodium();
#endif
}
PublicKeys getDefaultPublicKeys()
{
PublicKeys getDefaultPublicKeys() {
PublicKeys publicKeys;
// FIXME: filter duplicates
@ -123,4 +111,4 @@ PublicKeys getDefaultPublicKeys()
return publicKeys;
}
}
} // namespace nix

20
third_party/nix/src/libstore/crypto.hh vendored
View file

@ -1,13 +1,11 @@
#pragma once
#include "types.hh"
#include <map>
#include "types.hh"
namespace nix {
struct Key
{
struct Key {
std::string name;
std::string key;
@ -16,14 +14,12 @@ struct Key
Key(const std::string& s);
protected:
Key(const std::string & name, const std::string & key)
: name(name), key(key) { }
Key(const std::string& name, const std::string& key) : name(name), key(key) {}
};
struct PublicKey;
struct SecretKey : Key
{
struct SecretKey : Key {
SecretKey(const std::string& s);
/* Return a detached signature of the given string. */
@ -32,13 +28,11 @@ struct SecretKey : Key
PublicKey toPublicKey() const;
};
struct PublicKey : Key
{
struct PublicKey : Key {
PublicKey(const std::string& data);
private:
PublicKey(const std::string & name, const std::string & key)
: Key(name, key) { }
PublicKey(const std::string& name, const std::string& key) : Key(name, key) {}
friend struct SecretKey;
};
@ -51,4 +45,4 @@ bool verifyDetached(const std::string & data, const std::string & sig,
PublicKeys getDefaultPublicKeys();
}
} // namespace nix

265
third_party/nix/src/libstore/derivations.cc vendored
View file

@ -1,16 +1,14 @@
#include "derivations.hh"
#include "store-api.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "istringstream_nocopy.hh"
#include "store-api.hh"
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix {
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
{
void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
recursive = false;
string algo = hashAlgo;
@ -26,29 +24,22 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
hash = Hash(this->hash, hashType);
}
Path BasicDerivation::findOutput(const string & id) const
{
Path BasicDerivation::findOutput(const string& id) const {
auto i = outputs.find(id);
if (i == outputs.end())
throw Error(format("derivation has no output '%1%'") % id);
return i->second.path;
}
bool BasicDerivation::isBuiltin() const
{
bool BasicDerivation::isBuiltin() const {
return string(builder, 0, 8) == "builtin:";
}
Path writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair)
{
Path writeDerivation(ref<Store> store, const Derivation& drv,
const string& name, RepairFlag repair) {
PathSet references;
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
for (auto & i : drv.inputDrvs)
references.insert(i.first);
for (auto& i : drv.inputDrvs) references.insert(i.first);
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
@ -59,47 +50,43 @@ Path writeDerivation(ref<Store> store,
: store->addTextToStore(suffix, contents, references, repair);
}
/* Read string `s' from stream `str'. */
static void expect(std::istream & str, const string & s)
{
static void expect(std::istream& str, const string& s) {
char s2[s.size()];
str.read(s2, s.size());
if (string(s2, s.size()) != s)
throw FormatError(format("expected string '%1%'") % s);
}
/* Read a C-style string from stream `str'. */
static string parseString(std::istream & str)
{
static string parseString(std::istream& str) {
string res;
expect(str, "\"");
int c;
while ((c = str.get()) != '"')
if (c == '\\') {
c = str.get();
if (c == 'n') res += '\n';
else if (c == 'r') res += '\r';
else if (c == 't') res += '\t';
else res += c;
}
else res += c;
if (c == 'n')
res += '\n';
else if (c == 'r')
res += '\r';
else if (c == 't')
res += '\t';
else
res += c;
} else
res += c;
return res;
}
static Path parsePath(std::istream & str)
{
static Path parsePath(std::istream& str) {
string s = parseString(str);
if (s.size() == 0 || s[0] != '/')
throw FormatError(format("bad path '%1%' in derivation") % s);
return s;
}
static bool endOfList(std::istream & str)
{
static bool endOfList(std::istream& str) {
if (str.peek() == ',') {
str.get();
return false;
@ -111,18 +98,14 @@ static bool endOfList(std::istream & str)
return false;
}
static StringSet parseStrings(std::istream & str, bool arePaths)
{
static StringSet parseStrings(std::istream& str, bool arePaths) {
StringSet res;
while (!endOfList(str))
res.insert(arePaths ? parsePath(str) : parseString(str));
return res;
}
static Derivation parseDerivation(const string & s)
{
static Derivation parseDerivation(const string& s) {
Derivation drv;
istringstream_nocopy str(s);
expect(str, "Derive([");
@ -130,10 +113,14 @@ static Derivation parseDerivation(const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
DerivationOutput out;
expect(str, "("); string id = parseString(str);
expect(str, ","); out.path = parsePath(str);
expect(str, ","); out.hashAlgo = parseString(str);
expect(str, ","); out.hash = parseString(str);
expect(str, "(");
string id = parseString(str);
expect(str, ",");
out.path = parsePath(str);
expect(str, ",");
out.hashAlgo = parseString(str);
expect(str, ",");
out.hash = parseString(str);
expect(str, ")");
drv.outputs[id] = out;
}
@ -148,20 +135,24 @@ static Derivation parseDerivation(const string & s)
expect(str, ")");
}
expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
expect(str, ","); drv.platform = parseString(str);
expect(str, ","); drv.builder = parseString(str);
expect(str, ",[");
drv.inputSrcs = parseStrings(str, true);
expect(str, ",");
drv.platform = parseString(str);
expect(str, ",");
drv.builder = parseString(str);
/* Parse the builder arguments. */
expect(str, ",[");
while (!endOfList(str))
drv.args.push_back(parseString(str));
while (!endOfList(str)) drv.args.push_back(parseString(str));
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "("); string name = parseString(str);
expect(str, ","); string value = parseString(str);
expect(str, "(");
string name = parseString(str);
expect(str, ",");
string value = parseString(str);
expect(str, ")");
drv.env[name] = value;
}
@ -170,94 +161,115 @@ static Derivation parseDerivation(const string & s)
return drv;
}
Derivation readDerivation(const Path & drvPath)
{
Derivation readDerivation(const Path& drvPath) {
try {
return parseDerivation(readFile(drvPath));
} catch (FormatError& e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
e.msg());
}
}
Derivation Store::derivationFromPath(const Path & drvPath)
{
Derivation Store::derivationFromPath(const Path& drvPath) {
assertStorePath(drvPath);
ensurePath(drvPath);
auto accessor = getFSAccessor();
try {
return parseDerivation(accessor->readFile(drvPath));
} catch (FormatError& e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
e.msg());
}
}
static void printString(string & res, const string & s)
{
static void printString(string& res, const string& s) {
res += '"';
for (const char* i = s.c_str(); *i; i++)
if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
else if (*i == '\n') res += "\\n";
else if (*i == '\r') res += "\\r";
else if (*i == '\t') res += "\\t";
else res += *i;
if (*i == '\"' || *i == '\\') {
res += "\\";
res += *i;
} else if (*i == '\n')
res += "\\n";
else if (*i == '\r')
res += "\\r";
else if (*i == '\t')
res += "\\t";
else
res += *i;
res += '"';
}
template <class ForwardIterator>
static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
{
static void printStrings(string& res, ForwardIterator i, ForwardIterator j) {
res += '[';
bool first = true;
for (; i != j; ++i) {
if (first) first = false; else res += ',';
if (first)
first = false;
else
res += ',';
printString(res, *i);
}
res += ']';
}
string Derivation::unparse() const
{
string Derivation::unparse() const {
string s;
s.reserve(65536);
s += "Derive([";
bool first = true;
for (auto& i : outputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, i.second.path);
s += ','; printString(s, i.second.hashAlgo);
s += ','; printString(s, i.second.hash);
if (first)
first = false;
else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second.path);
s += ',';
printString(s, i.second.hashAlgo);
s += ',';
printString(s, i.second.hash);
s += ')';
}
s += "],[";
first = true;
for (auto& i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printStrings(s, i.second.begin(), i.second.end());
if (first)
first = false;
else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printStrings(s, i.second.begin(), i.second.end());
s += ')';
}
s += "],";
printStrings(s, inputSrcs.begin(), inputSrcs.end());
s += ','; printString(s, platform);
s += ','; printString(s, builder);
s += ','; printStrings(s, args.begin(), args.end());
s += ',';
printString(s, platform);
s += ',';
printString(s, builder);
s += ',';
printStrings(s, args.begin(), args.end());
s += ",[";
first = true;
for (auto& i : env) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, i.second);
if (first)
first = false;
else
s += ',';
s += '(';
printString(s, i.first);
s += ',';
printString(s, i.second);
s += ')';
}
@ -266,24 +278,17 @@ string Derivation::unparse() const
return s;
}
bool isDerivation(const string & fileName)
{
bool isDerivation(const string& fileName) {
return hasSuffix(fileName, drvExtension);
}
bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
bool BasicDerivation::isFixedOutput() const {
return outputs.size() == 1 && outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
}
DrvHashes drvHashes;
/* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are
@ -304,15 +309,12 @@ DrvHashes drvHashes;
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of
its output path. */
Hash hashDerivationModulo(Store & store, Derivation drv)
{
Hash hashDerivationModulo(Store& store, Derivation drv) {
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.path);
return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" +
i->second.hash + ":" + i->second.path);
}
/* For other derivations, replace the inputs paths with recursive
@ -333,41 +335,31 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
return hashString(htSHA256, drv.unparse());
}
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
{
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s) {
size_t n = s.find("!");
return n == s.npos
? DrvPathWithOutputs(s, std::set<string>())
: DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
return n == s.npos ? DrvPathWithOutputs(s, std::set<string>())
: DrvPathWithOutputs(string(s, 0, n),
tokenizeString<std::set<string> >(
string(s, n + 1), ","));
}
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
{
return outputs.empty()
? drvPath
Path makeDrvPathWithOutputs(const Path& drvPath,
const std::set<string>& outputs) {
return outputs.empty() ? drvPath
: drvPath + "!" + concatStringsSep(",", outputs);
}
bool wantOutput(const string & output, const std::set<string> & wanted)
{
bool wantOutput(const string& output, const std::set<string>& wanted) {
return wanted.empty() || wanted.find(output) != wanted.end();
}
PathSet BasicDerivation::outputPaths() const
{
PathSet BasicDerivation::outputPaths() const {
PathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path);
for (auto& i : outputs) paths.insert(i.second.path);
return paths;
}
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
{
Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) {
drv.outputs.clear();
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
@ -392,25 +384,20 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
return in;
}
Sink & operator << (Sink & out, const BasicDerivation & drv)
{
Sink& operator<<(Sink& out, const BasicDerivation& drv) {
out << drv.outputs.size();
for (auto& i : drv.outputs)
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
out << i.first << i.second;
for (auto& i : drv.env) out << i.first << i.second;
return out;
}
std::string hashPlaceholder(const std::string & outputName)
{
std::string hashPlaceholder(const std::string& outputName) {
// FIXME: memoize?
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
return "/" + hashString(htSHA256, "nix-output:" + outputName)
.to_string(Base32, false);
}
}
} // namespace nix

36
third_party/nix/src/libstore/derivations.hh vendored
View file

@ -1,31 +1,23 @@
#pragma once
#include "types.hh"
#include <map>
#include "hash.hh"
#include "store-api.hh"
#include <map>
#include "types.hh"
namespace nix {
/* Extension of derivations in the Nix store. */
const string drvExtension = ".drv";
/* Abstract syntax of derivations. */
struct DerivationOutput
{
struct DerivationOutput {
Path path;
string hashAlgo; /* hash used for expected hash computation */
string hash; /* expected hash, may be null */
DerivationOutput()
{
}
DerivationOutput(Path path, string hashAlgo, string hash)
{
DerivationOutput() {}
DerivationOutput(Path path, string hashAlgo, string hash) {
this->path = path;
this->hashAlgo = hashAlgo;
this->hash = hash;
@ -41,8 +33,7 @@ typedef std::map<Path, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs;
struct BasicDerivation
{
struct BasicDerivation {
DerivationOutputs outputs; /* keyed on symbolic IDs */
PathSet inputSrcs; /* inputs that are sources */
string platform;
@ -63,24 +54,20 @@ struct BasicDerivation
/* Return the output paths of a derivation. */
PathSet outputPaths() const;
};
struct Derivation : BasicDerivation
{
struct Derivation : BasicDerivation {
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
/* Print a derivation. */
std::string unparse() const;
};
class Store;
/* Write a derivation to the Nix store, and return its path. */
Path writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
Path writeDerivation(ref<Store> store, const Derivation& drv,
const string& name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
Derivation readDerivation(const Path& drvPath);
@ -102,7 +89,8 @@ extern DrvHashes drvHashes; // FIXME: global, not thread-safe
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s);
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
Path makeDrvPathWithOutputs(const Path& drvPath,
const std::set<string>& outputs);
bool wantOutput(const string& output, const std::set<string>& wanted);
@ -114,4 +102,4 @@ Sink & operator << (Sink & out, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string& outputName);
}
} // namespace nix

404
third_party/nix/src/libstore/download.cc vendored
View file

@ -1,23 +1,21 @@
#include "download.hh"
#include "util.hh"
#include "archive.hh"
#include "compression.hh"
#include "finally.hh"
#include "globals.hh"
#include "hash.hh"
#include "store-api.hh"
#include "archive.hh"
#include "s3.hh"
#include "compression.hh"
#include "pathlocks.hh"
#include "finally.hh"
#include "s3.hh"
#include "store-api.hh"
#include "util.hh"
#ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
#endif
#include <unistd.h>
#include <fcntl.h>
#include <curl/curl.h>
#include <fcntl.h>
#include <unistd.h>
#include <algorithm>
#include <cmath>
#include <cstring>
@ -34,31 +32,31 @@ DownloadSettings downloadSettings;
static GlobalConfig::Register r1(&downloadSettings);
std::string resolveUri(const std::string & uri)
{
std::string resolveUri(const std::string& uri) {
if (uri.compare(0, 8, "channel:") == 0)
return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz";
return "https://nixos.org/channels/" + std::string(uri, 8) +
"/nixexprs.tar.xz";
else
return uri;
}
struct CurlDownloader : public Downloader
{
struct CurlDownloader : public Downloader {
CURLM* curlm = 0;
std::random_device rd;
std::mt19937 mt19937;
struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
{
struct DownloadItem : public std::enable_shared_from_this<DownloadItem> {
CurlDownloader& downloader;
DownloadRequest request;
DownloadResult result;
Activity act;
bool done = false; // whether either the success or failure function has been called
bool done = false; // whether either the success or failure function has
// been called
Callback<DownloadResult> callback;
CURL* req = 0;
bool active = false; // whether the handle has been added to the multi object
bool active =
false; // whether the handle has been added to the multi object
std::string status;
unsigned int attempt = 0;
@ -75,62 +73,62 @@ struct CurlDownloader : public Downloader
curl_off_t writtenToSink = 0;
DownloadItem(CurlDownloader & downloader,
const DownloadRequest & request,
DownloadItem(CurlDownloader& downloader, const DownloadRequest& request,
Callback<DownloadResult>&& callback)
: downloader(downloader)
, request(request)
, act(*logger, lvlTalkative, actDownload,
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
: downloader(downloader),
request(request),
act(*logger, lvlTalkative, actDownload,
fmt(request.data ? "uploading '%s'" : "downloading '%s'",
request.uri),
{request.uri}, request.parentAct),
callback(std::move(callback)),
finalSink([this](const unsigned char* data, size_t len) {
if (this->request.dataCallback) {
long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 ||
httpStatus == 206) {
writtenToSink += len;
this->request.dataCallback((char*)data, len);
}
} else
this->result.data->append((char*)data, len);
})
{
}) {
if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
requestHeaders = curl_slist_append(
requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
if (!request.mimeType.empty())
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
requestHeaders = curl_slist_append(
requestHeaders, ("Content-Type: " + request.mimeType).c_str());
}
~DownloadItem()
{
~DownloadItem() {
if (req) {
if (active)
curl_multi_remove_handle(downloader.curlm, req);
if (active) curl_multi_remove_handle(downloader.curlm, req);
curl_easy_cleanup(req);
}
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
fail(DownloadError(
Interrupted,
format("download of '%s' was interrupted") % request.uri));
} catch (...) {
ignoreException();
}
}
void failEx(std::exception_ptr ex)
{
void failEx(std::exception_ptr ex) {
assert(!done);
done = true;
callback.rethrow(ex);
}
template <class T>
void fail(const T & e)
{
void fail(const T& e) {
failEx(std::make_exception_ptr(e));
}
@ -139,8 +137,7 @@ struct CurlDownloader : public Downloader
std::exception_ptr writeException;
size_t writeCallback(void * contents, size_t size, size_t nmemb)
{
size_t writeCallback(void* contents, size_t size, size_t nmemb) {
try {
size_t realSize = size * nmemb;
result.bodySize += realSize;
@ -157,16 +154,16 @@ struct CurlDownloader : public Downloader
}
}
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
static size_t writeCallbackWrapper(void* contents, size_t size,
size_t nmemb, void* userp) {
return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb);
}
size_t headerCallback(void * contents, size_t size, size_t nmemb)
{
size_t headerCallback(void* contents, size_t size, size_t nmemb) {
size_t realSize = size * nmemb;
std::string line((char*)contents, realSize);
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
printMsg(lvlVomit,
format("got header for '%s': %s") % request.uri % trim(line));
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
result.etag = "";
auto ss = tokenizeString<vector<string>>(line, " ");
@ -187,25 +184,26 @@ struct CurlDownloader : public Downloader
down the connection because we already have the
data. */
if (result.etag == request.expectedETag && status == "200") {
debug(format("shutting down on 200 HTTP response with expected ETag"));
debug(format(
"shutting down on 200 HTTP response with expected ETag"));
return 0;
}
} else if (name == "content-encoding")
encoding = trim(string(line, i + 1));
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
else if (name == "accept-ranges" &&
toLower(trim(std::string(line, i + 1))) == "bytes")
acceptRanges = true;
}
}
return realSize;
}
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
static size_t headerCallbackWrapper(void* contents, size_t size,
size_t nmemb, void* userp) {
return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb);
}
int progressCallback(double dltotal, double dlnow)
{
int progressCallback(double dltotal, double dlnow) {
try {
act.progress(dlnow, dltotal);
} catch (nix::Interrupted&) {
@ -214,23 +212,22 @@ struct CurlDownloader : public Downloader
return _isInterrupted;
}
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
{
static int progressCallbackWrapper(void* userp, double dltotal,
double dlnow, double ultotal,
double ulnow) {
return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow);
}
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
{
static int debugCallback(CURL* handle, curl_infotype type, char* data,
size_t size, void* userptr) {
if (type == CURLINFO_TEXT)
vomit("curl: %s", chomp(std::string(data, size)));
return 0;
}
size_t readOffset = 0;
size_t readCallback(char *buffer, size_t size, size_t nitems)
{
if (readOffset == request.data->length())
return 0;
size_t readCallback(char* buffer, size_t size, size_t nitems) {
if (readOffset == request.data->length()) return 0;
auto count = std::min(size * nitems, request.data->length() - readOffset);
assert(count);
memcpy(buffer, request.data->data() + readOffset, count);
@ -238,20 +235,20 @@ struct CurlDownloader : public Downloader
return count;
}
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{
static size_t readCallbackWrapper(char* buffer, size_t size, size_t nitems,
void* userp) {
return ((DownloadItem*)userp)->readCallback(buffer, size, nitems);
}
void init()
{
void init() {
if (!req) req = curl_easy_init();
curl_easy_reset(req);
if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION,
DownloadItem::debugCallback);
}
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@ -260,7 +257,10 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
(downloadSettings.userAgentSuffix != ""
? " " + downloadSettings.userAgentSuffix.get()
: ""))
.c_str());
#if LIBCURL_VERSION_NUM >= 0x072b00
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
#endif
@ -270,9 +270,11 @@ struct CurlDownloader : public Downloader
else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
#endif
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION,
DownloadItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION,
DownloadItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@ -281,14 +283,14 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
if (request.head)
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.data) {
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
curl_easy_setopt(req, CURLOPT_READDATA, this);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE,
(curl_off_t)request.data->length());
}
if (request.verifyTLS) {
@ -299,14 +301,17 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
}
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT,
downloadSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME,
downloadSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */
curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC_FILE,
settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
if (writtenToSink)
@ -316,17 +321,17 @@ struct CurlDownloader : public Downloader
result.bodySize = 0;
}
void finish(CURLcode code)
{
void finish(CURLcode code) {
long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
char* effectiveUriCStr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr)
result.effectiveUri = effectiveUriCStr;
if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr;
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
debug(
"finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d "
"bytes",
request.verb(), request.uri, code, httpStatus, result.bodySize);
if (decompressionSink) {
@ -346,8 +351,10 @@ struct CurlDownloader : public Downloader
failEx(writeException);
else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
{
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 ||
httpStatus == 206 || httpStatus == 304 ||
httpStatus == 226 /* FTP */ ||
httpStatus == 0 /* other protocol */)) {
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);
done = true;
@ -358,19 +365,25 @@ struct CurlDownloader : public Downloader
// We treat most errors as transient, but won't retry when hopeless
Error err = Transient;
if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
if (httpStatus == 404 || httpStatus == 410 ||
code == CURLE_FILE_COULDNT_READ_FILE) {
// The file is definitely not there
err = NotFound;
} else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
} else if (httpStatus == 401 || httpStatus == 403 ||
httpStatus == 407) {
// Don't retry on authentication/authorization failures
err = Forbidden;
} else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
// Most 4xx errors are client errors and are probably not worth retrying:
} else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 &&
httpStatus != 429) {
// Most 4xx errors are client errors and are probably not worth
// retrying:
// * 408 means the server timed out waiting for us, so we try again
// * 429 means too many requests, so we retry (with a delay)
err = Misc;
} else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
// Let's treat most 5xx (server) errors as transient, except for a handful:
} else if (httpStatus == 501 || httpStatus == 505 ||
httpStatus == 511) {
// Let's treat most 5xx (server) errors as transient, except for a
// handful:
// * 501 not implemented
// * 505 http version not supported
// * 511 we're behind a captive portal
@ -403,50 +416,57 @@ struct CurlDownloader : public Downloader
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted",
request.verb(), request.uri))
: httpStatus != 0
? DownloadError(err,
fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: DownloadError(err,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
? DownloadError(
err, fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus) +
(code == CURLE_OK
? ""
: fmt(" (curl error: %s)",
curl_easy_strerror(code))))
: DownloadError(err, fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri,
curl_easy_strerror(code), code));
/* If this is a transient error, then maybe retry the
download after a while. If we're writing to a
sink, we can only retry if the server supports
ranged requests. */
if (err == Transient
&& attempt < request.tries
&& (!this->request.dataCallback
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
{
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
if (err == Transient && attempt < request.tries &&
(!this->request.dataCallback || writtenToSink == 0 ||
(acceptRanges && encoding.empty()))) {
int ms = request.baseRetryTimeMs *
std::pow(2.0f, attempt - 1 +
std::uniform_real_distribution<>(
0.0, 0.5)(downloader.mt19937));
if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
warn("%s; retrying from offset %d in %d ms", exc.what(),
writtenToSink, ms);
else
warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
embargo =
std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this());
}
else
} else
fail(exc);
}
}
};
struct State
{
struct State {
struct EmbargoComparator {
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
bool operator()(const std::shared_ptr<DownloadItem>& i1,
const std::shared_ptr<DownloadItem>& i2) {
return i1->embargo > i2->embargo;
}
};
bool quit = false;
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
std::priority_queue<std::shared_ptr<DownloadItem>,
std::vector<std::shared_ptr<DownloadItem>>,
EmbargoComparator>
incoming;
};
Sync<State> state_;
@ -458,9 +478,7 @@ struct CurlDownloader : public Downloader
std::thread workerThread;
CurlDownloader()
: mt19937(rd())
{
CurlDownloader() : mt19937(rd()) {
static std::once_flag globalInit;
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
@ -480,8 +498,7 @@ struct CurlDownloader : public Downloader
workerThread = std::thread([&]() { workerThreadEntry(); });
}
~CurlDownloader()
{
~CurlDownloader() {
stopWorkerThread();
workerThread.join();
@ -489,8 +506,7 @@ struct CurlDownloader : public Downloader
if (curlm) curl_multi_cleanup(curlm);
}
void stopWorkerThread()
{
void stopWorkerThread() {
/* Signal the worker thread to exit. */
{
auto state(state_.lock());
@ -499,12 +515,9 @@ struct CurlDownloader : public Downloader
writeFull(wakeupPipe.writeSide.get(), " ", false);
}
void workerThreadMain()
{
void workerThreadMain() {
/* Cause this thread to be notified on SIGINT. */
auto callback = createInterruptCallback([&]() {
stopWorkerThread();
});
auto callback = createInterruptCallback([&]() { stopWorkerThread(); });
std::map<CURL*, std::shared_ptr<DownloadItem>> items;
@ -519,7 +532,9 @@ struct CurlDownloader : public Downloader
int running;
CURLMcode mc = curl_multi_perform(curlm, &running);
if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
throw nix::Error(
format("unexpected error from curl_multi_perform(): %s") %
curl_multi_strerror(mc));
/* Set the promises of any finished requests. */
CURLMsg* msg;
@ -544,12 +559,17 @@ struct CurlDownloader : public Downloader
long maxSleepTimeMs = items.empty() ? 10000 : 100;
auto sleepTimeMs =
nextWakeup != std::chrono::steady_clock::time_point()
? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
? std::max(
0,
(int)std::chrono::duration_cast<std::chrono::milliseconds>(
nextWakeup - std::chrono::steady_clock::now())
.count())
: maxSleepTimeMs;
vomit("download thread waiting for %d ms", sleepTimeMs);
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") %
curl_multi_strerror(mc));
nextWakeup = std::chrono::steady_clock::time_point();
@ -574,8 +594,8 @@ struct CurlDownloader : public Downloader
incoming.push_back(item);
state->incoming.pop();
} else {
if (nextWakeup == std::chrono::steady_clock::time_point()
|| item->embargo < nextWakeup)
if (nextWakeup == std::chrono::steady_clock::time_point() ||
item->embargo < nextWakeup)
nextWakeup = item->embargo;
break;
}
@ -595,8 +615,7 @@ struct CurlDownloader : public Downloader
debug("download thread shutting down");
}
void workerThreadEntry()
{
void workerThreadEntry() {
try {
workerThreadMain();
} catch (nix::Interrupted& e) {
@ -611,30 +630,29 @@ struct CurlDownloader : public Downloader
}
}
void enqueueItem(std::shared_ptr<DownloadItem> item)
{
if (item->request.data
&& !hasPrefix(item->request.uri, "http://")
&& !hasPrefix(item->request.uri, "https://"))
void enqueueItem(std::shared_ptr<DownloadItem> item) {
if (item->request.data && !hasPrefix(item->request.uri, "http://") &&
!hasPrefix(item->request.uri, "https://"))
throw nix::Error("uploading to '%s' is not supported", item->request.uri);
{
auto state(state_.lock());
if (state->quit)
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
throw nix::Error(
"cannot enqueue download request because the download thread is "
"shutting down");
state->incoming.push(item);
}
writeFull(wakeupPipe.writeSide.get(), " ");
}
#ifdef ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
{
std::tuple<std::string, std::string, Store::Params> parseS3Uri(
std::string uri) {
auto [path, params] = splitUriAndParams(uri);
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", path);
if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", path);
std::string bucketName(path, 5, slash - 5);
std::string key(path, slash + 1);
@ -644,8 +662,7 @@ struct CurlDownloader : public Downloader
#endif
void enqueueDownload(const DownloadRequest& request,
Callback<DownloadResult> callback) override
{
Callback<DownloadResult> callback) override {
/* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) {
// FIXME: do this on a worker thread
@ -664,36 +681,37 @@ struct CurlDownloader : public Downloader
auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res;
if (!s3Res.data)
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
throw DownloadError(
NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data;
callback(std::move(res));
#else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
throw nix::Error(
"cannot download '%s' because Nix is not built with S3 support",
request.uri);
#endif
} catch (...) { callback.rethrow(); }
} catch (...) {
callback.rethrow();
}
return;
}
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
enqueueItem(
std::make_shared<DownloadItem>(*this, request, std::move(callback)));
}
};
ref<Downloader> getDownloader()
{
ref<Downloader> getDownloader() {
static ref<Downloader> downloader = makeDownloader();
return downloader;
}
ref<Downloader> makeDownloader()
{
return make_ref<CurlDownloader>();
}
ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); }
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
{
std::future<DownloadResult> Downloader::enqueueDownload(
const DownloadRequest& request) {
auto promise = std::make_shared<std::promise<DownloadResult>>();
enqueueDownload(request,
{[promise](std::future<DownloadResult> fut) {
enqueueDownload(request, {[promise](std::future<DownloadResult> fut) {
try {
promise->set_value(fut.get());
} catch (...) {
@ -703,13 +721,11 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
return promise->get_future();
}
DownloadResult Downloader::download(const DownloadRequest & request)
{
DownloadResult Downloader::download(const DownloadRequest& request) {
return enqueueDownload(request).get();
}
void Downloader::download(DownloadRequest && request, Sink & sink)
{
void Downloader::download(DownloadRequest&& request, Sink& sink) {
/* Note: we can't call 'sink' via request.dataCallback, because
that would cause the sink to execute on the downloader
thread. If 'sink' is a coroutine, this will fail. Also, if the
@ -736,7 +752,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
});
request.dataCallback = [_state](char* buf, size_t len) {
auto state(_state->lock());
if (state->quit) return;
@ -757,8 +772,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
state->avail.notify_one();
};
enqueueDownload(request,
{[_state](std::future<DownloadResult> fut) {
enqueueDownload(request, {[_state](std::future<DownloadResult> fut) {
auto state(_state->lock());
state->quit = true;
try {
@ -781,7 +795,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
auto state(_state->lock());
while (state->data.empty()) {
if (state->quit) {
if (state->exc) std::rethrow_exception(state->exc);
return;
@ -804,8 +817,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}
CachedDownloadResult Downloader::downloadCached(
ref<Store> store, const CachedDownloadRequest & request)
{
ref<Store> store, const CachedDownloadRequest& request) {
auto url = resolveUri(request.uri);
auto name = request.name;
@ -816,7 +828,8 @@ CachedDownloadResult Downloader::downloadCached(
Path expectedStorePath;
if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
expectedStorePath =
store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(expectedStorePath)) {
CachedDownloadResult result;
result.storePath = expectedStorePath;
@ -828,7 +841,8 @@ CachedDownloadResult Downloader::downloadCached(
Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url)
.to_string(Base32, false);
Path dataFile = cacheDir + "/" + urlHash + ".info";
Path fileLink = cacheDir + "/" + urlHash + "-file";
@ -850,7 +864,8 @@ CachedDownloadResult Downloader::downloadCached(
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
if (string2Int(ss[2], lastChecked) &&
(uint64_t)lastChecked + request.ttl >= (uint64_t)time(0)) {
skip = true;
result.effectiveUri = request.uri;
result.etag = ss[1];
@ -864,7 +879,6 @@ CachedDownloadResult Downloader::downloadCached(
}
if (!skip) {
try {
DownloadRequest request2(url);
request2.expectedETag = expectedETag;
@ -876,7 +890,9 @@ CachedDownloadResult Downloader::downloadCached(
ValidPathInfo info;
StringSink sink;
dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
Hash hash = hashString(
request.expectedHash ? request.expectedHash.type : htSHA256,
*res.data);
info.path = store->makeFixedOutputPath(false, hash, name);
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
@ -888,7 +904,8 @@ CachedDownloadResult Downloader::downloadCached(
assert(!storePath.empty());
replaceSymlink(storePath, fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
writeFile(dataFile,
url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError& e) {
if (storePath.empty()) throw;
warn("warning: %s; using cached result", e.msg());
@ -898,21 +915,24 @@ CachedDownloadResult Downloader::downloadCached(
if (request.unpack) {
Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
PathLocks lock2({unpackedLink},
fmt("waiting for lock on '%1%'...", unpackedLink));
Path unpackedStorePath;
if (pathExists(unpackedLink)) {
unpackedStorePath = readLink(unpackedLink);
store->addTempRoot(unpackedStorePath);
if (!store->isValidPath(unpackedStorePath))
unpackedStorePath = "";
if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = "";
}
if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression.
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
runProgram("tar", true,
{"xf", store->toRealPath(storePath), "-C", tmpDir,
"--strip-components", "1"});
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256,
defaultPathFilter, NoRepair);
}
replaceSymlink(unpackedStorePath, unpackedLink);
storePath = unpackedStorePath;
@ -920,11 +940,16 @@ CachedDownloadResult Downloader::downloadCached(
if (expectedStorePath != "" && storePath != expectedStorePath) {
unsigned int statusCode = 102;
Hash gotHash = request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
Hash gotHash =
request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(storePath))
.first
: hashFile(request.expectedHash.type, store->toRealPath(storePath));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
url, request.expectedHash.to_string(), gotHash.to_string());
throw nix::Error(statusCode,
"hash mismatch in file downloaded from '%s':\n wanted: "
"%s\n got: %s",
url, request.expectedHash.to_string(),
gotHash.to_string());
}
result.storePath = storePath;
@ -932,15 +957,14 @@ CachedDownloadResult Downloader::downloadCached(
return result;
}
bool isUri(const string & s)
{
bool isUri(const string& s) {
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == string::npos) return false;
string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
return scheme == "http" || scheme == "https" || scheme == "file" ||
scheme == "channel" || scheme == "git" || scheme == "s3" ||
scheme == "ssh";
}
}
} // namespace nix

68
third_party/nix/src/libstore/download.hh vendored
View file

@ -1,40 +1,45 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "globals.hh"
#include <string>
#include <future>
#include <string>
#include "globals.hh"
#include "hash.hh"
#include "types.hh"
namespace nix {
struct DownloadSettings : Config
{
struct DownloadSettings : Config {
Setting<bool> enableHttp2{this, true, "http2",
"Whether to enable HTTP/2 support."};
Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
Setting<std::string> userAgentSuffix{
this, "", "user-agent-suffix",
"String appended to the user agent in HTTP requests."};
Setting<size_t> httpConnections{this, 25, "http-connections",
Setting<size_t> httpConnections{this,
25,
"http-connections",
"Number of parallel HTTP connections.",
{"binary-caches-parallel-connections"}};
Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
"Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
Setting<unsigned long> connectTimeout{
this, 0, "connect-timeout",
"Timeout for connecting to servers during downloads. 0 means use curl's "
"builtin default."};
Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
"Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
Setting<unsigned long> stalledDownloadTimeout{
this, 300, "stalled-download-timeout",
"Timeout (in seconds) for receiving data from servers during download. "
"Nix cancels idle downloads after this timeout's duration."};
Setting<unsigned int> tries{this, 5, "download-attempts",
Setting<unsigned int> tries{
this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."};
};
extern DownloadSettings downloadSettings;
struct DownloadRequest
{
struct DownloadRequest {
std::string uri;
std::string expectedETag;
bool verifyTLS = true;
@ -50,14 +55,10 @@ struct DownloadRequest
DownloadRequest(const std::string& uri)
: uri(uri), parentAct(getCurActivity()) {}
std::string verb()
{
return data ? "upload" : "download";
}
std::string verb() { return data ? "upload" : "download"; }
};
struct DownloadResult
{
struct DownloadResult {
bool cached = false;
std::string etag;
std::string effectiveUri;
@ -65,20 +66,17 @@ struct DownloadResult
uint64_t bodySize = 0;
};
struct CachedDownloadRequest
{
struct CachedDownloadRequest {
std::string uri;
bool unpack = false;
std::string name;
Hash expectedHash;
unsigned int ttl = settings.tarballTtl;
CachedDownloadRequest(const std::string & uri)
: uri(uri) { }
CachedDownloadRequest(const std::string& uri) : uri(uri) {}
};
struct CachedDownloadResult
{
struct CachedDownloadResult {
// Note: 'storePath' may be different from 'path' when using a
// chroot store.
Path storePath;
@ -89,8 +87,7 @@ struct CachedDownloadResult
class Store;
struct Downloader
{
struct Downloader {
virtual ~Downloader() {}
/* Enqueue a download request, returning a future to the result of
@ -112,7 +109,8 @@ struct Downloader
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
CachedDownloadResult downloadCached(ref<Store> store,
const CachedDownloadRequest& request);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};
@ -124,15 +122,13 @@ ref<Downloader> getDownloader();
/* Return a new Downloader object. */
ref<Downloader> makeDownloader();
class DownloadError : public Error
{
class DownloadError : public Error {
public:
Downloader::Error error;
DownloadError(Downloader::Error error, const FormatOrString& fs)
: Error(fs), error(error)
{ }
: Error(fs), error(error) {}
};
bool isUri(const string& s);
}
} // namespace nix

52
third_party/nix/src/libstore/export-import.cc vendored
View file

@ -1,31 +1,23 @@
#include "store-api.hh"
#include "archive.hh"
#include "worker-protocol.hh"
#include <algorithm>
#include "archive.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix {
struct HashAndWriteSink : Sink
{
struct HashAndWriteSink : Sink {
Sink& writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
HashAndWriteSink(Sink& writeSink)
: writeSink(writeSink), hashSink(htSHA256) {}
virtual void operator()(const unsigned char* data, size_t len) {
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
Hash currentHash() { return hashSink.currentHash().first; }
};
void Store::exportPaths(const Paths & paths, Sink & sink)
{
void Store::exportPaths(const Paths& paths, Sink& sink) {
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
std::reverse(sorted.begin(), sorted.end());
@ -42,8 +34,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
sink << 0;
}
void Store::exportPath(const Path & path, Sink & sink)
{
void Store::exportPath(const Path& path, Sink& sink) {
auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink);
@ -55,19 +46,22 @@ void Store::exportPath(const Path & path, Sink & sink)
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path
% info->narHash.to_string() % hash.to_string());
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") %
path % info->narHash.to_string() % hash.to_string());
hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
hashAndWriteSink << exportMagic << path << info->references << info->deriver
<< 0;
}
Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
{
Paths Store::importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs) {
Paths res;
while (true) {
auto n = readNum<uint64_t>(source);
if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
if (n != 1)
throw Error(
"input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */
TeeSink tee(source);
@ -81,7 +75,8 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
info.path = readStorePath(*this, source);
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
// Activity act(*logger, lvlInfo, format("importing path '%s'") %
// info.path);
info.references = readStorePaths<PathSet>(*this, source);
@ -92,8 +87,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
info.narSize = tee.source.data->size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
if (readInt(source) == 1) readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
@ -103,4 +97,4 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
return res;
}
}
} // namespace nix

8
third_party/nix/src/libstore/fs-accessor.hh vendored
View file

@ -6,13 +6,11 @@ namespace nix {
/* An abstract class for accessing a filesystem-like structure, such
as a (possibly remote) Nix store or the contents of a NAR file. */
class FSAccessor
{
class FSAccessor {
public:
enum Type { tMissing, tRegular, tSymlink, tDirectory };
struct Stat
{
struct Stat {
Type type = tMissing;
uint64_t fileSize = 0; // regular files only
bool isExecutable = false; // regular files only
@ -30,4 +28,4 @@ public:
virtual std::string readLink(const Path& path) = 0;
};
}
} // namespace nix

298
third_party/nix/src/libstore/gc.cc vendored
View file

@ -1,42 +1,37 @@
#include "derivations.hh"
#include "globals.hh"
#include "local-store.hh"
#include "finally.hh"
#include <functional>
#include <queue>
#include <algorithm>
#include <regex>
#include <random>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <climits>
#include <functional>
#include <queue>
#include <random>
#include <regex>
#include "derivations.hh"
#include "finally.hh"
#include "globals.hh"
#include "local-store.hh"
namespace nix {
static string gcLockName = "gc.lock";
static string gcRootsDir = "gcroots";
/* Acquire the global GC lock. This is used to prevent new Nix
processes from starting after the temporary root files have been
read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished /
yielded the GC lock. */
AutoCloseFD LocalStore::openGCLock(LockType lockType)
{
Path fnGCLock = (format("%1%/%2%")
% stateDir % gcLockName).str();
AutoCloseFD LocalStore::openGCLock(LockType lockType) {
Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str();
debug(format("acquiring global GC lock '%1%'") % fnGCLock);
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
AutoCloseFD fdGCLock =
open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
@ -52,69 +47,60 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
return fdGCLock;
}
static void makeSymlink(const Path & link, const Path & target)
{
static void makeSymlink(const Path& link, const Path& target) {
/* Create directories up to `gcRoot'. */
createDirs(dirOf(link));
/* Create the new symlink. */
Path tempLink = (format("%1%.tmp-%2%-%3%")
% link % getpid() % random()).str();
Path tempLink =
(format("%1%.tmp-%2%-%3%") % link % getpid() % random()).str();
createSymlink(target, tempLink);
/* Atomically replace the old one. */
if (rename(tempLink.c_str(), link.c_str()) == -1)
throw SysError(format("cannot rename '%1%' to '%2%'")
% tempLink % link);
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % link);
}
void LocalStore::syncWithGC() { AutoCloseFD fdGCLock = openGCLock(ltRead); }
void LocalStore::syncWithGC()
{
AutoCloseFD fdGCLock = openGCLock(ltRead);
}
void LocalStore::addIndirectRoot(const Path & path)
{
void LocalStore::addIndirectRoot(const Path& path) {
string hash = hashString(htSHA1, path).to_string(Base32, false);
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
% stateDir % gcRootsDir % hash).str());
Path realRoot = canonPath(
(format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str());
makeSymlink(realRoot, path);
}
Path LocalFSStore::addPermRoot(const Path & _storePath,
const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
{
Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot,
bool indirect, bool allowOutsideRootsDir) {
Path storePath(canonPath(_storePath));
Path gcRoot(canonPath(_gcRoot));
assertStorePath(storePath);
if (isInStore(gcRoot))
throw Error(format(
"creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)") % gcRoot);
throw Error(format("creating a garbage collector root (%1%) in the Nix "
"store is forbidden "
"(are you running nix-build inside the store?)") %
gcRoot);
if (indirect) {
/* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
throw Error(format("cannot create symlink '%1%'; already exists") %
gcRoot);
makeSymlink(gcRoot, storePath);
addIndirectRoot(gcRoot);
}
else {
if (!allowOutsideRootsDir) {
Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
Path rootsDir =
canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(format(
"path '%1%' is not a valid garbage collector root; "
"it's not in the directory '%2%'")
% gcRoot % rootsDir);
throw Error(format("path '%1%' is not a valid garbage collector root; "
"it's not in the directory '%2%'") %
gcRoot % rootsDir);
}
if (baseNameOf(gcRoot) == baseNameOf(storePath))
@ -132,10 +118,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0)
printError(
format(
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector")
% gcRoot % storePath);
format("warning: '%1%' is not in a directory where the garbage "
"collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector") %
gcRoot % storePath);
}
/* Grab the global GC root, causing us to block while a GC is in
@ -146,14 +132,11 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
return gcRoot;
}
void LocalStore::addTempRoot(const Path & path)
{
void LocalStore::addTempRoot(const Path& path) {
auto state(_state.lock());
/* Create the temporary roots file for this process. */
if (!state->fdTempRoots) {
while (1) {
AutoCloseFD fdGCLock = openGCLock(ltRead);
@ -180,7 +163,6 @@ void LocalStore::addTempRoot(const Path & path)
get a lock. (It won't delete the file after we get a
lock.) Try again. */
}
}
/* Upgrade the lock to a write lock. This will cause us to block
@ -196,12 +178,9 @@ void LocalStore::addTempRoot(const Path & path)
lockFile(state->fdTempRoots.get(), ltRead, true);
}
static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
{
void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto& i : readDirectory(tempRootsDir)) {
@ -255,21 +234,18 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
}
}
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
auto foundRoot = [&](const Path& path, const Path& target) {
Path storePath = toStorePath(target);
if (isStorePath(storePath) && isValidPath(storePath))
roots[storePath].emplace(path);
else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path %
storePath);
};
try {
if (type == DT_UNKNOWN)
type = getFileType(path);
if (type == DT_UNKNOWN) type = getFileType(path);
if (type == DT_DIR) {
for (auto& i : readDirectory(path))
@ -278,15 +254,15 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
else if (type == DT_LNK) {
Path target = readLink(path);
if (isInStore(target))
foundRoot(path, target);
if (isInStore(target)) foundRoot(path, target);
/* Handle indirect roots. */
else {
target = absPath(target, dirOf(path));
if (!pathExists(target)) {
if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
printInfo(format("removing stale link from '%1%' to '%2%'") % path % target);
printInfo(format("removing stale link from '%1%' to '%2%'") % path %
target);
unlink(path.c_str());
}
} else {
@ -315,9 +291,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
}
}
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
{
void LocalStore::findRootsNoTemp(Roots& roots, bool censor) {
/* Process direct roots in {gcroots,profiles}. */
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@ -328,9 +302,7 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
findRuntimeRoots(roots, censor);
}
Roots LocalStore::findRoots(bool censor)
{
Roots LocalStore::findRoots(bool censor) {
Roots roots;
findRootsNoTemp(roots, censor);
@ -340,55 +312,49 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}
static void readProcLink(const string & file, Roots & roots)
{
static void readProcLink(const string& file, Roots& roots) {
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
try_again:
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
return;
if (errno == ENOENT || errno == EACCES || errno == ESRCH) return;
throw SysError("reading symlink");
}
if (res == bufsiz) {
if (SSIZE_MAX / 2 < bufsiz)
throw Error("stupidly long symlink");
if (SSIZE_MAX / 2 < bufsiz) throw Error("stupidly long symlink");
bufsiz *= 2;
goto try_again;
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
roots[std::string(static_cast<char*>(buf), res)].emplace(file);
}
static string quoteRegexChars(const string & raw)
{
static string quoteRegexChars(const string& raw) {
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, Roots & roots)
{
static void readFileRoots(const char* path, Roots& roots) {
try {
roots[readFile(path)].emplace(path);
} catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != EACCES)
throw;
if (e.errNo != ENOENT && e.errNo != EACCES) throw;
}
}
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
Roots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent* ent;
auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
auto mapRegex =
std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = std::regex(quoteRegexChars(storeDir) +
R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
@ -398,8 +364,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
continue;
if (errno == ENOENT || errno == EACCES) continue;
throw SysError(format("opening %1%") % fdStr);
}
struct dirent* fd_ent;
@ -408,15 +373,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
continue;
if (errno == ESRCH) continue;
throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
}
fdDir.reset();
try {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
auto mapLines = tokenizeString<std::vector<string>>(
readFile(mapFile, true), "\n");
for (const auto& line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
@ -426,28 +391,28 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile, true);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
for (auto i = std::sregex_iterator{envString.begin(), envString.end(),
storePathRegex};
i != env_end; ++i)
unchecked[i->str()].emplace(envFile);
} catch (SysError& e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue;
throw;
}
}
}
if (errno)
throw SysError("iterating /proc");
if (errno) throw SysError("iterating /proc");
}
#if !defined(__linux__)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
// See: https://github.com/NixOS/nix/issues/3011
// Because of this we disable lsof when running the tests.
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test
// to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we
// disable lsof when running the tests.
if (getEnv("_NIX_TEST_NO_LSOF") == "") {
try {
std::regex lsofRegex(R"(^n(/.*)$)");
auto lsofLines =
tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
auto lsofLines = tokenizeString<std::vector<string>>(
runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n");
for (const auto& line : lsofLines) {
std::smatch match;
if (std::regex_match(line, match, lsofRegex))
@ -479,12 +444,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
}
}
struct GCLimitReached {};
struct LocalStore::GCState
{
struct LocalStore::GCState {
GCOptions options;
GCResults& results;
PathSet roots;
@ -499,25 +461,20 @@ struct LocalStore::GCState
GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {}
};
bool LocalStore::isActiveTempFile(const GCState & state,
const Path & path, const string & suffix)
{
return hasSuffix(path, suffix)
&& state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
bool LocalStore::isActiveTempFile(const GCState& state, const Path& path,
const string& suffix) {
return hasSuffix(path, suffix) &&
state.tempRoots.find(string(path, 0, path.size() - suffix.size())) !=
state.tempRoots.end();
}
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
void LocalStore::deleteGarbage(GCState& state, const Path& path) {
unsigned long long bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
checkInterrupt();
unsigned long long size = 0;
@ -557,26 +514,29 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
throw SysError(format("making '%1%' writable") % realPath);
Path tmp = trashDir + "/" + baseNameOf(path);
if (rename(realPath.c_str(), tmp.c_str()))
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath %
tmp);
state.bytesInvalidated += size;
} catch (SysError& e) {
if (e.errNo == ENOSPC) {
printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
printInfo(format("note: can't create move '%1%': %2%") % realPath %
e.msg());
deleteGarbage(state, realPath);
}
}
} else
deleteGarbage(state, realPath);
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
if (state.results.bytesFreed + state.bytesInvalidated >
state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") %
state.options.maxFreed);
throw GCLimitReached();
}
}
bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
{
bool LocalStore::canReachRoot(GCState& state, PathSet& visited,
const Path& path) {
if (visited.count(path)) return false;
if (state.alive.count(path)) return true;
@ -611,8 +571,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
are derivers of this path that are not garbage. */
if (state.gcKeepOutputs) {
PathSet derivers = queryValidDerivers(path);
for (auto & i : derivers)
incoming.insert(i);
for (auto& i : derivers) incoming.insert(i);
}
for (auto& i : incoming)
@ -625,15 +584,14 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
return false;
}
void LocalStore::tryToDelete(GCState & state, const Path & path)
{
void LocalStore::tryToDelete(GCState& state, const Path& path) {
checkInterrupt();
auto realPath = realStoreDir + "/" + baseNameOf(path);
if (realPath == linksDir || realPath == trashDir) return;
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
// Activity act(*logger, lvlDebug, format("considering whether to delete
// '%1%'") % path);
if (!isStorePath(path) || !isValidPath(path)) {
/* A lock file belonging to a path that we're building right
@ -660,19 +618,16 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
‘nix-store --delete’ doesn't have the unexpected effect of
recursing into derivations and outputs. */
state.dead.insert(visited.begin(), visited.end());
if (state.shouldDelete)
deletePathRecursive(state, path);
if (state.shouldDelete) deletePathRecursive(state, path);
}
}
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
void LocalStore::removeUnusedLinks(const GCState & state)
{
void LocalStore::removeUnusedLinks(const GCState& state) {
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
@ -708,13 +663,11 @@ void LocalStore::removeUnusedLinks(const GCState & state)
throw SysError(format("statting '%1%'") % linksDir);
long long overhead = st.st_blocks * 512ULL;
printInfo(format("note: currently hard linking saves %.2f MiB")
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
printInfo(format("note: currently hard linking saves %.2f MiB") %
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
GCState state(results);
state.options = options;
state.gcKeepOutputs = settings.gcKeepOutputs;
@ -729,10 +682,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
state.gcKeepDerivations = false;
}
state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
state.shouldDelete = options.action == GCOptions::gcDeleteDead ||
options.action == GCOptions::gcDeleteSpecific;
if (state.shouldDelete)
deletePath(reservedPath);
if (state.shouldDelete) deletePath(reservedPath);
/* Acquire the global GC root. This prevents
a) New roots from being added.
@ -743,8 +696,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
permanent roots cannot increase now. */
printError(format("finding garbage collector roots..."));
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true);
for (auto& i : rootMap) state.roots.insert(i.first);
@ -754,8 +706,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
FDs fds;
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots)
state.tempRoots.insert(root.first);
for (auto& root : tempRoots) state.tempRoots.insert(root.first);
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
/* After this point the set of roots or temporary roots cannot
@ -778,25 +729,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific) {
for (auto& i : options.pathsToDelete) {
assertStorePath(i);
tryToDelete(state, i);
if (state.dead.find(i) == state.dead.end())
throw Error(format("cannot delete path '%1%' since it is still alive") % i);
throw Error(format("cannot delete path '%1%' since it is still alive") %
i);
}
} else if (options.maxFreed > 0) {
if (state.shouldDelete)
printError(format("deleting garbage..."));
else
printError(format("determining live/dead paths..."));
try {
AutoCloseDir dir(opendir(realStoreDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
if (!dir)
throw SysError(format("opening directory '%1%'") % realStoreDir);
/* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting
@ -828,8 +778,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
std::mt19937 gen(1);
std::shuffle(entries_.begin(), entries_.end(), gen);
for (auto & i : entries_)
tryToDelete(state, i);
for (auto& i : entries_) tryToDelete(state, i);
} catch (GCLimitReached& e) {
}
@ -854,7 +803,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
deleteGarbage(state, trashDir);
/* Clean up the links directory. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
if (options.action == GCOptions::gcDeleteDead ||
options.action == GCOptions::gcDeleteSpecific) {
printError(format("deleting unused links..."));
removeUnusedLinks(state);
}
@ -863,9 +813,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
// if (options.action == GCOptions::gcDeleteDead) vacuumDB();
}
void LocalStore::autoGC(bool sync)
{
void LocalStore::autoGC(bool sync) {
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
auto getAvail = [this]() -> uint64_t {
@ -892,7 +840,9 @@ void LocalStore::autoGC(bool sync)
auto now = std::chrono::steady_clock::now();
if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return;
if (now < state->lastGCCheck +
std::chrono::seconds(settings.minFreeCheckInterval))
return;
auto avail = getAvail();
@ -908,9 +858,7 @@ void LocalStore::autoGC(bool sync)
future = state->gcFuture = promise.get_future().share();
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
try {
/* Wake up any threads waiting for the auto-GC to finish. */
Finally wakeup([&]() {
auto state(_state.lock());
@ -935,7 +883,6 @@ void LocalStore::autoGC(bool sync)
// future, but we don't really care.
ignoreException();
}
}).detach();
}
@ -944,5 +891,4 @@ void LocalStore::autoGC(bool sync)
if (sync) future.get();
}
}
} // namespace nix

123
third_party/nix/src/libstore/globals.cc vendored
View file

@ -1,17 +1,14 @@
#include "globals.hh"
#include "util.hh"
#include "archive.hh"
#include "args.hh"
#include <dlfcn.h>
#include <algorithm>
#include <map>
#include <thread>
#include <dlfcn.h>
#include "archive.hh"
#include "args.hh"
#include "util.hh"
namespace nix {
/* The default location of the daemon socket, relative to nixStateDir.
The socket is in a directory to allow you to control access to the
Nix daemon by setting the mode/ownership of the directory
@ -31,23 +28,25 @@ Settings settings;
static GlobalConfig::Register r1(&settings);
Settings::Settings()
: nixPrefix(NIX_PREFIX)
, nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
, nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)))
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR))
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
{
: nixPrefix(NIX_PREFIX),
nixStore(canonPath(
getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))),
nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))),
nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))),
nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))),
nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))),
nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))),
nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))),
nixManDir(canonPath(NIX_MAN_DIR)),
nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) {
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
if (caFile == "") {
for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
for (auto& fn :
{"/etc/ssl/certs/ca-certificates.crt",
"/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
if (pathExists(fn)) {
caFile = fn;
break;
@ -58,8 +57,7 @@ Settings::Settings()
auto s = getEnv("NIX_REMOTE_SYSTEMS");
if (s != "") {
Strings ss;
for (auto & p : tokenizeString<Strings>(s, ":"))
ss.push_back("@" + p);
for (auto& p : tokenizeString<Strings>(s, ":")) ss.push_back("@" + p);
builders = concatStringsSep(" ", ss);
}
@ -67,11 +65,11 @@ Settings::Settings()
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
#endif
allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
allowedImpureHostPrefixes =
tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
}
void loadConfFile()
{
void loadConfFile() {
globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
/* We only want to send overrides to the daemon, i.e. stuff from
@ -79,27 +77,25 @@ void loadConfFile()
globalConfig.resetOverriden();
auto dirs = getConfigDirs();
// Iterate over them in reverse so that the ones appearing first in the path take priority
// Iterate over them in reverse so that the ones appearing first in the path
// take priority
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
}
}
unsigned int Settings::getDefaultCores()
{
unsigned int Settings::getDefaultCores() {
return std::max(1U, std::thread::hardware_concurrency());
}
StringSet Settings::getDefaultSystemFeatures()
{
StringSet Settings::getDefaultSystemFeatures() {
/* For backwards compatibility, accept some "features" that are
used in Nixpkgs to route builds to certain machines but don't
actually require anything special on the machines. */
StringSet features{"nixos-test", "benchmark", "big-parallel"};
#if __linux__
if (access("/dev/kvm", R_OK | W_OK) == 0)
features.insert("kvm");
if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm");
#endif
return features;
@ -107,29 +103,38 @@ StringSet Settings::getDefaultSystemFeatures()
const string nixVersion = PACKAGE_VERSION;
template<> void BaseSetting<SandboxMode>::set(const std::string & str)
{
if (str == "true") value = smEnabled;
else if (str == "relaxed") value = smRelaxed;
else if (str == "false") value = smDisabled;
else throw UsageError("option '%s' has invalid value '%s'", name, str);
template <>
void BaseSetting<SandboxMode>::set(const std::string& str) {
if (str == "true")
value = smEnabled;
else if (str == "relaxed")
value = smRelaxed;
else if (str == "false")
value = smDisabled;
else
throw UsageError("option '%s' has invalid value '%s'", name, str);
}
template<> std::string BaseSetting<SandboxMode>::to_string()
{
if (value == smEnabled) return "true";
else if (value == smRelaxed) return "relaxed";
else if (value == smDisabled) return "false";
else abort();
template <>
std::string BaseSetting<SandboxMode>::to_string() {
if (value == smEnabled)
return "true";
else if (value == smRelaxed)
return "relaxed";
else if (value == smDisabled)
return "false";
else
abort();
}
template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
{
template <>
void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) {
AbstractSetting::toJSON(out);
}
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
{
template <>
void BaseSetting<SandboxMode>::convertToArg(Args& args,
const std::string& category) {
args.mkFlag()
.longName(name)
.description("Enable sandboxing.")
@ -147,16 +152,15 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
.category(category);
}
void MaxBuildJobsSetting::set(const std::string & str)
{
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
void MaxBuildJobsSetting::set(const std::string& str) {
if (str == "auto")
value = std::max(1U, std::thread::hardware_concurrency());
else if (!string2Int(str, value))
throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
throw UsageError(
"configuration setting '%s' should be 'auto' or an integer", name);
}
void initPlugins()
{
void initPlugins() {
for (const auto& pluginFile : settings.pluginFiles.get()) {
Paths pluginFiles;
try {
@ -164,17 +168,16 @@ void initPlugins()
for (const auto& ent : ents)
pluginFiles.emplace_back(pluginFile + "/" + ent.name);
} catch (SysError& e) {
if (e.errNo != ENOTDIR)
throw;
if (e.errNo != ENOTDIR) throw;
pluginFiles.emplace_back(pluginFile);
}
for (const auto& file : pluginFiles) {
/* handle is purposefully leaked as there may be state in the
DSO needed by the action of the plugin. */
void *handle =
dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle)
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
throw Error("could not dynamically open plugin file '%s': %s", file,
dlerror());
}
}
@ -184,4 +187,4 @@ void initPlugins()
globalConfig.warnUnknownSettings();
}
}
} // namespace nix

312
third_party/nix/src/libstore/globals.hh vendored
View file

@ -1,27 +1,21 @@
#pragma once
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include <map>
#include <limits>
#include <sys/types.h>
#include <limits>
#include <map>
#include "config.hh"
#include "types.hh"
#include "util.hh"
namespace nix {
typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
{
MaxBuildJobsSetting(Config * options,
unsigned int def,
const std::string & name,
const std::string & description,
struct MaxBuildJobsSetting : public BaseSetting<unsigned int> {
MaxBuildJobsSetting(Config* options, unsigned int def,
const std::string& name, const std::string& description,
const std::set<std::string>& aliases = {})
: BaseSetting<unsigned int>(def, name, description, aliases)
{
: BaseSetting<unsigned int>(def, name, description, aliases) {
options->addSetting(this);
}
@ -29,13 +23,11 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
};
class Settings : public Config {
unsigned int getDefaultCores();
StringSet getDefaultSystemFeatures();
public:
Settings();
Path nixPrefix;
@ -69,32 +61,45 @@ public:
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
"The default Nix store to use."};
Setting<bool> keepFailed{this, false, "keep-failed",
Setting<bool> keepFailed{
this, false, "keep-failed",
"Whether to keep temporary directories of failed builds."};
Setting<bool> keepGoing{this, false, "keep-going",
Setting<bool> keepGoing{
this, false, "keep-going",
"Whether to keep building derivations when another build fails."};
Setting<bool> tryFallback{this, false, "fallback",
Setting<bool> tryFallback{
this,
false,
"fallback",
"Whether to fall back to building when substitution fails.",
{"build-fallback"}};
/* Whether to show build log output in real time. */
bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines",
Setting<size_t> logLines{
this, 10, "log-lines",
"If verbose-build is false, the number of lines of the tail of "
"the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
"Maximum number of parallel build jobs. \"auto\" means use number of cores.",
MaxBuildJobsSetting maxBuildJobs{this,
1,
"max-jobs",
"Maximum number of parallel build jobs. "
"\"auto\" means use number of cores.",
{"build-max-jobs"}};
Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
Setting<unsigned int> buildCores{
this,
getDefaultCores(),
"cores",
"Number of CPU cores to utilize in parallel within a build, "
"i.e. by passing this number to Make via '-j'. 0 means that the "
"number of actual CPU cores on the local host ought to be "
"auto-detected.", {"build-cores"}};
"auto-detected.",
{"build-cores"}};
/* Read-only mode. Don't copy stuff to the store, don't change
the database. */
@ -103,28 +108,41 @@ public:
Setting<std::string> thisSystem{this, SYSTEM, "system",
"The canonical Nix system name."};
Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
Setting<time_t> maxSilentTime{
this,
0,
"max-silent-time",
"The maximum time in seconds that a builer can go without "
"producing any output on stdout/stderr before it is killed. "
"0 means infinity.",
{"build-max-silent-time"}};
Setting<time_t> buildTimeout{this, 0, "timeout",
Setting<time_t> buildTimeout{
this,
0,
"timeout",
"The maximum duration in seconds that a builder can run. "
"0 means infinity.", {"build-timeout"}};
"0 means infinity.",
{"build-timeout"}};
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
"The path of the helper program that executes builds to remote machines."};
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote",
"build-hook",
"The path of the helper program that executes builds "
"to remote machines."};
Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
"A semicolon-separated list of build machines, in the format of nix.machines."};
Setting<std::string> builders{this, "@" + nixConfDir + "/machines",
"builders",
"A semicolon-separated list of build machines, "
"in the format of nix.machines."};
Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
Setting<bool> buildersUseSubstitutes{
this, false, "builders-use-substitutes",
"Whether build machines should use their own substitutes for obtaining "
"build dependencies if possible, rather than waiting for this host to "
"upload them."};
Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
Setting<off_t> reservedSize{
this, 8 * 1024 * 1024, "gc-reserved-space",
"Amount of reserved disk space for the garbage collector."};
Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
@ -133,29 +151,43 @@ public:
Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
"Whether SQLite should use WAL mode."};
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
Setting<bool> syncBeforeRegistering{
this, false, "sync-before-registering",
"Whether to call sync() before registering a path as valid."};
Setting<bool> useSubstitutes{this, true, "substitute",
Setting<bool> useSubstitutes{this,
true,
"substitute",
"Whether to use substitutes.",
{"build-use-substitutes"}};
Setting<std::string> buildUsersGroup{this, "", "build-users-group",
Setting<std::string> buildUsersGroup{
this, "", "build-users-group",
"The Unix group that contains the build users."};
Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
Setting<bool> impersonateLinux26{
this,
false,
"impersonate-linux-26",
"Whether to impersonate a Linux 2.6 machine on newer kernels.",
{"build-impersonate-linux-26"}};
Setting<bool> keepLog{this, true, "keep-build-log",
Setting<bool> keepLog{this,
true,
"keep-build-log",
"Whether to store build logs.",
{"build-keep-log"}};
Setting<bool> compressLog{this, true, "compress-build-log",
Setting<bool> compressLog{this,
true,
"compress-build-log",
"Whether to compress logs.",
{"build-compress-log"}};
Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
Setting<unsigned long> maxLogSize{
this,
0,
"max-build-log-size",
"Maximum number of bytes a builder can write to stdout/stderr "
"before being killed (0 means no limit).",
{"build-max-log-size"}};
@ -165,25 +197,37 @@ public:
stderr. Hack to prevent Hydra logs from being polluted. */
bool printRepeatedBuilds = true;
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
Setting<unsigned int> pollInterval{
this, 5, "build-poll-interval",
"How often (in seconds) to poll for locks."};
Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
Setting<bool> checkRootReachability{
this, false, "gc-check-reachability",
"Whether to check if new GC roots can in fact be found by the "
"garbage collector."};
Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
Setting<bool> gcKeepOutputs{
this,
false,
"keep-outputs",
"Whether the garbage collector should keep outputs of live derivations.",
{"gc-keep-outputs"}};
Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
Setting<bool> gcKeepDerivations{
this,
true,
"keep-derivations",
"Whether the garbage collector should keep derivers of live paths.",
{"gc-keep-derivations"}};
Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
"Whether to automatically replace files with identical contents with hard links."};
"Whether to automatically replace files with "
"identical contents with hard links."};
Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
Setting<bool> envKeepDerivations{
this,
false,
"keep-env-derivations",
"Whether to add derivations as a dependency of user environments "
"(to prevent them from being GCed).",
{"env-keep-derivations"}};
@ -192,170 +236,240 @@ public:
bool lockCPU;
/* Whether to show a stack trace if Nix evaluation fails. */
Setting<bool> showTrace{this, false, "show-trace",
Setting<bool> showTrace{
this, false, "show-trace",
"Whether to show a stack trace on evaluation errors."};
Setting<SandboxMode> sandboxMode{this,
Setting<SandboxMode> sandboxMode {
this,
#if __linux__
smEnabled
#else
smDisabled
#endif
, "sandbox",
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
{"build-use-chroot", "build-use-sandbox"}};
,
"sandbox",
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or "
"\"relaxed\".",
{
"build-use-chroot", "build-use-sandbox"
}
};
Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
Setting<PathSet> sandboxPaths{
this,
{},
"sandbox-paths",
"The paths to make available inside the build sandbox.",
{"build-chroot-dirs", "build-sandbox-paths"}};
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
Setting<bool> sandboxFallback{
this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
Setting<PathSet> extraSandboxPaths{
this,
{},
"extra-sandbox-paths",
"Additional paths to make available inside the build sandbox.",
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
Setting<size_t> buildRepeat{this, 0, "repeat",
Setting<size_t> buildRepeat{
this,
0,
"repeat",
"The number of times to repeat a build in order to verify determinism.",
{"build-repeat"}};
#if __linux__
Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
Setting<std::string> sandboxShmSize{
this, "50%", "sandbox-dev-shm-size",
"The size of /dev/shm in the build sandbox."};
Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
"The build directory inside the sandbox."};
#endif
Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
"Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
Setting<PathSet> allowedImpureHostPrefixes{
this,
{},
"allowed-impure-host-deps",
"Which prefixes to allow derivations to ask for access to (primarily for "
"Darwin)."};
#if __APPLE__
Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations",
Setting<bool> darwinLogSandboxViolations{
this, false, "darwin-log-sandbox-violations",
"Whether to log Darwin sandbox access violations to the system log."};
#endif
Setting<bool> runDiffHook{this, false, "run-diff-hook",
Setting<bool> runDiffHook{
this, false, "run-diff-hook",
"Whether to run the program specified by the diff-hook setting "
"repeated builds produce a different result. Typically used to "
"plug in diffoscope."};
PathSetting diffHook{this, true, "", "diff-hook",
PathSetting diffHook{
this, true, "", "diff-hook",
"A program that prints out the differences between the two paths "
"specified on its command line."};
Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
Setting<bool> enforceDeterminism{
this, true, "enforce-determinism",
"Whether to fail if repeated builds produce different output."};
Setting<Strings> trustedPublicKeys{this,
Setting<Strings> trustedPublicKeys{
this,
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
"trusted-public-keys",
"Trusted public keys for secure substitution.",
{"binary-cache-public-keys"}};
Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
Setting<Strings> secretKeyFiles{
this,
{},
"secret-key-files",
"Secret keys with which to sign local builds."};
Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
Setting<unsigned int> tarballTtl{
this, 60 * 60, "tarball-ttl",
"How long downloaded files are considered up-to-date."};
Setting<bool> requireSigs{this, true, "require-sigs",
Setting<bool> requireSigs{
this, true, "require-sigs",
"Whether to check that any non-content-addressed path added to the "
"Nix store has a valid signature (that is, one signed using a key "
"listed in 'trusted-public-keys'."};
Setting<StringSet> extraPlatforms{this,
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
Setting<StringSet> extraPlatforms{
this,
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"}
: StringSet{},
"extra-platforms",
"Additional platforms that can be built on the local system. "
"These may be supported natively (e.g. armv7 on some aarch64 CPUs "
"or using hacks like qemu-user."};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features",
Setting<StringSet> systemFeatures{
this, getDefaultSystemFeatures(), "system-features",
"Optional features that this system implements (like \"kvm\")."};
Setting<Strings> substituters{this,
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
Setting<Strings> substituters{
this,
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"}
: Strings(),
"substituters",
"The URIs of substituters (such as https://cache.nixos.org/).",
{"binary-caches"}};
// FIXME: provide a way to add to option values.
Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
Setting<Strings> extraSubstituters{this,
{},
"extra-substituters",
"Additional URIs of substituters.",
{"extra-binary-caches"}};
Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
"Disabled substituters that may be enabled via the substituters option by untrusted users.",
Setting<StringSet> trustedSubstituters{
this,
{},
"trusted-substituters",
"Disabled substituters that may be enabled via the substituters option "
"by untrusted users.",
{"trusted-binary-caches"}};
Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
"Which users or groups are trusted to ask the daemon to do unsafe things."};
Setting<Strings> trustedUsers{this,
{"root"},
"trusted-users",
"Which users or groups are trusted to ask the "
"daemon to do unsafe things."};
Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
"The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
Setting<unsigned int> ttlNegativeNarInfoCache{
this, 3600, "narinfo-cache-negative-ttl",
"The TTL in seconds for negative lookups in the disk cache i.e binary "
"cache lookups that "
"return an invalid path result"};
Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
"The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
Setting<unsigned int> ttlPositiveNarInfoCache{
this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
"The TTL in seconds for positive lookups in the disk cache i.e binary "
"cache lookups that "
"return a valid path result."};
/* ?Who we trust to use the daemon in safe ways */
Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
Setting<Strings> allowedUsers{
this,
{"*"},
"allowed-users",
"Which users or groups are allowed to connect to the daemon."};
Setting<bool> printMissing{this, true, "print-missing",
Setting<bool> printMissing{
this, true, "print-missing",
"Whether to print what paths need to be built or downloaded."};
Setting<std::string> preBuildHook{this,
Setting<std::string> preBuildHook {
this,
#if __APPLE__
nixLibexecDir + "/nix/resolve-system-dependencies",
#else
"",
#endif
"pre-build-hook",
"A program to run just before a build to set derivation-specific build settings."};
"A program to run just before a build to set derivation-specific build "
"settings."
};
Setting<std::string> postBuildHook{this, "", "post-build-hook",
Setting<std::string> postBuildHook{
this, "", "post-build-hook",
"A program to run just after each successful build."};
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
"Path to the netrc file used to obtain usernames/passwords for downloads."};
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"),
"netrc-file",
"Path to the netrc file used to obtain "
"usernames/passwords for downloads."};
/* Path to the SSL CA file used */
Path caFile;
#if __linux__
Setting<bool> filterSyscalls{this, true, "filter-syscalls",
Setting<bool> filterSyscalls{
this, true, "filter-syscalls",
"Whether to prevent certain dangerous system calls, such as "
"creation of setuid/setgid files or adding ACLs or extended "
"attributes. Only disable this if you're aware of the "
"security implications."};
Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
Setting<bool> allowNewPrivileges{
this, false, "allow-new-privileges",
"Whether builders can acquire new privileges by calling programs with "
"setuid/setgid bits or with file capabilities."};
#endif
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
Setting<Strings> hashedMirrors{
this,
{"http://tarballs.nixos.org/"},
"hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
"Automatically run the garbage collector when free "
"disk space drops below the specified amount."};
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
"Stop deleting garbage when free disk space is above the specified amount."};
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(),
"max-free",
"Stop deleting garbage when free disk space is "
"above the specified amount."};
Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
Setting<uint64_t> minFreeCheckInterval{
this, 5, "min-free-check-interval",
"Number of seconds between checking free disk space."};
Setting<Paths> pluginFiles{this, {}, "plugin-files",
Setting<Paths> pluginFiles{
this,
{},
"plugin-files",
"Plugins to dynamically load at nix initialization time."};
};
// FIXME: don't use a global variable.
extern Settings settings;
@ -367,4 +481,4 @@ void loadConfFile();
extern const string nixVersion;
}
} // namespace nix

89
third_party/nix/src/libstore/http-binary-cache-store.cc vendored
View file

@ -7,14 +7,11 @@ namespace nix {
MakeError(UploadToHTTP, Error);
class HttpBinaryCacheStore : public BinaryCacheStore
{
class HttpBinaryCacheStore : public BinaryCacheStore {
private:
Path cacheUri;
struct State
{
struct State {
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
@ -22,25 +19,16 @@ private:
Sync<State> _state;
public:
HttpBinaryCacheStore(
const Params & params, const Path & _cacheUri)
: BinaryCacheStore(params)
, cacheUri(_cacheUri)
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
HttpBinaryCacheStore(const Params& params, const Path& _cacheUri)
: BinaryCacheStore(params), cacheUri(_cacheUri) {
if (cacheUri.back() == '/') cacheUri.pop_back();
diskCache = getNarInfoDiskCache();
}
std::string getUri() override
{
return cacheUri;
}
std::string getUri() override { return cacheUri; }
void init() override
{
void init() override {
// FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
try {
@ -53,20 +41,18 @@ public:
}
protected:
void maybeDisable()
{
void maybeDisable() {
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
state->disabledUntil =
std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
void checkEnabled()
{
void checkEnabled() {
auto state(_state.lock());
if (state->enabled) return;
if (std::chrono::steady_clock::now() > state->disabledUntil) {
@ -77,8 +63,7 @@ protected:
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
}
bool fileExists(const std::string & path) override
{
bool fileExists(const std::string& path) override {
checkEnabled();
try {
@ -96,55 +81,55 @@ protected:
}
}
void upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType) override
{
void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) override {
auto req = DownloadRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType;
try {
getDownloader()->download(req);
} catch (DownloadError& e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s",
cacheUri, e.msg());
}
}
DownloadRequest makeRequest(const std::string & path)
{
DownloadRequest makeRequest(const std::string& path) {
DownloadRequest request(cacheUri + "/" + path);
return request;
}
void getFile(const std::string & path, Sink & sink) override
{
void getFile(const std::string& path, Sink& sink) override {
checkEnabled();
auto request(makeRequest(path));
try {
getDownloader()->download(std::move(request), sink);
} catch (DownloadError& e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
}
}
void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept override
{
void getFile(
const std::string& path,
Callback<std::shared_ptr<std::string>> callback) noexcept override {
checkEnabled();
auto request(makeRequest(path));
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
auto callbackPtr =
std::make_shared<decltype(callback)>(std::move(callback));
getDownloader()->enqueueDownload(request,
{[callbackPtr, this](std::future<DownloadResult> result) {
getDownloader()->enqueueDownload(
request, {[callbackPtr, this](std::future<DownloadResult> result) {
try {
(*callbackPtr)(result.get().data);
} catch (DownloadError& e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
if (e.error == Downloader::NotFound ||
e.error == Downloader::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable();
callbackPtr->rethrow();
@ -153,21 +138,19 @@ protected:
}
}});
}
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore(
[](const std::string& uri,
const Store::Params& params) -> std::shared_ptr<Store> {
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" &&
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
) return 0;
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" ||
std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;
});
}
} // namespace nix

205
third_party/nix/src/libstore/legacy-ssh-store.cc vendored
View file

@ -1,29 +1,34 @@
#include "archive.hh"
#include "derivations.hh"
#include "pool.hh"
#include "remote-store.hh"
#include "serve-protocol.hh"
#include "ssh.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
namespace nix {
static std::string uriScheme = "ssh://";
struct LegacySSHStore : public Store
{
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
struct LegacySSHStore : public Store {
const Setting<int> maxConnections{
this, 1, "max-connections",
"maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
const Setting<bool> compress{this, false, "compress",
"whether to compress the connection"};
const Setting<Path> remoteProgram{
this, "nix-store", "remote-program",
"path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{
this, "", "remote-store", "URI of the store on the remote system"};
// Hack for getting remote build log output.
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
const Setting<int> logFD{
this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection
{
struct Connection {
std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to;
FdSource from;
@ -38,29 +43,23 @@ struct LegacySSHStore : public Store
SSHMaster master;
LegacySSHStore(const string& host, const Params& params)
: Store(params)
, host(host)
, connections(make_ref<Pool<Connection>>(
: Store(params),
host(host),
connections(make_ref<Pool<Connection>>(
std::max(1, (int)maxConnections),
[this]() { return openConnection(); },
[](const ref<Connection> & r) { return r->good; }
))
, master(
host,
sshKey,
[](const ref<Connection>& r) { return r->good; })),
master(host, sshKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress,
logFD)
{
}
connections->capacity() > 1, compress, logFD) {}
ref<Connection> openConnection()
{
ref<Connection> openConnection() {
auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
fmt("%s --serve --write", remoteProgram) +
(remoteStore.get() == ""
? ""
: " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
@ -73,7 +72,8 @@ struct LegacySSHStore : public Store
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
throw Error("unsupported 'nix-store --serve' protocol version on '%s'",
host);
} catch (EndOfFile& e) {
throw Error("cannot connect to '%1%'", host);
@ -82,14 +82,11 @@ struct LegacySSHStore : public Store
return conn;
};
string getUri() override
{
return uriScheme + host;
}
string getUri() override { return uriScheme + host; }
void queryPathInfoUncached(const Path & path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override
{
void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override {
try {
auto conn(connections->get());
@ -120,30 +117,23 @@ struct LegacySSHStore : public Store
assert(s == "");
callback(std::move(info));
} catch (...) { callback.rethrow(); }
} catch (...) {
callback.rethrow();
}
}
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
{
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override {
debug("adding path '%s' to remote host '%s'", info.path, host);
auto conn(connections->get());
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
conn->to
<< cmdAddToStoreNar
<< info.path
<< info.deriver
<< info.narHash.to_string(Base16, false)
<< info.references
<< info.registrationTime
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
conn->to << cmdAddToStoreNar << info.path << info.deriver
<< info.narHash.to_string(Base16, false) << info.references
<< info.registrationTime << info.narSize << info.ultimate
<< info.sigs << info.ca;
try {
copyNAR(source, conn->to);
} catch (...) {
@ -153,33 +143,24 @@ struct LegacySSHStore : public Store
conn->to.flush();
} else {
conn->to
<< cmdImportPaths
<< 1;
conn->to << cmdImportPaths << 1;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to
<< exportMagic
<< info.path
<< info.references
<< info.deriver
<< 0
<< 0;
conn->to << exportMagic << info.path << info.references << info.deriver
<< 0 << 0;
conn->to.flush();
}
if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s', info.path, host");
throw Error(
"failed to add path '%s' to remote host '%s', info.path, host");
}
void narFromPath(const Path & path, Sink & sink) override
{
void narFromPath(const Path& path, Sink& sink) override {
auto conn(connections->get());
conn->to << cmdDumpStorePath << path;
@ -187,36 +168,31 @@ struct LegacySSHStore : public Store
copyNAR(conn->from, sink);
}
Path queryPathFromHashPart(const string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
Path queryPathFromHashPart(const string& hashPart) override {
unsupported("queryPathFromHashPart");
}
Path addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
Path addToStore(const string& name, const Path& srcPath, bool recursive,
HashType hashAlgo, PathFilter& filter,
RepairFlag repair) override {
unsupported("addToStore");
}
Path addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair) override
{ unsupported("addTextToStore"); }
const PathSet& references, RepairFlag repair) override {
unsupported("addTextToStore");
}
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
BuildMode buildMode) override
{
BuildMode buildMode) override {
auto conn(connections->get());
conn->to
<< cmdBuildDerivation
<< drvPath
<< drv
<< settings.maxSilentTime
conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime
<< settings.buildTimeout;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
conn->to
<< settings.maxLogSize;
conn->to << settings.maxLogSize;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->to
<< settings.buildRepeat
<< settings.enforceDeterminism;
conn->to << settings.buildRepeat << settings.enforceDeterminism;
conn->to.flush();
@ -225,29 +201,26 @@ struct LegacySSHStore : public Store
conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
conn->from >> status.timesBuilt >> status.isNonDeterministic >>
status.startTime >> status.stopTime;
return status;
}
void ensurePath(const Path & path) override
{ unsupported("ensurePath"); }
void ensurePath(const Path& path) override { unsupported("ensurePath"); }
void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
{
void computeFSClosure(const PathSet& paths, PathSet& out,
bool flipDirection = false, bool includeOutputs = false,
bool includeDerivers = false) override {
if (flipDirection || includeDerivers) {
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
Store::computeFSClosure(paths, out, flipDirection, includeOutputs,
includeDerivers);
return;
}
auto conn(connections->get());
conn->to
<< cmdQueryClosure
<< includeOutputs
<< paths;
conn->to << cmdQueryClosure << includeOutputs << paths;
conn->to.flush();
auto res = readStorePaths<PathSet>(*this, conn->from);
@ -255,39 +228,31 @@ struct LegacySSHStore : public Store
out.insert(res.begin(), res.end());
}
PathSet queryValidPaths(const PathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override
{
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
NoSubstitute) override {
auto conn(connections->get());
conn->to
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute
<< paths;
conn->to << cmdQueryValidPaths << false // lock
<< maybeSubstitute << paths;
conn->to.flush();
return readStorePaths<PathSet>(*this, conn->from);
}
void connect() override
{
auto conn(connections->get());
}
void connect() override { auto conn(connections->get()); }
unsigned int getProtocol() override
{
unsigned int getProtocol() override {
auto conn(connections->get());
return conn->remoteVersion;
}
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore(
[](const std::string& uri,
const Store::Params& params) -> std::shared_ptr<Store> {
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
return std::make_shared<LegacySSHStore>(
std::string(uri, uriScheme.size()), params);
});
}
} // namespace nix

62
third_party/nix/src/libstore/local-binary-cache-store.cc vendored
View file

@ -4,70 +4,54 @@
namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore
{
class LocalBinaryCacheStore : public BinaryCacheStore {
private:
Path binaryCacheDir;
public:
LocalBinaryCacheStore(
const Params & params, const Path & binaryCacheDir)
: BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir)
{
}
LocalBinaryCacheStore(const Params& params, const Path& binaryCacheDir)
: BinaryCacheStore(params), binaryCacheDir(binaryCacheDir) {}
void init() override;
std::string getUri() override
{
return "file://" + binaryCacheDir;
}
std::string getUri() override { return "file://" + binaryCacheDir; }
protected:
bool fileExists(const std::string& path) override;
void upsertFile(const std::string & path,
const std::string & data,
void upsertFile(const std::string& path, const std::string& data,
const std::string& mimeType) override;
void getFile(const std::string & path, Sink & sink) override
{
void getFile(const std::string& path, Sink& sink) override {
try {
readFile(binaryCacheDir + "/" + path, sink);
} catch (SysError& e) {
if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache",
path);
}
}
PathSet queryAllValidPaths() override
{
PathSet queryAllValidPaths() override {
PathSet paths;
for (auto& entry : readDirectory(binaryCacheDir)) {
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8));
paths.insert(storeDir + "/" +
entry.name.substr(0, entry.name.size() - 8));
}
return paths;
}
};
void LocalBinaryCacheStore::init()
{
void LocalBinaryCacheStore::init() {
createDirs(binaryCacheDir + "/nar");
BinaryCacheStore::init();
}
static void atomicWrite(const Path & path, const std::string & s)
{
static void atomicWrite(const Path& path, const std::string& s) {
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
@ -76,28 +60,26 @@ static void atomicWrite(const Path & path, const std::string & s)
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
bool LocalBinaryCacheStore::fileExists(const std::string& path) {
return pathExists(binaryCacheDir + "/" + path);
}
void LocalBinaryCacheStore::upsertFile(const std::string& path,
const std::string& data,
const std::string & mimeType)
{
const std::string& mimeType) {
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore(
[](const std::string& uri,
const Store::Params& params) -> std::shared_ptr<Store> {
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
std::string(uri, 0, 7) != "file://")
return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
auto store =
std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
store->init();
return store;
});
}
} // namespace nix

72
third_party/nix/src/libstore/local-fs-store.cc vendored
View file

@ -1,100 +1,84 @@
#include "archive.hh"
#include "fs-accessor.hh"
#include "store-api.hh"
#include "globals.hh"
#include "compression.hh"
#include "derivations.hh"
#include "fs-accessor.hh"
#include "globals.hh"
#include "store-api.hh"
namespace nix {
LocalFSStore::LocalFSStore(const Params & params)
: Store(params)
{
}
LocalFSStore::LocalFSStore(const Params& params) : Store(params) {}
struct LocalStoreAccessor : public FSAccessor
{
struct LocalStoreAccessor : public FSAccessor {
ref<LocalFSStore> store;
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {}
Path toRealPath(const Path & path)
{
Path toRealPath(const Path& path) {
Path storePath = store->toStorePath(path);
if (!store->isValidPath(storePath))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
throw InvalidPath(format("path '%1%' is not a valid store path") %
storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
FSAccessor::Stat stat(const Path & path) override
{
FSAccessor::Stat stat(const Path& path) override {
auto realPath = toRealPath(path);
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
if (errno == ENOENT || errno == ENOTDIR)
return {Type::tMissing, 0, false};
throw SysError(format("getting status of '%1%'") % path);
}
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
throw Error(format("file '%1%' has unsupported type") % path);
return {
S_ISREG(st.st_mode) ? Type::tRegular :
S_ISLNK(st.st_mode) ? Type::tSymlink :
Type::tDirectory,
return {S_ISREG(st.st_mode)
? Type::tRegular
: S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory,
S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0,
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
}
StringSet readDirectory(const Path & path) override
{
StringSet readDirectory(const Path& path) override {
auto realPath = toRealPath(path);
auto entries = nix::readDirectory(realPath);
StringSet res;
for (auto & entry : entries)
res.insert(entry.name);
for (auto& entry : entries) res.insert(entry.name);
return res;
}
std::string readFile(const Path & path) override
{
std::string readFile(const Path& path) override {
return nix::readFile(toRealPath(path));
}
std::string readLink(const Path & path) override
{
std::string readLink(const Path& path) override {
return nix::readLink(toRealPath(path));
}
};
ref<FSAccessor> LocalFSStore::getFSAccessor()
{
ref<FSAccessor> LocalFSStore::getFSAccessor() {
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
}
void LocalFSStore::narFromPath(const Path & path, Sink & sink)
{
if (!isValidPath(path))
throw Error(format("path '%s' is not valid") % path);
void LocalFSStore::narFromPath(const Path& path, Sink& sink) {
if (!isValidPath(path)) throw Error(format("path '%s' is not valid") % path);
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
}
const string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
{
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) {
auto path(path_);
assertStorePath(path);
if (!isDerivation(path)) {
try {
path = queryPathInfo(path)->deriver;
@ -107,10 +91,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
string baseName = baseNameOf(path);
for (int j = 0; j < 2; j++) {
Path logPath =
j == 0
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
Path logPath = j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir,
string(baseName, 0, 2), string(baseName, 2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
@ -120,12 +102,12 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
else if (pathExists(logBz2Path)) {
try {
return decompress("bzip2", readFile(logBz2Path));
} catch (Error &) { }
} catch (Error&) {
}
}
}
return nullptr;
}
}
} // namespace nix

601
third_party/nix/src/libstore/local-store.cc vendored
View file

File diff suppressed because it is too large Load diff

83
third_party/nix/src/libstore/local-store.hh vendored
View file

@ -1,48 +1,37 @@
#pragma once
#include "sqlite.hh"
#include "pathlocks.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
#include <chrono>
#include <future>
#include <string>
#include <unordered_set>
#include "pathlocks.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
const int nixSchemaVersion = 10;
struct Derivation;
struct OptimiseStats
{
struct OptimiseStats {
unsigned long filesLinked = 0;
unsigned long long bytesFreed = 0;
unsigned long long blocksFreed = 0;
};
class LocalStore : public LocalFSStore
{
class LocalStore : public LocalFSStore {
private:
/* Lock file used for upgrading. */
AutoCloseFD globalLock;
struct State
{
struct State {
/* The SQLite database object. */
SQLite db;
@ -84,7 +73,6 @@ private:
Sync<State, std::recursive_mutex> _state;
public:
PathSetting realStoreDir_;
const Path realStoreDir;
@ -97,15 +85,13 @@ public:
const Path fnTempRoots;
private:
Setting<bool> requireSigs{(Store*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
Setting<bool> requireSigs{
(Store*)this, settings.requireSigs, "require-sigs",
"whether store paths should have a trusted signature on import"};
const PublicKeys& getPublicKeys();
public:
// Hack for build-remote.cc.
PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
@ -121,12 +107,13 @@ public:
bool isValidPathUncached(const Path& path) override;
PathSet queryValidPaths(const PathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
NoSubstitute) override;
PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path& path, PathSet& referrers) override;
@ -144,20 +131,21 @@ public:
void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
Path addToStore(const string& name, const Path& srcPath, bool recursive,
HashType hashAlgo, PathFilter& filter,
RepairFlag repair) override;
/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
Path addToStoreFromDump(const string& dump, const string& name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair);
bool recursive = true, HashType hashAlgo = htSHA256,
RepairFlag repair = NoRepair);
Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override;
@ -176,14 +164,12 @@ public:
void syncWithGC() override;
private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
void findTempRoots(FDs& fds, Roots& roots, bool censor);
public:
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions& options, GCResults& results) override;
@ -224,7 +210,6 @@ public:
void autoGC(bool sync = true);
private:
int getSchema();
void openDB(State& state, bool create);
@ -233,15 +218,16 @@ private:
uint64_t queryValidPathId(State& state, const Path& path);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
uint64_t addValidPath(State& state, const ValidPathInfo& info,
bool checkOutputs = true);
void invalidatePath(State& state, const Path& path);
/* Delete a path from the Nix store. */
void invalidatePathChecked(const Path& path);
void verifyPath(const Path & path, const PathSet & store,
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors);
void verifyPath(const Path& path, const PathSet& store, PathSet& done,
PathSet& validPaths, RepairFlag repair, bool& errors);
void updatePathInfo(State& state, const ValidPathInfo& info);
@ -260,8 +246,8 @@ private:
void deletePathRecursive(GCState& state, const Path& path);
bool isActiveTempFile(const GCState & state,
const Path & path, const string & suffix);
bool isActiveTempFile(const GCState& state, const Path& path,
const string& suffix);
AutoCloseFD openGCLock(LockType lockType);
@ -280,8 +266,10 @@ private:
typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
Strings readDirectoryIgnoringInodes(const Path& path,
const InodeHash& inodeHash);
void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path,
InodeHash& inodeHash);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State& state, const Path& path);
@ -299,11 +287,9 @@ private:
friend class SubstitutionGoal;
};
typedef std::pair<dev_t, ino_t> Inode;
typedef set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular:
- the last modification date on each file is set to 1 (i.e.,
@ -312,11 +298,12 @@ typedef set<Inode> InodesSeen;
without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're
running as root. */
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen);
void canonicalisePathMetaData(const Path& path, uid_t fromUid,
InodesSeen& inodesSeen);
void canonicalisePathMetaData(const Path& path, uid_t fromUid);
void canonicaliseTimestampAndPermissions(const Path& path);
MakeError(PathInUse, Error);
}
} // namespace nix

63
third_party/nix/src/libstore/machines.cc vendored
View file

@ -1,27 +1,23 @@
#include "machines.hh"
#include "util.hh"
#include "globals.hh"
#include <algorithm>
#include "globals.hh"
#include "util.hh"
namespace nix {
Machine::Machine(decltype(storeUri) storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
Machine::Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
decltype(speedFactor) speedFactor,
decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) :
storeUri(
decltype(sshPublicHostKey) sshPublicHostKey)
: storeUri(
// Backwards compatibility: if the URI is a hostname,
// prepend ssh://.
storeUri.find("://") != std::string::npos
|| hasPrefix(storeUri, "local")
|| hasPrefix(storeUri, "remote")
|| hasPrefix(storeUri, "auto")
|| hasPrefix(storeUri, "/")
storeUri.find("://") != std::string::npos ||
hasPrefix(storeUri, "local") ||
hasPrefix(storeUri, "remote") ||
hasPrefix(storeUri, "auto") || hasPrefix(storeUri, "/")
? storeUri
: "ssh://" + storeUri),
systemTypes(systemTypes),
@ -30,8 +26,7 @@ Machine::Machine(decltype(storeUri) storeUri,
speedFactor(std::max(1U, speedFactor)),
supportedFeatures(supportedFeatures),
mandatoryFeatures(mandatoryFeatures),
sshPublicHostKey(sshPublicHostKey)
{}
sshPublicHostKey(sshPublicHostKey) {}
bool Machine::allSupported(const std::set<string>& features) const {
return std::all_of(features.begin(), features.end(),
@ -42,14 +37,12 @@ bool Machine::allSupported(const std::set<string> & features) const {
}
bool Machine::mandatoryMet(const std::set<string>& features) const {
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
[&](const string & feature) {
return features.count(feature);
});
return std::all_of(
mandatoryFeatures.begin(), mandatoryFeatures.end(),
[&](const string& feature) { return features.count(feature); });
}
void parseMachines(const std::string & s, Machines & machines)
{
void parseMachines(const std::string& s, Machines& machines) {
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
trim(line);
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
@ -60,8 +53,7 @@ void parseMachines(const std::string & s, Machines & machines)
try {
parseMachines(readFile(file), machines);
} catch (const SysError& e) {
if (e.errNo != ENOENT)
throw;
if (e.errNo != ENOENT) throw;
debug("cannot find machines file '%s'", file);
}
continue;
@ -69,26 +61,27 @@ void parseMachines(const std::string & s, Machines & machines)
auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size();
if (sz < 1)
throw FormatError("bad machine specification '%s'", line);
if (sz < 1) throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
};
machines.emplace_back(tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? std::stoull(tokens[3]) : 1LL,
machines.emplace_back(
tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",")
: std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",")
: std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",")
: std::set<string>{},
isSet(7) ? tokens[7] : "");
}
}
Machines getMachines()
{
Machines getMachines() {
static auto machines = [&]() {
Machines machines;
parseMachines(settings.builders, machines);
@ -97,4 +90,4 @@ Machines getMachines()
return machines;
}
}
} // namespace nix

9
third_party/nix/src/libstore/machines.hh vendored
View file

@ -5,7 +5,6 @@
namespace nix {
struct Machine {
const string storeUri;
const std::vector<string> systemTypes;
const string sshKey;
@ -20,10 +19,8 @@ struct Machine {
bool mandatoryMet(const std::set<string>& features) const;
Machine(decltype(storeUri) storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
decltype(speedFactor) speedFactor,
decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures,
@ -36,4 +33,4 @@ void parseMachines(const std::string & s, Machines & machines);
Machines getMachines();
}
} // namespace nix

98
third_party/nix/src/libstore/misc.cc vendored
View file

@ -1,19 +1,16 @@
#include "derivations.hh"
#include "parsed-derivations.hh"
#include "globals.hh"
#include "local-store.hh"
#include "parsed-derivations.hh"
#include "store-api.hh"
#include "thread-pool.hh"
namespace nix {
void Store::computeFSClosure(const PathSet & startPaths,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
struct State
{
void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_,
bool flipDirection, bool includeOutputs,
bool includeDerivers) {
struct State {
size_t pending;
PathSet& paths;
std::exception_ptr exc;
@ -34,23 +31,21 @@ void Store::computeFSClosure(const PathSet & startPaths,
state->pending++;
}
queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
queryPathInfo(
path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
if (flipDirection) {
PathSet referrers;
queryReferrers(path, referrers);
for (auto& ref : referrers)
if (ref != path)
enqueue(ref);
if (ref != path) enqueue(ref);
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
enqueue(i);
for (auto& i : queryValidDerivers(path)) enqueue(i);
if (includeDerivers && isDerivation(path))
for (auto& i : queryDerivationOutputs(path))
@ -58,10 +53,8 @@ void Store::computeFSClosure(const PathSet & startPaths,
enqueue(i);
} else {
for (auto& ref : info->references)
if (ref != path)
enqueue(ref);
if (ref != path) enqueue(ref);
if (includeOutputs && isDerivation(path))
for (auto& i : queryDerivationOutputs(path))
@ -69,7 +62,6 @@ void Store::computeFSClosure(const PathSet & startPaths,
if (includeDerivers && isValidPath(info->deriver))
enqueue(info->deriver);
}
{
@ -87,8 +79,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
}});
};
for (auto & startPath : startPaths)
enqueue(startPath);
for (auto& startPath : startPaths) enqueue(startPath);
{
auto state(state_.lock());
@ -97,41 +88,40 @@ void Store::computeFSClosure(const PathSet & startPaths,
}
}
void Store::computeFSClosure(const Path & startPath,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
void Store::computeFSClosure(const Path& startPath, PathSet& paths_,
bool flipDirection, bool includeOutputs,
bool includeDerivers) {
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs,
includeDerivers);
}
void Store::queryMissing(const PathSet & targets,
PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
unsigned long long & downloadSize_, unsigned long long & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
void Store::queryMissing(const PathSet& targets, PathSet& willBuild_,
PathSet& willSubstitute_, PathSet& unknown_,
unsigned long long& downloadSize_,
unsigned long long& narSize_) {
Activity act(*logger, lvlDebug, actUnknown,
"querying info about missing paths");
downloadSize_ = narSize_ = 0;
ThreadPool pool;
struct State
{
struct State {
PathSet done;
PathSet &unknown, &willSubstitute, &willBuild;
unsigned long long& downloadSize;
unsigned long long& narSize;
};
struct DrvState
{
struct DrvState {
size_t left;
bool done = false;
PathSet outPaths;
DrvState(size_t left) : left(left) {}
};
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_,
downloadSize_, narSize_});
std::function<void(Path)> doPath;
@ -142,12 +132,12 @@ void Store::queryMissing(const PathSet & targets,
}
for (auto& i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
pool.enqueue(
std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
};
auto checkOutput = [&](
const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_)
{
auto checkOutput = [&](const Path& drvPath, ref<Derivation> drv,
const Path& outPath, ref<Sync<DrvState>> drvState_) {
if (drvState_->lock()->done) return;
SubstitutablePathInfos infos;
@ -172,7 +162,6 @@ void Store::queryMissing(const PathSet & targets,
};
doPath = [&](const Path& path) {
{
auto state(state_.lock());
if (state->done.count(path)) return;
@ -194,20 +183,19 @@ void Store::queryMissing(const PathSet & targets,
PathSet invalid;
for (auto& j : drv.outputs)
if (wantOutput(j.first, i2.second)
&& !isValidPath(j.second.path))
if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path))
invalid.insert(j.second.path);
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto& output : invalid)
pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
pool.enqueue(std::bind(checkOutput, i2.first,
make_ref<Derivation>(drv), output, drvState));
} else
mustBuildDrv(i2.first, drv);
} else {
if (isValidPath(path)) return;
SubstitutablePathInfos infos;
@ -234,15 +222,12 @@ void Store::queryMissing(const PathSet & targets,
}
};
for (auto & path : targets)
pool.enqueue(std::bind(doPath, path));
for (auto& path : targets) pool.enqueue(std::bind(doPath, path));
pool.process();
}
Paths Store::topoSortPaths(const PathSet & paths)
{
Paths Store::topoSortPaths(const PathSet& paths) {
Paths sorted;
PathSet visited, parents;
@ -250,7 +235,9 @@ Paths Store::topoSortPaths(const PathSet & paths)
dfsVisit = [&](const Path& path, const Path* parent) {
if (parents.find(path) != parents.end())
throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
throw BuildError(
format("cycle detected in the references of '%1%' from '%2%'") %
path % *parent);
if (visited.find(path) != visited.end()) return;
visited.insert(path);
@ -265,18 +252,15 @@ Paths Store::topoSortPaths(const PathSet & paths)
for (auto& i : references)
/* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */
if (i != path && paths.find(i) != paths.end())
dfsVisit(i, &path);
if (i != path && paths.find(i) != paths.end()) dfsVisit(i, &path);
sorted.push_front(path);
parents.erase(path);
};
for (auto & i : paths)
dfsVisit(i, nullptr);
for (auto& i : paths) dfsVisit(i, nullptr);
return sorted;
}
}
} // namespace nix

100
third_party/nix/src/libstore/nar-accessor.cc vendored
View file

@ -1,17 +1,14 @@
#include "nar-accessor.hh"
#include <algorithm>
#include <map>
#include <nlohmann/json.hpp>
#include <stack>
#include "archive.hh"
#include "json.hh"
#include <map>
#include <stack>
#include <algorithm>
#include <nlohmann/json.hpp>
namespace nix {
struct NarMember
{
struct NarMember {
FSAccessor::Type type = FSAccessor::Type::tMissing;
bool isExecutable = false;
@ -26,16 +23,14 @@ struct NarMember
std::map<std::string, NarMember> children;
};
struct NarAccessor : public FSAccessor
{
struct NarAccessor : public FSAccessor {
std::shared_ptr<const std::string> nar;
GetNarBytes getNarBytes;
NarMember root;
struct NarIndexer : ParseSink, StringSource
{
struct NarIndexer : ParseSink, StringSource {
NarAccessor& acc;
std::stack<NarMember*> parents;
@ -44,8 +39,7 @@ struct NarAccessor : public FSAccessor
bool isExec = false;
NarIndexer(NarAccessor& acc, const std::string& nar)
: StringSource(nar), acc(acc)
{ }
: StringSource(nar), acc(acc) {}
void createMember(const Path& path, NarMember member) {
size_t level = std::count(path.begin(), path.end(), '/');
@ -57,36 +51,30 @@ struct NarAccessor : public FSAccessor
} else {
if (parents.top()->type != FSAccessor::Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
auto result = parents.top()->children.emplace(baseNameOf(path),
std::move(member));
parents.push(&result.first->second);
}
}
void createDirectory(const Path & path) override
{
void createDirectory(const Path& path) override {
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
}
void createRegularFile(const Path & path) override
{
void createRegularFile(const Path& path) override {
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
}
void isExecutable() override
{
parents.top()->isExecutable = true;
}
void isExecutable() override { parents.top()->isExecutable = true; }
void preallocateContents(unsigned long long size) override
{
void preallocateContents(unsigned long long size) override {
currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, unsigned int len) override
{
void receiveContents(unsigned char* data, unsigned int len) override {
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char*)data, 16));
@ -94,22 +82,19 @@ struct NarAccessor : public FSAccessor
}
}
void createSymlink(const Path & path, const string & target) override
{
void createSymlink(const Path& path, const string& target) override {
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
{
NarAccessor(ref<const std::string> nar) : nar(nar) {
NarIndexer indexer(*this, *nar);
parseDump(indexer, indexer);
}
NarAccessor(const std::string& listing, GetNarBytes getNarBytes)
: getNarBytes(getNarBytes)
{
: getNarBytes(getNarBytes) {
using json = nlohmann::json;
std::function<void(NarMember&, json&)> recurse;
@ -131,15 +116,15 @@ struct NarAccessor : public FSAccessor
} else if (type == "symlink") {
member.type = FSAccessor::Type::tSymlink;
member.target = v.value("target", "");
} else return;
} else
return;
};
json v = json::parse(listing);
recurse(root, v);
}
NarMember * find(const Path & path)
{
NarMember* find(const Path& path) {
Path canon = path == "" ? "" : canonPath(path);
NarMember* current = &root;
auto end = path.end();
@ -171,33 +156,30 @@ struct NarAccessor : public FSAccessor
return *result;
}
Stat stat(const Path & path) override
{
Stat stat(const Path& path) override {
auto i = find(path);
if (i == nullptr)
return {FSAccessor::Type::tMissing, 0, false};
if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false};
return {i->type, i->size, i->isExecutable, i->start};
}
StringSet readDirectory(const Path & path) override
{
StringSet readDirectory(const Path& path) override {
auto i = get(path);
if (i.type != FSAccessor::Type::tDirectory)
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
throw Error(format("path '%1%' inside NAR file is not a directory") %
path);
StringSet res;
for (auto & child : i.children)
res.insert(child.first);
for (auto& child : i.children) res.insert(child.first);
return res;
}
std::string readFile(const Path & path) override
{
std::string readFile(const Path& path) override {
auto i = get(path);
if (i.type != FSAccessor::Type::tRegular)
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
throw Error(format("path '%1%' inside NAR file is not a regular file") %
path);
if (getNarBytes) return getNarBytes(i.start, i.size);
@ -205,8 +187,7 @@ struct NarAccessor : public FSAccessor
return std::string(*nar, i.start, i.size);
}
std::string readLink(const Path & path) override
{
std::string readLink(const Path& path) override {
auto i = get(path);
if (i.type != FSAccessor::Type::tSymlink)
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
@ -214,20 +195,17 @@ struct NarAccessor : public FSAccessor
}
};
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
{
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) {
return make_ref<NarAccessor>(nar);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
GetNarBytes getNarBytes)
{
GetNarBytes getNarBytes) {
return make_ref<NarAccessor>(listing, getNarBytes);
}
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
const Path & path, bool recurse)
{
void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
bool recurse) {
auto st = accessor->stat(path);
auto obj = res.object();
@ -236,10 +214,8 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
case FSAccessor::Type::tRegular:
obj.attr("type", "regular");
obj.attr("size", st.fileSize);
if (st.isExecutable)
obj.attr("executable", true);
if (st.narOffset)
obj.attr("narOffset", st.narOffset);
if (st.isExecutable) obj.attr("executable", true);
if (st.narOffset) obj.attr("narOffset", st.narOffset);
break;
case FSAccessor::Type::tDirectory:
obj.attr("type", "directory");
@ -263,4 +239,4 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
}
}
}
} // namespace nix

10
third_party/nix/src/libstore/nar-accessor.hh vendored
View file

@ -1,7 +1,6 @@
#pragma once
#include <functional>
#include "fs-accessor.hh"
namespace nix {
@ -16,15 +15,14 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
inside the NAR. */
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor(
const std::string & listing,
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
GetNarBytes getNarBytes);
class JSONPlaceholder;
/* Write a JSON representation of the contents of a NAR (except file
contents). */
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
const Path & path, bool recurse);
void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
bool recurse);
}
} // namespace nix

168
third_party/nix/src/libstore/nar-info-disk-cache.cc vendored
View file

@ -1,9 +1,8 @@
#include "nar-info-disk-cache.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"
#include <sqlite3.h>
#include "globals.hh"
#include "sqlite.hh"
#include "sync.hh"
namespace nix {
@ -45,32 +44,28 @@ create table if not exists LastPurge (
)sql";
class NarInfoDiskCacheImpl : public NarInfoDiskCache
{
class NarInfoDiskCacheImpl : public NarInfoDiskCache {
public:
/* How often to purge expired entries from the cache. */
const int purgeInterval = 24 * 3600;
struct Cache
{
struct Cache {
int id;
Path storeDir;
bool wantMassQuery;
int priority;
};
struct State
{
struct State {
SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR,
purgeCache;
std::map<std::string, Cache> caches;
};
Sync<State> _state;
NarInfoDiskCacheImpl()
{
NarInfoDiskCacheImpl() {
auto state(_state.lock());
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
@ -87,21 +82,33 @@ public:
state->db.exec(schema);
state->insertCache.create(state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
state->insertCache.create(
state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, "
"wantMassQuery, priority) values (?, ?, ?, ?, ?)");
state->queryCache.create(state->db,
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
"select id, storeDir, wantMassQuery, priority "
"from BinaryCaches where url = ?");
state->insertNAR.create(state->db,
"insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
state->insertNAR.create(
state->db,
"insert or replace into NARs(cache, hashPart, namePart, url, "
"compression, fileHash, fileSize, narHash, "
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, "
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
state->insertMissingNAR.create(state->db,
"insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
state->insertMissingNAR.create(
state->db,
"insert or replace into NARs(cache, hashPart, timestamp, present) "
"values (?, ?, ?, 0)");
state->queryNAR.create(state->db,
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
state->queryNAR.create(
state->db,
"select present, namePart, url, compression, fileHash, fileSize, "
"narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? "
"and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 "
"and timestamp > ?))");
/* Periodically purge expired entries from the database. */
retrySQLite<void>([&]() {
@ -110,46 +117,50 @@ public:
SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
auto queryLastPurge_(queryLastPurge.use());
if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) {
if (!queryLastPurge_.next() ||
queryLastPurge_.getInt(0) < now - purgeInterval) {
SQLiteStmt(state->db,
"delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
.use()
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache)
"delete from NARs where ((present = 0 and timestamp < ?) or "
"(present = 1 and timestamp < ?))")
.use()(now - settings.ttlNegativeNarInfoCache)(
now - settings.ttlPositiveNarInfoCache)
.exec();
debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
debug("deleted %d entries from the NAR info disk cache",
sqlite3_changes(state->db));
SQLiteStmt(state->db,
SQLiteStmt(
state->db,
"insert or replace into LastPurge(dummy, value) values ('', ?)")
.use()(now).exec();
.use()(now)
.exec();
}
});
}
Cache & getCache(State & state, const std::string & uri)
{
Cache& getCache(State& state, const std::string& uri) {
auto i = state.caches.find(uri);
if (i == state.caches.end()) abort();
return i->second;
}
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
{
void createCache(const std::string& uri, const Path& storeDir,
bool wantMassQuery, int priority) override {
retrySQLite<void>([&]() {
auto state(_state.lock());
// FIXME: race
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)
.exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
state->caches[uri] = Cache{(int)sqlite3_last_insert_rowid(state->db),
storeDir, wantMassQuery, priority};
});
}
bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) override
{
bool cacheExists(const std::string& uri, bool& wantMassQuery,
int& priority) override {
return retrySQLite<bool>([&]() {
auto state(_state.lock());
@ -157,8 +168,9 @@ public:
if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri));
if (!queryCache.next()) return false;
state->caches.emplace(uri,
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
state->caches.emplace(
uri, Cache{(int)queryCache.getInt(0), queryCache.getStr(1),
queryCache.getInt(2) != 0, (int)queryCache.getInt(3)});
}
auto& cache(getCache(*state, uri));
@ -171,8 +183,7 @@ public:
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) override
{
const std::string& uri, const std::string& hashPart) override {
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
auto state(_state.lock());
@ -181,27 +192,22 @@ public:
auto now = time(0);
auto queryNAR(state->queryNAR.use()
(cache.id)
(hashPart)
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache));
auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)(
now - settings.ttlNegativeNarInfoCache)(
now - settings.ttlPositiveNarInfoCache));
if (!queryNAR.next())
return {oUnknown, 0};
if (!queryNAR.next()) return {oUnknown, 0};
if (!queryNAR.getInt(0))
return {oInvalid, 0};
if (!queryNAR.getInt(0)) return {oInvalid, 0};
auto narInfo = make_ref<NarInfo>();
auto namePart = queryNAR.getStr(1);
narInfo->path = cache.storeDir + "/" +
hashPart + (namePart.empty() ? "" : "-" + namePart);
narInfo->path = cache.storeDir + "/" + hashPart +
(namePart.empty() ? "" : "-" + namePart);
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
narInfo->fileHash = Hash(queryNAR.getStr(4));
if (!queryNAR.isNull(4)) narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
@ -217,51 +223,43 @@ public:
});
}
void upsertNarInfo(
const std::string & uri, const std::string & hashPart,
std::shared_ptr<ValidPathInfo> info) override
{
void upsertNarInfo(const std::string& uri, const std::string& hashPart,
std::shared_ptr<ValidPathInfo> info) override {
retrySQLite<void>([&]() {
auto state(_state.lock());
auto& cache(getCache(*state, uri));
if (info) {
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
assert(hashPart == storePathToHash(info->path));
state->insertNAR.use()
(cache.id)
(hashPart)
(storePathToName(info->path))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string())
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
(concatStringsSep(" ", info->sigs))
(info->ca)
(time(0)).exec();
state->insertNAR
.use()(cache.id)(hashPart)(storePathToName(info->path))(
narInfo ? narInfo->url : "", narInfo != 0)(
narInfo ? narInfo->compression : "", narInfo != 0)(
narInfo && narInfo->fileHash ? narInfo->fileHash.to_string()
: "",
narInfo && narInfo->fileHash)(
narInfo ? narInfo->fileSize : 0,
narInfo != 0 && narInfo->fileSize)(info->narHash.to_string())(
info->narSize)(concatStringsSep(" ", info->shortRefs()))(
info->deriver != "" ? baseNameOf(info->deriver) : "",
info->deriver !=
"")(concatStringsSep(" ", info->sigs))(info->ca)(time(0))
.exec();
} else {
state->insertMissingNAR.use()
(cache.id)
(hashPart)
(time(0)).exec();
state->insertMissingNAR.use()(cache.id)(hashPart)(time(0)).exec();
}
});
}
};
ref<NarInfoDiskCache> getNarInfoDiskCache()
{
ref<NarInfoDiskCache> getNarInfoDiskCache() {
static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
return cache;
}
}
} // namespace nix

15
third_party/nix/src/libstore/nar-info-disk-cache.hh vendored
View file

@ -1,26 +1,25 @@
#pragma once
#include "ref.hh"
#include "nar-info.hh"
#include "ref.hh"
namespace nix {
class NarInfoDiskCache
{
class NarInfoDiskCache {
public:
typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual void createCache(const std::string& uri, const Path& storeDir,
bool wantMassQuery, int priority) = 0;
virtual bool cacheExists(const std::string & uri,
bool & wantMassQuery, int & priority) = 0;
virtual bool cacheExists(const std::string& uri, bool& wantMassQuery,
int& priority) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string& uri, const std::string& hashPart) = 0;
virtual void upsertNarInfo(
const std::string & uri, const std::string & hashPart,
virtual void upsertNarInfo(const std::string& uri,
const std::string& hashPart,
std::shared_ptr<ValidPathInfo> info) = 0;
};
@ -28,4 +27,4 @@ public:
multiple threads. */
ref<NarInfoDiskCache> getNarInfoDiskCache();
}
} // namespace nix

39
third_party/nix/src/libstore/nar-info.cc vendored
View file

@ -1,10 +1,10 @@
#include "globals.hh"
#include "nar-info.hh"
#include "globals.hh"
namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
{
NarInfo::NarInfo(const Store& store, const std::string& s,
const std::string& whence) {
auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence);
};
@ -20,7 +20,6 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
size_t pos = 0;
while (pos < s.size()) {
size_t colon = s.find(':', pos);
if (colon == std::string::npos) corrupt();
@ -34,8 +33,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (name == "StorePath") {
if (!store.isStorePath(value)) corrupt();
path = value;
}
else if (name == "URL")
} else if (name == "URL")
url = value;
else if (name == "Compression")
compression = value;
@ -43,13 +41,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
fileHash = parseHashField(value);
else if (name == "FileSize") {
if (!string2Int(value, fileSize)) corrupt();
}
else if (name == "NarHash")
} else if (name == "NarHash")
narHash = parseHashField(value);
else if (name == "NarSize") {
if (!string2Int(value, narSize)) corrupt();
}
else if (name == "References") {
} else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt();
for (auto& r : refs) {
@ -57,15 +53,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (!store.isStorePath(r2)) corrupt();
references.insert(r2);
}
}
else if (name == "Deriver") {
} else if (name == "Deriver") {
if (value != "unknown-deriver") {
auto p = store.storeDir + "/" + value;
if (!store.isStorePath(p)) corrupt();
deriver = p;
}
}
else if (name == "System")
} else if (name == "System")
system = value;
else if (name == "Sig")
sigs.insert(value);
@ -82,8 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
}
std::string NarInfo::to_string() const
{
std::string NarInfo::to_string() const {
std::string res;
res += "StorePath: " + path + "\n";
res += "URL: " + url + "\n";
@ -98,19 +91,15 @@ std::string NarInfo::to_string() const
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
if (!deriver.empty())
res += "Deriver: " + baseNameOf(deriver) + "\n";
if (!deriver.empty()) res += "Deriver: " + baseNameOf(deriver) + "\n";
if (!system.empty())
res += "System: " + system + "\n";
if (!system.empty()) res += "System: " + system + "\n";
for (auto sig : sigs)
res += "Sig: " + sig + "\n";
for (auto sig : sigs) res += "Sig: " + sig + "\n";
if (!ca.empty())
res += "CA: " + ca + "\n";
if (!ca.empty()) res += "CA: " + ca + "\n";
return res;
}
}
} // namespace nix

7
third_party/nix/src/libstore/nar-info.hh vendored
View file

@ -1,13 +1,12 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "store-api.hh"
#include "types.hh"
namespace nix {
struct NarInfo : ValidPathInfo
{
struct NarInfo : ValidPathInfo {
std::string url;
std::string compression;
Hash fileHash;
@ -21,4 +20,4 @@ struct NarInfo : ValidPathInfo
std::string to_string() const;
};
}
} // namespace nix

93
third_party/nix/src/libstore/optimise-store.cc vendored
View file

@ -1,22 +1,18 @@
#include "util.hh"
#include "local-store.hh"
#include "globals.hh"
#include <cstdlib>
#include <cstring>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <cstring>
#include <regex>
#include "globals.hh"
#include "local-store.hh"
#include "util.hh"
namespace nix {
static void makeWritable(const Path & path)
{
static void makeWritable(const Path& path) {
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
@ -24,13 +20,10 @@ static void makeWritable(const Path & path)
throw SysError(format("changing writability of '%1%'") % path);
}
struct MakeReadOnly
{
struct MakeReadOnly {
Path path;
MakeReadOnly(const Path& path) : path(path) {}
~MakeReadOnly()
{
~MakeReadOnly() {
try {
/* This will make the path read-only. */
if (path != "") canonicaliseTimestampAndPermissions(path);
@ -40,9 +33,7 @@ struct MakeReadOnly
}
};
LocalStore::InodeHash LocalStore::loadInodeHash()
{
LocalStore::InodeHash LocalStore::loadInodeHash() {
debug("loading hash inodes in memory");
InodeHash inodeHash;
@ -62,9 +53,8 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
return inodeHash;
}
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
{
Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
const InodeHash& inodeHash) {
Strings names;
AutoCloseDir dir(opendir(path.c_str()));
@ -88,10 +78,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
return names;
}
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
const Path & path, InodeHash & inodeHash)
{
const Path& path, InodeHash& inodeHash) {
checkInterrupt();
struct stat st;
@ -104,8 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
https://github.com/NixOS/nix/issues/1443 for more discussion. */
if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
{
if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) {
debug(format("'%1%' is not allowed to be linked in macOS") % path);
return;
}
@ -113,8 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names)
optimisePath_(act, stats, path + "/" + i, inodeHash);
for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash);
return;
}
@ -123,7 +109,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
#if CAN_LINK_SYMLINK
&& !S_ISLNK(st.st_mode)
#endif
) return;
)
return;
/* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under
@ -136,7 +123,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* This can still happen on top-level files. */
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
debug(format("'%1%' is already linked, with %2% other file(s)") % path %
(st.st_nlink - 2));
return;
}
@ -174,7 +162,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this
file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
printInfo("cannot link '%s' to '%s': %s", linkPath, path,
strerror(errno));
return;
default:
@ -211,8 +200,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% realStoreDir % getpid() % random()).str();
Path tempLink =
(format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random())
.str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
@ -245,13 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
if (act)
act->result(resFileLinked, st.st_size, st.st_blocks);
if (act) act->result(resFileLinked, st.st_size, st.st_blocks);
}
void LocalStore::optimiseStore(OptimiseStats & stats)
{
void LocalStore::optimiseStore(OptimiseStats& stats) {
Activity act(*logger, actOptimiseStore);
PathSet paths = queryAllValidPaths();
@ -265,7 +252,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
addTempRoot(i);
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i));
Activity act(*logger, lvlTalkative, actUnknown,
fmt("optimising path '%s'", i));
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
}
done++;
@ -273,30 +261,25 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
static string showBytes(unsigned long long bytes)
{
static string showBytes(unsigned long long bytes) {
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
void LocalStore::optimiseStore()
{
void LocalStore::optimiseStore() {
OptimiseStats stats;
optimiseStore(stats);
printInfo(
format("%1% freed by hard-linking %2% files")
% showBytes(stats.bytesFreed)
% stats.filesLinked);
printInfo(format("%1% freed by hard-linking %2% files") %
showBytes(stats.bytesFreed) % stats.filesLinked);
}
void LocalStore::optimisePath(const Path & path)
{
void LocalStore::optimisePath(const Path& path) {
OptimiseStats stats;
InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
if (settings.autoOptimiseStore)
optimisePath_(nullptr, stats, path, inodeHash);
}
}
} // namespace nix

50
third_party/nix/src/libstore/parsed-derivations.cc vendored
View file

@ -3,28 +3,29 @@
namespace nix {
ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
: drvPath(drvPath), drv(drv)
{
: drvPath(drvPath), drv(drv) {
/* Parse the __json attribute, if any. */
auto jsonAttr = drv.env.find("__json");
if (jsonAttr != drv.env.end()) {
try {
structuredAttrs = nlohmann::json::parse(jsonAttr->second);
} catch (std::exception& e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
throw Error("cannot process __json attribute of '%s': %s", drvPath,
e.what());
}
}
}
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
{
std::optional<std::string> ParsedDerivation::getStringAttr(
const std::string& name) const {
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return {};
else {
if (!i->is_string())
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a string", name,
drvPath);
return i->get<std::string>();
}
} else {
@ -36,15 +37,15 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n
}
}
bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
{
bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const {
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return def;
else {
if (!i->is_boolean())
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name,
drvPath);
return i->get<bool>();
}
} else {
@ -56,19 +57,23 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
}
}
std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
{
std::optional<Strings> ParsedDerivation::getStringsAttr(
const std::string& name) const {
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return {};
else {
if (!i->is_array())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
throw Error(
"attribute '%s' of derivation '%s' must be a list of strings", name,
drvPath);
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
throw Error(
"attribute '%s' of derivation '%s' must be a list of strings",
name, drvPath);
res.push_back(j->get<std::string>());
}
return res;
@ -82,19 +87,16 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
}
}
StringSet ParsedDerivation::getRequiredSystemFeatures() const
{
StringSet ParsedDerivation::getRequiredSystemFeatures() const {
StringSet res;
for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i);
return res;
}
bool ParsedDerivation::canBuildLocally() const
{
if (drv.platform != settings.thisSystem.get()
&& !settings.extraPlatforms.get().count(drv.platform)
&& !drv.isBuiltin())
bool ParsedDerivation::canBuildLocally() const {
if (drv.platform != settings.thisSystem.get() &&
!settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin())
return false;
for (auto& feature : getRequiredSystemFeatures())
@ -103,14 +105,12 @@ bool ParsedDerivation::canBuildLocally() const
return true;
}
bool ParsedDerivation::willBuildLocally() const
{
bool ParsedDerivation::willBuildLocally() const {
return getBoolAttr("preferLocalBuild") && canBuildLocally();
}
bool ParsedDerivation::substitutesAllowed() const
{
bool ParsedDerivation::substitutesAllowed() const {
return getBoolAttr("allowSubstitutes", true);
}
}
} // namespace nix

12
third_party/nix/src/libstore/parsed-derivations.hh vendored
View file

@ -1,21 +1,17 @@
#include "derivations.hh"
#include <nlohmann/json.hpp>
#include "derivations.hh"
namespace nix {
class ParsedDerivation
{
class ParsedDerivation {
Path drvPath;
BasicDerivation& drv;
std::optional<nlohmann::json> structuredAttrs;
public:
ParsedDerivation(const Path& drvPath, BasicDerivation& drv);
const std::optional<nlohmann::json> & getStructuredAttrs() const
{
const std::optional<nlohmann::json>& getStructuredAttrs() const {
return structuredAttrs;
}
@ -34,4 +30,4 @@ public:
bool substitutesAllowed() const;
};
}
} // namespace nix

78
third_party/nix/src/libstore/pathlocks.cc vendored
View file

@ -1,21 +1,16 @@
#include "pathlocks.hh"
#include "util.hh"
#include "sync.hh"
#include <fcntl.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <cerrno>
#include <cstdlib>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/file.h>
#include "sync.hh"
#include "util.hh"
namespace nix {
AutoCloseFD openLockFile(const Path & path, bool create)
{
AutoCloseFD openLockFile(const Path& path, bool create) {
AutoCloseFD fd;
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
@ -25,9 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
return fd;
}
void deleteLockFile(const Path & path, int fd)
{
void deleteLockFile(const Path& path, int fd) {
/* Get rid of the lock file. Have to be careful not to introduce
races. Write a (meaningless) token to the file to indicate to
other processes waiting on this lock that the lock is stale
@ -38,14 +31,16 @@ void deleteLockFile(const Path & path, int fd)
file is an optimisation, not a necessity. */
}
bool lockFile(int fd, LockType lockType, bool wait)
{
bool lockFile(int fd, LockType lockType, bool wait) {
int type;
if (lockType == ltRead) type = LOCK_SH;
else if (lockType == ltWrite) type = LOCK_EX;
else if (lockType == ltNone) type = LOCK_UN;
else abort();
if (lockType == ltRead)
type = LOCK_SH;
else if (lockType == ltWrite)
type = LOCK_EX;
else if (lockType == ltNone)
type = LOCK_UN;
else
abort();
if (wait) {
while (flock(fd, type) != 0) {
@ -59,31 +54,22 @@ bool lockFile(int fd, LockType lockType, bool wait)
while (flock(fd, type | LOCK_NB) != 0) {
checkInterrupt();
if (errno == EWOULDBLOCK) return false;
if (errno != EINTR)
throw SysError(format("acquiring/releasing lock"));
if (errno != EINTR) throw SysError(format("acquiring/releasing lock"));
}
}
return true;
}
PathLocks::PathLocks()
: deletePaths(false)
{
}
PathLocks::PathLocks() : deletePaths(false) {}
PathLocks::PathLocks(const PathSet& paths, const string& waitMsg)
: deletePaths(false)
{
: deletePaths(false) {
lockPaths(paths, waitMsg);
}
bool PathLocks::lockPaths(const PathSet & paths,
const string & waitMsg, bool wait)
{
bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
bool wait) {
assert(fds.empty());
/* Note that `fds' is built incrementally so that the destructor
@ -101,7 +87,6 @@ bool PathLocks::lockPaths(const PathSet & paths,
AutoCloseFD fd;
while (1) {
/* Open/create the lock file. */
fd = openLockFile(lockPath, true);
@ -142,9 +127,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
return true;
}
PathLocks::~PathLocks()
{
PathLocks::~PathLocks() {
try {
unlock();
} catch (...) {
@ -152,15 +135,13 @@ PathLocks::~PathLocks()
}
}
void PathLocks::unlock()
{
void PathLocks::unlock() {
for (auto& i : fds) {
if (deletePaths) deleteLockFile(i.second, i.first);
if (close(i.first) == -1)
printError(
format("error (ignored): cannot close lock file on '%1%'") % i.second);
printError(format("error (ignored): cannot close lock file on '%1%'") %
i.second);
debug(format("lock released on '%1%'") % i.second);
}
@ -168,11 +149,8 @@ void PathLocks::unlock()
fds.clear();
}
void PathLocks::setDeletion(bool deletePaths)
{
void PathLocks::setDeletion(bool deletePaths) {
this->deletePaths = deletePaths;
}
}
} // namespace nix

11
third_party/nix/src/libstore/pathlocks.hh vendored
View file

@ -16,8 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(int fd, LockType lockType, bool wait);
class PathLocks
{
class PathLocks {
private:
typedef std::pair<int, Path> FDPair;
list<FDPair> fds;
@ -25,14 +24,12 @@ private:
public:
PathLocks();
PathLocks(const PathSet & paths,
const string & waitMsg = "");
bool lockPaths(const PathSet & _paths,
const string & waitMsg = "",
PathLocks(const PathSet& paths, const string& waitMsg = "");
bool lockPaths(const PathSet& _paths, const string& waitMsg = "",
bool wait = true);
~PathLocks();
void unlock();
void setDeletion(bool deletePaths);
};
}
} // namespace nix

99
third_party/nix/src/libstore/profiles.cc vendored
View file

@ -1,27 +1,21 @@
#include "profiles.hh"
#include <errno.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "store-api.hh"
#include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
namespace nix {
static bool cmpGensByNumber(const Generation & a, const Generation & b)
{
static bool cmpGensByNumber(const Generation& a, const Generation& b) {
return a.number < b.number;
}
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
static int parseName(const string & profileName, const string & name)
{
static int parseName(const string& profileName, const string& name) {
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
@ -33,10 +27,7 @@ static int parseName(const string & profileName, const string & name)
return -1;
}
Generations findGenerations(Path profile, int & curGen)
{
Generations findGenerations(Path profile, int& curGen) {
Generations gens;
Path profileDir = dirOf(profile);
@ -58,24 +49,17 @@ Generations findGenerations(Path profile, int & curGen)
gens.sort(cmpGensByNumber);
curGen = pathExists(profile)
? parseName(profileName, readLink(profile))
: -1;
curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1;
return gens;
}
static void makeName(const Path & profile, unsigned int num,
Path & outLink)
{
static void makeName(const Path& profile, unsigned int num, Path& outLink) {
Path prefix = (format("%1%-%2%") % profile % num).str();
outLink = prefix + "-link";
}
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
{
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) {
/* The new generation number should be higher than old the
previous ones. */
int dummy;
@ -113,24 +97,19 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
return generation;
}
static void removeFile(const Path & path)
{
static void removeFile(const Path& path) {
if (remove(path.c_str()) == -1)
throw SysError(format("cannot unlink '%1%'") % path);
}
void deleteGeneration(const Path & profile, unsigned int gen)
{
void deleteGeneration(const Path& profile, unsigned int gen) {
Path generation;
makeName(profile, gen, generation);
removeFile(generation);
}
static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
{
static void deleteGeneration2(const Path& profile, unsigned int gen,
bool dryRun) {
if (dryRun)
printInfo(format("would remove generation %1%") % gen);
else {
@ -139,9 +118,9 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
}
}
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
{
void deleteGenerations(const Path& profile,
const std::set<unsigned int>& gensToDelete,
bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
@ -149,7 +128,8 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
Generations gens = findGenerations(profile, curGen);
if (gensToDelete.find(curGen) != gensToDelete.end())
throw Error(format("cannot delete current generation of profile %1%'") % profile);
throw Error(format("cannot delete current generation of profile %1%'") %
profile);
for (auto& i : gens) {
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@ -157,8 +137,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
}
}
void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
{
void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
@ -181,8 +160,7 @@ void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
}
}
void deleteOldGenerations(const Path & profile, bool dryRun)
{
void deleteOldGenerations(const Path& profile, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
@ -190,13 +168,10 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
Generations gens = findGenerations(profile, curGen);
for (auto& i : gens)
if (i.number != curGen)
deleteGeneration2(profile, i.number, dryRun);
if (i.number != curGen) deleteGeneration2(profile, i.number, dryRun);
}
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
{
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) {
PathLocks lock;
lockProfile(lock, profile);
@ -207,8 +182,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
if (canDelete) {
assert(i->creationTime < t);
if (i->number != curGen)
deleteGeneration2(profile, i->number, dryRun);
if (i->number != curGen) deleteGeneration2(profile, i->number, dryRun);
} else if (i->creationTime < t) {
/* We may now start deleting generations, but we don't
delete this generation yet, because this generation was
@ -218,9 +192,8 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
}
}
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
{
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
bool dryRun) {
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
int days;
@ -233,27 +206,21 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
deleteGenerationsOlderThan(profile, oldTime, dryRun);
}
void switchLink(Path link, Path target)
{
void switchLink(Path link, Path target) {
/* Hacky. */
if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
replaceSymlink(target, link);
}
void lockProfile(PathLocks & lock, const Path & profile)
{
lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());
void lockProfile(PathLocks& lock, const Path& profile) {
lock.lockPaths({profile},
(format("waiting for lock on profile '%1%'") % profile).str());
lock.setDeletion(true);
}
string optimisticLockProfile(const Path & profile)
{
string optimisticLockProfile(const Path& profile) {
return pathExists(profile) ? readLink(profile) : "";
}
}
} // namespace nix

32
third_party/nix/src/libstore/profiles.hh vendored
View file

@ -1,32 +1,21 @@
#pragma once
#include "types.hh"
#include "pathlocks.hh"
#include <time.h>
#include "pathlocks.hh"
#include "types.hh"
namespace nix {
struct Generation
{
struct Generation {
int number;
Path path;
time_t creationTime;
Generation()
{
number = -1;
}
operator bool() const
{
return number != -1;
}
Generation() { number = -1; }
operator bool() const { return number != -1; }
};
typedef list<Generation> Generations;
/* Returns the list of currently present generations for the specified
profile, sorted by generation number. */
Generations findGenerations(Path profile, int& curGen);
@ -37,15 +26,18 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
void deleteGeneration(const Path& profile, unsigned int gen);
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
void deleteGenerations(const Path& profile,
const std::set<unsigned int>& gensToDelete, bool dryRun);
void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
void deleteGenerationsGreaterThan(const Path& profile, const int max,
bool dryRun);
void deleteOldGenerations(const Path& profile, bool dryRun);
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun);
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
bool dryRun);
void switchLink(Path link, Path target);
@ -64,4 +56,4 @@ void lockProfile(PathLocks & lock, const Path & profile);
rebuilt. */
string optimisticLockProfile(const Path& profile);
}
} // namespace nix

46
third_party/nix/src/libstore/references.cc vendored
View file

@ -1,21 +1,16 @@
#include "references.hh"
#include <cstdlib>
#include <map>
#include "archive.hh"
#include "hash.hh"
#include "util.hh"
#include "archive.hh"
#include <map>
#include <cstdlib>
namespace nix {
static unsigned int refLength = 32; /* characters */
static void search(const unsigned char * s, size_t len,
StringSet & hashes, StringSet & seen)
{
static void search(const unsigned char* s, size_t len, StringSet& hashes,
StringSet& seen) {
static bool initialised = false;
static bool isBase32[256];
if (!initialised) {
@ -37,8 +32,7 @@ static void search(const unsigned char * s, size_t len,
if (!match) continue;
string ref((const char*)s + i, refLength);
if (hashes.find(ref) != hashes.end()) {
debug(format("found reference to '%1%' at offset '%2%'")
% ref % i);
debug(format("found reference to '%1%' at offset '%2%'") % ref % i);
seen.insert(ref);
hashes.erase(ref);
}
@ -46,9 +40,7 @@ static void search(const unsigned char * s, size_t len,
}
}
struct RefScanSink : Sink
{
struct RefScanSink : Sink {
HashSink hashSink;
StringSet hashes;
StringSet seen;
@ -60,29 +52,27 @@ struct RefScanSink : Sink
void operator()(const unsigned char* data, size_t len);
};
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
void RefScanSink::operator()(const unsigned char* data, size_t len) {
hashSink(data, len);
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + string((const char *) data, len > refLength ? refLength : len);
string s =
tail + string((const char*)data, len > refLength ? refLength : len);
search((const unsigned char*)s.data(), s.size(), hashes, seen);
search(data, len, hashes, seen);
size_t tailLen = len <= refLength ? len : refLength;
tail =
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
tail = string(tail, tail.size() < refLength - tailLen
? 0
: tail.size() - (refLength - tailLen)) +
string((const char*)data + len - tailLen, tailLen);
}
PathSet scanForReferences(const string & path,
const PathSet & refs, HashResult & hash)
{
PathSet scanForReferences(const string& path, const PathSet& refs,
HashResult& hash) {
RefScanSink sink;
std::map<string, Path> backMap;
@ -92,8 +82,7 @@ PathSet scanForReferences(const string & path,
for (auto& i : refs) {
string baseName = baseNameOf(i);
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error(format("bad reference '%1%'") % i);
if (pos == string::npos) throw Error(format("bad reference '%1%'") % i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
@ -118,5 +107,4 @@ PathSet scanForReferences(const string & path,
return found;
}
}
} // namespace nix

2
third_party/nix/src/libstore/references.hh vendored
View file

@ -1,7 +1,7 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "types.hh"
namespace nix {

61
third_party/nix/src/libstore/remote-fs-accessor.cc vendored
View file

@ -1,30 +1,25 @@
#include "remote-fs-accessor.hh"
#include "nar-accessor.hh"
#include "json.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "json.hh"
#include "nar-accessor.hh"
namespace nix {
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir)
: store(store)
, cacheDir(cacheDir)
{
if (cacheDir != "")
createDirs(cacheDir);
: store(store), cacheDir(cacheDir) {
if (cacheDir != "") createDirs(cacheDir);
}
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
{
Path RemoteFSAccessor::makeCacheFile(const Path& storePath,
const std::string& ext) {
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
}
void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar,
ref<FSAccessor> narAccessor)
{
ref<FSAccessor> narAccessor) {
nars.emplace(storePath, narAccessor);
if (cacheDir != "") {
@ -43,15 +38,15 @@ void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & na
}
}
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) {
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
throw InvalidPath(format("path '%1%' is not a valid store path") %
storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};
@ -60,17 +55,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
std::string listing;
Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
if (cacheDir != "" &&
pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath, "ls"));
auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) {
auto narAccessor = makeLazyNarAccessor(
listing, [cacheFile](uint64_t offset, uint64_t length) {
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening NAR cache file '%s'", cacheFile);
if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile);
if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset)
throw SysError("seeking in '%s'", cacheFile);
@ -84,7 +77,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
nars.emplace(storePath, narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
} catch (SysError&) {
}
try {
*sink.s = nix::readFile(cacheFile);
@ -93,7 +87,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
nars.emplace(storePath, narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
} catch (SysError&) {
}
}
store->narFromPath(storePath, sink);
@ -102,28 +97,24 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return {narAccessor, restPath};
}
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
{
FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) {
auto res = fetch(path);
return res.first->stat(res.second);
}
StringSet RemoteFSAccessor::readDirectory(const Path & path)
{
StringSet RemoteFSAccessor::readDirectory(const Path& path) {
auto res = fetch(path);
return res.first->readDirectory(res.second);
}
std::string RemoteFSAccessor::readFile(const Path & path)
{
std::string RemoteFSAccessor::readFile(const Path& path) {
auto res = fetch(path);
return res.first->readFile(res.second);
}
std::string RemoteFSAccessor::readLink(const Path & path)
{
std::string RemoteFSAccessor::readLink(const Path& path) {
auto res = fetch(path);
return res.first->readLink(res.second);
}
}
} // namespace nix

6
third_party/nix/src/libstore/remote-fs-accessor.hh vendored
View file

@ -6,8 +6,7 @@
namespace nix {
class RemoteFSAccessor : public FSAccessor
{
class RemoteFSAccessor : public FSAccessor {
ref<Store> store;
std::map<Path, ref<FSAccessor>> nars;
@ -24,7 +23,6 @@ class RemoteFSAccessor : public FSAccessor
ref<FSAccessor> narAccessor);
public:
RemoteFSAccessor(ref<Store> store,
const /* FIXME: use std::optional */ Path& cacheDir = "");
@ -37,4 +35,4 @@ public:
std::string readLink(const Path& path) override;
};
}
} // namespace nix

380
third_party/nix/src/libstore/remote-store.cc vendored
View file

@ -1,37 +1,32 @@
#include "serialise.hh"
#include "util.hh"
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "archive.hh"
#include "affinity.hh"
#include "globals.hh"
#include "derivations.hh"
#include "pool.hh"
#include "finally.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <errno.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h>
#include <cstring>
#include "affinity.hh"
#include "archive.hh"
#include "derivations.hh"
#include "finally.hh"
#include "globals.hh"
#include "pool.hh"
#include "serialise.hh"
#include "util.hh"
#include "worker-protocol.hh"
namespace nix {
Path readStorePath(Store & store, Source & from)
{
Path readStorePath(Store& store, Source& from) {
Path path = readString(from);
store.assertStorePath(path);
return path;
}
template<class T> T readStorePaths(Store & store, Source & from)
{
template <class T>
T readStorePaths(Store& store, Source& from) {
T paths = readStrings<T>(from);
for (auto& i : paths) store.assertStorePath(i);
return paths;
@ -40,28 +35,24 @@ template<class T> T readStorePaths(Store & store, Source & from)
template PathSet readStorePaths(Store& store, Source& from);
template Paths readStorePaths(Store& store, Source& from);
/* TODO: Separate these store impls into different files, give them better names */
/* TODO: Separate these store impls into different files, give them better names
*/
RemoteStore::RemoteStore(const Params& params)
: Store(params)
, connections(make_ref<Pool<Connection>>(
: Store(params),
connections(make_ref<Pool<Connection>>(
std::max(1, (int)maxConnections),
[this]() { return openConnectionWrapper(); },
[this](const ref<Connection>& r) {
return
r->to.good()
&& r->from.good()
&& std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
}
))
{
}
return r->to.good() && r->from.good() &&
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::steady_clock::now() - r->startTime)
.count() < maxConnectionAge;
})) {}
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
{
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() {
if (failed)
throw Error("opening a connection to remote store '%s' previously failed", getUri());
throw Error("opening a connection to remote store '%s' previously failed",
getUri());
try {
return openConnection();
} catch (...) {
@ -70,26 +61,16 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
}
}
UDSRemoteStore::UDSRemoteStore(const Params& params)
: Store(params)
, LocalFSStore(params)
, RemoteStore(params)
{
}
: Store(params), LocalFSStore(params), RemoteStore(params) {}
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params)
: Store(params)
, LocalFSStore(params)
, RemoteStore(params)
, path(socket_path)
{
}
: Store(params),
LocalFSStore(params),
RemoteStore(params),
path(socket_path) {}
std::string UDSRemoteStore::getUri()
{
std::string UDSRemoteStore::getUri() {
if (path) {
return std::string("unix://") + *path;
} else {
@ -97,19 +78,18 @@ std::string UDSRemoteStore::getUri()
}
}
ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
{
ref<RemoteStore::Connection> UDSRemoteStore::openConnection() {
auto conn = make_ref<Connection>();
/* Connect to a daemon that does the privileged work for us. */
conn->fd = socket(PF_UNIX, SOCK_STREAM
conn->fd = socket(PF_UNIX,
SOCK_STREAM
#ifdef SOCK_CLOEXEC
| SOCK_CLOEXEC
#endif
, 0);
if (!conn->fd)
throw SysError("cannot create Unix domain socket");
,
0);
if (!conn->fd) throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get());
string socketPath = path ? *path : settings.nixDaemonSocketFile;
@ -133,9 +113,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
return conn;
}
void RemoteStore::initConnection(Connection & conn)
{
void RemoteStore::initConnection(Connection& conn) {
/* Send the magic greeting, check for the reply. */
try {
conn.to << WORKER_MAGIC_1;
@ -144,7 +122,8 @@ void RemoteStore::initConnection(Connection & conn)
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) !=
GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported");
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
throw Error("the Nix daemon version is too old");
@ -158,35 +137,26 @@ void RemoteStore::initConnection(Connection & conn)
conn.to << 0;
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
conn.to << false;
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false;
auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex);
}
catch (Error & e) {
throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
} catch (Error& e) {
throw Error("cannot open connection to remote store '%s': %s", getUri(),
e.what());
}
setOptions(conn);
}
void RemoteStore::setOptions(Connection & conn)
{
conn.to << wopSetOptions
<< settings.keepFailed
<< settings.keepGoing
<< settings.tryFallback
<< verbosity
<< settings.maxBuildJobs
<< settings.maxSilentTime
<< true
void RemoteStore::setOptions(Connection& conn) {
conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing
<< settings.tryFallback << verbosity << settings.maxBuildJobs
<< settings.maxSilentTime << true
<< (settings.verboseBuild ? lvlError : lvlVomit)
<< 0 // obsolete log type
<< 0 /* obsolete print build trace */
<< settings.buildCores
<< settings.useSubstitutes;
<< settings.buildCores << settings.useSubstitutes;
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
std::map<std::string, Config::SettingInfo> overrides;
@ -200,35 +170,28 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.useSubstitutes.name);
overrides.erase(settings.showTrace.name);
conn.to << overrides.size();
for (auto & i : overrides)
conn.to << i.first << i.second.value;
for (auto& i : overrides) conn.to << i.first << i.second.value;
}
auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex);
}
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
the connection as bad (causing it to be closed) if a non-daemon
exception is thrown before the handle is closed. Such an exception
causes a deviation from the expected protocol and therefore a
desynchronization between the client and daemon. */
struct ConnectionHandle
{
struct ConnectionHandle {
Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle)
: handle(std::move(handle))
{ }
: handle(std::move(handle)) {}
ConnectionHandle(ConnectionHandle && h)
: handle(std::move(h.handle))
{ }
ConnectionHandle(ConnectionHandle&& h) : handle(std::move(h.handle)) {}
~ConnectionHandle()
{
~ConnectionHandle() {
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
@ -237,8 +200,7 @@ struct ConnectionHandle
RemoteStore::Connection* operator->() { return &*handle; }
void processStderr(Sink * sink = 0, Source * source = 0)
{
void processStderr(Sink* sink = 0, Source* source = 0) {
auto ex = handle->processStderr(sink, source);
if (ex) {
daemonException = true;
@ -247,24 +209,19 @@ struct ConnectionHandle
}
};
ConnectionHandle RemoteStore::getConnection()
{
ConnectionHandle RemoteStore::getConnection() {
return ConnectionHandle(connections->get());
}
bool RemoteStore::isValidPathUncached(const Path & path)
{
bool RemoteStore::isValidPathUncached(const Path& path) {
auto conn(getConnection());
conn->to << wopIsValidPath << path;
conn.processStderr();
return readInt(conn->from);
}
PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
{
PathSet RemoteStore::queryValidPaths(const PathSet& paths,
SubstituteFlag maybeSubstitute) {
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res;
@ -278,18 +235,14 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
}
}
PathSet RemoteStore::queryAllValidPaths()
{
PathSet RemoteStore::queryAllValidPaths() {
auto conn(getConnection());
conn->to << wopQueryAllValidPaths;
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
}
PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
{
PathSet RemoteStore::querySubstitutablePaths(const PathSet& paths) {
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res;
@ -306,16 +259,13 @@ PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
}
}
void RemoteStore::querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos & infos)
{
SubstitutablePathInfos& infos) {
if (paths.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
for (auto& i : paths) {
SubstitutablePathInfo info;
conn->to << wopQuerySubstitutablePathInfo << i;
@ -331,7 +281,6 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
}
} else {
conn->to << wopQuerySubstitutablePathInfos << paths;
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
@ -344,14 +293,12 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
}
}
}
void RemoteStore::queryPathInfoUncached(const Path & path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
{
void RemoteStore::queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
try {
std::shared_ptr<ValidPathInfo> info;
{
@ -366,7 +313,8 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
throw;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid; conn->from >> valid;
bool valid;
conn->from >> valid;
if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
}
info = std::make_shared<ValidPathInfo>();
@ -383,13 +331,12 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
}
}
callback(std::move(info));
} catch (...) { callback.rethrow(); }
} catch (...) {
callback.rethrow();
}
}
void RemoteStore::queryReferrers(const Path & path,
PathSet & referrers)
{
void RemoteStore::queryReferrers(const Path& path, PathSet& referrers) {
auto conn(getConnection());
conn->to << wopQueryReferrers << path;
conn.processStderr();
@ -397,36 +344,28 @@ void RemoteStore::queryReferrers(const Path & path,
referrers.insert(referrers2.begin(), referrers2.end());
}
PathSet RemoteStore::queryValidDerivers(const Path & path)
{
PathSet RemoteStore::queryValidDerivers(const Path& path) {
auto conn(getConnection());
conn->to << wopQueryValidDerivers << path;
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
}
PathSet RemoteStore::queryDerivationOutputs(const Path & path)
{
PathSet RemoteStore::queryDerivationOutputs(const Path& path) {
auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << path;
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
}
PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
{
PathSet RemoteStore::queryDerivationOutputNames(const Path& path) {
auto conn(getConnection());
conn->to << wopQueryDerivationOutputNames << path;
conn.processStderr();
return readStrings<PathSet>(conn->from);
}
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
{
Path RemoteStore::queryPathFromHashPart(const string& hashPart) {
auto conn(getConnection());
conn->to << wopQueryPathFromHashPart << hashPart;
conn.processStderr();
@ -435,10 +374,9 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
return path;
}
void RemoteStore::addToStore(const ValidPathInfo& info, Source& source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) {
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
@ -448,11 +386,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
sink << 1 // == path follows
;
copyNAR(source, sink);
sink
<< exportMagic
<< info.path
<< info.references
<< info.deriver
sink << exportMagic << info.path << info.references << info.deriver
<< 0 // == no legacy signature
<< 0 // == no path follows
;
@ -465,31 +399,32 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
}
else {
conn->to << wopAddToStoreNar
<< info.path << info.deriver << info.narHash.to_string(Base16, false)
<< info.references << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << info.ca
<< repair << !checkSigs;
conn->to << wopAddToStoreNar << info.path << info.deriver
<< info.narHash.to_string(Base16, false) << info.references
<< info.registrationTime << info.narSize << info.ultimate
<< info.sigs << info.ca << repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to);
conn.processStderr(0, tunnel ? &source : nullptr);
}
}
Path RemoteStore::addToStore(const string& name, const Path& _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
bool recursive, HashType hashAlgo,
PathFilter& filter, RepairFlag repair) {
if (repair)
throw Error(
"repairing is not supported when building through the Nix daemon");
auto conn(getConnection());
Path srcPath(absPath(_srcPath));
conn->to << wopAddToStore << name
<< ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
<< (recursive ? 1 : 0)
<< printHashType(hashAlgo);
<< ((hashAlgo == htSHA256 && recursive)
? 0
: 1) /* backwards compatibility hack */
<< (recursive ? 1 : 0) << printHashType(hashAlgo);
try {
conn->to.written = 0;
@ -504,21 +439,21 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
} catch (SysError& e) {
/* Daemon closed while we were sending the path. Probably OOM
or I/O error. */
if (e.errNo == EPIPE)
try {
if (e.errNo == EPIPE) try {
conn.processStderr();
} catch (EndOfFile & e) { }
} catch (EndOfFile& e) {
}
throw;
}
return readStorePath(*this, conn->from);
}
Path RemoteStore::addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
const PathSet& references, RepairFlag repair) {
if (repair)
throw Error(
"repairing is not supported when building through the Nix daemon");
auto conn(getConnection());
conn->to << wopAddTextToStore << name << s << references;
@ -527,9 +462,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
return readStorePath(*this, conn->from);
}
void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
{
void RemoteStore::buildPaths(const PathSet& drvPaths, BuildMode buildMode) {
auto conn(getConnection());
conn->to << wopBuildPaths;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
@ -540,23 +473,23 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
/* Old daemons did not take a 'buildMode' parameter, so we
need to validate it here on the client side. */
if (buildMode != bmNormal)
throw Error("repairing or checking is not supported when building through the Nix daemon");
throw Error(
"repairing or checking is not supported when building through the "
"Nix daemon");
} else {
/* For backwards compatibility with old daemons, strip output
identifiers. */
PathSet drvPaths2;
for (auto & i : drvPaths)
drvPaths2.insert(string(i, 0, i.find('!')));
for (auto& i : drvPaths) drvPaths2.insert(string(i, 0, i.find('!')));
conn->to << drvPaths2;
}
conn.processStderr();
readInt(conn->from);
}
BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
BuildResult RemoteStore::buildDerivation(const Path& drvPath,
const BasicDerivation& drv,
BuildMode buildMode) {
auto conn(getConnection());
conn->to << wopBuildDerivation << drvPath << drv << buildMode;
conn.processStderr();
@ -567,45 +500,35 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
return res;
}
void RemoteStore::ensurePath(const Path & path)
{
void RemoteStore::ensurePath(const Path& path) {
auto conn(getConnection());
conn->to << wopEnsurePath << path;
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::addTempRoot(const Path & path)
{
void RemoteStore::addTempRoot(const Path& path) {
auto conn(getConnection());
conn->to << wopAddTempRoot << path;
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::addIndirectRoot(const Path & path)
{
void RemoteStore::addIndirectRoot(const Path& path) {
auto conn(getConnection());
conn->to << wopAddIndirectRoot << path;
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::syncWithGC()
{
void RemoteStore::syncWithGC() {
auto conn(getConnection());
conn->to << wopSyncWithGC;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
Roots RemoteStore::findRoots(bool censor) {
auto conn(getConnection());
conn->to << wopFindRoots;
conn.processStderr();
@ -619,13 +542,11 @@ Roots RemoteStore::findRoots(bool censor)
return result;
}
void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
{
void RemoteStore::collectGarbage(const GCOptions& options, GCResults& results) {
auto conn(getConnection());
conn->to
<< wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
conn->to << wopCollectGarbage << options.action << options.pathsToDelete
<< options.ignoreLiveness
<< options.maxFreed
/* removed options */
<< 0 << 0 << 0;
@ -642,38 +563,31 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
}
}
void RemoteStore::optimiseStore()
{
void RemoteStore::optimiseStore() {
auto conn(getConnection());
conn->to << wopOptimiseStore;
conn.processStderr();
readInt(conn->from);
}
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
{
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) {
auto conn(getConnection());
conn->to << wopVerifyStore << checkContents << repair;
conn.processStderr();
return readInt(conn->from);
}
void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
{
void RemoteStore::addSignatures(const Path& storePath, const StringSet& sigs) {
auto conn(getConnection());
conn->to << wopAddSignatures << storePath << sigs;
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize)
{
void RemoteStore::queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet& willSubstitute, PathSet& unknown,
unsigned long long& downloadSize,
unsigned long long& narSize) {
{
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
@ -690,32 +604,20 @@ void RemoteStore::queryMissing(const PathSet & targets,
}
fallback:
return Store::queryMissing(targets, willBuild, willSubstitute,
unknown, downloadSize, narSize);
return Store::queryMissing(targets, willBuild, willSubstitute, unknown,
downloadSize, narSize);
}
void RemoteStore::connect() { auto conn(getConnection()); }
void RemoteStore::connect()
{
auto conn(getConnection());
}
unsigned int RemoteStore::getProtocol()
{
unsigned int RemoteStore::getProtocol() {
auto conn(connections->get());
return conn->daemonVersion;
}
void RemoteStore::flushBadConnections() { connections->flushBad(); }
void RemoteStore::flushBadConnections()
{
connections->flushBad();
}
RemoteStore::Connection::~Connection()
{
RemoteStore::Connection::~Connection() {
try {
to.flush();
} catch (...) {
@ -723,9 +625,7 @@ RemoteStore::Connection::~Connection()
}
}
static Logger::Fields readFields(Source & from)
{
static Logger::Fields readFields(Source& from) {
Logger::Fields fields;
size_t size = readInt(from);
for (size_t n = 0; n < size; n++) {
@ -740,13 +640,11 @@ static Logger::Fields readFields(Source & from)
return fields;
}
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
{
std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink,
Source* source) {
to.flush();
while (true) {
auto msg = readNum<uint64_t>(from);
if (msg == STDERR_WRITE) {
@ -806,12 +704,12 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
static std::string uriScheme = "unix://";
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore(
[](const std::string& uri,
const Store::Params& params) -> std::shared_ptr<Store> {
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
return std::make_shared<UDSRemoteStore>(
std::string(uri, uriScheme.size()), params);
});
}
} // namespace nix

57
third_party/nix/src/libstore/remote-store.hh vendored
View file

@ -2,31 +2,28 @@
#include <limits>
#include <string>
#include "store-api.hh"
namespace nix {
class Pipe;
class Pid;
struct FdSink;
struct FdSource;
template<typename T> class Pool;
template <typename T>
class Pool;
struct ConnectionHandle;
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
class RemoteStore : public virtual Store
{
class RemoteStore : public virtual Store {
public:
const Setting<int> maxConnections{
(Store*)this, 1, "max-connections",
"maximum number of concurrent connections to the Nix daemon"};
const Setting<int> maxConnections{(Store*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
const Setting<unsigned int> maxConnectionAge{
(Store*)this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
virtual bool sameMachine() = 0;
@ -37,12 +34,13 @@ public:
bool isValidPathUncached(const Path& path) override;
PathSet queryValidPaths(const PathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
NoSubstitute) override;
PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path& path, PathSet& referrers) override;
@ -60,13 +58,14 @@ public:
void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos& infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
void addToStore(const ValidPathInfo& info, Source& nar, RepairFlag repair,
CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string& name, const Path& srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
PathFilter& filter = defaultPathFilter,
RepairFlag repair = NoRepair) override;
Path addTextToStore(const string& name, const string& s,
const PathSet& references, RepairFlag repair) override;
@ -94,9 +93,10 @@ public:
void addSignatures(const Path& storePath, const StringSet& sigs) override;
void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override;
void queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet& willSubstitute, PathSet& unknown,
unsigned long long& downloadSize,
unsigned long long& narSize) override;
void connect() override;
@ -105,9 +105,7 @@ public:
void flushBadConnections();
protected:
struct Connection
{
struct Connection {
AutoCloseFD fd;
FdSink to;
FdSource from;
@ -134,28 +132,21 @@ protected:
friend struct ConnectionHandle;
private:
std::atomic_bool failed{false};
};
class UDSRemoteStore : public LocalFSStore, public RemoteStore
{
class UDSRemoteStore : public LocalFSStore, public RemoteStore {
public:
UDSRemoteStore(const Params& params);
UDSRemoteStore(std::string path, const Params& params);
std::string getUri() override;
bool sameMachine()
{ return true; }
bool sameMachine() { return true; }
private:
ref<RemoteStore::Connection> openConnection() override;
std::optional<std::string> path;
};
}
} // namespace nix

249
third_party/nix/src/libstore/s3-binary-cache-store.cc vendored
View file

@ -1,14 +1,6 @@
#if ENABLE_S3
#include "s3.hh"
#include "s3-binary-cache-store.hh"
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
@ -24,13 +16,19 @@
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h>
#include "compression.hh"
#include "download.hh"
#include "globals.hh"
#include "istringstream_nocopy.hh"
#include "nar-info-disk-cache.hh"
#include "nar-info.hh"
#include "s3.hh"
using namespace Aws::Transfer;
namespace nix {
struct S3Error : public Error
{
struct S3Error : public Error {
Aws::S3::S3Errors err;
S3Error(Aws::S3::S3Errors err, const FormatOrString& fs)
: Error(fs), err(err){};
@ -39,27 +37,22 @@ struct S3Error : public Error
/* Helper: given an Outcome<R, E>, return R in case of success, or
throw an exception in case of an error. */
template <typename R, typename E>
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
{
R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) {
if (!outcome.IsSuccess())
throw S3Error(
outcome.GetError().GetErrorType(),
throw S3Error(outcome.GetError().GetErrorType(),
fs.s + ": " + outcome.GetError().GetMessage());
return outcome.GetResultWithOwnership();
}
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
{
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem {
using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
void ProcessFormattedStatement(Aws::String && statement) override
{
void ProcessFormattedStatement(Aws::String&& statement) override {
debug("AWS: %s", chomp(statement));
}
};
static void initAWS()
{
static void initAWS() {
static std::once_flag flag;
std::call_once(flag, []() {
Aws::SDKOptions options;
@ -70,8 +63,7 @@ static void initAWS()
if (verbosity >= lvlDebug) {
options.loggingOptions.logLevel =
verbosity == lvlDebug
? Aws::Utils::Logging::LogLevel::Debug
verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug
: Aws::Utils::Logging::LogLevel::Trace;
options.loggingOptions.logger_create_fn = [options]() {
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
@ -82,14 +74,18 @@ static void initAWS()
});
}
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
: config(makeConfig(region, scheme, endpoint))
, client(make_ref<Aws::S3::S3Client>(
S3Helper::S3Helper(const string& profile, const string& region,
const string& scheme, const string& endpoint)
: config(makeConfig(region, scheme, endpoint)),
client(make_ref<Aws::S3::S3Client>(
profile == ""
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
std::make_shared<
Aws::Auth::DefaultAWSCredentialsProviderChain>())
: std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
std::make_shared<
Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(
profile.c_str())),
*config,
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
@ -97,25 +93,25 @@ S3Helper::S3Helper(const string & profile, const string & region, const string &
#else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif
endpoint.empty()))
{
endpoint.empty())) {
}
/* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy
{
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
{
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
class RetryStrategy : public Aws::Client::DefaultRetryStrategy {
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
long attemptedRetries) const override {
auto retry =
Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
if (retry)
printError("AWS error '%s' (%s), will retry in %d ms",
error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
error.GetExceptionName(), error.GetMessage(),
CalculateDelayBeforeNextRetry(error, attemptedRetries));
return retry;
}
};
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
{
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
const string& region, const string& scheme, const string& endpoint) {
initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = region;
@ -132,30 +128,26 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
return res;
}
S3Helper::DownloadResult S3Helper::getObject(
const std::string & bucketName, const std::string & key)
{
S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName,
const std::string& key) {
debug("fetching 's3://%s/%s'...", bucketName, key);
auto request =
Aws::S3::Model::GetObjectRequest()
.WithBucket(bucketName)
.WithKey(key);
Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key);
request.SetResponseStreamFactory([&]() {
return Aws::New<std::stringstream>("STRINGSTREAM");
});
request.SetResponseStreamFactory(
[&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); });
DownloadResult res;
auto now1 = std::chrono::steady_clock::now();
try {
auto result = checkAws(fmt("AWS error fetching '%s'", key),
client->GetObject(request));
res.data = decompress(result.GetContentEncoding(),
res.data =
decompress(result.GetContentEncoding(),
dynamic_cast<std::stringstream&>(result.GetBody()).str());
} catch (S3Error& e) {
@ -164,24 +156,35 @@ S3Helper::DownloadResult S3Helper::getObject(
auto now2 = std::chrono::steady_clock::now();
res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
res.durationMs =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
return res;
}
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
{
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{
this, false, "multipart-upload", "whether to use multi-part uploads"};
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore {
const Setting<std::string> profile{
this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{
this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{
this, "", "scheme",
"The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{
this, "", "endpoint",
"An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression",
"compression method for .ls files"};
const Setting<std::string> logCompression{
this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{this, false, "multipart-upload",
"whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
this, 5 * 1024 * 1024, "buffer-size",
"size (in bytes) of each part in multi-part uploads"};
std::string bucketName;
@ -189,41 +192,30 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
S3Helper s3Helper;
S3BinaryCacheStoreImpl(
const Params & params, const std::string & bucketName)
: S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint)
{
S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName)
: S3BinaryCacheStore(params),
bucketName(bucketName),
s3Helper(profile, region, scheme, endpoint) {
diskCache = getNarInfoDiskCache();
}
std::string getUri() override
{
return "s3://" + bucketName;
}
std::string getUri() override { return "s3://" + bucketName; }
void init() override
{
void init() override {
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
BinaryCacheStore::init();
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
}
}
const Stats & getS3Stats() override
{
return stats;
}
const Stats& getS3Stats() override { return stats; }
/* This is a specialisation of isValidPath() that optimistically
fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing
a GET is unlikely to be slower than HEAD. */
bool isValidPathUncached(const Path & storePath) override
{
bool isValidPathUncached(const Path& storePath) override {
try {
queryPathInfo(storePath);
return true;
@ -232,23 +224,22 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
}
}
bool fileExists(const std::string & path) override
{
bool fileExists(const std::string& path) override {
stats.head++;
auto res = s3Helper.client->HeadObject(
Aws::S3::Model::HeadObjectRequest()
auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest()
.WithBucket(bucketName)
.WithKey(path));
if (!res.IsSuccess()) {
auto& error = res.GetError();
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND ||
error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
// If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
return false;
throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
throw Error(format("AWS error fetching '%s': %s") % path %
error.GetMessage());
}
return true;
@ -259,17 +250,17 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
void uploadFile(const std::string& path, const std::string& data,
const std::string& mimeType,
const std::string & contentEncoding)
{
const std::string& contentEncoding) {
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
executor =
std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(
maxThreads);
std::call_once(transferManagerCreated, [&]()
{
std::call_once(transferManagerCreated, [&]() {
if (multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get());
@ -278,9 +269,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
transferConfig.uploadProgressCallback =
[](const TransferManager* transferManager,
const std::shared_ptr<const TransferHandle>
&transferHandle)
{
const std::shared_ptr<const TransferHandle>& transferHandle) {
// FIXME: find a way to properly abort the multipart upload.
// checkInterrupt();
debug("upload progress ('%s'): '%d' of '%d' bytes",
@ -296,37 +285,35 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
auto now1 = std::chrono::steady_clock::now();
if (transferManager) {
if (contentEncoding != "")
throw Error("setting a content encoding is not supported with S3 multi-part uploads");
throw Error(
"setting a content encoding is not supported with S3 multi-part "
"uploads");
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
stream, bucketName, path, mimeType,
transferManager->UploadFile(stream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
transferHandle->WaitUntilFinished();
if (transferHandle->GetStatus() == TransferStatus::FAILED)
throw Error("AWS error: failed to upload 's3://%s/%s': %s",
bucketName, path, transferHandle->GetLastError().GetMessage());
throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName,
path, transferHandle->GetLastError().GetMessage());
if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
throw Error(
"AWS error: transfer status of 's3://%s/%s' in unexpected state",
bucketName, path);
} else {
auto request =
Aws::S3::Model::PutObjectRequest()
auto request = Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);
request.SetContentType(mimeType);
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
if (contentEncoding != "") request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
@ -351,20 +338,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
}
void upsertFile(const std::string& path, const std::string& data,
const std::string & mimeType) override
{
const std::string& mimeType) override {
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
uploadFile(path, *compress(narinfoCompression, data), mimeType,
narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
uploadFile(path, *compress(logCompression, data), mimeType,
logCompression);
else
uploadFile(path, data, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
{
void getFile(const std::string& path, Sink& sink) override {
stats.get++;
// FIXME: stream output to sink.
@ -374,33 +361,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
stats.getTimeMs += res.durationMs;
if (res.data) {
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName,
path, res.data->size(), res.durationMs);
sink((unsigned char*)res.data->data(), res.data->size());
} else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri());
}
PathSet queryAllValidPaths() override
{
PathSet queryAllValidPaths() override {
PathSet paths;
std::string marker;
do {
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName %
marker);
auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
s3Helper.client->ListObjects(
Aws::S3::Model::ListObjectsRequest()
auto res = checkAws(
format("AWS error listing bucket '%s'") % bucketName,
s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest()
.WithBucket(bucketName)
.WithDelimiter("/")
.WithMarker(marker)));
auto& contents = res.GetContents();
debug(format("got %d keys, next marker '%s'")
% contents.size() % res.GetNextMarker());
debug(format("got %d keys, next marker '%s'") % contents.size() %
res.GetNextMarker());
for (auto object : contents) {
auto& key = object.GetKey();
@ -413,19 +401,18 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return paths;
}
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore(
[](const std::string& uri,
const Store::Params& params) -> std::shared_ptr<Store> {
if (std::string(uri, 0, 5) != "s3://") return 0;
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
auto store =
std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
store->init();
return store;
});
}
} // namespace nix
#endif

17
third_party/nix/src/libstore/s3-binary-cache-store.hh vendored
View file

@ -1,23 +1,16 @@
#pragma once
#include "binary-cache-store.hh"
#include <atomic>
#include "binary-cache-store.hh"
namespace nix {
class S3BinaryCacheStore : public BinaryCacheStore
{
class S3BinaryCacheStore : public BinaryCacheStore {
protected:
S3BinaryCacheStore(const Params & params)
: BinaryCacheStore(params)
{ }
S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {}
public:
struct Stats
{
struct Stats {
std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
@ -30,4 +23,4 @@ public:
virtual const Stats& getS3Stats() = 0;
};
}
} // namespace nix

31
third_party/nix/src/libstore/s3.hh vendored
View file

@ -4,30 +4,39 @@
#include "ref.hh"
namespace Aws { namespace Client { class ClientConfiguration; } }
namespace Aws { namespace S3 { class S3Client; } }
namespace Aws {
namespace Client {
class ClientConfiguration;
}
} // namespace Aws
namespace Aws {
namespace S3 {
class S3Client;
}
} // namespace Aws
namespace nix {
struct S3Helper
{
struct S3Helper {
ref<Aws::Client::ClientConfiguration> config;
ref<Aws::S3::S3Client> client;
S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
S3Helper(const std::string& profile, const std::string& region,
const std::string& scheme, const std::string& endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region,
const std::string& scheme,
const std::string& endpoint);
struct DownloadResult
{
struct DownloadResult {
std::shared_ptr<std::string> data;
unsigned int durationMs;
};
DownloadResult getObject(
const std::string & bucketName, const std::string & key);
DownloadResult getObject(const std::string& bucketName,
const std::string& key);
};
}
} // namespace nix
#endif

2
third_party/nix/src/libstore/serve-protocol.hh vendored
View file

@ -21,4 +21,4 @@ typedef enum {
cmdAddToStoreNar = 9,
} ServeCommand;
}
} // namespace nix

84
third_party/nix/src/libstore/sqlite.cc vendored
View file

@ -1,14 +1,11 @@
#include "sqlite.hh"
#include "util.hh"
#include <sqlite3.h>
#include <atomic>
#include "util.hh"
namespace nix {
[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs)
{
[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) {
int err = sqlite3_errcode(db);
int exterr = sqlite3_extended_errcode(db);
@ -20,20 +17,18 @@ namespace nix {
err == SQLITE_PROTOCOL
? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
: fmt("SQLite database '%s' is busy", path));
}
else
} else
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
}
SQLite::SQLite(const Path & path)
{
SQLite::SQLite(const Path& path) {
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path);
}
SQLite::~SQLite()
{
SQLite::~SQLite() {
try {
if (db && sqlite3_close(db) != SQLITE_OK)
throwSQLiteError(db, "closing database");
@ -42,16 +37,14 @@ SQLite::~SQLite()
}
}
void SQLite::exec(const std::string & stmt)
{
void SQLite::exec(const std::string& stmt) {
retrySQLite<void>([&]() {
if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
});
}
void SQLiteStmt::create(sqlite3 * db, const string & sql)
{
void SQLiteStmt::create(sqlite3* db, const string& sql) {
checkInterrupt();
assert(!stmt);
if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
@ -60,8 +53,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & sql)
this->sql = sql;
}
SQLiteStmt::~SQLiteStmt()
{
SQLiteStmt::~SQLiteStmt() {
try {
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
@ -70,32 +62,27 @@ SQLiteStmt::~SQLiteStmt()
}
}
SQLiteStmt::Use::Use(SQLiteStmt & stmt)
: stmt(stmt)
{
SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) {
assert(stmt.stmt);
/* Note: sqlite3_reset() returns the error code for the most
recent call to sqlite3_step(). So ignore it. */
sqlite3_reset(stmt);
}
SQLiteStmt::Use::~Use()
{
sqlite3_reset(stmt);
}
SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
{
SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value,
bool notNull) {
if (notNull) {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1,
SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) {
if (notNull) {
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
@ -104,69 +91,57 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::bind()
{
SQLiteStmt::Use& SQLiteStmt::Use::bind() {
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
return *this;
}
int SQLiteStmt::Use::step()
{
return sqlite3_step(stmt);
}
int SQLiteStmt::Use::step() { return sqlite3_step(stmt); }
void SQLiteStmt::Use::exec()
{
void SQLiteStmt::Use::exec() {
int r = step();
assert(r != SQLITE_ROW);
if (r != SQLITE_DONE)
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
}
bool SQLiteStmt::Use::next()
{
bool SQLiteStmt::Use::next() {
int r = step();
if (r != SQLITE_DONE && r != SQLITE_ROW)
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
return r == SQLITE_ROW;
}
std::string SQLiteStmt::Use::getStr(int col)
{
std::string SQLiteStmt::Use::getStr(int col) {
auto s = (const char*)sqlite3_column_text(stmt, col);
assert(s);
return s;
}
int64_t SQLiteStmt::Use::getInt(int col)
{
int64_t SQLiteStmt::Use::getInt(int col) {
// FIXME: detect nulls?
return sqlite3_column_int64(stmt, col);
}
bool SQLiteStmt::Use::isNull(int col)
{
bool SQLiteStmt::Use::isNull(int col) {
return sqlite3_column_type(stmt, col) == SQLITE_NULL;
}
SQLiteTxn::SQLiteTxn(sqlite3 * db)
{
SQLiteTxn::SQLiteTxn(sqlite3* db) {
this->db = db;
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "starting transaction");
active = true;
}
void SQLiteTxn::commit()
{
void SQLiteTxn::commit() {
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "committing transaction");
active = false;
}
SQLiteTxn::~SQLiteTxn()
{
SQLiteTxn::~SQLiteTxn() {
try {
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "aborting transaction");
@ -175,8 +150,7 @@ SQLiteTxn::~SQLiteTxn()
}
}
void handleSQLiteBusy(const SQLiteBusy & e)
{
void handleSQLiteBusy(const SQLiteBusy& e) {
static std::atomic<time_t> lastWarned{0};
time_t now = time(0);
@ -195,4 +169,4 @@ void handleSQLiteBusy(const SQLiteBusy & e)
nanosleep(&t, 0);
}
}
} // namespace nix

32
third_party/nix/src/libstore/sqlite.hh vendored
View file

@ -2,7 +2,6 @@
#include <functional>
#include <string>
#include "types.hh"
class sqlite3;
@ -11,14 +10,17 @@ class sqlite3_stmt;
namespace nix {
/* RAII wrapper to close a SQLite database automatically. */
struct SQLite
{
struct SQLite {
sqlite3* db = 0;
SQLite() {}
SQLite(const Path& path);
SQLite(const SQLite& from) = delete;
SQLite& operator=(const SQLite& from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
SQLite& operator=(SQLite&& from) {
db = from.db;
from.db = 0;
return *this;
}
~SQLite();
operator sqlite3*() { return db; }
@ -26,8 +28,7 @@ struct SQLite
};
/* RAII wrapper to create and destroy SQLite prepared statements. */
struct SQLiteStmt
{
struct SQLiteStmt {
sqlite3* db = 0;
sqlite3_stmt* stmt = 0;
std::string sql;
@ -38,16 +39,15 @@ struct SQLiteStmt
operator sqlite3_stmt*() { return stmt; }
/* Helper for binding / executing statements. */
class Use
{
class Use {
friend struct SQLiteStmt;
private:
SQLiteStmt& stmt;
unsigned int curArg = 1;
Use(SQLiteStmt& stmt);
public:
~Use();
/* Bind the next parameter. */
@ -69,16 +69,12 @@ struct SQLiteStmt
bool isNull(int col);
};
Use use()
{
return Use(*this);
}
Use use() { return Use(*this); }
};
/* RAII helper that ensures transactions are aborted unless explicitly
committed. */
struct SQLiteTxn
{
struct SQLiteTxn {
bool active = false;
sqlite3* db;
@ -89,7 +85,6 @@ struct SQLiteTxn
~SQLiteTxn();
};
MakeError(SQLiteError, Error);
MakeError(SQLiteBusy, SQLiteError);
@ -100,8 +95,7 @@ void handleSQLiteBusy(const SQLiteBusy & e);
/* Convenience function for retrying a SQLite transaction when the
database is busy. */
template <typename T>
T retrySQLite(std::function<T()> fun)
{
T retrySQLite(std::function<T()> fun) {
while (true) {
try {
return fun();
@ -111,4 +105,4 @@ T retrySQLite(std::function<T()> fun)
}
}
}
} // namespace nix

66
third_party/nix/src/libstore/ssh-store.cc vendored
View file

@ -1,51 +1,40 @@
#include "store-api.hh"
#include "remote-store.hh"
#include "remote-fs-accessor.hh"
#include "archive.hh"
#include "worker-protocol.hh"
#include "pool.hh"
#include "remote-fs-accessor.hh"
#include "remote-store.hh"
#include "ssh.hh"
#include "store-api.hh"
#include "worker-protocol.hh"
namespace nix {
static std::string uriScheme = "ssh-ng://";
class SSHStore : public RemoteStore
{
class SSHStore : public RemoteStore {
public:
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> sshKey{(Store*)this, "", "ssh-key",
"path to an SSH private key"};
const Setting<bool> compress{(Store*)this, false, "compress",
"whether to compress the connection"};
SSHStore(const std::string& host, const Params& params)
: Store(params)
, RemoteStore(params)
, host(host)
, master(
host,
sshKey,
: Store(params),
RemoteStore(params),
host(host),
master(host, sshKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress)
{
}
connections->capacity() > 1, compress) {}
std::string getUri() override
{
return uriScheme + host;
}
std::string getUri() override { return uriScheme + host; }
bool sameMachine()
{ return false; }
bool sameMachine() { return false; }
void narFromPath(const Path& path, Sink& sink) override;
ref<FSAccessor> getFSAccessor() override;
private:
struct Connection : RemoteStore::Connection
{
struct Connection : RemoteStore::Connection {
std::unique_ptr<SSHMaster::Connection> sshConn;
};
@ -55,8 +44,7 @@ private:
SSHMaster master;
void setOptions(RemoteStore::Connection & conn) override
{
void setOptions(RemoteStore::Connection& conn) override{
/* TODO Add a way to explicitly ask for some options to be
forwarded. One option: A way to query the daemon for its
settings, and then a series of params to SSHStore like
@ -66,21 +54,18 @@ private:
};
};
void SSHStore::narFromPath(const Path & path, Sink & sink)
{
void SSHStore::narFromPath(const Path& path, Sink& sink) {
auto conn(connections->get());
conn->to << wopNarFromPath << path;
conn->processStderr();
copyNAR(conn->from, sink);
}
ref<FSAccessor> SSHStore::getFSAccessor()
{
ref<FSAccessor> SSHStore::getFSAccessor() {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
}
ref<RemoteStore::Connection> SSHStore::openConnection()
{
ref<RemoteStore::Connection> SSHStore::openConnection() {
auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand("nix-daemon --stdio");
conn->to = FdSink(conn->sshConn->in.get());
@ -89,12 +74,11 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
return conn;
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore([](const std::string& uri,
const Store::Params& params)
-> std::shared_ptr<Store> {
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
});
}
} // namespace nix

72
third_party/nix/src/libstore/ssh.cc vendored
View file

@ -2,30 +2,27 @@
namespace nix {
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
: host(host)
, fakeSSH(host == "localhost")
, keyFile(keyFile)
, useMaster(useMaster && !fakeSSH)
, compress(compress)
, logFD(logFD)
{
SSHMaster::SSHMaster(const std::string& host, const std::string& keyFile,
bool useMaster, bool compress, int logFD)
: host(host),
fakeSSH(host == "localhost"),
keyFile(keyFile),
useMaster(useMaster && !fakeSSH),
compress(compress),
logFD(logFD) {
if (host == "" || hasPrefix(host, "-"))
throw Error("invalid SSH host name '%s'", host);
}
void SSHMaster::addCommonSSHOpts(Strings & args)
{
void SSHMaster::addCommonSSHOpts(Strings& args) {
for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
args.push_back(i);
if (!keyFile.empty())
args.insert(args.end(), {"-i", keyFile});
if (compress)
args.push_back("-C");
if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile});
if (compress) args.push_back("-C");
}
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
{
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
const std::string& command) {
Path socketPath = startMaster();
Pipe in, out;
@ -36,7 +33,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
ProcessOptions options;
options.dieWithParent = false;
conn->sshPid = startProcess([&]() {
conn->sshPid = startProcess(
[&]() {
restoreSignals();
close(in.writeSide.get());
@ -56,10 +54,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
} else {
args = {"ssh", host.c_str(), "-x", "-a"};
addCommonSSHOpts(args);
if (socketPath != "")
args.insert(args.end(), {"-S", socketPath});
if (verbosity >= lvlChatty)
args.push_back("-v");
if (socketPath != "") args.insert(args.end(), {"-S", socketPath});
if (verbosity >= lvlChatty) args.push_back("-v");
}
args.push_back(command);
@ -67,8 +63,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
// could not exec ssh/bash
throw SysError("unable to execute '%s'", args.front());
}, options);
},
options);
in.readSide = -1;
out.writeSide = -1;
@ -79,15 +75,15 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
return conn;
}
Path SSHMaster::startMaster()
{
Path SSHMaster::startMaster() {
if (!useMaster) return "";
auto state(state_.lock());
if (state->sshMaster != -1) return state->socketPath;
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->tmpDir =
std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->socketPath = (Path)*state->tmpDir + "/ssh.sock";
@ -97,7 +93,8 @@ Path SSHMaster::startMaster()
ProcessOptions options;
options.dieWithParent = false;
state->sshMaster = startProcess([&]() {
state->sshMaster = startProcess(
[&]() {
restoreSignals();
close(out.readSide.get());
@ -105,25 +102,26 @@ Path SSHMaster::startMaster()
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("duping over stdout");
Strings args =
{ "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath
, "-o", "LocalCommand=echo started"
, "-o", "PermitLocalCommand=yes"
};
if (verbosity >= lvlChatty)
args.push_back("-v");
Strings args = {"ssh", host.c_str(),
"-M", "-N",
"-S", state->socketPath,
"-o", "LocalCommand=echo started",
"-o", "PermitLocalCommand=yes"};
if (verbosity >= lvlChatty) args.push_back("-v");
addCommonSSHOpts(args);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
throw SysError("unable to execute '%s'", args.front());
}, options);
},
options);
out.writeSide = -1;
std::string reply;
try {
reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { }
} catch (EndOfFile& e) {
}
if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host);
@ -131,4 +129,4 @@ Path SSHMaster::startMaster()
return state->socketPath;
}
}
} // namespace nix

18
third_party/nix/src/libstore/ssh.hh vendored
View file

@ -1,14 +1,12 @@
#pragma once
#include "util.hh"
#include "sync.hh"
#include "util.hh"
namespace nix {
class SSHMaster
{
class SSHMaster {
private:
const std::string host;
bool fakeSSH;
const std::string keyFile;
@ -16,8 +14,7 @@ private:
const bool compress;
const int logFD;
struct State
{
struct State {
Pid sshMaster;
std::unique_ptr<AutoDelete> tmpDir;
Path socketPath;
@ -28,11 +25,10 @@ private:
void addCommonSSHOpts(Strings& args);
public:
SSHMaster(const std::string& host, const std::string& keyFile, bool useMaster,
bool compress, int logFD = -1);
SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
struct Connection
{
struct Connection {
Pid sshPid;
AutoCloseFD out, in;
};
@ -42,4 +38,4 @@ public:
Path startMaster();
};
}
} // namespace nix

427
third_party/nix/src/libstore/store-api.cc vendored
View file

@ -1,41 +1,31 @@
#include "crypto.hh"
#include "globals.hh"
#include "store-api.hh"
#include "util.hh"
#include <future>
#include "crypto.hh"
#include "derivations.hh"
#include "globals.hh"
#include "json.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
#include "json.hh"
#include "derivations.hh"
#include <future>
#include "util.hh"
namespace nix {
bool Store::isInStore(const Path & path) const
{
bool Store::isInStore(const Path& path) const {
return isInDir(path, storeDir);
}
bool Store::isStorePath(const Path & path) const
{
return isInStore(path)
&& path.size() >= storeDir.size() + 1 + storePathHashLen
&& path.find('/', storeDir.size() + 1) == Path::npos;
bool Store::isStorePath(const Path& path) const {
return isInStore(path) &&
path.size() >= storeDir.size() + 1 + storePathHashLen &&
path.find('/', storeDir.size() + 1) == Path::npos;
}
void Store::assertStorePath(const Path & path) const
{
void Store::assertStorePath(const Path& path) const {
if (!isStorePath(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
}
Path Store::toStorePath(const Path & path) const
{
Path Store::toStorePath(const Path& path) const {
if (!isInStore(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
@ -45,9 +35,7 @@ Path Store::toStorePath(const Path & path) const
return Path(path, 0, slash);
}
Path Store::followLinksToStore(const Path & _path) const
{
Path Store::followLinksToStore(const Path& _path) const {
Path path = absPath(_path);
while (!isInStore(path)) {
if (!isLink(path)) break;
@ -59,39 +47,36 @@ Path Store::followLinksToStore(const Path & _path) const
return path;
}
Path Store::followLinksToStorePath(const Path & path) const
{
Path Store::followLinksToStorePath(const Path& path) const {
return toStorePath(followLinksToStore(path));
}
string storePathToName(const Path & path)
{
string storePathToName(const Path& path) {
auto base = baseNameOf(path);
assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
assert(base.size() == storePathHashLen ||
(base.size() > storePathHashLen && base[storePathHashLen] == '-'));
return base.size() == storePathHashLen ? ""
: string(base, storePathHashLen + 1);
}
string storePathToHash(const Path & path)
{
string storePathToHash(const Path& path) {
auto base = baseNameOf(path);
assert(base.size() >= storePathHashLen);
return string(base, 0, storePathHashLen);
}
void checkStoreName(const string & name)
{
void checkStoreName(const string& name) {
string validChars = "+-._?=";
auto baseError = format("The path name '%2%' is invalid: %3%. "
auto baseError =
format(
"The path name '%2%' is invalid: %3%. "
"Path names are alphanumeric and can include the symbols %1% "
"and must not begin with a period. "
"Note: If '%2%' is a source file and you cannot rename it on "
"disk, builtins.path { name = ... } can be used to give it an "
"alternative name.") % validChars % name;
"alternative name.") %
validChars % name;
/* Disallow names starting with a dot for possible security
reasons (e.g., "." and ".."). */
@ -102,16 +87,12 @@ void checkStoreName(const string & name)
if (name.length() > 211)
throw Error(baseError % "name must be less than 212 characters");
for (auto& i : name)
if (!((i >= 'A' && i <= 'Z') ||
(i >= 'a' && i <= 'z') ||
(i >= '0' && i <= '9') ||
validChars.find(i) != string::npos))
{
if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') ||
(i >= '0' && i <= '9') || validChars.find(i) != string::npos)) {
throw Error(baseError % (format("the '%1%' character is invalid") % i));
}
}
/* Store paths have the following form:
<store>/<h>-<name>
@ -182,44 +163,38 @@ void checkStoreName(const string & name)
"source:".
*/
Path Store::makeStorePath(const string & type,
const Hash & hash, const string & name) const
{
Path Store::makeStorePath(const string& type, const Hash& hash,
const string& name) const {
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
checkStoreName(name);
return storeDir + "/"
+ compressHash(hashString(htSHA256, s), 20).to_string(Base32, false)
+ "-" + name;
return storeDir + "/" +
compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) +
"-" + name;
}
Path Store::makeOutputPath(const string & id,
const Hash & hash, const string & name) const
{
Path Store::makeOutputPath(const string& id, const Hash& hash,
const string& name) const {
return makeStorePath("output:" + id, hash,
name + (id == "out" ? "" : "-" + id));
}
Path Store::makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const
{
Path Store::makeFixedOutputPath(bool recursive, const Hash& hash,
const string& name) const {
return hash.type == htSHA256 && recursive
? makeStorePath("source", hash, name)
: makeStorePath("output:out", hashString(htSHA256,
: makeStorePath(
"output:out",
hashString(htSHA256,
"fixed:out:" + (recursive ? (string) "r:" : "") +
hash.to_string(Base16) + ":"),
name);
}
Path Store::makeTextPath(const string& name, const Hash& hash,
const PathSet & references) const
{
const PathSet& references) const {
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
@ -232,38 +207,28 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
return makeStorePath(type, hash, name);
}
std::pair<Path, Hash> Store::computeStorePathForPath(const string& name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
{
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
const Path& srcPath,
bool recursive,
HashType hashAlgo,
PathFilter& filter) const {
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
Path dstPath = makeFixedOutputPath(recursive, h, name);
return std::pair<Path, Hash>(dstPath, h);
}
Path Store::computeStorePathForText(const string& name, const string& s,
const PathSet & references) const
{
const PathSet& references) const {
return makeTextPath(name, hashString(htSHA256, s), references);
}
Store::Store(const Params& params)
: Config(params)
, state({(size_t) pathInfoCacheSize})
{
}
: Config(params), state({(size_t)pathInfoCacheSize}) {}
std::string Store::getUri() { return ""; }
std::string Store::getUri()
{
return "";
}
bool Store::isValidPath(const Path & storePath)
{
bool Store::isValidPath(const Path& storePath) {
assertStorePath(storePath);
auto hashPart = storePathToHash(storePath);
@ -282,8 +247,8 @@ bool Store::isValidPath(const Path & storePath)
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
state_->pathInfoCache.upsert(
hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
return res.first == NarInfoDiskCache::oValid;
}
}
@ -297,11 +262,9 @@ bool Store::isValidPath(const Path & storePath)
return valid;
}
/* Default implementation for stores that only implement
queryPathInfoUncached(). */
bool Store::isValidPathUncached(const Path & path)
{
bool Store::isValidPathUncached(const Path& path) {
try {
queryPathInfo(path);
return true;
@ -310,13 +273,10 @@ bool Store::isValidPathUncached(const Path & path)
}
}
ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
{
ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) {
std::promise<ref<ValidPathInfo>> promise;
queryPathInfo(storePath,
{[&](std::future<ref<ValidPathInfo>> result) {
queryPathInfo(storePath, {[&](std::future<ref<ValidPathInfo>> result) {
try {
promise.set_value(result.get());
} catch (...) {
@ -327,10 +287,8 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
return promise.get_future().get();
}
void Store::queryPathInfo(const Path& storePath,
Callback<ref<ValidPathInfo>> callback) noexcept
{
Callback<ref<ValidPathInfo>> callback) noexcept {
std::string hashPart;
try {
@ -354,51 +312,53 @@ void Store::queryPathInfo(const Path & storePath,
stats.narInfoReadAverted++;
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
state_->pathInfoCache.upsert(
hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
if (res.first == NarInfoDiskCache::oInvalid ||
(res.second->path != storePath && storePathToName(storePath) != ""))
(res.second->path != storePath &&
storePathToName(storePath) != ""))
throw InvalidPath(format("path '%s' is not valid") % storePath);
}
return callback(ref<ValidPathInfo>(res.second));
}
}
} catch (...) { return callback.rethrow(); }
} catch (...) {
return callback.rethrow();
}
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) {
queryPathInfoUncached(
storePath, {[this, storePath, hashPart, callbackPtr](
std::future<std::shared_ptr<ValidPathInfo>> fut) {
try {
auto info = fut.get();
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, info);
if (diskCache) diskCache->upsertNarInfo(getUri(), hashPart, info);
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, info);
}
if (!info
|| (info->path != storePath && storePathToName(storePath) != ""))
{
if (!info ||
(info->path != storePath && storePathToName(storePath) != "")) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
}
(*callbackPtr)(ref<ValidPathInfo>(info));
} catch (...) { callbackPtr->rethrow(); }
} catch (...) {
callbackPtr->rethrow();
}
}});
}
PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
{
struct State
{
PathSet Store::queryValidPaths(const PathSet& paths,
SubstituteFlag maybeSubstitute) {
struct State {
size_t left;
PathSet valid;
std::exception_ptr exc;
@ -411,7 +371,8 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
auto doQuery = [&](const Path& path) {
checkInterrupt();
queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
queryPathInfo(
path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
auto state(state_.lock());
try {
auto info = fut.get();
@ -421,13 +382,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
state->exc = std::current_exception();
}
assert(state->left);
if (!--state->left)
wakeup.notify_one();
if (!--state->left) wakeup.notify_one();
}});
};
for (auto & path : paths)
pool.enqueue(std::bind(doQuery, path));
for (auto& path : paths) pool.enqueue(std::bind(doQuery, path));
pool.process();
@ -441,13 +400,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
}
}
/* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */
string Store::makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash)
{
string Store::makeValidityRegistration(const PathSet& paths, bool showDerivers,
bool showHash) {
string s = "";
for (auto& i : paths) {
@ -465,17 +422,15 @@ string Store::makeValidityRegistration(const PathSet & paths,
s += (format("%1%\n") % info->references.size()).str();
for (auto & j : info->references)
s += j + "\n";
for (auto& j : info->references) s += j + "\n";
}
return s;
}
void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths,
bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
{
bool includeImpureInfo, bool showClosureSize,
AllowInvalidFlag allowInvalid) {
auto jsonList = jsonOut.list();
for (auto storePath : storePaths) {
@ -486,18 +441,15 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
auto info = queryPathInfo(storePath);
storePath = info->path;
jsonPath
.attr("narHash", info->narHash.to_string())
jsonPath.attr("narHash", info->narHash.to_string())
.attr("narSize", info->narSize);
{
auto jsonRefs = jsonPath.list("references");
for (auto & ref : info->references)
jsonRefs.elem(ref);
for (auto& ref : info->references) jsonRefs.elem(ref);
}
if (info->ca != "")
jsonPath.attr("ca", info->ca);
if (info->ca != "") jsonPath.attr("ca", info->ca);
std::pair<uint64_t, uint64_t> closureSizes;
@ -507,28 +459,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
}
if (includeImpureInfo) {
if (info->deriver != "")
jsonPath.attr("deriver", info->deriver);
if (info->deriver != "") jsonPath.attr("deriver", info->deriver);
if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime);
if (info->ultimate)
jsonPath.attr("ultimate", info->ultimate);
if (info->ultimate) jsonPath.attr("ultimate", info->ultimate);
if (!info->sigs.empty()) {
auto jsonSigs = jsonPath.list("signatures");
for (auto & sig : info->sigs)
jsonSigs.elem(sig);
for (auto& sig : info->sigs) jsonSigs.elem(sig);
}
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
if (narInfo) {
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
if (narInfo->fileSize)
@ -544,9 +491,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
}
}
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
{
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) {
uint64_t totalNarSize = 0, totalDownloadSize = 0;
PathSet closure;
computeFSClosure(storePath, closure, false, false);
@ -555,15 +500,12 @@ std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
totalNarSize += info->narSize;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
if (narInfo)
totalDownloadSize += narInfo->fileSize;
if (narInfo) totalDownloadSize += narInfo->fileSize;
}
return {totalNarSize, totalDownloadSize};
}
const Store::Stats & Store::getStats()
{
const Store::Stats& Store::getStats() {
{
auto state_(state.lock());
stats.pathInfoCacheSize = state_->pathInfoCache.size();
@ -571,21 +513,16 @@ const Store::Stats & Store::getStats()
return stats;
}
void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
{
void Store::buildPaths(const PathSet& paths, BuildMode buildMode) {
for (auto& path : paths)
if (isDerivation(path))
unsupported("buildPaths");
if (isDerivation(path)) unsupported("buildPaths");
if (queryValidPaths(paths).size() != paths.size())
unsupported("buildPaths");
if (queryValidPaths(paths).size() != paths.size()) unsupported("buildPaths");
}
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
{
const Path& storePath, RepairFlag repair,
CheckSigsFlag checkSigs) {
auto srcUri = srcStore->getUri();
auto dstUri = dstStore->getUri();
@ -594,7 +531,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
? fmt("copying path '%s' to '%s'", storePath, dstUri)
: dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", storePath, srcUri)
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
: fmt("copying path '%s' from '%s' to '%s'", storePath,
srcUri, dstUri),
{storePath, srcUri, dstUri});
PushActivity pact(act.id);
@ -622,24 +560,26 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
info = info2;
}
auto source = sinkToSource([&](Sink & sink) {
auto source = sinkToSource(
[&](Sink& sink) {
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
sink(data, len);
total += len;
act.progress(total, info->narSize);
});
srcStore->narFromPath({storePath}, wrapperSink);
}, [&]() {
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
},
[&]() {
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete",
storePath, srcStore->getUri());
});
dstStore->addToStore(*info, *source, repair, checkSigs);
}
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const PathSet& storePaths, RepairFlag repair,
CheckSigsFlag checkSigs, SubstituteFlag substitute) {
PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
PathSet missing;
@ -648,7 +588,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
if (missing.empty()) return;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
Activity act(*logger, lvlInfo, actCopyPaths,
fmt("copying %d paths", missing.size()));
std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0};
@ -661,8 +602,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
ThreadPool pool;
processGraph<Path>(pool,
PathSet(missing.begin(), missing.end()),
processGraph<Path>(
pool, PathSet(missing.begin(), missing.end()),
[&](const Path& storePath) {
if (dstStore->isValidPath(storePath)) {
@ -689,9 +630,9 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
} catch (Error& e) {
nrFailed++;
if (!settings.keepGoing)
throw e;
logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
if (!settings.keepGoing) throw e;
logger->log(lvlError,
format("could not copy %s: %s") % storePath % e.what());
showProgress();
return;
}
@ -702,22 +643,21 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
});
}
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
const PathSet& storePaths, RepairFlag repair,
CheckSigsFlag checkSigs, SubstituteFlag substitute) {
PathSet closure;
srcStore->computeFSClosure({storePaths}, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
{
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) {
ValidPathInfo info;
getline(str, info.path);
if (str.eof()) { info.path = ""; return info; }
if (str.eof()) {
info.path = "";
return info;
}
if (hashGiven) {
string s;
getline(str, s);
@ -726,7 +666,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
if (!string2Int(s, info.narSize)) throw Error("number expected");
}
getline(str, info.deriver);
string s; int n;
string s;
int n;
getline(str, s);
if (!string2Int(s, n)) throw Error("number expected");
while (n--) {
@ -737,9 +678,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
return info;
}
string showPaths(const PathSet & paths)
{
string showPaths(const PathSet& paths) {
string s;
for (auto& i : paths) {
if (s.size() != 0) s += ", ";
@ -748,30 +687,24 @@ string showPaths(const PathSet & paths)
return s;
}
std::string ValidPathInfo::fingerprint() const
{
std::string ValidPathInfo::fingerprint() const {
if (narSize == 0 || !narHash)
throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known")
% path);
return
"1;" + path + ";"
+ narHash.to_string(Base32) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", references);
throw Error(format("cannot calculate fingerprint of path '%s' because its "
"size/hash is not known") %
path);
return "1;" + path + ";" + narHash.to_string(Base32) + ";" +
std::to_string(narSize) + ";" + concatStringsSep(",", references);
}
void ValidPathInfo::sign(const SecretKey & secretKey)
{
void ValidPathInfo::sign(const SecretKey& secretKey) {
sigs.insert(secretKey.signDetached(fingerprint()));
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
bool ValidPathInfo::isContentAddressed(const Store& store) const {
auto warn = [&]() {
printError(format("warning: path '%s' claims to be content-addressed but isn't") % path);
printError(
format("warning: path '%s' claims to be content-addressed but isn't") %
path);
};
if (hasPrefix(ca, "text:")) {
@ -786,7 +719,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6));
if (references.empty() &&
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) ==
path)
return true;
else
warn();
@ -795,70 +729,58 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
return false;
}
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
size_t ValidPathInfo::checkSignatures(const Store& store,
const PublicKeys& publicKeys) const {
if (isContentAddressed(store)) return maxSigs;
size_t good = 0;
for (auto& sig : sigs)
if (checkSignature(publicKeys, sig))
good++;
if (checkSignature(publicKeys, sig)) good++;
return good;
}
bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
{
bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys,
const std::string& sig) const {
return verifyDetached(fingerprint(), sig, publicKeys);
}
Strings ValidPathInfo::shortRefs() const
{
Strings ValidPathInfo::shortRefs() const {
Strings refs;
for (auto & r : references)
refs.push_back(baseNameOf(r));
for (auto& r : references) refs.push_back(baseNameOf(r));
return refs;
}
std::string makeFixedOutputCA(bool recursive, const Hash & hash)
{
std::string makeFixedOutputCA(bool recursive, const Hash& hash) {
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
}
void Store::addToStore(const ValidPathInfo& info, Source& narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
std::shared_ptr<FSAccessor> accessor) {
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs,
accessor);
}
void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor)
{
std::shared_ptr<FSAccessor> accessor) {
StringSource source(*nar);
addToStore(info, source, repair, checkSigs, accessor);
}
}
} // namespace nix
#include "local-store.hh"
#include "remote-store.hh"
namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
RegisterStoreImplementation::Implementations*
RegisterStoreImplementation::implementations = 0;
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
std::pair<std::string, Store::Params> splitUriAndParams(
const std::string& uri_) {
auto uri(uri_);
Store::Params params;
auto q = uri.find('?');
@ -890,8 +812,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
}
ref<Store> openStore(const std::string& uri_,
const Store::Params & extraParams)
{
const Store::Params& extraParams) {
auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end());
@ -907,9 +828,7 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri);
}
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
StoreType getStoreType(const std::string& uri, const std::string& stateDir) {
if (uri == "daemon") {
return tDaemon;
} else if (uri == "local" || hasPrefix(uri, "/")) {
@ -926,18 +845,15 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
}
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static RegisterStoreImplementation regStore([](const std::string& uri,
const Store::Params& params)
-> std::shared_ptr<Store> {
switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
case tDaemon:
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {
Store::Params params2 = params;
if (hasPrefix(uri, "/"))
params2["root"] = uri;
if (hasPrefix(uri, "/")) params2["root"] = uri;
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
}
default:
@ -945,9 +861,7 @@ static RegisterStoreImplementation regStore([](
}
});
std::list<ref<Store>> getDefaultSubstituters()
{
std::list<ref<Store>> getDefaultSubstituters() {
static auto stores([]() {
std::list<ref<Store>> stores;
@ -963,11 +877,9 @@ std::list<ref<Store>> getDefaultSubstituters()
}
};
for (auto uri : settings.substituters.get())
addStore(uri);
for (auto uri : settings.substituters.get()) addStore(uri);
for (auto uri : settings.extraSubstituters.get())
addStore(uri);
for (auto uri : settings.extraSubstituters.get()) addStore(uri);
stores.sort([](ref<Store>& a, ref<Store>& b) {
return a->getPriority() < b->getPriority();
@ -979,5 +891,4 @@ std::list<ref<Store>> getDefaultSubstituters()
return stores;
}
}
} // namespace nix

324
third_party/nix/src/libstore/store-api.hh vendored
View file

@ -1,32 +1,26 @@
#pragma once
#include "hash.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
#include <atomic>
#include <limits>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include "config.hh"
#include "crypto.hh"
#include "globals.hh"
#include "hash.hh"
#include "lru-cache.hh"
#include "serialise.hh"
#include "sync.hh"
namespace nix {
MakeError(SubstError, Error)
MakeError(BuildError, Error) /* denotes a permanent build failure */
MakeError(InvalidPath, Error)
MakeError(Unsupported, Error)
MakeError(SubstituteGone, Error)
MakeError(SubstituterDisabled, Error)
MakeError(InvalidPath, Error) MakeError(Unsupported, Error)
MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error)
struct BasicDerivation;
struct Derivation;
@ -35,25 +29,20 @@ class NarInfoDiskCache;
class Store;
class JSONPlaceholder;
enum RepairFlag : bool { NoRepair = false, Repair = true };
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
/* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
struct GCOptions
{
struct GCOptions {
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
@ -89,9 +78,7 @@ struct GCOptions
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
};
struct GCResults
{
struct GCResults {
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
PathSet paths;
@ -101,9 +88,7 @@ struct GCResults
unsigned long long bytesFreed = 0;
};
struct SubstitutablePathInfo
{
struct SubstitutablePathInfo {
Path deriver;
PathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
@ -112,9 +97,7 @@ struct SubstitutablePathInfo
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
struct ValidPathInfo {
Path path;
Path deriver;
Hash narHash;
@ -157,12 +140,8 @@ struct ValidPathInfo
*/
std::string ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
bool operator==(const ValidPathInfo& i) const {
return path == i.path && narHash == i.narHash && references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
@ -183,10 +162,12 @@ struct ValidPathInfo
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
size_t checkSignatures(const Store& store,
const PublicKeys& publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
bool checkSignature(const PublicKeys& publicKeys,
const std::string& sig) const;
Strings shortRefs() const;
@ -195,12 +176,9 @@ struct ValidPathInfo
typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck };
struct BuildResult
{
struct BuildResult {
/* Note: don't remove status codes, and only add new status codes
at the end of the list, to prevent client/server
incompatibilities in the nix-store --serve protocol. */
@ -239,25 +217,25 @@ struct BuildResult
}
};
class Store : public std::enable_shared_from_this<Store>, public Config
{
class Store : public std::enable_shared_from_this<Store>, public Config {
public:
typedef std::map<std::string, std::string> Params;
const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"};
const PathSetting storeDir_{this, false, settings.nixStore, "store",
"path to the Nix store"};
const Path storeDir = storeDir_;
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
const Setting<int> pathInfoCacheSize{
this, 65536, "path-info-cache-size",
"size of the in-memory store path information cache"};
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
const Setting<bool> isTrusted{
this, false, "trusted",
"whether paths from this store can be used as substitutes even when they "
"lack trusted signatures"};
protected:
struct State
{
struct State {
LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
};
@ -268,7 +246,6 @@ protected:
Store(const Params& params);
public:
virtual ~Store() {}
virtual std::string getUri() = 0;
@ -296,14 +273,14 @@ public:
Path followLinksToStorePath(const Path& path) const;
/* Constructs a unique store path name. */
Path makeStorePath(const string & type,
const Hash & hash, const string & name) const;
Path makeStorePath(const string& type, const Hash& hash,
const string& name) const;
Path makeOutputPath(const string & id,
const Hash & hash, const string & name) const;
Path makeOutputPath(const string& id, const Hash& hash,
const string& name) const;
Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const;
Path makeFixedOutputPath(bool recursive, const Hash& hash,
const string& name) const;
Path makeTextPath(const string& name, const Hash& hash,
const PathSet& references) const;
@ -311,9 +288,10 @@ public:
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const string & name,
const Path & srcPath, bool recursive = true,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
std::pair<Path, Hash> computeStorePathForPath(
const string& name, const Path& srcPath, bool recursive = true,
HashType hashAlgo = htSHA256,
PathFilter& filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore().
@ -336,23 +314,20 @@ public:
bool isValidPath(const Path& path);
protected:
virtual bool isValidPathUncached(const Path& path);
public:
/* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */
virtual PathSet queryValidPaths(const PathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute);
virtual PathSet queryValidPaths(
const PathSet& paths, SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash> rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
virtual PathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
virtual PathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); }
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
@ -363,16 +338,16 @@ public:
Callback<ref<ValidPathInfo>> callback) noexcept;
protected:
virtual void queryPathInfoUncached(const Path & path,
virtual void queryPathInfoUncached(
const Path& path,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
public:
/* Queries the set of incoming FS references for a store path.
The result is not cleared. */
virtual void queryReferrers(const Path & path, PathSet & referrers)
{ unsupported("queryReferrers"); }
virtual void queryReferrers(const Path& path, PathSet& referrers) {
unsupported("queryReferrers");
}
/* Return all currently valid derivations that have `path' as an
output. (Note that the result of `queryDeriver()' is the
@ -381,12 +356,14 @@ public:
virtual PathSet queryValidDerivers(const Path& path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */
virtual PathSet queryDerivationOutputs(const Path & path)
{ unsupported("queryDerivationOutputs"); }
virtual PathSet queryDerivationOutputs(const Path& path) {
unsupported("queryDerivationOutputs");
}
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path)
{ unsupported("queryDerivationOutputNames"); }
virtual StringSet queryDerivationOutputNames(const Path& path) {
unsupported("queryDerivationOutputNames");
}
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
@ -399,18 +376,23 @@ public:
sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting ‘infos’ map. */
virtual void querySubstitutablePathInfos(const PathSet& paths,
SubstitutablePathInfos & infos) { return; };
SubstitutablePathInfos& infos) {
return;
};
virtual bool wantMassQuery() { return false; }
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo& info, Source& narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0);
// FIXME: remove
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
virtual void addToStore(const ValidPathInfo& info,
const ref<std::string>& nar,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0);
/* Copy the contents of a path to the store and register the
@ -419,12 +401,14 @@ public:
libutil/archive.hh). */
virtual Path addToStore(const string& name, const Path& srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
PathFilter& filter = defaultPathFilter,
RepairFlag repair = NoRepair) = 0;
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual Path addTextToStore(const string& name, const string& s,
const PathSet & references, RepairFlag repair = NoRepair) = 0;
const PathSet& references,
RepairFlag repair = NoRepair) = 0;
/* Write a NAR dump of a store path. */
virtual void narFromPath(const Path& path, Sink& sink) = 0;
@ -442,7 +426,8 @@ public:
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). Note that ‘drvPath’ is only used for
informational purposes. */
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
virtual BuildResult buildDerivation(const Path& drvPath,
const BasicDerivation& drv,
BuildMode buildMode = bmNormal) = 0;
/* Ensure that a path is valid. If it is not currently valid, it
@ -452,16 +437,16 @@ public:
/* Add a store path as a temporary root of the garbage collector.
The root disappears as soon as we exit. */
virtual void addTempRoot(const Path & path)
{ unsupported("addTempRoot"); }
virtual void addTempRoot(const Path& path) { unsupported("addTempRoot"); }
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
virtual void addIndirectRoot(const Path & path)
{ unsupported("addIndirectRoot"); }
virtual void addIndirectRoot(const Path& path) {
unsupported("addIndirectRoot");
}
/* Acquire the global GC lock, then immediately release it. This
function must be called after registering a new permanent root,
@ -488,18 +473,18 @@ public:
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor)
{ unsupported("findRoots"); }
virtual Roots findRoots(bool censor) { unsupported("findRoots"); }
/* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results)
{ unsupported("collectGarbage"); }
virtual void collectGarbage(const GCOptions& options, GCResults& results) {
unsupported("collectGarbage");
}
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
string makeValidityRegistration(const PathSet& paths, bool showDerivers,
bool showHash);
/* Write a JSON representation of store path metadata, such as the
hash and the references. If ‘includeImpureInfo’ is true,
@ -521,16 +506,18 @@ public:
/* Check the integrity of the Nix store. Returns true if errors
remain. */
virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; };
virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) {
return false;
};
/* Return an object to access files in the Nix store. */
virtual ref<FSAccessor> getFSAccessor()
{ unsupported("getFSAccessor"); }
virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); }
/* Add signatures to the specified store path. The signatures are
not verified. */
virtual void addSignatures(const Path & storePath, const StringSet & sigs)
{ unsupported("addSignatures"); }
virtual void addSignatures(const Path& storePath, const StringSet& sigs) {
unsupported("addSignatures");
}
/* Utility functions. */
@ -545,20 +532,22 @@ public:
`storePath' is returned; that is, the closures under the
`referrers' relation instead of the `references' relation is
returned. */
virtual void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
virtual void computeFSClosure(const PathSet& paths, PathSet& out,
bool flipDirection = false,
bool includeOutputs = false,
bool includeDerivers = false);
void computeFSClosure(const Path & path,
PathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
void computeFSClosure(const Path& path, PathSet& out,
bool flipDirection = false, bool includeOutputs = false,
bool includeDerivers = false);
/* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths
that will be substituted. */
virtual void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize);
virtual void queryMissing(const PathSet& targets, PathSet& willBuild,
PathSet& willSubstitute, PathSet& unknown,
unsigned long long& downloadSize,
unsigned long long& narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
@ -577,8 +566,7 @@ public:
Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
struct Stats
{
struct Stats {
std::atomic<uint64_t> narInfoRead{0};
std::atomic<uint64_t> narInfoReadAverted{0};
std::atomic<uint64_t> narInfoMissing{0};
@ -598,67 +586,55 @@ public:
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
{ return nullptr; }
virtual std::shared_ptr<std::string> getBuildLog(const Path& path) {
return nullptr;
}
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
void clearPathInfoCache()
{
state.lock()->pathInfoCache.clear();
}
void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); }
/* Establish a connection to the store, for store types that have
a notion of connection. Otherwise this is a no-op. */
virtual void connect(){};
/* Get the protocol version of this store or it's connection. */
virtual unsigned int getProtocol()
{
return 0;
};
virtual unsigned int getProtocol() { return 0; };
/* Get the priority of the store, used to order substituters. In
particular, binary caches can specify a priority field in their
"nix-cache-info" file. Lower value means higher priority. */
virtual int getPriority() { return 0; }
virtual Path toRealPath(const Path & storePath)
{
return storePath;
}
virtual Path toRealPath(const Path& storePath) { return storePath; }
virtual void createUser(const std::string & userName, uid_t userId)
{ }
virtual void createUser(const std::string& userName, uid_t userId) {}
protected:
Stats stats;
/* Unsupported methods. */
[[noreturn]] void unsupported(const std::string & op)
{
throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri());
[[noreturn]] void unsupported(const std::string& op) {
throw Unsupported("operation '%s' is not supported by store '%s'", op,
getUri());
}
};
class LocalFSStore : public virtual Store
{
class LocalFSStore : public virtual Store {
public:
// FIXME: the (Store*) cast works around a bug in gcc that causes
// it to emit the call to the Option constructor. Clang works fine
// either way.
const PathSetting rootDir{(Store*) this, true, "",
"root", "directory prefixed to all other paths"};
const PathSetting stateDir{(Store*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"};
const PathSetting logDir{(Store*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"};
const PathSetting rootDir{(Store*)this, true, "", "root",
"directory prefixed to all other paths"};
const PathSetting stateDir{
(Store*)this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state",
"directory where Nix will store state"};
const PathSetting logDir{
(Store*)this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log",
"directory where Nix will store state"};
const static string drvsLogDir;
@ -668,21 +644,20 @@ public:
ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */
Path addPermRoot(const Path & storePath,
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
Path addPermRoot(const Path& storePath, const Path& gcRoot, bool indirect,
bool allowOutsideRootsDir = false);
virtual Path getRealStoreDir() { return storeDir; }
Path toRealPath(const Path & storePath) override
{
Path toRealPath(const Path& storePath) override {
assert(isInStore(storePath));
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
return getRealStoreDir() + "/" +
std::string(storePath, storeDir.size() + 1);
}
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
};
/* Extract the name part of the given store path. */
string storePathToName(const Path& path);
@ -694,37 +669,32 @@ string storePathToHash(const Path & path);
a dot. */
void checkStoreName(const string& name);
/* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
const Path& storePath, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
/* Copy store paths from one store to another. The paths may be copied
in parallel. They are copied in a topologically sorted order
(i.e. if A is a reference of B, then A is copied before B), but
the set of store paths is not automatically closed; use
copyClosure() for that. */
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
RepairFlag repair = NoRepair,
void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
const PathSet& storePaths, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Copy the closure of the specified paths from one store to another. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths,
RepairFlag repair = NoRepair,
const PathSet& storePaths, RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered
as a (permanent) root. */
void removeTempRoots();
/* Return a Store object to access the Nix store denoted by
‘uri’ (slight misnomer...). Supported values are:
@ -757,13 +727,7 @@ void removeTempRoots();
ref<Store> openStore(const std::string& uri = settings.storeUri.get(),
const Store::Params& extraParams = Store::Params());
enum StoreType {
tDaemon,
tLocal,
tOther
};
enum StoreType { tDaemon, tLocal, tOther };
StoreType getStoreType(const std::string& uri = settings.storeUri.get(),
const std::string& stateDir = settings.nixStateDir);
@ -772,40 +736,32 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
‘substituters’ option and various legacy options. */
std::list<ref<Store>> getDefaultSubstituters();
/* Store implementation registration. */
typedef std::function<std::shared_ptr<Store>(
const std::string & uri, const Store::Params & params)> OpenStore;
typedef std::function<std::shared_ptr<Store>(const std::string& uri,
const Store::Params& params)>
OpenStore;
struct RegisterStoreImplementation
{
struct RegisterStoreImplementation {
typedef std::vector<OpenStore> Implementations;
static Implementations* implementations;
RegisterStoreImplementation(OpenStore fun)
{
RegisterStoreImplementation(OpenStore fun) {
if (!implementations) implementations = new Implementations;
implementations->push_back(fun);
}
};
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
string showPaths(const PathSet& paths);
ValidPathInfo decodeValidPathInfo(std::istream & str,
bool hashGiven = false);
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven = false);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash& hash);
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri);
}
} // namespace nix

10
third_party/nix/src/libstore/worker-protocol.hh vendored
View file

@ -2,7 +2,6 @@
namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
@ -10,7 +9,6 @@ namespace nix {
#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
typedef enum {
wopIsValidPath = 1,
wopHasSubstitutes = 3,
@ -51,7 +49,6 @@ typedef enum {
wopQueryMissing = 40,
} WorkerOp;
#define STDERR_NEXT 0x6f6c6d67
#define STDERR_READ 0x64617461 // data needed from source
#define STDERR_WRITE 0x64617416 // data for sink
@ -61,9 +58,8 @@ typedef enum {
#define STDERR_STOP_ACTIVITY 0x53544f50
#define STDERR_RESULT 0x52534c54
Path readStorePath(Store& store, Source& from);
template<class T> T readStorePaths(Store & store, Source & from);
template <class T>
T readStorePaths(Store& store, Source& from);
}
} // namespace nix

18
third_party/nix/src/libutil/affinity.cc vendored
View file

@ -1,6 +1,6 @@
#include "affinity.hh"
#include "types.hh"
#include "util.hh"
#include "affinity.hh"
#if __linux__
#include <sched.h>
@ -8,15 +8,12 @@
namespace nix {
#if __linux__
static bool didSaveAffinity = false;
static cpu_set_t savedAffinity;
#endif
void setAffinityTo(int cpu)
{
void setAffinityTo(int cpu) {
#if __linux__
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true;
@ -29,9 +26,7 @@ void setAffinityTo(int cpu)
#endif
}
int lockToCurrentCPU()
{
int lockToCurrentCPU() {
#if __linux__
int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu);
@ -41,9 +36,7 @@ int lockToCurrentCPU()
#endif
}
void restoreAffinity()
{
void restoreAffinity() {
#if __linux__
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
@ -51,5 +44,4 @@ void restoreAffinity()
#endif
}
}
} // namespace nix

2
third_party/nix/src/libutil/affinity.hh vendored
View file

@ -6,4 +6,4 @@ void setAffinityTo(int cpu);
int lockToCurrentCPU();
void restoreAffinity();
}
} // namespace nix

158
third_party/nix/src/libutil/archive.cc vendored
View file

@ -1,32 +1,31 @@
#include <cerrno>
#include <algorithm>
#include <vector>
#include <map>
#include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "archive.hh"
#include <dirent.h>
#include <fcntl.h>
#include "archive.hh"
#include "util.hh"
#include <strings.h> // for strcasecmp
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <cerrno>
#include <map>
#include <vector>
#include "config.hh"
#include "util.hh"
namespace nix {
struct ArchiveSettings : Config
{
Setting<bool> useCaseHack{this,
struct ArchiveSettings : Config {
Setting<bool> useCaseHack {
this,
#if __APPLE__
true,
#else
false,
#endif
"use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."};
"Whether to enable a Darwin-specific hack for dealing with file name "
"collisions."
};
};
static ArchiveSettings archiveSettings;
@ -39,10 +38,7 @@ static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path&) { return true; };
static void dumpContents(const Path & path, size_t size,
Sink & sink)
{
static void dumpContents(const Path& path, size_t size, Sink& sink) {
sink << "contents" << size;
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
@ -61,9 +57,7 @@ static void dumpContents(const Path & path, size_t size,
writePadding(size, sink);
}
static void dump(const Path & path, Sink & sink, PathFilter & filter)
{
static void dump(const Path& path, Sink& sink, PathFilter& filter) {
checkInterrupt();
struct stat st;
@ -73,14 +67,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "(";
if (S_ISREG(st.st_mode)) {
sink << "type" << "regular";
sink << "type"
<< "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
sink << "executable"
<< "";
dumpContents(path, (size_t)st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {
sink << "type" << "directory";
sink << "type"
<< "directory";
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
@ -90,52 +87,54 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
string name(i.name);
size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) {
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name));
debug(format("removing case hack suffix from '%1%'") %
(path + "/" + i.name));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'")
% (path + "/" + unhacked[name]) % (path + "/" + i.name));
throw Error(format("file name collision in between '%1%' and '%2%'") %
(path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
for (auto& i : unhacked)
if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node";
sink << "entry"
<< "("
<< "name" << i.first << "node";
dump(path + "/" + i.second, sink, filter);
sink << ")";
}
}
else if (S_ISLNK(st.st_mode))
sink << "type" << "symlink" << "target" << readLink(path);
sink << "type"
<< "symlink"
<< "target" << readLink(path);
else throw Error(format("file '%1%' has an unsupported type") % path);
else
throw Error(format("file '%1%' has an unsupported type") % path);
sink << ")";
}
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
void dumpPath(const Path& path, Sink& sink, PathFilter& filter) {
sink << narVersionMagic1;
dump(path, sink, filter);
}
void dumpString(const std::string & s, Sink & sink)
{
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
void dumpString(const std::string& s, Sink& sink) {
sink << narVersionMagic1 << "("
<< "type"
<< "regular"
<< "contents" << s << ")";
}
static SerialisationError badArchive(string s)
{
static SerialisationError badArchive(string s) {
return SerialisationError("bad archive: " + s);
}
#if 0
static void skipGeneric(Source & source)
{
@ -146,9 +145,7 @@ static void skipGeneric(Source & source)
}
#endif
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
static void parseContents(ParseSink& sink, Source& source, const Path& path) {
unsigned long long size = readLongLong(source);
sink.preallocateContents(size);
@ -168,18 +165,13 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
readPadding(size, source);
}
struct CaseInsensitiveCompare
{
bool operator() (const string & a, const string & b) const
{
struct CaseInsensitiveCompare {
bool operator()(const string& a, const string& b) const {
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};
static void parse(ParseSink & sink, Source & source, const Path & path)
{
static void parse(ParseSink& sink, Source& source, const Path& path) {
string s;
s = readString(source);
@ -199,8 +191,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
else if (s == "type") {
if (type != tpUnknown)
throw badArchive("multiple type fields");
if (type != tpUnknown) throw badArchive("multiple type fields");
string t = readString(source);
if (t == "regular") {
@ -217,7 +208,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
type = tpSymlink;
}
else throw badArchive("unknown file type " + t);
else
throw badArchive("unknown file type " + t);
}
@ -246,15 +238,17 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
if (name.empty() || name == "." || name == ".." ||
name.find('/') != string::npos ||
name.find((char)0) != string::npos)
throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
if (name <= prevName) throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") % i->first % name);
debug(format("case collision between '%1%' and '%2%'") %
i->first % name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
@ -278,9 +272,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
}
void parseDump(ParseSink & sink, Source & source)
{
void parseDump(ParseSink& sink, Source& source) {
string version;
try {
version = readString(source, narVersionMagic1.size());
@ -293,37 +285,30 @@ void parseDump(ParseSink & sink, Source & source)
parse(sink, source, "");
}
struct RestoreSink : ParseSink
{
struct RestoreSink : ParseSink {
Path dstPath;
AutoCloseFD fd;
void createDirectory(const Path & path)
{
void createDirectory(const Path& path) {
Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1)
throw SysError(format("creating directory '%1%'") % p);
};
void createRegularFile(const Path & path)
{
void createRegularFile(const Path& path) {
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError(format("creating file '%1%'") % p);
}
void isExecutable()
{
void isExecutable() {
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("fstat");
if (fstat(fd.get(), &st) == -1) throw SysError("fstat");
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
throw SysError("fchmod");
}
void preallocateContents(unsigned long long len)
{
void preallocateContents(unsigned long long len) {
#if HAVE_POSIX_FALLOCATE
if (len) {
errno = posix_fallocate(fd.get(), 0, len);
@ -337,29 +322,23 @@ struct RestoreSink : ParseSink
#endif
}
void receiveContents(unsigned char * data, unsigned int len)
{
void receiveContents(unsigned char* data, unsigned int len) {
writeFull(fd.get(), data, len);
}
void createSymlink(const Path & path, const string & target)
{
void createSymlink(const Path& path, const string& target) {
Path p = dstPath + path;
nix::createSymlink(target, p);
}
};
void restorePath(const Path & path, Source & source)
{
void restorePath(const Path& path, Source& source) {
RestoreSink sink;
sink.dstPath = path;
parseDump(sink, source);
}
void copyNAR(Source & source, Sink & sink)
{
void copyNAR(Source& source, Sink& sink) {
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
@ -374,5 +353,4 @@ void copyNAR(Source & source, Sink & sink)
parseDump(parseSink, wrapper);
}
}
} // namespace nix

15
third_party/nix/src/libutil/archive.hh vendored
View file

@ -1,12 +1,10 @@
#pragma once
#include "types.hh"
#include "serialise.hh"
#include "types.hh"
namespace nix {
/* dumpPath creates a Nix archive of the specified path. The format
is as follows:
@ -44,15 +42,13 @@ namespace nix {
`+' denotes string concatenation. */
void dumpPath(const Path& path, Sink& sink,
PathFilter& filter = defaultPathFilter);
void dumpString(const std::string& s, Sink& sink);
/* FIXME: fix this API, it sucks. */
struct ParseSink
{
struct ParseSink {
virtual void createDirectory(const Path& path){};
virtual void createRegularFile(const Path& path){};
@ -63,8 +59,7 @@ struct ParseSink
virtual void createSymlink(const Path& path, const string& target){};
};
struct TeeSink : ParseSink
{
struct TeeSink : ParseSink {
TeeSource source;
TeeSink(Source& source) : source(source) {}
@ -77,8 +72,6 @@ void restorePath(const Path & path, Source & source);
/* Read a NAR from 'source' and write it to 'sink'. */
void copyNAR(Source& source, Sink& sink);
extern const std::string narVersionMagic1;
}
} // namespace nix

81
third_party/nix/src/libutil/args.cc vendored
View file

@ -3,34 +3,30 @@
namespace nix {
Args::FlagMaker Args::mkFlag()
{
return FlagMaker(*this);
}
Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); }
Args::FlagMaker::~FlagMaker()
{
Args::FlagMaker::~FlagMaker() {
assert(flag->longName != "");
args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
}
void Args::parseCmdline(const Strings & _cmdline)
{
void Args::parseCmdline(const Strings& _cmdline) {
Strings pendingArgs;
bool dashDash = false;
Strings cmdline(_cmdline);
for (auto pos = cmdline.begin(); pos != cmdline.end();) {
auto arg = *pos;
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) {
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' &&
isalpha(arg[1])) {
*pos = (string) "-" + arg[1];
auto next = pos; ++next;
auto next = pos;
++next;
for (unsigned int j = 2; j < arg.length(); j++)
if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]);
@ -44,23 +40,19 @@ void Args::parseCmdline(const Strings & _cmdline)
if (!dashDash && arg == "--") {
dashDash = true;
++pos;
}
else if (!dashDash && std::string(arg, 0, 1) == "-") {
} else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg);
}
else {
} else {
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false))
pendingArgs.clear();
if (processArgs(pendingArgs, false)) pendingArgs.clear();
}
}
processArgs(pendingArgs, true);
}
void Args::printHelp(const string & programName, std::ostream & out)
{
void Args::printHelp(const string& programName, std::ostream& out) {
std::cout << "Usage: " << programName << " <FLAGS>...";
for (auto& exp : expectedArgs) {
std::cout << renderLabels({exp.label});
@ -71,8 +63,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
std::cout << "\n";
auto s = description();
if (s != "")
std::cout << "\nSummary: " << s << ".\n";
if (s != "") std::cout << "\nSummary: " << s << ".\n";
if (longFlags.size()) {
std::cout << "\n";
@ -81,21 +72,21 @@ void Args::printHelp(const string & programName, std::ostream & out)
}
}
void Args::printFlags(std::ostream & out)
{
void Args::printFlags(std::ostream& out) {
Table2 table;
for (auto& flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair(
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
+ "--" + flag.first + renderLabels(flag.second->labels),
(flag.second->shortName
? std::string("-") + flag.second->shortName + ", "
: " ") +
"--" + flag.first + renderLabels(flag.second->labels),
flag.second->description));
}
printTable(out, table);
}
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
{
bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) {
assert(pos != end);
auto process = [&](const std::string& name, const Flag& flag) -> bool {
@ -104,8 +95,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
for (size_t n = 0; n < flag.arity; ++n) {
if (pos == end) {
if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)")
% name % flag.arity);
throw UsageError(format("flag '%1%' requires %2% argument(s)") % name %
flag.arity);
}
args.push_back(*pos++);
}
@ -129,8 +120,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
return false;
}
bool Args::processArgs(const Strings & args, bool finish)
{
bool Args::processArgs(const Strings& args, bool finish) {
if (expectedArgs.empty()) {
if (!args.empty())
throw UsageError(format("unexpected argument '%1%'") % args.front());
@ -142,8 +132,7 @@ bool Args::processArgs(const Strings & args, bool finish)
bool res = false;
if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity))
{
(exp.arity > 0 && args.size() == exp.arity)) {
std::vector<std::string> ss;
for (auto& s : args) ss.push_back(s);
exp.handler(std::move(ss));
@ -157,29 +146,26 @@ bool Args::processArgs(const Strings & args, bool finish)
return res;
}
Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
{
Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) {
arity(1);
label("type");
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
handler([ht](std::string s) {
*ht = parseHashType(s);
if (*ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s);
});
return *this;
}
Strings argvToStrings(int argc, char * * argv)
{
Strings argvToStrings(int argc, char** argv) {
Strings args;
argc--; argv++;
argc--;
argv++;
while (argc--) args.push_back(*argv++);
return args;
}
std::string renderLabels(const Strings & labels)
{
std::string renderLabels(const Strings& labels) {
std::string res;
for (auto label : labels) {
for (auto& c : label) c = std::toupper(c);
@ -188,16 +174,13 @@ std::string renderLabels(const Strings & labels)
return res;
}
void printTable(std::ostream & out, const Table2 & table)
{
void printTable(std::ostream& out, const Table2& table) {
size_t max = 0;
for (auto & row : table)
max = std::max(max, row.first.size());
for (auto& row : table) max = std::max(max, row.first.size());
for (auto& row : table) {
out << " " << row.first
<< std::string(max - row.first.size() + 2, ' ')
out << " " << row.first << std::string(max - row.first.size() + 2, ' ')
<< row.second << "\n";
}
}
}
} // namespace nix

123
third_party/nix/src/libutil/args.hh vendored
View file

@ -3,7 +3,6 @@
#include <iostream>
#include <map>
#include <memory>
#include "util.hh"
namespace nix {
@ -12,10 +11,8 @@ MakeError(UsageError, Error);
enum HashType : char;
class Args
{
class Args {
public:
/* Parse the command line, throwing a UsageError if something goes
wrong. */
void parseCmdline(const Strings& cmdline);
@ -25,12 +22,10 @@ public:
virtual std::string description() { return ""; }
protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max();
/* Flags. */
struct Flag
{
struct Flag {
typedef std::shared_ptr<Flag> ptr;
std::string longName;
char shortName = 0;
@ -49,8 +44,7 @@ protected:
virtual void printFlags(std::ostream& out);
/* Positional arguments. */
struct ExpectedArg
{
struct ExpectedArg {
std::string label;
size_t arity; // 0 = any
bool optional;
@ -64,41 +58,69 @@ protected:
std::set<std::string> hiddenCategories;
public:
class FlagMaker
{
class FlagMaker {
Args& args;
Flag::ptr flag;
friend class Args;
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
public:
~FlagMaker();
FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
FlagMaker & handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
FlagMaker& longName(const std::string& s) {
flag->longName = s;
return *this;
};
FlagMaker& shortName(char s) {
flag->shortName = s;
return *this;
};
FlagMaker& description(const std::string& s) {
flag->description = s;
return *this;
};
FlagMaker& label(const std::string& l) {
flag->arity = 1;
flag->labels = {l};
return *this;
};
FlagMaker& labels(const Strings& ls) {
flag->arity = ls.size();
flag->labels = ls;
return *this;
};
FlagMaker& arity(size_t arity) {
flag->arity = arity;
return *this;
};
FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) {
flag->handler = handler;
return *this;
};
FlagMaker& handler(std::function<void()> handler) {
flag->handler = [handler](std::vector<std::string>) { handler(); };
return *this;
};
FlagMaker& handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) {
handler(std::move(ss[0]));
};
return *this;
};
FlagMaker& category(const std::string& s) {
flag->category = s;
return *this;
};
FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
template <class T>
FlagMaker & dest(T * dest)
{
FlagMaker& dest(T* dest) {
flag->arity = 1;
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this;
};
template <class T>
FlagMaker & set(T * dest, const T & val)
{
FlagMaker& set(T* dest, const T& val) {
flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
return *this;
@ -114,8 +136,7 @@ public:
void mkFlag1(char shortName, const std::string& longName,
const std::string& label, const std::string& description,
std::function<void(std::string)> fun)
{
std::function<void(std::string)> fun) {
mkFlag()
.shortName(shortName)
.longName(longName)
@ -126,15 +147,13 @@ public:
}
void mkFlag(char shortName, const std::string& name,
const std::string & description, bool * dest)
{
const std::string& description, bool* dest) {
mkFlag(shortName, name, description, dest, true);
}
template <class T>
void mkFlag(char shortName, const std::string & longName, const std::string & description,
T * dest, const T & value)
{
void mkFlag(char shortName, const std::string& longName,
const std::string& description, T* dest, const T& value) {
mkFlag()
.shortName(shortName)
.longName(longName)
@ -144,17 +163,13 @@ public:
template <class I>
void mkIntFlag(char shortName, const std::string& longName,
const std::string & description, I * dest)
{
mkFlag<I>(shortName, longName, description, [=](I n) {
*dest = n;
});
const std::string& description, I* dest) {
mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; });
}
template <class I>
void mkFlag(char shortName, const std::string& longName,
const std::string & description, std::function<void(I)> fun)
{
const std::string& description, std::function<void(I)> fun) {
mkFlag()
.shortName(shortName)
.longName(longName)
@ -164,25 +179,25 @@ public:
.handler([=](std::vector<std::string> ss) {
I n;
if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument", longName);
throw UsageError("flag '--%s' requires a integer argument",
longName);
fun(n);
});
}
/* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false)
{
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
*dest = ss[0];
}});
void expectArg(const std::string& label, string* dest,
bool optional = false) {
expectedArgs.push_back(
ExpectedArg{label, 1, optional,
[=](std::vector<std::string> ss) { *dest = ss[0]; }});
}
/* Expect 0 or more arguments. */
void expectArgs(const std::string & label, std::vector<std::string> * dest)
{
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
*dest = std::move(ss);
}});
void expectArgs(const std::string& label, std::vector<std::string>* dest) {
expectedArgs.push_back(ExpectedArg{
label, 0, false,
[=](std::vector<std::string> ss) { *dest = std::move(ss); }});
}
friend class MultiCommand;
@ -198,4 +213,4 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream& out, const Table2& table);
}
} // namespace nix

Some files were not shown because too many files have changed in this diff Show more