style(3p/nix): Reformat project in Google C++ style
Reformatted with:
fd . -e hh -e cc | xargs clang-format -i
This commit is contained in:
parent
65a1aae98c
commit
0f2cf531f7
175 changed files with 32126 additions and 34689 deletions
103
third_party/nix/src/build-remote/build-remote.cc
vendored
103
third_party/nix/src/build-remote/build-remote.cc
vendored
|
|
@ -1,41 +1,39 @@
|
||||||
|
#include <algorithm>
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <algorithm>
|
|
||||||
#include <set>
|
|
||||||
#include <memory>
|
|
||||||
#include <tuple>
|
|
||||||
#include <iomanip>
|
#include <iomanip>
|
||||||
|
#include <memory>
|
||||||
|
#include <set>
|
||||||
|
#include <tuple>
|
||||||
#if __APPLE__
|
#if __APPLE__
|
||||||
#include <sys/time.h>
|
#include <sys/time.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include "machines.hh"
|
|
||||||
#include "shared.hh"
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "serialise.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
#include "local-store.hh"
|
#include "globals.hh"
|
||||||
#include "legacy.hh"
|
#include "legacy.hh"
|
||||||
|
#include "local-store.hh"
|
||||||
|
#include "machines.hh"
|
||||||
|
#include "pathlocks.hh"
|
||||||
|
#include "serialise.hh"
|
||||||
|
#include "shared.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
using std::cin;
|
using std::cin;
|
||||||
|
|
||||||
static void handleAlarm(int sig) {
|
static void handleAlarm(int sig) {}
|
||||||
}
|
|
||||||
|
|
||||||
std::string escapeUri(std::string uri)
|
std::string escapeUri(std::string uri) {
|
||||||
{
|
|
||||||
std::replace(uri.begin(), uri.end(), '/', '_');
|
std::replace(uri.begin(), uri.end(), '/', '_');
|
||||||
return uri;
|
return uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
static string currentLoad;
|
static string currentLoad;
|
||||||
|
|
||||||
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
|
static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) {
|
||||||
{
|
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot),
|
||||||
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
|
true);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
|
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
|
||||||
|
|
@ -44,8 +42,7 @@ static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int _main(int argc, char * * argv)
|
static int _main(int argc, char** argv) {
|
||||||
{
|
|
||||||
{
|
{
|
||||||
logger = makeJSONLogger(*logger);
|
logger = makeJSONLogger(*logger);
|
||||||
|
|
||||||
|
|
@ -53,8 +50,7 @@ static int _main(int argc, char * * argv)
|
||||||
unsetenv("DISPLAY");
|
unsetenv("DISPLAY");
|
||||||
unsetenv("SSH_ASKPASS");
|
unsetenv("SSH_ASKPASS");
|
||||||
|
|
||||||
if (argc != 2)
|
if (argc != 2) throw UsageError("called without required arguments");
|
||||||
throw UsageError("called without required arguments");
|
|
||||||
|
|
||||||
verbosity = (Verbosity)std::stoll(argv[1]);
|
verbosity = (Verbosity)std::stoll(argv[1]);
|
||||||
|
|
||||||
|
|
@ -92,21 +88,23 @@ static int _main(int argc, char * * argv)
|
||||||
string storeUri;
|
string storeUri;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto s = readString(source);
|
auto s = readString(source);
|
||||||
if (s != "try") return 0;
|
if (s != "try") return 0;
|
||||||
} catch (EndOfFile &) { return 0; }
|
} catch (EndOfFile&) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
auto amWilling = readInt(source);
|
auto amWilling = readInt(source);
|
||||||
auto neededSystem = readString(source);
|
auto neededSystem = readString(source);
|
||||||
source >> drvPath;
|
source >> drvPath;
|
||||||
auto requiredFeatures = readStrings<std::set<std::string>>(source);
|
auto requiredFeatures = readStrings<std::set<std::string>>(source);
|
||||||
|
|
||||||
auto canBuildLocally = amWilling
|
auto canBuildLocally =
|
||||||
&& ( neededSystem == settings.thisSystem
|
amWilling &&
|
||||||
|| settings.extraPlatforms.get().count(neededSystem) > 0)
|
(neededSystem == settings.thisSystem ||
|
||||||
&& allSupportedLocally(requiredFeatures);
|
settings.extraPlatforms.get().count(neededSystem) > 0) &&
|
||||||
|
allSupportedLocally(requiredFeatures);
|
||||||
|
|
||||||
/* Error ignored here, will be caught later */
|
/* Error ignored here, will be caught later */
|
||||||
mkdir(currentLoad.c_str(), 0777);
|
mkdir(currentLoad.c_str(), 0777);
|
||||||
|
|
@ -123,8 +121,8 @@ static int _main(int argc, char * * argv)
|
||||||
for (auto& m : machines) {
|
for (auto& m : machines) {
|
||||||
debug("considering building on remote machine '%s'", m.storeUri);
|
debug("considering building on remote machine '%s'", m.storeUri);
|
||||||
|
|
||||||
if (m.enabled && std::find(m.systemTypes.begin(),
|
if (m.enabled &&
|
||||||
m.systemTypes.end(),
|
std::find(m.systemTypes.begin(), m.systemTypes.end(),
|
||||||
neededSystem) != m.systemTypes.end() &&
|
neededSystem) != m.systemTypes.end() &&
|
||||||
m.allSupported(requiredFeatures) &&
|
m.allSupported(requiredFeatures) &&
|
||||||
m.mandatoryMet(requiredFeatures)) {
|
m.mandatoryMet(requiredFeatures)) {
|
||||||
|
|
@ -147,9 +145,11 @@ static int _main(int argc, char * * argv)
|
||||||
bool best = false;
|
bool best = false;
|
||||||
if (!bestSlotLock) {
|
if (!bestSlotLock) {
|
||||||
best = true;
|
best = true;
|
||||||
} else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) {
|
} else if (load / m.speedFactor <
|
||||||
|
bestLoad / bestMachine->speedFactor) {
|
||||||
best = true;
|
best = true;
|
||||||
} else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) {
|
} else if (load / m.speedFactor ==
|
||||||
|
bestLoad / bestMachine->speedFactor) {
|
||||||
if (m.speedFactor > bestMachine->speedFactor) {
|
if (m.speedFactor > bestMachine->speedFactor) {
|
||||||
best = true;
|
best = true;
|
||||||
} else if (m.speedFactor == bestMachine->speedFactor) {
|
} else if (m.speedFactor == bestMachine->speedFactor) {
|
||||||
|
|
@ -183,8 +183,8 @@ static int _main(int argc, char * * argv)
|
||||||
lock = -1;
|
lock = -1;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
|
fmt("connecting to '%s'", bestMachine->storeUri));
|
||||||
|
|
||||||
Store::Params storeParams;
|
Store::Params storeParams;
|
||||||
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
|
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
|
||||||
|
|
@ -200,9 +200,8 @@ static int _main(int argc, char * * argv)
|
||||||
|
|
||||||
} catch (std::exception& e) {
|
} catch (std::exception& e) {
|
||||||
auto msg = chomp(drainFD(5, false));
|
auto msg = chomp(drainFD(5, false));
|
||||||
printError("cannot build on '%s': %s%s",
|
printError("cannot build on '%s': %s%s", bestMachine->storeUri,
|
||||||
bestMachine->storeUri, e.what(),
|
e.what(), (msg.empty() ? "" : ": " + msg));
|
||||||
(msg.empty() ? "" : ": " + msg));
|
|
||||||
bestMachine->enabled = false;
|
bestMachine->enabled = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -219,44 +218,54 @@ connected:
|
||||||
auto inputs = readStrings<PathSet>(source);
|
auto inputs = readStrings<PathSet>(source);
|
||||||
auto outputs = readStrings<PathSet>(source);
|
auto outputs = readStrings<PathSet>(source);
|
||||||
|
|
||||||
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
|
AutoCloseFD uploadLock = openLockFile(
|
||||||
|
currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
|
||||||
|
|
||||||
{
|
{
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
|
fmt("waiting for the upload lock to '%s'", storeUri));
|
||||||
|
|
||||||
auto old = signal(SIGALRM, handleAlarm);
|
auto old = signal(SIGALRM, handleAlarm);
|
||||||
alarm(15 * 60);
|
alarm(15 * 60);
|
||||||
if (!lockFile(uploadLock.get(), ltWrite, true))
|
if (!lockFile(uploadLock.get(), ltWrite, true))
|
||||||
printError("somebody is hogging the upload lock for '%s', continuing...");
|
printError(
|
||||||
|
"somebody is hogging the upload lock for '%s', continuing...");
|
||||||
alarm(0);
|
alarm(0);
|
||||||
signal(SIGALRM, old);
|
signal(SIGALRM, old);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
|
auto substitute =
|
||||||
|
settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
|
||||||
|
|
||||||
{
|
{
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
|
fmt("copying dependencies to '%s'", storeUri));
|
||||||
|
copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs,
|
||||||
|
substitute);
|
||||||
}
|
}
|
||||||
|
|
||||||
uploadLock = -1;
|
uploadLock = -1;
|
||||||
|
|
||||||
BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
|
BasicDerivation drv(
|
||||||
|
readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
|
||||||
drv.inputSrcs = inputs;
|
drv.inputSrcs = inputs;
|
||||||
|
|
||||||
auto result = sshStore->buildDerivation(drvPath, drv);
|
auto result = sshStore->buildDerivation(drvPath, drv);
|
||||||
|
|
||||||
if (!result.success())
|
if (!result.success())
|
||||||
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
|
throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri,
|
||||||
|
result.errorMsg);
|
||||||
|
|
||||||
PathSet missing;
|
PathSet missing;
|
||||||
for (auto& path : outputs)
|
for (auto& path : outputs)
|
||||||
if (!store->isValidPath(path)) missing.insert(path);
|
if (!store->isValidPath(path)) missing.insert(path);
|
||||||
|
|
||||||
if (!missing.empty()) {
|
if (!missing.empty()) {
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
|
fmt("copying outputs from '%s'", storeUri));
|
||||||
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
|
store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
|
||||||
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
|
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs,
|
||||||
|
NoSubstitute);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
||||||
45
third_party/nix/src/libexpr/attr-path.cc
vendored
45
third_party/nix/src/libexpr/attr-path.cc
vendored
|
|
@ -2,12 +2,9 @@
|
||||||
#include "eval-inline.hh"
|
#include "eval-inline.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static Strings parseAttrPath(const string& s) {
|
||||||
static Strings parseAttrPath(const string & s)
|
|
||||||
{
|
|
||||||
Strings res;
|
Strings res;
|
||||||
string cur;
|
string cur;
|
||||||
string::const_iterator i = s.begin();
|
string::const_iterator i = s.begin();
|
||||||
|
|
@ -19,7 +16,8 @@ static Strings parseAttrPath(const string & s)
|
||||||
++i;
|
++i;
|
||||||
while (1) {
|
while (1) {
|
||||||
if (i == s.end())
|
if (i == s.end())
|
||||||
throw Error(format("missing closing quote in selection path '%1%'") % s);
|
throw Error(format("missing closing quote in selection path '%1%'") %
|
||||||
|
s);
|
||||||
if (*i == '"') break;
|
if (*i == '"') break;
|
||||||
cur.push_back(*i++);
|
cur.push_back(*i++);
|
||||||
}
|
}
|
||||||
|
|
@ -31,19 +29,17 @@ static Strings parseAttrPath(const string & s)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Value* findAlongAttrPath(EvalState& state, const string& attrPath,
|
Value* findAlongAttrPath(EvalState& state, const string& attrPath,
|
||||||
Bindings & autoArgs, Value & vIn)
|
Bindings& autoArgs, Value& vIn) {
|
||||||
{
|
|
||||||
Strings tokens = parseAttrPath(attrPath);
|
Strings tokens = parseAttrPath(attrPath);
|
||||||
|
|
||||||
Error attrError =
|
Error attrError =
|
||||||
Error(format("attribute selection path '%1%' does not match expression") % attrPath);
|
Error(format("attribute selection path '%1%' does not match expression") %
|
||||||
|
attrPath);
|
||||||
|
|
||||||
Value* v = &vIn;
|
Value* v = &vIn;
|
||||||
|
|
||||||
for (auto& attr : tokens) {
|
for (auto& attr : tokens) {
|
||||||
|
|
||||||
/* Is i an index (integer) or a normal attribute name? */
|
/* Is i an index (integer) or a normal attribute name? */
|
||||||
enum { apAttr, apIndex } apType = apAttr;
|
enum { apAttr, apIndex } apType = apAttr;
|
||||||
unsigned int attrIndex;
|
unsigned int attrIndex;
|
||||||
|
|
@ -59,38 +55,39 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
|
||||||
according to what is specified in the attrPath. */
|
according to what is specified in the attrPath. */
|
||||||
|
|
||||||
if (apType == apAttr) {
|
if (apType == apAttr) {
|
||||||
|
|
||||||
if (v->type != tAttrs)
|
if (v->type != tAttrs)
|
||||||
throw TypeError(
|
throw TypeError(format("the expression selected by the selection path "
|
||||||
format("the expression selected by the selection path '%1%' should be a set but is %2%")
|
"'%1%' should be a set but is %2%") %
|
||||||
% attrPath % showType(*v));
|
attrPath % showType(*v));
|
||||||
|
|
||||||
if (attr.empty())
|
if (attr.empty())
|
||||||
throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
|
throw Error(format("empty attribute name in selection path '%1%'") %
|
||||||
|
attrPath);
|
||||||
|
|
||||||
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
|
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
|
||||||
if (a == v->attrs->end())
|
if (a == v->attrs->end())
|
||||||
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
|
throw Error(
|
||||||
|
format("attribute '%1%' in selection path '%2%' not found") % attr %
|
||||||
|
attrPath);
|
||||||
v = &*a->value;
|
v = &*a->value;
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (apType == apIndex) {
|
else if (apType == apIndex) {
|
||||||
|
|
||||||
if (!v->isList())
|
if (!v->isList())
|
||||||
throw TypeError(
|
throw TypeError(format("the expression selected by the selection path "
|
||||||
format("the expression selected by the selection path '%1%' should be a list but is %2%")
|
"'%1%' should be a list but is %2%") %
|
||||||
% attrPath % showType(*v));
|
attrPath % showType(*v));
|
||||||
|
|
||||||
if (attrIndex >= v->listSize())
|
if (attrIndex >= v->listSize())
|
||||||
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
|
throw Error(
|
||||||
|
format("list index %1% in selection path '%2%' is out of range") %
|
||||||
|
attrIndex % attrPath);
|
||||||
|
|
||||||
v = v->listElems()[attrIndex];
|
v = v->listElems()[attrIndex];
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
5
third_party/nix/src/libexpr/attr-path.hh
vendored
5
third_party/nix/src/libexpr/attr-path.hh
vendored
|
|
@ -1,9 +1,8 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include "eval.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
|
||||||
28
third_party/nix/src/libexpr/attr-set.cc
vendored
28
third_party/nix/src/libexpr/attr-set.cc
vendored
|
|
@ -1,25 +1,20 @@
|
||||||
#include "attr-set.hh"
|
#include "attr-set.hh"
|
||||||
|
#include <algorithm>
|
||||||
#include "eval-inline.hh"
|
#include "eval-inline.hh"
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* Allocate a new array of attributes for an attribute set with a specific
|
/* Allocate a new array of attributes for an attribute set with a specific
|
||||||
capacity. The space is implicitly reserved after the Bindings
|
capacity. The space is implicitly reserved after the Bindings
|
||||||
structure. */
|
structure. */
|
||||||
Bindings * EvalState::allocBindings(size_t capacity)
|
Bindings* EvalState::allocBindings(size_t capacity) {
|
||||||
{
|
|
||||||
if (capacity > std::numeric_limits<Bindings::size_t>::max())
|
if (capacity > std::numeric_limits<Bindings::size_t>::max())
|
||||||
throw Error("attribute set of size %d is too big", capacity);
|
throw Error("attribute set of size %d is too big", capacity);
|
||||||
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
|
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity))
|
||||||
|
Bindings((Bindings::size_t)capacity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void EvalState::mkAttrs(Value& v, size_t capacity) {
|
||||||
void EvalState::mkAttrs(Value & v, size_t capacity)
|
|
||||||
{
|
|
||||||
if (capacity == 0) {
|
if (capacity == 0) {
|
||||||
v = vEmptySet;
|
v = vEmptySet;
|
||||||
return;
|
return;
|
||||||
|
|
@ -31,22 +26,15 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
|
||||||
nrAttrsInAttrsets += capacity;
|
nrAttrsInAttrsets += capacity;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Create a new attribute named 'name' on an existing attribute set stored
|
/* Create a new attribute named 'name' on an existing attribute set stored
|
||||||
in 'vAttrs' and return the newly allocated Value which is associated with
|
in 'vAttrs' and return the newly allocated Value which is associated with
|
||||||
this attribute. */
|
this attribute. */
|
||||||
Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
|
Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) {
|
||||||
{
|
|
||||||
Value* v = allocValue();
|
Value* v = allocValue();
|
||||||
vAttrs.attrs->push_back(Attr(name, v));
|
vAttrs.attrs->push_back(Attr(name, v));
|
||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Bindings::sort() { std::sort(begin(), end()); }
|
||||||
|
|
||||||
void Bindings::sort()
|
} // namespace nix
|
||||||
{
|
|
||||||
std::sort(begin(), end());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
||||||
35
third_party/nix/src/libexpr/attr-set.hh
vendored
35
third_party/nix/src/libexpr/attr-set.hh
vendored
|
|
@ -1,37 +1,30 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
#include "nixexpr.hh"
|
#include "nixexpr.hh"
|
||||||
#include "symbol-table.hh"
|
#include "symbol-table.hh"
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
class EvalState;
|
class EvalState;
|
||||||
struct Value;
|
struct Value;
|
||||||
|
|
||||||
/* Map one attribute name to its value. */
|
/* Map one attribute name to its value. */
|
||||||
struct Attr
|
struct Attr {
|
||||||
{
|
|
||||||
Symbol name;
|
Symbol name;
|
||||||
Value* value;
|
Value* value;
|
||||||
Pos* pos;
|
Pos* pos;
|
||||||
Attr(Symbol name, Value* value, Pos* pos = &noPos)
|
Attr(Symbol name, Value* value, Pos* pos = &noPos)
|
||||||
: name(name), value(value), pos(pos){};
|
: name(name), value(value), pos(pos){};
|
||||||
Attr() : pos(&noPos){};
|
Attr() : pos(&noPos){};
|
||||||
bool operator < (const Attr & a) const
|
bool operator<(const Attr& a) const { return name < a.name; }
|
||||||
{
|
|
||||||
return name < a.name;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Bindings contains all the attributes of an attribute set. It is defined
|
/* Bindings contains all the attributes of an attribute set. It is defined
|
||||||
by its size and its capacity, the capacity being the number of Attr
|
by its size and its capacity, the capacity being the number of Attr
|
||||||
elements allocated after this structure, while the size corresponds to
|
elements allocated after this structure, while the size corresponds to
|
||||||
the number of elements already inserted in this structure. */
|
the number of elements already inserted in this structure. */
|
||||||
class Bindings
|
class Bindings {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
typedef uint32_t size_t;
|
typedef uint32_t size_t;
|
||||||
|
|
||||||
|
|
@ -49,14 +42,12 @@ public:
|
||||||
|
|
||||||
typedef Attr* iterator;
|
typedef Attr* iterator;
|
||||||
|
|
||||||
void push_back(const Attr & attr)
|
void push_back(const Attr& attr) {
|
||||||
{
|
|
||||||
assert(size_ < capacity_);
|
assert(size_ < capacity_);
|
||||||
attrs[size_++] = attr;
|
attrs[size_++] = attr;
|
||||||
}
|
}
|
||||||
|
|
||||||
iterator find(const Symbol & name)
|
iterator find(const Symbol& name) {
|
||||||
{
|
|
||||||
Attr key(name, 0);
|
Attr key(name, 0);
|
||||||
iterator i = std::lower_bound(begin(), end(), key);
|
iterator i = std::lower_bound(begin(), end(), key);
|
||||||
if (i != end() && i->name == name) return i;
|
if (i != end() && i->name == name) return i;
|
||||||
|
|
@ -66,22 +57,17 @@ public:
|
||||||
iterator begin() { return &attrs[0]; }
|
iterator begin() { return &attrs[0]; }
|
||||||
iterator end() { return &attrs[size_]; }
|
iterator end() { return &attrs[size_]; }
|
||||||
|
|
||||||
Attr & operator[](size_t pos)
|
Attr& operator[](size_t pos) { return attrs[pos]; }
|
||||||
{
|
|
||||||
return attrs[pos];
|
|
||||||
}
|
|
||||||
|
|
||||||
void sort();
|
void sort();
|
||||||
|
|
||||||
size_t capacity() { return capacity_; }
|
size_t capacity() { return capacity_; }
|
||||||
|
|
||||||
/* Returns the attributes in lexicographically sorted order. */
|
/* Returns the attributes in lexicographically sorted order. */
|
||||||
std::vector<const Attr *> lexicographicOrder() const
|
std::vector<const Attr*> lexicographicOrder() const {
|
||||||
{
|
|
||||||
std::vector<const Attr*> res;
|
std::vector<const Attr*> res;
|
||||||
res.reserve(size_);
|
res.reserve(size_);
|
||||||
for (size_t n = 0; n < size_; n++)
|
for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]);
|
||||||
res.emplace_back(&attrs[n]);
|
|
||||||
std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) {
|
std::sort(res.begin(), res.end(), [](const Attr* a, const Attr* b) {
|
||||||
return (const string&)a->name < (const string&)b->name;
|
return (const string&)a->name < (const string&)b->name;
|
||||||
});
|
});
|
||||||
|
|
@ -91,5 +77,4 @@ public:
|
||||||
friend class EvalState;
|
friend class EvalState;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
28
third_party/nix/src/libexpr/common-eval-args.cc
vendored
28
third_party/nix/src/libexpr/common-eval-args.cc
vendored
|
|
@ -1,40 +1,43 @@
|
||||||
#include "common-eval-args.hh"
|
#include "common-eval-args.hh"
|
||||||
#include "shared.hh"
|
|
||||||
#include "download.hh"
|
#include "download.hh"
|
||||||
#include "util.hh"
|
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
|
#include "shared.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
MixEvalArgs::MixEvalArgs()
|
MixEvalArgs::MixEvalArgs() {
|
||||||
{
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("arg")
|
.longName("arg")
|
||||||
.description("argument to be passed to Nix functions")
|
.description("argument to be passed to Nix functions")
|
||||||
.labels({"name", "expr"})
|
.labels({"name", "expr"})
|
||||||
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
|
.handler(
|
||||||
|
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
|
||||||
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("argstr")
|
.longName("argstr")
|
||||||
.description("string-valued argument to be passed to Nix functions")
|
.description("string-valued argument to be passed to Nix functions")
|
||||||
.labels({"name", "string"})
|
.labels({"name", "string"})
|
||||||
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
|
.handler(
|
||||||
|
[&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
|
||||||
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.shortName('I')
|
.shortName('I')
|
||||||
.longName("include")
|
.longName("include")
|
||||||
.description("add a path to the list of locations used to look up <...> file names")
|
.description(
|
||||||
|
"add a path to the list of locations used to look up <...> file "
|
||||||
|
"names")
|
||||||
.label("path")
|
.label("path")
|
||||||
.handler([&](std::string s) { searchPath.push_back(s); });
|
.handler([&](std::string s) { searchPath.push_back(s); });
|
||||||
}
|
}
|
||||||
|
|
||||||
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
|
Bindings* MixEvalArgs::getAutoArgs(EvalState& state) {
|
||||||
{
|
|
||||||
Bindings* res = state.allocBindings(autoArgs.size());
|
Bindings* res = state.allocBindings(autoArgs.size());
|
||||||
for (auto& i : autoArgs) {
|
for (auto& i : autoArgs) {
|
||||||
Value* v = state.allocValue();
|
Value* v = state.allocValue();
|
||||||
if (i.second[0] == 'E')
|
if (i.second[0] == 'E')
|
||||||
state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
|
state.mkThunk_(
|
||||||
|
*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
|
||||||
else
|
else
|
||||||
mkString(*v, string(i.second, 1));
|
mkString(*v, string(i.second, 1));
|
||||||
res->push_back(Attr(state.symbols.create(i.first), v));
|
res->push_back(Attr(state.symbols.create(i.first), v));
|
||||||
|
|
@ -43,8 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
Path lookupFileArg(EvalState & state, string s)
|
Path lookupFileArg(EvalState& state, string s) {
|
||||||
{
|
|
||||||
if (isUri(s)) {
|
if (isUri(s)) {
|
||||||
CachedDownloadRequest request(s);
|
CachedDownloadRequest request(s);
|
||||||
request.unpack = true;
|
request.unpack = true;
|
||||||
|
|
@ -56,4 +58,4 @@ Path lookupFileArg(EvalState & state, string s)
|
||||||
return absPath(s);
|
return absPath(s);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,7 @@ class Store;
|
||||||
class EvalState;
|
class EvalState;
|
||||||
class Bindings;
|
class Bindings;
|
||||||
|
|
||||||
struct MixEvalArgs : virtual Args
|
struct MixEvalArgs : virtual Args {
|
||||||
{
|
|
||||||
MixEvalArgs();
|
MixEvalArgs();
|
||||||
|
|
||||||
Bindings* getAutoArgs(EvalState& state);
|
Bindings* getAutoArgs(EvalState& state);
|
||||||
|
|
@ -17,10 +16,9 @@ struct MixEvalArgs : virtual Args
|
||||||
Strings searchPath;
|
Strings searchPath;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
std::map<std::string, std::string> autoArgs;
|
std::map<std::string, std::string> autoArgs;
|
||||||
};
|
};
|
||||||
|
|
||||||
Path lookupFileArg(EvalState& state, string s);
|
Path lookupFileArg(EvalState& state, string s);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
51
third_party/nix/src/libexpr/eval-inline.hh
vendored
51
third_party/nix/src/libexpr/eval-inline.hh
vendored
|
|
@ -2,30 +2,29 @@
|
||||||
|
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
|
|
||||||
#define LocalNoInline(f) static f __attribute__((noinline)); f
|
#define LocalNoInline(f) \
|
||||||
#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f
|
static f __attribute__((noinline)); \
|
||||||
|
f
|
||||||
|
#define LocalNoInlineNoReturn(f) \
|
||||||
|
static f __attribute__((noinline, noreturn)); \
|
||||||
|
f
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
|
LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) {
|
||||||
{
|
|
||||||
throw EvalError(format(s) % pos);
|
throw EvalError(format(s) % pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
|
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) {
|
||||||
{
|
|
||||||
throw TypeError(format(s) % showType(v));
|
throw TypeError(format(s) % showType(v));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v,
|
||||||
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
|
const Pos& pos)) {
|
||||||
{
|
|
||||||
throw TypeError(format(s) % showType(v) % pos);
|
throw TypeError(format(s) % showType(v) % pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void EvalState::forceValue(Value& v, const Pos& pos) {
|
||||||
void EvalState::forceValue(Value & v, const Pos & pos)
|
|
||||||
{
|
|
||||||
if (v.type == tThunk) {
|
if (v.type == tThunk) {
|
||||||
Env* env = v.thunk.env;
|
Env* env = v.thunk.env;
|
||||||
Expr* expr = v.thunk.expr;
|
Expr* expr = v.thunk.expr;
|
||||||
|
|
@ -39,48 +38,37 @@ void EvalState::forceValue(Value & v, const Pos & pos)
|
||||||
v.thunk.expr = expr;
|
v.thunk.expr = expr;
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
} else if (v.type == tApp)
|
||||||
else if (v.type == tApp)
|
|
||||||
callFunction(*v.app.left, *v.app.right, v, noPos);
|
callFunction(*v.app.left, *v.app.right, v, noPos);
|
||||||
else if (v.type == tBlackhole)
|
else if (v.type == tBlackhole)
|
||||||
throwEvalError("infinite recursion encountered, at %1%", pos);
|
throwEvalError("infinite recursion encountered, at %1%", pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline void EvalState::forceAttrs(Value& v) {
|
||||||
inline void EvalState::forceAttrs(Value & v)
|
|
||||||
{
|
|
||||||
forceValue(v);
|
forceValue(v);
|
||||||
if (v.type != tAttrs)
|
if (v.type != tAttrs)
|
||||||
throwTypeError("value is %1% while a set was expected", v);
|
throwTypeError("value is %1% while a set was expected", v);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline void EvalState::forceAttrs(Value& v, const Pos& pos) {
|
||||||
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
|
|
||||||
{
|
|
||||||
forceValue(v);
|
forceValue(v);
|
||||||
if (v.type != tAttrs)
|
if (v.type != tAttrs)
|
||||||
throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
|
throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline void EvalState::forceList(Value& v) {
|
||||||
inline void EvalState::forceList(Value & v)
|
|
||||||
{
|
|
||||||
forceValue(v);
|
forceValue(v);
|
||||||
if (!v.isList())
|
if (!v.isList()) throwTypeError("value is %1% while a list was expected", v);
|
||||||
throwTypeError("value is %1% while a list was expected", v);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline void EvalState::forceList(Value& v, const Pos& pos) {
|
||||||
inline void EvalState::forceList(Value & v, const Pos & pos)
|
|
||||||
{
|
|
||||||
forceValue(v);
|
forceValue(v);
|
||||||
if (!v.isList())
|
if (!v.isList())
|
||||||
throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
|
throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Note: Various places expect the allocated memory to be zeroed. */
|
/* Note: Various places expect the allocated memory to be zeroed. */
|
||||||
inline void * allocBytes(size_t n)
|
inline void* allocBytes(size_t n) {
|
||||||
{
|
|
||||||
void* p;
|
void* p;
|
||||||
#if HAVE_BOEHMGC
|
#if HAVE_BOEHMGC
|
||||||
p = GC_MALLOC(n);
|
p = GC_MALLOC(n);
|
||||||
|
|
@ -91,5 +79,4 @@ inline void * allocBytes(size_t n)
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
873
third_party/nix/src/libexpr/eval.cc
vendored
873
third_party/nix/src/libexpr/eval.cc
vendored
File diff suppressed because it is too large
Load diff
116
third_party/nix/src/libexpr/eval.hh
vendored
116
third_party/nix/src/libexpr/eval.hh
vendored
|
|
@ -1,30 +1,25 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "attr-set.hh"
|
|
||||||
#include "value.hh"
|
|
||||||
#include "nixexpr.hh"
|
|
||||||
#include "symbol-table.hh"
|
|
||||||
#include "hash.hh"
|
|
||||||
#include "config.hh"
|
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
|
#include "attr-set.hh"
|
||||||
|
#include "config.hh"
|
||||||
|
#include "hash.hh"
|
||||||
|
#include "nixexpr.hh"
|
||||||
|
#include "symbol-table.hh"
|
||||||
|
#include "value.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
class Store;
|
class Store;
|
||||||
class EvalState;
|
class EvalState;
|
||||||
enum RepairFlag : bool;
|
enum RepairFlag : bool;
|
||||||
|
|
||||||
|
typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args,
|
||||||
|
Value& v);
|
||||||
|
|
||||||
typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v);
|
struct PrimOp {
|
||||||
|
|
||||||
|
|
||||||
struct PrimOp
|
|
||||||
{
|
|
||||||
PrimOpFun fun;
|
PrimOpFun fun;
|
||||||
size_t arity;
|
size_t arity;
|
||||||
Symbol name;
|
Symbol name;
|
||||||
|
|
@ -32,9 +27,7 @@ struct PrimOp
|
||||||
: fun(fun), arity(arity), name(name) {}
|
: fun(fun), arity(arity), name(name) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct Env {
|
||||||
struct Env
|
|
||||||
{
|
|
||||||
Env* up;
|
Env* up;
|
||||||
unsigned short size; // used by ‘valueSize’
|
unsigned short size; // used by ‘valueSize’
|
||||||
unsigned short prevWith : 14; // nr of levels up to next `with' environment
|
unsigned short prevWith : 14; // nr of levels up to next `with' environment
|
||||||
|
|
@ -42,37 +35,29 @@ struct Env
|
||||||
Value* values[0];
|
Value* values[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
Value& mkString(Value& v, const string& s, const PathSet& context = PathSet());
|
Value& mkString(Value& v, const string& s, const PathSet& context = PathSet());
|
||||||
|
|
||||||
void copyContext(const Value& v, PathSet& context);
|
void copyContext(const Value& v, PathSet& context);
|
||||||
|
|
||||||
|
|
||||||
/* Cache for calls to addToStore(); maps source paths to the store
|
/* Cache for calls to addToStore(); maps source paths to the store
|
||||||
paths. */
|
paths. */
|
||||||
typedef std::map<Path, Path> SrcToStore;
|
typedef std::map<Path, Path> SrcToStore;
|
||||||
|
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& str, const Value& v);
|
std::ostream& operator<<(std::ostream& str, const Value& v);
|
||||||
|
|
||||||
|
|
||||||
typedef std::pair<std::string, std::string> SearchPathElem;
|
typedef std::pair<std::string, std::string> SearchPathElem;
|
||||||
typedef std::list<SearchPathElem> SearchPath;
|
typedef std::list<SearchPathElem> SearchPath;
|
||||||
|
|
||||||
|
|
||||||
/* Initialise the Boehm GC, if applicable. */
|
/* Initialise the Boehm GC, if applicable. */
|
||||||
void initGC();
|
void initGC();
|
||||||
|
|
||||||
|
class EvalState {
|
||||||
class EvalState
|
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
SymbolTable symbols;
|
SymbolTable symbols;
|
||||||
|
|
||||||
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
|
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem,
|
||||||
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
|
sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn,
|
||||||
sFile, sLine, sColumn, sFunctor, sToString,
|
sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
|
||||||
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
|
|
||||||
sOutputHash, sOutputHashAlgo, sOutputHashMode;
|
sOutputHash, sOutputHashAlgo, sOutputHashMode;
|
||||||
Symbol sDerivationNix;
|
Symbol sDerivationNix;
|
||||||
|
|
||||||
|
|
@ -93,7 +78,9 @@ private:
|
||||||
|
|
||||||
/* A cache from path names to parse trees. */
|
/* A cache from path names to parse trees. */
|
||||||
#if HAVE_BOEHMGC
|
#if HAVE_BOEHMGC
|
||||||
typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
|
typedef std::map<Path, Expr*, std::less<Path>,
|
||||||
|
traceable_allocator<std::pair<const Path, Expr*>>>
|
||||||
|
FileParseCache;
|
||||||
#else
|
#else
|
||||||
typedef std::map<Path, Expr*> FileParseCache;
|
typedef std::map<Path, Expr*> FileParseCache;
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -101,7 +88,9 @@ private:
|
||||||
|
|
||||||
/* A cache from path names to values. */
|
/* A cache from path names to values. */
|
||||||
#if HAVE_BOEHMGC
|
#if HAVE_BOEHMGC
|
||||||
typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
|
typedef std::map<Path, Value, std::less<Path>,
|
||||||
|
traceable_allocator<std::pair<const Path, Value>>>
|
||||||
|
FileEvalCache;
|
||||||
#else
|
#else
|
||||||
typedef std::map<Path, Value> FileEvalCache;
|
typedef std::map<Path, Value> FileEvalCache;
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -115,7 +104,6 @@ private:
|
||||||
std::unordered_map<Path, Path> resolvedPaths;
|
std::unordered_map<Path, Path> resolvedPaths;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
EvalState(const Strings& _searchPath, ref<Store> store);
|
EvalState(const Strings& _searchPath, ref<Store> store);
|
||||||
~EvalState();
|
~EvalState();
|
||||||
|
|
||||||
|
|
@ -141,7 +129,8 @@ public:
|
||||||
Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv);
|
Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv);
|
||||||
|
|
||||||
/* Parse a Nix expression from the specified string. */
|
/* Parse a Nix expression from the specified string. */
|
||||||
Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
|
Expr* parseExprFromString(const string& s, const Path& basePath,
|
||||||
|
StaticEnv& staticEnv);
|
||||||
Expr* parseExprFromString(const string& s, const Path& basePath);
|
Expr* parseExprFromString(const string& s, const Path& basePath);
|
||||||
|
|
||||||
Expr* parseStdin();
|
Expr* parseStdin();
|
||||||
|
|
@ -154,10 +143,12 @@ public:
|
||||||
|
|
||||||
/* Look up a file in the search path. */
|
/* Look up a file in the search path. */
|
||||||
Path findFile(const string& path);
|
Path findFile(const string& path);
|
||||||
Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos);
|
Path findFile(SearchPath& searchPath, const string& path,
|
||||||
|
const Pos& pos = noPos);
|
||||||
|
|
||||||
/* If the specified search path element is a URI, download it. */
|
/* If the specified search path element is a URI, download it. */
|
||||||
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
|
std::pair<bool, std::string> resolveSearchPathElem(
|
||||||
|
const SearchPathElem& elem);
|
||||||
|
|
||||||
/* Evaluate an expression to normal form, storing the result in
|
/* Evaluate an expression to normal form, storing the result in
|
||||||
value `v'. */
|
value `v'. */
|
||||||
|
|
@ -197,7 +188,9 @@ public:
|
||||||
bool isDerivation(Value& v);
|
bool isDerivation(Value& v);
|
||||||
|
|
||||||
std::optional<string> tryAttrsToString(const Pos& pos, Value& v,
|
std::optional<string> tryAttrsToString(const Pos& pos, Value& v,
|
||||||
PathSet & context, bool coerceMore = false, bool copyToStore = true);
|
PathSet& context,
|
||||||
|
bool coerceMore = false,
|
||||||
|
bool copyToStore = true);
|
||||||
|
|
||||||
/* String coercion. Converts strings, paths and derivations to a
|
/* String coercion. Converts strings, paths and derivations to a
|
||||||
string. If `coerceMore' is set, also converts nulls, integers,
|
string. If `coerceMore' is set, also converts nulls, integers,
|
||||||
|
|
@ -214,7 +207,6 @@ public:
|
||||||
Path coerceToPath(const Pos& pos, Value& v, PathSet& context);
|
Path coerceToPath(const Pos& pos, Value& v, PathSet& context);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* The base environment, containing the builtin functions and
|
/* The base environment, containing the builtin functions and
|
||||||
values. */
|
values. */
|
||||||
Env& baseEnv;
|
Env& baseEnv;
|
||||||
|
|
@ -223,33 +215,28 @@ public:
|
||||||
StaticEnv staticBaseEnv; // !!! should be private
|
StaticEnv staticBaseEnv; // !!! should be private
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
unsigned int baseEnvDispl = 0;
|
unsigned int baseEnvDispl = 0;
|
||||||
|
|
||||||
void createBaseEnv();
|
void createBaseEnv();
|
||||||
|
|
||||||
Value* addConstant(const string& name, Value& v);
|
Value* addConstant(const string& name, Value& v);
|
||||||
|
|
||||||
Value * addPrimOp(const string & name,
|
Value* addPrimOp(const string& name, size_t arity, PrimOpFun primOp);
|
||||||
size_t arity, PrimOpFun primOp);
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
Value& getBuiltin(const string& name);
|
Value& getBuiltin(const string& name);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval);
|
inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval);
|
||||||
|
|
||||||
friend struct ExprVar;
|
friend struct ExprVar;
|
||||||
friend struct ExprAttrs;
|
friend struct ExprAttrs;
|
||||||
friend struct ExprLet;
|
friend struct ExprLet;
|
||||||
|
|
||||||
Expr * parse(const char * text, const Path & path,
|
Expr* parse(const char* text, const Path& path, const Path& basePath,
|
||||||
const Path & basePath, StaticEnv & staticEnv);
|
StaticEnv& staticEnv);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* Do a deep equality test between two values. That is, list
|
/* Do a deep equality test between two values. That is, list
|
||||||
elements and attributes are compared recursively. */
|
elements and attributes are compared recursively. */
|
||||||
bool eqValues(Value& v1, Value& v2);
|
bool eqValues(Value& v1, Value& v2);
|
||||||
|
|
@ -284,7 +271,6 @@ public:
|
||||||
void realiseContext(const PathSet& context);
|
void realiseContext(const PathSet& context);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
unsigned long nrEnvs = 0;
|
unsigned long nrEnvs = 0;
|
||||||
unsigned long nrValuesInEnvs = 0;
|
unsigned long nrValuesInEnvs = 0;
|
||||||
unsigned long nrValues = 0;
|
unsigned long nrValues = 0;
|
||||||
|
|
@ -313,10 +299,10 @@ private:
|
||||||
friend struct ExprOpUpdate;
|
friend struct ExprOpUpdate;
|
||||||
friend struct ExprOpConcatLists;
|
friend struct ExprOpConcatLists;
|
||||||
friend struct ExprSelect;
|
friend struct ExprSelect;
|
||||||
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
|
friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args,
|
||||||
|
Value& v);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Return a string representing the type of the value `v'. */
|
/* Return a string representing the type of the value `v'. */
|
||||||
string showType(const Value& v);
|
string showType(const Value& v);
|
||||||
|
|
||||||
|
|
@ -327,8 +313,7 @@ std::pair<string, string> decodeContext(const string & s);
|
||||||
/* If `path' refers to a directory, then append "/default.nix". */
|
/* If `path' refers to a directory, then append "/default.nix". */
|
||||||
Path resolveExprPath(Path path);
|
Path resolveExprPath(Path path);
|
||||||
|
|
||||||
struct InvalidPathError : EvalError
|
struct InvalidPathError : EvalError {
|
||||||
{
|
|
||||||
Path path;
|
Path path;
|
||||||
InvalidPathError(const Path& path);
|
InvalidPathError(const Path& path);
|
||||||
#ifdef EXCEPTION_NEEDS_THROW_SPEC
|
#ifdef EXCEPTION_NEEDS_THROW_SPEC
|
||||||
|
|
@ -336,28 +321,37 @@ struct InvalidPathError : EvalError
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
struct EvalSettings : Config
|
struct EvalSettings : Config {
|
||||||
{
|
Setting<bool> enableNativeCode{this, false,
|
||||||
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
|
"allow-unsafe-native-code-during-evaluation",
|
||||||
"Whether builtin functions that allow executing native code should be enabled."};
|
"Whether builtin functions that allow "
|
||||||
|
"executing native code should be enabled."};
|
||||||
|
|
||||||
Setting<bool> restrictEval{this, false, "restrict-eval",
|
Setting<bool> restrictEval{
|
||||||
|
this, false, "restrict-eval",
|
||||||
"Whether to restrict file system access to paths in $NIX_PATH, "
|
"Whether to restrict file system access to paths in $NIX_PATH, "
|
||||||
"and network access to the URI prefixes listed in 'allowed-uris'."};
|
"and network access to the URI prefixes listed in 'allowed-uris'."};
|
||||||
|
|
||||||
Setting<bool> pureEval{this, false, "pure-eval",
|
Setting<bool> pureEval{this, false, "pure-eval",
|
||||||
"Whether to restrict file system and network access to files specified by cryptographic hash."};
|
"Whether to restrict file system and network access "
|
||||||
|
"to files specified by cryptographic hash."};
|
||||||
|
|
||||||
Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
|
Setting<bool> enableImportFromDerivation{
|
||||||
|
this, true, "allow-import-from-derivation",
|
||||||
"Whether the evaluator allows importing the result of a derivation."};
|
"Whether the evaluator allows importing the result of a derivation."};
|
||||||
|
|
||||||
Setting<Strings> allowedUris{this, {}, "allowed-uris",
|
Setting<Strings> allowedUris{
|
||||||
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
|
this,
|
||||||
|
{},
|
||||||
|
"allowed-uris",
|
||||||
|
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit "
|
||||||
|
"are allowed to fetch."};
|
||||||
|
|
||||||
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
|
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
|
||||||
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
|
"Emit log messages for each function entry "
|
||||||
|
"and exit at the 'vomit' log level (-vvvv)"};
|
||||||
};
|
};
|
||||||
|
|
||||||
extern EvalSettings evalSettings;
|
extern EvalSettings evalSettings;
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -14,4 +14,4 @@ FunctionCallTrace::~FunctionCallTrace() {
|
||||||
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
|
printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,13 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
#include "eval.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct FunctionCallTrace
|
struct FunctionCallTrace {
|
||||||
{
|
|
||||||
const Pos& pos;
|
const Pos& pos;
|
||||||
FunctionCallTrace(const Pos& pos);
|
FunctionCallTrace(const Pos& pos);
|
||||||
~FunctionCallTrace();
|
~FunctionCallTrace();
|
||||||
};
|
};
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
199
third_party/nix/src/libexpr/get-drvs.cc
vendored
199
third_party/nix/src/libexpr/get-drvs.cc
vendored
|
|
@ -1,24 +1,18 @@
|
||||||
#include "get-drvs.hh"
|
#include "get-drvs.hh"
|
||||||
#include "util.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
#include "derivations.hh"
|
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <regex>
|
#include <regex>
|
||||||
|
#include "derivations.hh"
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs)
|
DrvInfo::DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs)
|
||||||
: state(&state), attrs(attrs), attrPath(attrPath)
|
: state(&state), attrs(attrs), attrPath(attrPath) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
DrvInfo::DrvInfo(EvalState& state, ref<Store> store,
|
||||||
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
|
const std::string& drvPathWithOutputs)
|
||||||
: state(&state), attrs(nullptr), attrPath("")
|
: state(&state), attrs(nullptr), attrPath("") {
|
||||||
{
|
|
||||||
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
|
auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
|
||||||
|
|
||||||
drvPath = spec.first;
|
drvPath = spec.first;
|
||||||
|
|
@ -28,23 +22,22 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
|
||||||
name = storePathToName(drvPath);
|
name = storePathToName(drvPath);
|
||||||
|
|
||||||
if (spec.second.size() > 1)
|
if (spec.second.size() > 1)
|
||||||
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
|
throw Error(
|
||||||
|
"building more than one derivation output is not supported, in '%s'",
|
||||||
|
drvPathWithOutputs);
|
||||||
|
|
||||||
outputName =
|
outputName = spec.second.empty() ? get(drv.env, "outputName", "out")
|
||||||
spec.second.empty()
|
|
||||||
? get(drv.env, "outputName", "out")
|
|
||||||
: *spec.second.begin();
|
: *spec.second.begin();
|
||||||
|
|
||||||
auto i = drv.outputs.find(outputName);
|
auto i = drv.outputs.find(outputName);
|
||||||
if (i == drv.outputs.end())
|
if (i == drv.outputs.end())
|
||||||
throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
|
throw Error("derivation '%s' does not have output '%s'", drvPath,
|
||||||
|
outputName);
|
||||||
|
|
||||||
outPath = i->second.path;
|
outPath = i->second.path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::queryName() const {
|
||||||
string DrvInfo::queryName() const
|
|
||||||
{
|
|
||||||
if (name == "" && attrs) {
|
if (name == "" && attrs) {
|
||||||
auto i = attrs->find(state->sName);
|
auto i = attrs->find(state->sName);
|
||||||
if (i == attrs->end()) throw TypeError("derivation name missing");
|
if (i == attrs->end()) throw TypeError("derivation name missing");
|
||||||
|
|
@ -53,41 +46,38 @@ string DrvInfo::queryName() const
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::querySystem() const {
|
||||||
string DrvInfo::querySystem() const
|
|
||||||
{
|
|
||||||
if (system == "" && attrs) {
|
if (system == "" && attrs) {
|
||||||
auto i = attrs->find(state->sSystem);
|
auto i = attrs->find(state->sSystem);
|
||||||
system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
|
system = i == attrs->end() ? "unknown"
|
||||||
|
: state->forceStringNoCtx(*i->value, *i->pos);
|
||||||
}
|
}
|
||||||
return system;
|
return system;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::queryDrvPath() const {
|
||||||
string DrvInfo::queryDrvPath() const
|
|
||||||
{
|
|
||||||
if (drvPath == "" && attrs) {
|
if (drvPath == "" && attrs) {
|
||||||
Bindings::iterator i = attrs->find(state->sDrvPath);
|
Bindings::iterator i = attrs->find(state->sDrvPath);
|
||||||
PathSet context;
|
PathSet context;
|
||||||
drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
|
drvPath = i != attrs->end()
|
||||||
|
? state->coerceToPath(*i->pos, *i->value, context)
|
||||||
|
: "";
|
||||||
}
|
}
|
||||||
return drvPath;
|
return drvPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::queryOutPath() const {
|
||||||
string DrvInfo::queryOutPath() const
|
|
||||||
{
|
|
||||||
if (outPath == "" && attrs) {
|
if (outPath == "" && attrs) {
|
||||||
Bindings::iterator i = attrs->find(state->sOutPath);
|
Bindings::iterator i = attrs->find(state->sOutPath);
|
||||||
PathSet context;
|
PathSet context;
|
||||||
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
|
outPath = i != attrs->end()
|
||||||
|
? state->coerceToPath(*i->pos, *i->value, context)
|
||||||
|
: "";
|
||||||
}
|
}
|
||||||
return outPath;
|
return outPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) {
|
||||||
DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
|
|
||||||
{
|
|
||||||
if (outputs.empty()) {
|
if (outputs.empty()) {
|
||||||
/* Get the ‘outputs’ list. */
|
/* Get the ‘outputs’ list. */
|
||||||
Bindings::iterator i;
|
Bindings::iterator i;
|
||||||
|
|
@ -97,22 +87,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
|
||||||
/* For each output... */
|
/* For each output... */
|
||||||
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
|
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
|
||||||
/* Evaluate the corresponding set. */
|
/* Evaluate the corresponding set. */
|
||||||
string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
|
string name =
|
||||||
|
state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
|
||||||
Bindings::iterator out = attrs->find(state->symbols.create(name));
|
Bindings::iterator out = attrs->find(state->symbols.create(name));
|
||||||
if (out == attrs->end()) continue; // FIXME: throw error?
|
if (out == attrs->end()) continue; // FIXME: throw error?
|
||||||
state->forceAttrs(*out->value);
|
state->forceAttrs(*out->value);
|
||||||
|
|
||||||
/* And evaluate its ‘outPath’ attribute. */
|
/* And evaluate its ‘outPath’ attribute. */
|
||||||
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
|
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
|
||||||
if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
|
if (outPath == out->value->attrs->end())
|
||||||
|
continue; // FIXME: throw error?
|
||||||
PathSet context;
|
PathSet context;
|
||||||
outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
|
outputs[name] =
|
||||||
|
state->coerceToPath(*outPath->pos, *outPath->value, context);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
outputs["out"] = queryOutPath();
|
outputs["out"] = queryOutPath();
|
||||||
}
|
}
|
||||||
if (!onlyOutputsToInstall || !attrs)
|
if (!onlyOutputsToInstall || !attrs) return outputs;
|
||||||
return outputs;
|
|
||||||
|
|
||||||
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
|
/* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
|
||||||
const Value* outTI = queryMeta("outputsToInstall");
|
const Value* outTI = queryMeta("outputsToInstall");
|
||||||
|
|
@ -121,7 +113,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
|
||||||
/* ^ this shows during `nix-env -i` right under the bad derivation */
|
/* ^ this shows during `nix-env -i` right under the bad derivation */
|
||||||
if (!outTI->isList()) throw errMsg;
|
if (!outTI->isList()) throw errMsg;
|
||||||
Outputs result;
|
Outputs result;
|
||||||
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
|
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize();
|
||||||
|
++i) {
|
||||||
if ((*i)->type != tString) throw errMsg;
|
if ((*i)->type != tString) throw errMsg;
|
||||||
auto out = outputs.find((*i)->string.s);
|
auto out = outputs.find((*i)->string.s);
|
||||||
if (out == outputs.end()) throw errMsg;
|
if (out == outputs.end()) throw errMsg;
|
||||||
|
|
@ -130,9 +123,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::queryOutputName() const {
|
||||||
string DrvInfo::queryOutputName() const
|
|
||||||
{
|
|
||||||
if (outputName == "" && attrs) {
|
if (outputName == "" && attrs) {
|
||||||
Bindings::iterator i = attrs->find(state->sOutputName);
|
Bindings::iterator i = attrs->find(state->sOutputName);
|
||||||
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
|
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
|
||||||
|
|
@ -140,9 +131,7 @@ string DrvInfo::queryOutputName() const
|
||||||
return outputName;
|
return outputName;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Bindings* DrvInfo::getMeta() {
|
||||||
Bindings * DrvInfo::getMeta()
|
|
||||||
{
|
|
||||||
if (meta) return meta;
|
if (meta) return meta;
|
||||||
if (!attrs) return 0;
|
if (!attrs) return 0;
|
||||||
Bindings::iterator a = attrs->find(state->sMeta);
|
Bindings::iterator a = attrs->find(state->sMeta);
|
||||||
|
|
@ -152,56 +141,44 @@ Bindings * DrvInfo::getMeta()
|
||||||
return meta;
|
return meta;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
StringSet DrvInfo::queryMetaNames() {
|
||||||
StringSet DrvInfo::queryMetaNames()
|
|
||||||
{
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
if (!getMeta()) return res;
|
if (!getMeta()) return res;
|
||||||
for (auto & i : *meta)
|
for (auto& i : *meta) res.insert(i.name);
|
||||||
res.insert(i.name);
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool DrvInfo::checkMeta(Value& v) {
|
||||||
bool DrvInfo::checkMeta(Value & v)
|
|
||||||
{
|
|
||||||
state->forceValue(v);
|
state->forceValue(v);
|
||||||
if (v.isList()) {
|
if (v.isList()) {
|
||||||
for (unsigned int n = 0; n < v.listSize(); ++n)
|
for (unsigned int n = 0; n < v.listSize(); ++n)
|
||||||
if (!checkMeta(*v.listElems()[n])) return false;
|
if (!checkMeta(*v.listElems()[n])) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
} else if (v.type == tAttrs) {
|
||||||
else if (v.type == tAttrs) {
|
|
||||||
Bindings::iterator i = v.attrs->find(state->sOutPath);
|
Bindings::iterator i = v.attrs->find(state->sOutPath);
|
||||||
if (i != v.attrs->end()) return false;
|
if (i != v.attrs->end()) return false;
|
||||||
for (auto& i : *v.attrs)
|
for (auto& i : *v.attrs)
|
||||||
if (!checkMeta(*i.value)) return false;
|
if (!checkMeta(*i.value)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
} else
|
||||||
else return v.type == tInt || v.type == tBool || v.type == tString ||
|
return v.type == tInt || v.type == tBool || v.type == tString ||
|
||||||
v.type == tFloat;
|
v.type == tFloat;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Value* DrvInfo::queryMeta(const string& name) {
|
||||||
Value * DrvInfo::queryMeta(const string & name)
|
|
||||||
{
|
|
||||||
if (!getMeta()) return 0;
|
if (!getMeta()) return 0;
|
||||||
Bindings::iterator a = meta->find(state->symbols.create(name));
|
Bindings::iterator a = meta->find(state->symbols.create(name));
|
||||||
if (a == meta->end() || !checkMeta(*a->value)) return 0;
|
if (a == meta->end() || !checkMeta(*a->value)) return 0;
|
||||||
return a->value;
|
return a->value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string DrvInfo::queryMetaString(const string& name) {
|
||||||
string DrvInfo::queryMetaString(const string & name)
|
|
||||||
{
|
|
||||||
Value* v = queryMeta(name);
|
Value* v = queryMeta(name);
|
||||||
if (!v || v->type != tString) return "";
|
if (!v || v->type != tString) return "";
|
||||||
return v->string.s;
|
return v->string.s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
NixInt DrvInfo::queryMetaInt(const string& name, NixInt def) {
|
||||||
NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
|
|
||||||
{
|
|
||||||
Value* v = queryMeta(name);
|
Value* v = queryMeta(name);
|
||||||
if (!v) return def;
|
if (!v) return def;
|
||||||
if (v->type == tInt) return v->integer;
|
if (v->type == tInt) return v->integer;
|
||||||
|
|
@ -214,8 +191,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
|
||||||
return def;
|
return def;
|
||||||
}
|
}
|
||||||
|
|
||||||
NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
|
NixFloat DrvInfo::queryMetaFloat(const string& name, NixFloat def) {
|
||||||
{
|
|
||||||
Value* v = queryMeta(name);
|
Value* v = queryMeta(name);
|
||||||
if (!v) return def;
|
if (!v) return def;
|
||||||
if (v->type == tFloat) return v->fpoint;
|
if (v->type == tFloat) return v->fpoint;
|
||||||
|
|
@ -228,9 +204,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
|
||||||
return def;
|
return def;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool DrvInfo::queryMetaBool(const string& name, bool def) {
|
||||||
bool DrvInfo::queryMetaBool(const string & name, bool def)
|
|
||||||
{
|
|
||||||
Value* v = queryMeta(name);
|
Value* v = queryMeta(name);
|
||||||
if (!v) return def;
|
if (!v) return def;
|
||||||
if (v->type == tBool) return v->boolean;
|
if (v->type == tBool) return v->boolean;
|
||||||
|
|
@ -243,34 +217,28 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
|
||||||
return def;
|
return def;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void DrvInfo::setMeta(const string& name, Value* v) {
|
||||||
void DrvInfo::setMeta(const string & name, Value * v)
|
|
||||||
{
|
|
||||||
getMeta();
|
getMeta();
|
||||||
Bindings* old = meta;
|
Bindings* old = meta;
|
||||||
meta = state->allocBindings(1 + (old ? old->size() : 0));
|
meta = state->allocBindings(1 + (old ? old->size() : 0));
|
||||||
Symbol sym = state->symbols.create(name);
|
Symbol sym = state->symbols.create(name);
|
||||||
if (old)
|
if (old)
|
||||||
for (auto i : *old)
|
for (auto i : *old)
|
||||||
if (i.name != sym)
|
if (i.name != sym) meta->push_back(i);
|
||||||
meta->push_back(i);
|
|
||||||
if (v) meta->push_back(Attr(sym, v));
|
if (v) meta->push_back(Attr(sym, v));
|
||||||
meta->sort();
|
meta->sort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Cache for already considered attrsets. */
|
/* Cache for already considered attrsets. */
|
||||||
typedef set<Bindings*> Done;
|
typedef set<Bindings*> Done;
|
||||||
|
|
||||||
|
|
||||||
/* Evaluate value `v'. If it evaluates to a set of type `derivation',
|
/* Evaluate value `v'. If it evaluates to a set of type `derivation',
|
||||||
then put information about it in `drvs' (unless it's already in `done').
|
then put information about it in `drvs' (unless it's already in `done').
|
||||||
The result boolean indicates whether it makes sense
|
The result boolean indicates whether it makes sense
|
||||||
for the caller to recursively search for derivations in `v'. */
|
for the caller to recursively search for derivations in `v'. */
|
||||||
static bool getDerivation(EvalState & state, Value & v,
|
static bool getDerivation(EvalState& state, Value& v, const string& attrPath,
|
||||||
const string & attrPath, DrvInfos & drvs, Done & done,
|
DrvInfos& drvs, Done& done,
|
||||||
bool ignoreAssertionFailures)
|
bool ignoreAssertionFailures) {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
state.forceValue(v);
|
state.forceValue(v);
|
||||||
if (!state.isDerivation(v)) return true;
|
if (!state.isDerivation(v)) return true;
|
||||||
|
|
@ -294,10 +262,8 @@ static bool getDerivation(EvalState & state, Value & v,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
|
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
|
||||||
bool ignoreAssertionFailures)
|
bool ignoreAssertionFailures) {
|
||||||
{
|
|
||||||
Done done;
|
Done done;
|
||||||
DrvInfos drvs;
|
DrvInfos drvs;
|
||||||
getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
|
getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
|
||||||
|
|
@ -305,32 +271,29 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
|
||||||
return std::move(drvs.front());
|
return std::move(drvs.front());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static string addToPath(const string& s1, const string& s2) {
|
||||||
static string addToPath(const string & s1, const string & s2)
|
|
||||||
{
|
|
||||||
return s1.empty() ? s2 : s1 + "." + s2;
|
return s1.empty() ? s2 : s1 + "." + s2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
|
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
|
||||||
|
|
||||||
|
|
||||||
static void getDerivations(EvalState& state, Value& vIn,
|
static void getDerivations(EvalState& state, Value& vIn,
|
||||||
const string& pathPrefix, Bindings& autoArgs,
|
const string& pathPrefix, Bindings& autoArgs,
|
||||||
DrvInfos& drvs, Done& done,
|
DrvInfos& drvs, Done& done,
|
||||||
bool ignoreAssertionFailures)
|
bool ignoreAssertionFailures) {
|
||||||
{
|
|
||||||
Value v;
|
Value v;
|
||||||
state.autoCallFunction(autoArgs, vIn, v);
|
state.autoCallFunction(autoArgs, vIn, v);
|
||||||
|
|
||||||
/* Process the expression. */
|
/* Process the expression. */
|
||||||
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
|
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures))
|
||||||
|
;
|
||||||
|
|
||||||
else if (v.type == tAttrs) {
|
else if (v.type == tAttrs) {
|
||||||
|
|
||||||
/* !!! undocumented hackery to support combining channels in
|
/* !!! undocumented hackery to support combining channels in
|
||||||
nix-env.cc. */
|
nix-env.cc. */
|
||||||
bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
|
bool combineChannels =
|
||||||
|
v.attrs->find(state.symbols.create("_combineChannels")) !=
|
||||||
|
v.attrs->end();
|
||||||
|
|
||||||
/* Consider the attributes in sorted order to get more
|
/* Consider the attributes in sorted order to get more
|
||||||
deterministic behaviour in nix-env operations (e.g. when
|
deterministic behaviour in nix-env operations (e.g. when
|
||||||
|
|
@ -339,19 +302,23 @@ static void getDerivations(EvalState & state, Value & vIn,
|
||||||
precedence). */
|
precedence). */
|
||||||
for (auto& i : v.attrs->lexicographicOrder()) {
|
for (auto& i : v.attrs->lexicographicOrder()) {
|
||||||
debug("evaluating attribute '%1%'", i->name);
|
debug("evaluating attribute '%1%'", i->name);
|
||||||
if (!std::regex_match(std::string(i->name), attrRegex))
|
if (!std::regex_match(std::string(i->name), attrRegex)) continue;
|
||||||
continue;
|
|
||||||
string pathPrefix2 = addToPath(pathPrefix, i->name);
|
string pathPrefix2 = addToPath(pathPrefix, i->name);
|
||||||
if (combineChannels)
|
if (combineChannels)
|
||||||
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
|
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
|
||||||
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
|
ignoreAssertionFailures);
|
||||||
|
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done,
|
||||||
|
ignoreAssertionFailures)) {
|
||||||
/* If the value of this attribute is itself a set,
|
/* If the value of this attribute is itself a set,
|
||||||
should we recurse into it? => Only if it has a
|
should we recurse into it? => Only if it has a
|
||||||
`recurseForDerivations = true' attribute. */
|
`recurseForDerivations = true' attribute. */
|
||||||
if (i->value->type == tAttrs) {
|
if (i->value->type == tAttrs) {
|
||||||
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
|
Bindings::iterator j = i->value->attrs->find(
|
||||||
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
|
state.symbols.create("recurseForDerivations"));
|
||||||
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
|
if (j != i->value->attrs->end() &&
|
||||||
|
state.forceBool(*j->value, *j->pos))
|
||||||
|
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done,
|
||||||
|
ignoreAssertionFailures);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -360,21 +327,25 @@ static void getDerivations(EvalState & state, Value & vIn,
|
||||||
else if (v.isList()) {
|
else if (v.isList()) {
|
||||||
for (unsigned int n = 0; n < v.listSize(); ++n) {
|
for (unsigned int n = 0; n < v.listSize(); ++n) {
|
||||||
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
|
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
|
||||||
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
|
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done,
|
||||||
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
|
ignoreAssertionFailures))
|
||||||
|
getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs,
|
||||||
|
done, ignoreAssertionFailures);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
|
else
|
||||||
|
throw TypeError(
|
||||||
|
"expression does not evaluate to a derivation (or a set or list of "
|
||||||
|
"those)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
|
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
|
||||||
Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
|
Bindings& autoArgs, DrvInfos& drvs,
|
||||||
{
|
bool ignoreAssertionFailures) {
|
||||||
Done done;
|
Done done;
|
||||||
getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures);
|
getDerivations(state, v, pathPrefix, autoArgs, drvs, done,
|
||||||
|
ignoreAssertionFailures);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
25
third_party/nix/src/libexpr/get-drvs.hh
vendored
25
third_party/nix/src/libexpr/get-drvs.hh
vendored
|
|
@ -1,16 +1,12 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include "eval.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
struct DrvInfo {
|
||||||
struct DrvInfo
|
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
typedef std::map<string, Path> Outputs;
|
typedef std::map<string, Path> Outputs;
|
||||||
|
|
||||||
|
|
@ -37,14 +33,16 @@ public:
|
||||||
|
|
||||||
DrvInfo(EvalState& state) : state(&state){};
|
DrvInfo(EvalState& state) : state(&state){};
|
||||||
DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs);
|
DrvInfo(EvalState& state, const string& attrPath, Bindings* attrs);
|
||||||
DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
|
DrvInfo(EvalState& state, ref<Store> store,
|
||||||
|
const std::string& drvPathWithOutputs);
|
||||||
|
|
||||||
string queryName() const;
|
string queryName() const;
|
||||||
string querySystem() const;
|
string querySystem() const;
|
||||||
string queryDrvPath() const;
|
string queryDrvPath() const;
|
||||||
string queryOutPath() const;
|
string queryOutPath() const;
|
||||||
string queryOutputName() const;
|
string queryOutputName() const;
|
||||||
/** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
|
/** Return the list of outputs. The "outputs to install" are determined by
|
||||||
|
* `meta.outputsToInstall`. */
|
||||||
Outputs queryOutputs(bool onlyOutputsToInstall = false);
|
Outputs queryOutputs(bool onlyOutputsToInstall = false);
|
||||||
|
|
||||||
StringSet queryMetaNames();
|
StringSet queryMetaNames();
|
||||||
|
|
@ -68,22 +66,19 @@ public:
|
||||||
bool hasFailed() { return failed; };
|
bool hasFailed() { return failed; };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
#if HAVE_BOEHMGC
|
#if HAVE_BOEHMGC
|
||||||
typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
|
typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
|
||||||
#else
|
#else
|
||||||
typedef list<DrvInfo> DrvInfos;
|
typedef list<DrvInfo> DrvInfos;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
/* If value `v' denotes a derivation, return a DrvInfo object
|
/* If value `v' denotes a derivation, return a DrvInfo object
|
||||||
describing it. Otherwise return nothing. */
|
describing it. Otherwise return nothing. */
|
||||||
std::optional<DrvInfo> getDerivation(EvalState & state,
|
std::optional<DrvInfo> getDerivation(EvalState& state, Value& v,
|
||||||
Value & v, bool ignoreAssertionFailures);
|
bool ignoreAssertionFailures);
|
||||||
|
|
||||||
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
|
void getDerivations(EvalState& state, Value& v, const string& pathPrefix,
|
||||||
Bindings& autoArgs, DrvInfos& drvs,
|
Bindings& autoArgs, DrvInfos& drvs,
|
||||||
bool ignoreAssertionFailures);
|
bool ignoreAssertionFailures);
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
76
third_party/nix/src/libexpr/json-to-value.cc
vendored
76
third_party/nix/src/libexpr/json-to-value.cc
vendored
|
|
@ -1,35 +1,42 @@
|
||||||
#include "json-to-value.hh"
|
#include "json-to-value.hh"
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static void skipWhitespace(const char*& s) {
|
||||||
static void skipWhitespace(const char * & s)
|
|
||||||
{
|
|
||||||
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
|
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static string parseJSONString(const char*& s) {
|
||||||
static string parseJSONString(const char * & s)
|
|
||||||
{
|
|
||||||
string res;
|
string res;
|
||||||
if (*s++ != '"') throw JSONParseError("expected JSON string");
|
if (*s++ != '"') throw JSONParseError("expected JSON string");
|
||||||
while (*s != '"') {
|
while (*s != '"') {
|
||||||
if (!*s) throw JSONParseError("got end-of-string in JSON string");
|
if (!*s) throw JSONParseError("got end-of-string in JSON string");
|
||||||
if (*s == '\\') {
|
if (*s == '\\') {
|
||||||
s++;
|
s++;
|
||||||
if (*s == '"') res += '"';
|
if (*s == '"')
|
||||||
else if (*s == '\\') res += '\\';
|
res += '"';
|
||||||
else if (*s == '/') res += '/';
|
else if (*s == '\\')
|
||||||
else if (*s == '/') res += '/';
|
res += '\\';
|
||||||
else if (*s == 'b') res += '\b';
|
else if (*s == '/')
|
||||||
else if (*s == 'f') res += '\f';
|
res += '/';
|
||||||
else if (*s == 'n') res += '\n';
|
else if (*s == '/')
|
||||||
else if (*s == 'r') res += '\r';
|
res += '/';
|
||||||
else if (*s == 't') res += '\t';
|
else if (*s == 'b')
|
||||||
else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
|
res += '\b';
|
||||||
else throw JSONParseError("invalid escaped character in JSON string");
|
else if (*s == 'f')
|
||||||
|
res += '\f';
|
||||||
|
else if (*s == 'n')
|
||||||
|
res += '\n';
|
||||||
|
else if (*s == 'r')
|
||||||
|
res += '\r';
|
||||||
|
else if (*s == 't')
|
||||||
|
res += '\t';
|
||||||
|
else if (*s == 'u')
|
||||||
|
throw JSONParseError(
|
||||||
|
"\\u characters in JSON strings are currently not supported");
|
||||||
|
else
|
||||||
|
throw JSONParseError("invalid escaped character in JSON string");
|
||||||
s++;
|
s++;
|
||||||
} else
|
} else
|
||||||
res += *s++;
|
res += *s++;
|
||||||
|
|
@ -38,9 +45,7 @@ static string parseJSONString(const char * & s)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void parseJSON(EvalState& state, const char*& s, Value& v) {
|
||||||
static void parseJSON(EvalState & state, const char * & s, Value & v)
|
|
||||||
{
|
|
||||||
skipWhitespace(s);
|
skipWhitespace(s);
|
||||||
|
|
||||||
if (!*s) throw JSONParseError("expected JSON value");
|
if (!*s) throw JSONParseError("expected JSON value");
|
||||||
|
|
@ -57,13 +62,13 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
|
||||||
values.push_back(v2);
|
values.push_back(v2);
|
||||||
skipWhitespace(s);
|
skipWhitespace(s);
|
||||||
if (*s == ']') break;
|
if (*s == ']') break;
|
||||||
if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
|
if (*s != ',')
|
||||||
|
throw JSONParseError("expected ',' or ']' after JSON array element");
|
||||||
s++;
|
s++;
|
||||||
}
|
}
|
||||||
s++;
|
s++;
|
||||||
state.mkList(v, values.size());
|
state.mkList(v, values.size());
|
||||||
for (size_t n = 0; n < values.size(); ++n)
|
for (size_t n = 0; n < values.size(); ++n) v.listElems()[n] = values[n];
|
||||||
v.listElems()[n] = values[n];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (*s == '{') {
|
else if (*s == '{') {
|
||||||
|
|
@ -81,12 +86,12 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
|
||||||
attrs[state.symbols.create(name)] = v2;
|
attrs[state.symbols.create(name)] = v2;
|
||||||
skipWhitespace(s);
|
skipWhitespace(s);
|
||||||
if (*s == '}') break;
|
if (*s == '}') break;
|
||||||
if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
|
if (*s != ',')
|
||||||
|
throw JSONParseError("expected ',' or '}' after JSON member");
|
||||||
s++;
|
s++;
|
||||||
}
|
}
|
||||||
state.mkAttrs(v, attrs.size());
|
state.mkAttrs(v, attrs.size());
|
||||||
for (auto & i : attrs)
|
for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second));
|
||||||
v.attrs->push_back(Attr(i.first, i.second));
|
|
||||||
v.attrs->sort();
|
v.attrs->sort();
|
||||||
s++;
|
s++;
|
||||||
}
|
}
|
||||||
|
|
@ -101,8 +106,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
|
||||||
ValueType number_type = tInt;
|
ValueType number_type = tInt;
|
||||||
|
|
||||||
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
|
while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
|
||||||
if (*s == '.' || *s == 'e' || *s == 'E')
|
if (*s == '.' || *s == 'e' || *s == 'E') number_type = tFloat;
|
||||||
number_type = tFloat;
|
|
||||||
tmp_number += *s++;
|
tmp_number += *s++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -133,17 +137,17 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
|
||||||
mkNull(v);
|
mkNull(v);
|
||||||
}
|
}
|
||||||
|
|
||||||
else throw JSONParseError("unrecognised JSON value");
|
else
|
||||||
|
throw JSONParseError("unrecognised JSON value");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void parseJSON(EvalState& state, const string& s_, Value& v) {
|
||||||
void parseJSON(EvalState & state, const string & s_, Value & v)
|
|
||||||
{
|
|
||||||
const char* s = s_.c_str();
|
const char* s = s_.c_str();
|
||||||
parseJSON(state, s, v);
|
parseJSON(state, s, v);
|
||||||
skipWhitespace(s);
|
skipWhitespace(s);
|
||||||
if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
|
if (*s)
|
||||||
|
throw JSONParseError(
|
||||||
|
format("expected end-of-string while parsing JSON value: %1%") % s);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
3
third_party/nix/src/libexpr/json-to-value.hh
vendored
3
third_party/nix/src/libexpr/json-to-value.hh
vendored
|
|
@ -1,8 +1,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include "eval.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
|
||||||
71
third_party/nix/src/libexpr/names.cc
vendored
71
third_party/nix/src/libexpr/names.cc
vendored
|
|
@ -1,23 +1,16 @@
|
||||||
#include "names.hh"
|
#include "names.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
DrvName::DrvName() { name = ""; }
|
||||||
DrvName::DrvName()
|
|
||||||
{
|
|
||||||
name = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/* Parse a derivation name. The `name' part of a derivation name is
|
/* Parse a derivation name. The `name' part of a derivation name is
|
||||||
everything up to but not including the first dash *not* followed by
|
everything up to but not including the first dash *not* followed by
|
||||||
a letter. The `version' part is the rest (excluding the separating
|
a letter. The `version' part is the rest (excluding the separating
|
||||||
dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
|
dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
|
||||||
'2.0.48'). */
|
'2.0.48'). */
|
||||||
DrvName::DrvName(const string & s) : hits(0)
|
DrvName::DrvName(const string& s) : hits(0) {
|
||||||
{
|
|
||||||
name = fullName = s;
|
name = fullName = s;
|
||||||
for (unsigned int i = 0; i < s.size(); ++i) {
|
for (unsigned int i = 0; i < s.size(); ++i) {
|
||||||
/* !!! isalpha/isdigit are affected by the locale. */
|
/* !!! isalpha/isdigit are affected by the locale. */
|
||||||
|
|
@ -29,21 +22,19 @@ DrvName::DrvName(const string & s) : hits(0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool DrvName::matches(DrvName& n) {
|
||||||
bool DrvName::matches(DrvName & n)
|
|
||||||
{
|
|
||||||
if (name != "*") {
|
if (name != "*") {
|
||||||
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
|
if (!regex)
|
||||||
|
regex = std::unique_ptr<std::regex>(
|
||||||
|
new std::regex(name, std::regex::extended));
|
||||||
if (!std::regex_match(n.name, *regex)) return false;
|
if (!std::regex_match(n.name, *regex)) return false;
|
||||||
}
|
}
|
||||||
if (version != "" && version != n.version) return false;
|
if (version != "" && version != n.version) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
string nextComponent(string::const_iterator& p,
|
string nextComponent(string::const_iterator& p,
|
||||||
const string::const_iterator end)
|
const string::const_iterator end) {
|
||||||
{
|
|
||||||
/* Skip any dots and dashes (component separators). */
|
/* Skip any dots and dashes (component separators). */
|
||||||
while (p != end && (*p == '.' || *p == '-')) ++p;
|
while (p != end && (*p == '.' || *p == '-')) ++p;
|
||||||
|
|
||||||
|
|
@ -56,52 +47,52 @@ string nextComponent(string::const_iterator & p,
|
||||||
if (isdigit(*p))
|
if (isdigit(*p))
|
||||||
while (p != end && isdigit(*p)) s += *p++;
|
while (p != end && isdigit(*p)) s += *p++;
|
||||||
else
|
else
|
||||||
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
|
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) s += *p++;
|
||||||
s += *p++;
|
|
||||||
|
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool componentsLT(const string& c1, const string& c2) {
|
||||||
static bool componentsLT(const string & c1, const string & c2)
|
|
||||||
{
|
|
||||||
int n1, n2;
|
int n1, n2;
|
||||||
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
|
bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
|
||||||
|
|
||||||
if (c1Num && c2Num) return n1 < n2;
|
if (c1Num && c2Num)
|
||||||
else if (c1 == "" && c2Num) return true;
|
return n1 < n2;
|
||||||
else if (c1 == "pre" && c2 != "pre") return true;
|
else if (c1 == "" && c2Num)
|
||||||
else if (c2 == "pre") return false;
|
return true;
|
||||||
|
else if (c1 == "pre" && c2 != "pre")
|
||||||
|
return true;
|
||||||
|
else if (c2 == "pre")
|
||||||
|
return false;
|
||||||
/* Assume that `2.3a' < `2.3.1'. */
|
/* Assume that `2.3a' < `2.3.1'. */
|
||||||
else if (c2Num) return true;
|
else if (c2Num)
|
||||||
else if (c1Num) return false;
|
return true;
|
||||||
else return c1 < c2;
|
else if (c1Num)
|
||||||
|
return false;
|
||||||
|
else
|
||||||
|
return c1 < c2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int compareVersions(const string& v1, const string& v2) {
|
||||||
int compareVersions(const string & v1, const string & v2)
|
|
||||||
{
|
|
||||||
string::const_iterator p1 = v1.begin();
|
string::const_iterator p1 = v1.begin();
|
||||||
string::const_iterator p2 = v2.begin();
|
string::const_iterator p2 = v2.begin();
|
||||||
|
|
||||||
while (p1 != v1.end() || p2 != v2.end()) {
|
while (p1 != v1.end() || p2 != v2.end()) {
|
||||||
string c1 = nextComponent(p1, v1.end());
|
string c1 = nextComponent(p1, v1.end());
|
||||||
string c2 = nextComponent(p2, v2.end());
|
string c2 = nextComponent(p2, v2.end());
|
||||||
if (componentsLT(c1, c2)) return -1;
|
if (componentsLT(c1, c2))
|
||||||
else if (componentsLT(c2, c1)) return 1;
|
return -1;
|
||||||
|
else if (componentsLT(c2, c1))
|
||||||
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DrvNames drvNamesFromArgs(const Strings& opArgs) {
|
||||||
DrvNames drvNamesFromArgs(const Strings & opArgs)
|
|
||||||
{
|
|
||||||
DrvNames result;
|
DrvNames result;
|
||||||
for (auto & i : opArgs)
|
for (auto& i : opArgs) result.push_back(DrvName(i));
|
||||||
result.push_back(DrvName(i));
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
8
third_party/nix/src/libexpr/names.hh
vendored
8
third_party/nix/src/libexpr/names.hh
vendored
|
|
@ -1,14 +1,12 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include <regex>
|
#include <regex>
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct DrvName
|
struct DrvName {
|
||||||
{
|
|
||||||
string fullName;
|
string fullName;
|
||||||
string name;
|
string name;
|
||||||
string version;
|
string version;
|
||||||
|
|
@ -29,4 +27,4 @@ string nextComponent(string::const_iterator & p,
|
||||||
int compareVersions(const string& v1, const string& v2);
|
int compareVersions(const string& v1, const string& v2);
|
||||||
DrvNames drvNamesFromArgs(const Strings& opArgs);
|
DrvNames drvNamesFromArgs(const Strings& opArgs);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
267
third_party/nix/src/libexpr/nixexpr.cc
vendored
267
third_party/nix/src/libexpr/nixexpr.cc
vendored
|
|
@ -1,35 +1,34 @@
|
||||||
#include "nixexpr.hh"
|
#include "nixexpr.hh"
|
||||||
|
#include <cstdlib>
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
|
||||||
#include <cstdlib>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* Displaying abstract syntax trees. */
|
/* Displaying abstract syntax trees. */
|
||||||
|
|
||||||
std::ostream & operator << (std::ostream & str, const Expr & e)
|
std::ostream& operator<<(std::ostream& str, const Expr& e) {
|
||||||
{
|
|
||||||
e.show(str);
|
e.show(str);
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void showString(std::ostream & str, const string & s)
|
static void showString(std::ostream& str, const string& s) {
|
||||||
{
|
|
||||||
str << '"';
|
str << '"';
|
||||||
for (auto c : (string)s)
|
for (auto c : (string)s)
|
||||||
if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
|
if (c == '"' || c == '\\' || c == '$')
|
||||||
else if (c == '\n') str << "\\n";
|
str << "\\" << c;
|
||||||
else if (c == '\r') str << "\\r";
|
else if (c == '\n')
|
||||||
else if (c == '\t') str << "\\t";
|
str << "\\n";
|
||||||
else str << c;
|
else if (c == '\r')
|
||||||
|
str << "\\r";
|
||||||
|
else if (c == '\t')
|
||||||
|
str << "\\t";
|
||||||
|
else
|
||||||
|
str << c;
|
||||||
str << '"';
|
str << '"';
|
||||||
}
|
}
|
||||||
|
|
||||||
static void showId(std::ostream & str, const string & s)
|
static void showId(std::ostream& str, const string& s) {
|
||||||
{
|
|
||||||
if (s.empty())
|
if (s.empty())
|
||||||
str << "\"\"";
|
str << "\"\"";
|
||||||
else if (s == "if") // FIXME: handle other keywords
|
else if (s == "if") // FIXME: handle other keywords
|
||||||
|
|
@ -41,10 +40,8 @@ static void showId(std::ostream & str, const string & s)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (auto c : s)
|
for (auto c : s)
|
||||||
if (!((c >= 'a' && c <= 'z') ||
|
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
||||||
(c >= 'A' && c <= 'Z') ||
|
(c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) {
|
||||||
(c >= '0' && c <= '9') ||
|
|
||||||
c == '_' || c == '\'' || c == '-')) {
|
|
||||||
showString(str, s);
|
showString(str, s);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -52,60 +49,39 @@ static void showId(std::ostream & str, const string & s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::ostream & operator << (std::ostream & str, const Symbol & sym)
|
std::ostream& operator<<(std::ostream& str, const Symbol& sym) {
|
||||||
{
|
|
||||||
showId(str, *sym.s);
|
showId(str, *sym.s);
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Expr::show(std::ostream & str) const
|
void Expr::show(std::ostream& str) const { abort(); }
|
||||||
{
|
|
||||||
abort();
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprInt::show(std::ostream & str) const
|
void ExprInt::show(std::ostream& str) const { str << n; }
|
||||||
{
|
|
||||||
str << n;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprFloat::show(std::ostream & str) const
|
void ExprFloat::show(std::ostream& str) const { str << nf; }
|
||||||
{
|
|
||||||
str << nf;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprString::show(std::ostream & str) const
|
void ExprString::show(std::ostream& str) const { showString(str, s); }
|
||||||
{
|
|
||||||
showString(str, s);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprPath::show(std::ostream & str) const
|
void ExprPath::show(std::ostream& str) const { str << s; }
|
||||||
{
|
|
||||||
str << s;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprVar::show(std::ostream & str) const
|
void ExprVar::show(std::ostream& str) const { str << name; }
|
||||||
{
|
|
||||||
str << name;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprSelect::show(std::ostream & str) const
|
void ExprSelect::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "(" << *e << ")." << showAttrPath(attrPath);
|
str << "(" << *e << ")." << showAttrPath(attrPath);
|
||||||
if (def) str << " or (" << *def << ")";
|
if (def) str << " or (" << *def << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprOpHasAttr::show(std::ostream & str) const
|
void ExprOpHasAttr::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
|
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprAttrs::show(std::ostream & str) const
|
void ExprAttrs::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
if (recursive) str << "rec ";
|
if (recursive) str << "rec ";
|
||||||
str << "{ ";
|
str << "{ ";
|
||||||
for (auto& i : attrs)
|
for (auto& i : attrs)
|
||||||
if (i.second.inherited)
|
if (i.second.inherited)
|
||||||
str << "inherit " << i.first << " " << "; ";
|
str << "inherit " << i.first << " "
|
||||||
|
<< "; ";
|
||||||
else
|
else
|
||||||
str << i.first << " = " << *i.second.e << "; ";
|
str << i.first << " = " << *i.second.e << "; ";
|
||||||
for (auto& i : dynamicAttrs)
|
for (auto& i : dynamicAttrs)
|
||||||
|
|
@ -113,22 +89,22 @@ void ExprAttrs::show(std::ostream & str) const
|
||||||
str << "}";
|
str << "}";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprList::show(std::ostream & str) const
|
void ExprList::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "[ ";
|
str << "[ ";
|
||||||
for (auto & i : elems)
|
for (auto& i : elems) str << "(" << *i << ") ";
|
||||||
str << "(" << *i << ") ";
|
|
||||||
str << "]";
|
str << "]";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprLambda::show(std::ostream & str) const
|
void ExprLambda::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "(";
|
str << "(";
|
||||||
if (matchAttrs) {
|
if (matchAttrs) {
|
||||||
str << "{ ";
|
str << "{ ";
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (auto& i : formals->formals) {
|
for (auto& i : formals->formals) {
|
||||||
if (first) first = false; else str << ", ";
|
if (first)
|
||||||
|
first = false;
|
||||||
|
else
|
||||||
|
str << ", ";
|
||||||
str << i.name;
|
str << i.name;
|
||||||
if (i.def) str << " ? " << *i.def;
|
if (i.def) str << " ? " << *i.def;
|
||||||
}
|
}
|
||||||
|
|
@ -143,71 +119,63 @@ void ExprLambda::show(std::ostream & str) const
|
||||||
str << ": " << *body << ")";
|
str << ": " << *body << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprLet::show(std::ostream & str) const
|
void ExprLet::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "(let ";
|
str << "(let ";
|
||||||
for (auto& i : attrs->attrs)
|
for (auto& i : attrs->attrs)
|
||||||
if (i.second.inherited) {
|
if (i.second.inherited) {
|
||||||
str << "inherit " << i.first << "; ";
|
str << "inherit " << i.first << "; ";
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
str << i.first << " = " << *i.second.e << "; ";
|
str << i.first << " = " << *i.second.e << "; ";
|
||||||
str << "in " << *body << ")";
|
str << "in " << *body << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprWith::show(std::ostream & str) const
|
void ExprWith::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "(with " << *attrs << "; " << *body << ")";
|
str << "(with " << *attrs << "; " << *body << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprIf::show(std::ostream & str) const
|
void ExprIf::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
|
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprAssert::show(std::ostream & str) const
|
void ExprAssert::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
str << "assert " << *cond << "; " << *body;
|
str << "assert " << *cond << "; " << *body;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprOpNot::show(std::ostream & str) const
|
void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; }
|
||||||
{
|
|
||||||
str << "(! " << *e << ")";
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprConcatStrings::show(std::ostream & str) const
|
void ExprConcatStrings::show(std::ostream& str) const {
|
||||||
{
|
|
||||||
bool first = true;
|
bool first = true;
|
||||||
str << "(";
|
str << "(";
|
||||||
for (auto& i : *es) {
|
for (auto& i : *es) {
|
||||||
if (first) first = false; else str << " + ";
|
if (first)
|
||||||
|
first = false;
|
||||||
|
else
|
||||||
|
str << " + ";
|
||||||
str << *i;
|
str << *i;
|
||||||
}
|
}
|
||||||
str << ")";
|
str << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprPos::show(std::ostream & str) const
|
void ExprPos::show(std::ostream& str) const { str << "__curPos"; }
|
||||||
{
|
|
||||||
str << "__curPos";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
std::ostream& operator<<(std::ostream& str, const Pos& pos) {
|
||||||
std::ostream & operator << (std::ostream & str, const Pos & pos)
|
|
||||||
{
|
|
||||||
if (!pos)
|
if (!pos)
|
||||||
str << "undefined position";
|
str << "undefined position";
|
||||||
else
|
else
|
||||||
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
|
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string)pos.file %
|
||||||
|
pos.line % pos.column)
|
||||||
|
.str();
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string showAttrPath(const AttrPath& attrPath) {
|
||||||
string showAttrPath(const AttrPath & attrPath)
|
|
||||||
{
|
|
||||||
std::ostringstream out;
|
std::ostringstream out;
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (auto& i : attrPath) {
|
for (auto& i : attrPath) {
|
||||||
if (!first) out << '.'; else first = false;
|
if (!first)
|
||||||
|
out << '.';
|
||||||
|
else
|
||||||
|
first = false;
|
||||||
if (i.symbol.set())
|
if (i.symbol.set())
|
||||||
out << i.symbol;
|
out << i.symbol;
|
||||||
else
|
else
|
||||||
|
|
@ -216,35 +184,21 @@ string showAttrPath(const AttrPath & attrPath)
|
||||||
return out.str();
|
return out.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Pos noPos;
|
Pos noPos;
|
||||||
|
|
||||||
|
|
||||||
/* Computing levels/displacements for variables. */
|
/* Computing levels/displacements for variables. */
|
||||||
|
|
||||||
void Expr::bindVars(const StaticEnv & env)
|
void Expr::bindVars(const StaticEnv& env) { abort(); }
|
||||||
{
|
|
||||||
abort();
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprInt::bindVars(const StaticEnv & env)
|
void ExprInt::bindVars(const StaticEnv& env) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprFloat::bindVars(const StaticEnv & env)
|
void ExprFloat::bindVars(const StaticEnv& env) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprString::bindVars(const StaticEnv & env)
|
void ExprString::bindVars(const StaticEnv& env) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprPath::bindVars(const StaticEnv & env)
|
void ExprPath::bindVars(const StaticEnv& env) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprVar::bindVars(const StaticEnv & env)
|
void ExprVar::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
/* Check whether the variable appears in the environment. If so,
|
/* Check whether the variable appears in the environment. If so,
|
||||||
set its level and displacement. */
|
set its level and displacement. */
|
||||||
const StaticEnv* curEnv;
|
const StaticEnv* curEnv;
|
||||||
|
|
@ -267,31 +221,28 @@ void ExprVar::bindVars(const StaticEnv & env)
|
||||||
/* Otherwise, the variable must be obtained from the nearest
|
/* Otherwise, the variable must be obtained from the nearest
|
||||||
enclosing `with'. If there is no `with', then we can issue an
|
enclosing `with'. If there is no `with', then we can issue an
|
||||||
"undefined variable" error now. */
|
"undefined variable" error now. */
|
||||||
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
|
if (withLevel == -1)
|
||||||
|
throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name %
|
||||||
|
pos);
|
||||||
|
|
||||||
fromWith = true;
|
fromWith = true;
|
||||||
this->level = withLevel;
|
this->level = withLevel;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprSelect::bindVars(const StaticEnv & env)
|
void ExprSelect::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
e->bindVars(env);
|
e->bindVars(env);
|
||||||
if (def) def->bindVars(env);
|
if (def) def->bindVars(env);
|
||||||
for (auto& i : attrPath)
|
for (auto& i : attrPath)
|
||||||
if (!i.symbol.set())
|
if (!i.symbol.set()) i.expr->bindVars(env);
|
||||||
i.expr->bindVars(env);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprOpHasAttr::bindVars(const StaticEnv & env)
|
void ExprOpHasAttr::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
e->bindVars(env);
|
e->bindVars(env);
|
||||||
for (auto& i : attrPath)
|
for (auto& i : attrPath)
|
||||||
if (!i.symbol.set())
|
if (!i.symbol.set()) i.expr->bindVars(env);
|
||||||
i.expr->bindVars(env);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprAttrs::bindVars(const StaticEnv & env)
|
void ExprAttrs::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
const StaticEnv* dynamicEnv = &env;
|
const StaticEnv* dynamicEnv = &env;
|
||||||
StaticEnv newEnv(false, &env);
|
StaticEnv newEnv(false, &env);
|
||||||
|
|
||||||
|
|
@ -299,16 +250,14 @@ void ExprAttrs::bindVars(const StaticEnv & env)
|
||||||
dynamicEnv = &newEnv;
|
dynamicEnv = &newEnv;
|
||||||
|
|
||||||
unsigned int displ = 0;
|
unsigned int displ = 0;
|
||||||
for (auto & i : attrs)
|
for (auto& i : attrs) newEnv.vars[i.first] = i.second.displ = displ++;
|
||||||
newEnv.vars[i.first] = i.second.displ = displ++;
|
|
||||||
|
|
||||||
for (auto& i : attrs)
|
for (auto& i : attrs)
|
||||||
i.second.e->bindVars(i.second.inherited ? env : newEnv);
|
i.second.e->bindVars(i.second.inherited ? env : newEnv);
|
||||||
}
|
}
|
||||||
|
|
||||||
else
|
else
|
||||||
for (auto & i : attrs)
|
for (auto& i : attrs) i.second.e->bindVars(env);
|
||||||
i.second.e->bindVars(env);
|
|
||||||
|
|
||||||
for (auto& i : dynamicAttrs) {
|
for (auto& i : dynamicAttrs) {
|
||||||
i.nameExpr->bindVars(*dynamicEnv);
|
i.nameExpr->bindVars(*dynamicEnv);
|
||||||
|
|
@ -316,14 +265,11 @@ void ExprAttrs::bindVars(const StaticEnv & env)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprList::bindVars(const StaticEnv & env)
|
void ExprList::bindVars(const StaticEnv& env) {
|
||||||
{
|
for (auto& i : elems) i->bindVars(env);
|
||||||
for (auto & i : elems)
|
|
||||||
i->bindVars(env);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprLambda::bindVars(const StaticEnv & env)
|
void ExprLambda::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
StaticEnv newEnv(false, &env);
|
StaticEnv newEnv(false, &env);
|
||||||
|
|
||||||
unsigned int displ = 0;
|
unsigned int displ = 0;
|
||||||
|
|
@ -331,8 +277,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
|
||||||
if (!arg.empty()) newEnv.vars[arg] = displ++;
|
if (!arg.empty()) newEnv.vars[arg] = displ++;
|
||||||
|
|
||||||
if (matchAttrs) {
|
if (matchAttrs) {
|
||||||
for (auto & i : formals->formals)
|
for (auto& i : formals->formals) newEnv.vars[i.name] = displ++;
|
||||||
newEnv.vars[i.name] = displ++;
|
|
||||||
|
|
||||||
for (auto& i : formals->formals)
|
for (auto& i : formals->formals)
|
||||||
if (i.def) i.def->bindVars(newEnv);
|
if (i.def) i.def->bindVars(newEnv);
|
||||||
|
|
@ -341,13 +286,11 @@ void ExprLambda::bindVars(const StaticEnv & env)
|
||||||
body->bindVars(newEnv);
|
body->bindVars(newEnv);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprLet::bindVars(const StaticEnv & env)
|
void ExprLet::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
StaticEnv newEnv(false, &env);
|
StaticEnv newEnv(false, &env);
|
||||||
|
|
||||||
unsigned int displ = 0;
|
unsigned int displ = 0;
|
||||||
for (auto & i : attrs->attrs)
|
for (auto& i : attrs->attrs) newEnv.vars[i.first] = i.second.displ = displ++;
|
||||||
newEnv.vars[i.first] = i.second.displ = displ++;
|
|
||||||
|
|
||||||
for (auto& i : attrs->attrs)
|
for (auto& i : attrs->attrs)
|
||||||
i.second.e->bindVars(i.second.inherited ? env : newEnv);
|
i.second.e->bindVars(i.second.inherited ? env : newEnv);
|
||||||
|
|
@ -355,8 +298,7 @@ void ExprLet::bindVars(const StaticEnv & env)
|
||||||
body->bindVars(newEnv);
|
body->bindVars(newEnv);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprWith::bindVars(const StaticEnv & env)
|
void ExprWith::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
/* Does this `with' have an enclosing `with'? If so, record its
|
/* Does this `with' have an enclosing `with'? If so, record its
|
||||||
level so that `lookupVar' can look up variables in the previous
|
level so that `lookupVar' can look up variables in the previous
|
||||||
`with' if this one doesn't contain the desired attribute. */
|
`with' if this one doesn't contain the desired attribute. */
|
||||||
|
|
@ -374,65 +316,46 @@ void ExprWith::bindVars(const StaticEnv & env)
|
||||||
body->bindVars(newEnv);
|
body->bindVars(newEnv);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprIf::bindVars(const StaticEnv & env)
|
void ExprIf::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
cond->bindVars(env);
|
cond->bindVars(env);
|
||||||
then->bindVars(env);
|
then->bindVars(env);
|
||||||
else_->bindVars(env);
|
else_->bindVars(env);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprAssert::bindVars(const StaticEnv & env)
|
void ExprAssert::bindVars(const StaticEnv& env) {
|
||||||
{
|
|
||||||
cond->bindVars(env);
|
cond->bindVars(env);
|
||||||
body->bindVars(env);
|
body->bindVars(env);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExprOpNot::bindVars(const StaticEnv & env)
|
void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); }
|
||||||
{
|
|
||||||
e->bindVars(env);
|
void ExprConcatStrings::bindVars(const StaticEnv& env) {
|
||||||
}
|
for (auto& i : *es) i->bindVars(env);
|
||||||
|
|
||||||
void ExprConcatStrings::bindVars(const StaticEnv & env)
|
|
||||||
{
|
|
||||||
for (auto & i : *es)
|
|
||||||
i->bindVars(env);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ExprPos::bindVars(const StaticEnv & env)
|
|
||||||
{
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ExprPos::bindVars(const StaticEnv& env) {}
|
||||||
|
|
||||||
/* Storing function names. */
|
/* Storing function names. */
|
||||||
|
|
||||||
void Expr::setName(Symbol & name)
|
void Expr::setName(Symbol& name) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
void ExprLambda::setName(Symbol& name) {
|
||||||
void ExprLambda::setName(Symbol & name)
|
|
||||||
{
|
|
||||||
this->name = name;
|
this->name = name;
|
||||||
body->setName(name);
|
body->setName(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string ExprLambda::showNamePos() const {
|
||||||
string ExprLambda::showNamePos() const
|
return (format("%1% at %2%") %
|
||||||
{
|
(name.set() ? "'" + (string)name + "'" : "anonymous function") % pos)
|
||||||
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
|
.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* Symbol table. */
|
/* Symbol table. */
|
||||||
|
|
||||||
size_t SymbolTable::totalSize() const
|
size_t SymbolTable::totalSize() const {
|
||||||
{
|
|
||||||
size_t n = 0;
|
size_t n = 0;
|
||||||
for (auto & i : symbols)
|
for (auto& i : symbols) n += i.size();
|
||||||
n += i.size();
|
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
161
third_party/nix/src/libexpr/nixexpr.hh
vendored
161
third_party/nix/src/libexpr/nixexpr.hh
vendored
|
|
@ -1,39 +1,27 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "value.hh"
|
|
||||||
#include "symbol-table.hh"
|
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include "symbol-table.hh"
|
||||||
|
#include "value.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
MakeError(EvalError, Error) MakeError(ParseError, Error)
|
||||||
MakeError(EvalError, Error)
|
MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError)
|
||||||
MakeError(ParseError, Error)
|
MakeError(Abort, EvalError) MakeError(TypeError, EvalError)
|
||||||
MakeError(AssertionError, EvalError)
|
|
||||||
MakeError(ThrownError, AssertionError)
|
|
||||||
MakeError(Abort, EvalError)
|
|
||||||
MakeError(TypeError, EvalError)
|
|
||||||
MakeError(UndefinedVarError, Error)
|
MakeError(UndefinedVarError, Error)
|
||||||
MakeError(RestrictedPathError, Error)
|
MakeError(RestrictedPathError, Error)
|
||||||
|
|
||||||
|
|
||||||
/* Position objects. */
|
/* Position objects. */
|
||||||
|
|
||||||
struct Pos
|
struct Pos {
|
||||||
{
|
|
||||||
Symbol file;
|
Symbol file;
|
||||||
unsigned int line, column;
|
unsigned int line, column;
|
||||||
Pos() : line(0), column(0){};
|
Pos() : line(0), column(0){};
|
||||||
Pos(const Symbol& file, unsigned int line, unsigned int column)
|
Pos(const Symbol& file, unsigned int line, unsigned int column)
|
||||||
: file(file), line(line), column(column){};
|
: file(file), line(line), column(column){};
|
||||||
operator bool() const
|
operator bool() const { return line != 0; }
|
||||||
{
|
bool operator<(const Pos& p2) const {
|
||||||
return line != 0;
|
|
||||||
}
|
|
||||||
bool operator < (const Pos & p2) const
|
|
||||||
{
|
|
||||||
if (!line) return p2.line;
|
if (!line) return p2.line;
|
||||||
if (!p2.line) return false;
|
if (!p2.line) return false;
|
||||||
int d = ((string)file).compare((string)p2.file);
|
int d = ((string)file).compare((string)p2.file);
|
||||||
|
|
@ -49,16 +37,13 @@ extern Pos noPos;
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& str, const Pos& pos);
|
std::ostream& operator<<(std::ostream& str, const Pos& pos);
|
||||||
|
|
||||||
|
|
||||||
struct Env;
|
struct Env;
|
||||||
struct Value;
|
struct Value;
|
||||||
class EvalState;
|
class EvalState;
|
||||||
struct StaticEnv;
|
struct StaticEnv;
|
||||||
|
|
||||||
|
|
||||||
/* An attribute path is a sequence of attribute names. */
|
/* An attribute path is a sequence of attribute names. */
|
||||||
struct AttrName
|
struct AttrName {
|
||||||
{
|
|
||||||
Symbol symbol;
|
Symbol symbol;
|
||||||
Expr* expr;
|
Expr* expr;
|
||||||
AttrName(const Symbol& s) : symbol(s){};
|
AttrName(const Symbol& s) : symbol(s){};
|
||||||
|
|
@ -69,11 +54,9 @@ typedef std::vector<AttrName> AttrPath;
|
||||||
|
|
||||||
string showAttrPath(const AttrPath& attrPath);
|
string showAttrPath(const AttrPath& attrPath);
|
||||||
|
|
||||||
|
|
||||||
/* Abstract syntax of Nix expressions. */
|
/* Abstract syntax of Nix expressions. */
|
||||||
|
|
||||||
struct Expr
|
struct Expr {
|
||||||
{
|
|
||||||
virtual ~Expr(){};
|
virtual ~Expr(){};
|
||||||
virtual void show(std::ostream& str) const;
|
virtual void show(std::ostream& str) const;
|
||||||
virtual void bindVars(const StaticEnv& env);
|
virtual void bindVars(const StaticEnv& env);
|
||||||
|
|
@ -89,8 +72,7 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
|
||||||
void eval(EvalState& state, Env& env, Value& v); \
|
void eval(EvalState& state, Env& env, Value& v); \
|
||||||
void bindVars(const StaticEnv& env);
|
void bindVars(const StaticEnv& env);
|
||||||
|
|
||||||
struct ExprInt : Expr
|
struct ExprInt : Expr {
|
||||||
{
|
|
||||||
NixInt n;
|
NixInt n;
|
||||||
Value v;
|
Value v;
|
||||||
ExprInt(NixInt n) : n(n) { mkInt(v, n); };
|
ExprInt(NixInt n) : n(n) { mkInt(v, n); };
|
||||||
|
|
@ -98,8 +80,7 @@ struct ExprInt : Expr
|
||||||
Value* maybeThunk(EvalState& state, Env& env);
|
Value* maybeThunk(EvalState& state, Env& env);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprFloat : Expr
|
struct ExprFloat : Expr {
|
||||||
{
|
|
||||||
NixFloat nf;
|
NixFloat nf;
|
||||||
Value v;
|
Value v;
|
||||||
ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
|
ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
|
||||||
|
|
@ -107,8 +88,7 @@ struct ExprFloat : Expr
|
||||||
Value* maybeThunk(EvalState& state, Env& env);
|
Value* maybeThunk(EvalState& state, Env& env);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprString : Expr
|
struct ExprString : Expr {
|
||||||
{
|
|
||||||
Symbol s;
|
Symbol s;
|
||||||
Value v;
|
Value v;
|
||||||
ExprString(const Symbol& s) : s(s) { mkString(v, s); };
|
ExprString(const Symbol& s) : s(s) { mkString(v, s); };
|
||||||
|
|
@ -117,14 +97,12 @@ struct ExprString : Expr
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Temporary class used during parsing of indented strings. */
|
/* Temporary class used during parsing of indented strings. */
|
||||||
struct ExprIndStr : Expr
|
struct ExprIndStr : Expr {
|
||||||
{
|
|
||||||
string s;
|
string s;
|
||||||
ExprIndStr(const string& s) : s(s){};
|
ExprIndStr(const string& s) : s(s){};
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprPath : Expr
|
struct ExprPath : Expr {
|
||||||
{
|
|
||||||
string s;
|
string s;
|
||||||
Value v;
|
Value v;
|
||||||
ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
|
ExprPath(const string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
|
||||||
|
|
@ -132,8 +110,7 @@ struct ExprPath : Expr
|
||||||
Value* maybeThunk(EvalState& state, Env& env);
|
Value* maybeThunk(EvalState& state, Env& env);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprVar : Expr
|
struct ExprVar : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
Symbol name;
|
Symbol name;
|
||||||
|
|
||||||
|
|
@ -156,26 +133,27 @@ struct ExprVar : Expr
|
||||||
Value* maybeThunk(EvalState& state, Env& env);
|
Value* maybeThunk(EvalState& state, Env& env);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprSelect : Expr
|
struct ExprSelect : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
Expr *e, *def;
|
Expr *e, *def;
|
||||||
AttrPath attrPath;
|
AttrPath attrPath;
|
||||||
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
|
ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def)
|
||||||
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
|
: pos(pos), e(e), def(def), attrPath(attrPath){};
|
||||||
|
ExprSelect(const Pos& pos, Expr* e, const Symbol& name)
|
||||||
|
: pos(pos), e(e), def(0) {
|
||||||
|
attrPath.push_back(AttrName(name));
|
||||||
|
};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprOpHasAttr : Expr
|
struct ExprOpHasAttr : Expr {
|
||||||
{
|
|
||||||
Expr* e;
|
Expr* e;
|
||||||
AttrPath attrPath;
|
AttrPath attrPath;
|
||||||
ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){};
|
ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprAttrs : Expr
|
struct ExprAttrs : Expr {
|
||||||
{
|
|
||||||
bool recursive;
|
bool recursive;
|
||||||
struct AttrDef {
|
struct AttrDef {
|
||||||
bool inherited;
|
bool inherited;
|
||||||
|
|
@ -200,116 +178,109 @@ struct ExprAttrs : Expr
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprList : Expr
|
struct ExprList : Expr {
|
||||||
{
|
|
||||||
std::vector<Expr*> elems;
|
std::vector<Expr*> elems;
|
||||||
ExprList(){};
|
ExprList(){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Formal
|
struct Formal {
|
||||||
{
|
|
||||||
Symbol name;
|
Symbol name;
|
||||||
Expr* def;
|
Expr* def;
|
||||||
Formal(const Symbol& name, Expr* def) : name(name), def(def){};
|
Formal(const Symbol& name, Expr* def) : name(name), def(def){};
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Formals
|
struct Formals {
|
||||||
{
|
|
||||||
typedef std::list<Formal> Formals_;
|
typedef std::list<Formal> Formals_;
|
||||||
Formals_ formals;
|
Formals_ formals;
|
||||||
std::set<Symbol> argNames; // used during parsing
|
std::set<Symbol> argNames; // used during parsing
|
||||||
bool ellipsis;
|
bool ellipsis;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprLambda : Expr
|
struct ExprLambda : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
Symbol name;
|
Symbol name;
|
||||||
Symbol arg;
|
Symbol arg;
|
||||||
bool matchAttrs;
|
bool matchAttrs;
|
||||||
Formals* formals;
|
Formals* formals;
|
||||||
Expr* body;
|
Expr* body;
|
||||||
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
|
ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs,
|
||||||
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
|
Formals* formals, Expr* body)
|
||||||
{
|
: pos(pos),
|
||||||
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
|
arg(arg),
|
||||||
throw ParseError(format("duplicate formal function argument '%1%' at %2%")
|
matchAttrs(matchAttrs),
|
||||||
% arg % pos);
|
formals(formals),
|
||||||
|
body(body) {
|
||||||
|
if (!arg.empty() && formals &&
|
||||||
|
formals->argNames.find(arg) != formals->argNames.end())
|
||||||
|
throw ParseError(
|
||||||
|
format("duplicate formal function argument '%1%' at %2%") % arg %
|
||||||
|
pos);
|
||||||
};
|
};
|
||||||
void setName(Symbol& name);
|
void setName(Symbol& name);
|
||||||
string showNamePos() const;
|
string showNamePos() const;
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprLet : Expr
|
struct ExprLet : Expr {
|
||||||
{
|
|
||||||
ExprAttrs* attrs;
|
ExprAttrs* attrs;
|
||||||
Expr* body;
|
Expr* body;
|
||||||
ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){};
|
ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprWith : Expr
|
struct ExprWith : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
Expr *attrs, *body;
|
Expr *attrs, *body;
|
||||||
size_t prevWith;
|
size_t prevWith;
|
||||||
ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
|
ExprWith(const Pos& pos, Expr* attrs, Expr* body)
|
||||||
|
: pos(pos), attrs(attrs), body(body){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprIf : Expr
|
struct ExprIf : Expr {
|
||||||
{
|
|
||||||
Expr *cond, *then, *else_;
|
Expr *cond, *then, *else_;
|
||||||
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
|
ExprIf(Expr* cond, Expr* then, Expr* else_)
|
||||||
|
: cond(cond), then(then), else_(else_){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprAssert : Expr
|
struct ExprAssert : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
Expr *cond, *body;
|
Expr *cond, *body;
|
||||||
ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
|
ExprAssert(const Pos& pos, Expr* cond, Expr* body)
|
||||||
|
: pos(pos), cond(cond), body(body){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprOpNot : Expr
|
struct ExprOpNot : Expr {
|
||||||
{
|
|
||||||
Expr* e;
|
Expr* e;
|
||||||
ExprOpNot(Expr* e) : e(e){};
|
ExprOpNot(Expr* e) : e(e){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
#define MakeBinOp(name, s) \
|
#define MakeBinOp(name, s) \
|
||||||
struct name : Expr \
|
struct name : Expr { \
|
||||||
{ \
|
|
||||||
Pos pos; \
|
Pos pos; \
|
||||||
Expr *e1, *e2; \
|
Expr *e1, *e2; \
|
||||||
name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \
|
name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \
|
||||||
name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \
|
name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \
|
||||||
void show(std::ostream & str) const \
|
void show(std::ostream& str) const { \
|
||||||
{ \
|
|
||||||
str << "(" << *e1 << " " s " " << *e2 << ")"; \
|
str << "(" << *e1 << " " s " " << *e2 << ")"; \
|
||||||
} \
|
} \
|
||||||
void bindVars(const StaticEnv & env) \
|
void bindVars(const StaticEnv& env) { \
|
||||||
{ \
|
e1->bindVars(env); \
|
||||||
e1->bindVars(env); e2->bindVars(env); \
|
e2->bindVars(env); \
|
||||||
} \
|
} \
|
||||||
void eval(EvalState& state, Env& env, Value& v); \
|
void eval(EvalState& state, Env& env, Value& v); \
|
||||||
};
|
};
|
||||||
|
|
||||||
MakeBinOp(ExprApp, "")
|
MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=")
|
||||||
MakeBinOp(ExprOpEq, "==")
|
MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||")
|
||||||
MakeBinOp(ExprOpNEq, "!=")
|
MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//")
|
||||||
MakeBinOp(ExprOpAnd, "&&")
|
|
||||||
MakeBinOp(ExprOpOr, "||")
|
|
||||||
MakeBinOp(ExprOpImpl, "->")
|
|
||||||
MakeBinOp(ExprOpUpdate, "//")
|
|
||||||
MakeBinOp(ExprOpConcatLists, "++")
|
MakeBinOp(ExprOpConcatLists, "++")
|
||||||
|
|
||||||
struct ExprConcatStrings : Expr
|
struct ExprConcatStrings : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
bool forceString;
|
bool forceString;
|
||||||
vector<Expr*>* es;
|
vector<Expr*>* es;
|
||||||
|
|
@ -318,19 +289,16 @@ struct ExprConcatStrings : Expr
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ExprPos : Expr
|
struct ExprPos : Expr {
|
||||||
{
|
|
||||||
Pos pos;
|
Pos pos;
|
||||||
ExprPos(const Pos& pos) : pos(pos){};
|
ExprPos(const Pos& pos) : pos(pos){};
|
||||||
COMMON_METHODS
|
COMMON_METHODS
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Static environments are used to map variable names onto (level,
|
/* Static environments are used to map variable names onto (level,
|
||||||
displacement) pairs used to obtain the value of the variable at
|
displacement) pairs used to obtain the value of the variable at
|
||||||
runtime. */
|
runtime. */
|
||||||
struct StaticEnv
|
struct StaticEnv {
|
||||||
{
|
|
||||||
bool isWith;
|
bool isWith;
|
||||||
const StaticEnv* up;
|
const StaticEnv* up;
|
||||||
typedef std::map<Symbol, unsigned int> Vars;
|
typedef std::map<Symbol, unsigned int> Vars;
|
||||||
|
|
@ -338,5 +306,4 @@ struct StaticEnv
|
||||||
StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){};
|
StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
980
third_party/nix/src/libexpr/primops.cc
vendored
980
third_party/nix/src/libexpr/primops.cc
vendored
File diff suppressed because it is too large
Load diff
11
third_party/nix/src/libexpr/primops.hh
vendored
11
third_party/nix/src/libexpr/primops.hh
vendored
|
|
@ -1,12 +1,10 @@
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <tuple>
|
#include <tuple>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
#include "eval.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct RegisterPrimOp
|
struct RegisterPrimOp {
|
||||||
{
|
|
||||||
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
|
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
|
||||||
static PrimOps* primOps;
|
static PrimOps* primOps;
|
||||||
/* You can register a constant by passing an arity of 0. fun
|
/* You can register a constant by passing an arity of 0. fun
|
||||||
|
|
@ -19,8 +17,9 @@ struct RegisterPrimOp
|
||||||
may wish to use them in limited contexts without globally enabling
|
may wish to use them in limited contexts without globally enabling
|
||||||
them. */
|
them. */
|
||||||
/* Load a ValueInitializer from a DSO and return whatever it initializes */
|
/* Load a ValueInitializer from a DSO and return whatever it initializes */
|
||||||
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
|
void prim_importNative(EvalState& state, const Pos& pos, Value** args,
|
||||||
|
Value& v);
|
||||||
/* Execute a program and parse its output */
|
/* Execute a program and parse its output */
|
||||||
void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v);
|
void prim_exec(EvalState& state, const Pos& pos, Value** args, Value& v);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
69
third_party/nix/src/libexpr/primops/context.cc
vendored
69
third_party/nix/src/libexpr/primops/context.cc
vendored
|
|
@ -1,21 +1,21 @@
|
||||||
#include "primops.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "primops.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos,
|
||||||
{
|
Value** args, Value& v) {
|
||||||
PathSet context;
|
PathSet context;
|
||||||
string s = state.coerceToString(pos, *args[0], context);
|
string s = state.coerceToString(pos, *args[0], context);
|
||||||
mkString(v, s, PathSet());
|
mkString(v, s, PathSet());
|
||||||
}
|
}
|
||||||
|
|
||||||
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
|
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1,
|
||||||
|
prim_unsafeDiscardStringContext);
|
||||||
|
|
||||||
|
static void prim_hasContext(EvalState& state, const Pos& pos, Value** args,
|
||||||
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
Value& v) {
|
||||||
{
|
|
||||||
PathSet context;
|
PathSet context;
|
||||||
state.forceString(*args[0], context, pos);
|
state.forceString(*args[0], context, pos);
|
||||||
mkBool(v, !context.empty());
|
mkBool(v, !context.empty());
|
||||||
|
|
@ -23,27 +23,25 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
|
||||||
|
|
||||||
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
|
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
|
||||||
|
|
||||||
|
|
||||||
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
|
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
|
||||||
builder without causing the derivation to be built (for instance,
|
builder without causing the derivation to be built (for instance,
|
||||||
in the derivation that builds NARs in nix-push, when doing
|
in the derivation that builds NARs in nix-push, when doing
|
||||||
source-only deployment). This primop marks the string context so
|
source-only deployment). This primop marks the string context so
|
||||||
that builtins.derivation adds the path to drv.inputSrcs rather than
|
that builtins.derivation adds the path to drv.inputSrcs rather than
|
||||||
drv.inputDrvs. */
|
drv.inputDrvs. */
|
||||||
static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos,
|
||||||
{
|
Value** args, Value& v) {
|
||||||
PathSet context;
|
PathSet context;
|
||||||
string s = state.coerceToString(pos, *args[0], context);
|
string s = state.coerceToString(pos, *args[0], context);
|
||||||
|
|
||||||
PathSet context2;
|
PathSet context2;
|
||||||
for (auto & p : context)
|
for (auto& p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p);
|
||||||
context2.insert(p.at(0) == '=' ? string(p, 1) : p);
|
|
||||||
|
|
||||||
mkString(v, s, context2);
|
mkString(v, s, context2);
|
||||||
}
|
}
|
||||||
|
|
||||||
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
|
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1,
|
||||||
|
prim_unsafeDiscardOutputDependency);
|
||||||
|
|
||||||
/* Extract the context of a string as a structured Nix value.
|
/* Extract the context of a string as a structured Nix value.
|
||||||
|
|
||||||
|
|
@ -64,8 +62,8 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar
|
||||||
Note that for a given path any combination of the above attributes
|
Note that for a given path any combination of the above attributes
|
||||||
may be present.
|
may be present.
|
||||||
*/
|
*/
|
||||||
static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_getContext(EvalState& state, const Pos& pos, Value** args,
|
||||||
{
|
Value& v) {
|
||||||
struct ContextInfo {
|
struct ContextInfo {
|
||||||
bool path = false;
|
bool path = false;
|
||||||
bool allOutputs = false;
|
bool allOutputs = false;
|
||||||
|
|
@ -92,7 +90,10 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
|
||||||
|
|
||||||
auto iter = contextInfos.find(*path);
|
auto iter = contextInfos.find(*path);
|
||||||
if (iter == contextInfos.end()) {
|
if (iter == contextInfos.end()) {
|
||||||
contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
|
contextInfos.emplace(
|
||||||
|
*path,
|
||||||
|
ContextInfo{isPath, isAllOutputs,
|
||||||
|
output.empty() ? Strings{} : Strings{std::move(output)}});
|
||||||
} else {
|
} else {
|
||||||
if (isPath)
|
if (isPath)
|
||||||
iter->second.path = true;
|
iter->second.path = true;
|
||||||
|
|
@ -110,8 +111,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
|
||||||
for (const auto& info : contextInfos) {
|
for (const auto& info : contextInfos) {
|
||||||
auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first));
|
auto& infoVal = *state.allocAttr(v, state.symbols.create(info.first));
|
||||||
state.mkAttrs(infoVal, 3);
|
state.mkAttrs(infoVal, 3);
|
||||||
if (info.second.path)
|
if (info.second.path) mkBool(*state.allocAttr(infoVal, sPath), true);
|
||||||
mkBool(*state.allocAttr(infoVal, sPath), true);
|
|
||||||
if (info.second.allOutputs)
|
if (info.second.allOutputs)
|
||||||
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
|
mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
|
||||||
if (!info.second.outputs.empty()) {
|
if (!info.second.outputs.empty()) {
|
||||||
|
|
@ -129,14 +129,13 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
|
||||||
|
|
||||||
static RegisterPrimOp r4("__getContext", 1, prim_getContext);
|
static RegisterPrimOp r4("__getContext", 1, prim_getContext);
|
||||||
|
|
||||||
|
|
||||||
/* Append the given context to a given string.
|
/* Append the given context to a given string.
|
||||||
|
|
||||||
See the commentary above unsafeGetContext for details of the
|
See the commentary above unsafeGetContext for details of the
|
||||||
context representation.
|
context representation.
|
||||||
*/
|
*/
|
||||||
static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_appendContext(EvalState& state, const Pos& pos, Value** args,
|
||||||
{
|
Value& v) {
|
||||||
PathSet context;
|
PathSet context;
|
||||||
auto orig = state.forceString(*args[0], context, pos);
|
auto orig = state.forceString(*args[0], context, pos);
|
||||||
|
|
||||||
|
|
@ -146,21 +145,23 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||||
auto sAllOutputs = state.symbols.create("allOutputs");
|
auto sAllOutputs = state.symbols.create("allOutputs");
|
||||||
for (auto& i : *args[1]->attrs) {
|
for (auto& i : *args[1]->attrs) {
|
||||||
if (!state.store->isStorePath(i.name))
|
if (!state.store->isStorePath(i.name))
|
||||||
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
|
throw EvalError("Context key '%s' is not a store path, at %s", i.name,
|
||||||
if (!settings.readOnlyMode)
|
i.pos);
|
||||||
state.store->ensurePath(i.name);
|
if (!settings.readOnlyMode) state.store->ensurePath(i.name);
|
||||||
state.forceAttrs(*i.value, *i.pos);
|
state.forceAttrs(*i.value, *i.pos);
|
||||||
auto iter = i.value->attrs->find(sPath);
|
auto iter = i.value->attrs->find(sPath);
|
||||||
if (iter != i.value->attrs->end()) {
|
if (iter != i.value->attrs->end()) {
|
||||||
if (state.forceBool(*iter->value, *iter->pos))
|
if (state.forceBool(*iter->value, *iter->pos)) context.insert(i.name);
|
||||||
context.insert(i.name);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
iter = i.value->attrs->find(sAllOutputs);
|
iter = i.value->attrs->find(sAllOutputs);
|
||||||
if (iter != i.value->attrs->end()) {
|
if (iter != i.value->attrs->end()) {
|
||||||
if (state.forceBool(*iter->value, *iter->pos)) {
|
if (state.forceBool(*iter->value, *iter->pos)) {
|
||||||
if (!isDerivation(i.name)) {
|
if (!isDerivation(i.name)) {
|
||||||
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
|
throw EvalError(
|
||||||
|
"Tried to add all-outputs context of %s, which is not a "
|
||||||
|
"derivation, to a string, at %s",
|
||||||
|
i.name, i.pos);
|
||||||
}
|
}
|
||||||
context.insert("=" + string(i.name));
|
context.insert("=" + string(i.name));
|
||||||
}
|
}
|
||||||
|
|
@ -170,10 +171,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||||
if (iter != i.value->attrs->end()) {
|
if (iter != i.value->attrs->end()) {
|
||||||
state.forceList(*iter->value, *iter->pos);
|
state.forceList(*iter->value, *iter->pos);
|
||||||
if (iter->value->listSize() && !isDerivation(i.name)) {
|
if (iter->value->listSize() && !isDerivation(i.name)) {
|
||||||
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
|
throw EvalError(
|
||||||
|
"Tried to add derivation output context of %s, which is not a "
|
||||||
|
"derivation, to a string, at %s",
|
||||||
|
i.name, i.pos);
|
||||||
}
|
}
|
||||||
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
|
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
|
||||||
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
|
auto name =
|
||||||
|
state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
|
||||||
context.insert("!" + name + "!" + string(i.name));
|
context.insert("!" + name + "!" + string(i.name));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -184,4 +189,4 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||||
|
|
||||||
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
|
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
76
third_party/nix/src/libexpr/primops/fetchGit.cc
vendored
76
third_party/nix/src/libexpr/primops/fetchGit.cc
vendored
|
|
@ -1,22 +1,18 @@
|
||||||
#include "primops.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
#include "download.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
#include "hash.hh"
|
|
||||||
|
|
||||||
#include <sys/time.h>
|
#include <sys/time.h>
|
||||||
|
|
||||||
#include <regex>
|
|
||||||
|
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
|
#include <regex>
|
||||||
|
#include "download.hh"
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "hash.hh"
|
||||||
|
#include "pathlocks.hh"
|
||||||
|
#include "primops.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
using namespace std::string_literals;
|
using namespace std::string_literals;
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct GitInfo
|
struct GitInfo {
|
||||||
{
|
|
||||||
Path storePath;
|
Path storePath;
|
||||||
std::string rev;
|
std::string rev;
|
||||||
std::string shortRev;
|
std::string shortRev;
|
||||||
|
|
@ -27,24 +23,22 @@ std::regex revRegex("^[0-9a-fA-F]{40}$");
|
||||||
|
|
||||||
GitInfo exportGit(ref<Store> store, const std::string& uri,
|
GitInfo exportGit(ref<Store> store, const std::string& uri,
|
||||||
std::optional<std::string> ref, std::string rev,
|
std::optional<std::string> ref, std::string rev,
|
||||||
const std::string & name)
|
const std::string& name) {
|
||||||
{
|
|
||||||
if (evalSettings.pureEval && rev == "")
|
if (evalSettings.pureEval && rev == "")
|
||||||
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
|
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
|
||||||
|
|
||||||
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
|
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
|
||||||
|
|
||||||
bool clean = true;
|
bool clean = true;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
|
runProgram("git", true,
|
||||||
|
{"-C", uri, "diff-index", "--quiet", "HEAD", "--"});
|
||||||
} catch (ExecError& e) {
|
} catch (ExecError& e) {
|
||||||
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
|
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
|
||||||
clean = false;
|
clean = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!clean) {
|
if (!clean) {
|
||||||
|
|
||||||
/* This is an unclean working tree. So copy all tracked
|
/* This is an unclean working tree. So copy all tracked
|
||||||
files. */
|
files. */
|
||||||
|
|
||||||
|
|
@ -70,7 +64,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
return files.count(file);
|
return files.count(file);
|
||||||
};
|
};
|
||||||
|
|
||||||
gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
|
gitInfo.storePath =
|
||||||
|
store->addToStore("source", uri, true, htSHA256, filter);
|
||||||
|
|
||||||
return gitInfo;
|
return gitInfo;
|
||||||
}
|
}
|
||||||
|
|
@ -87,7 +82,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
|
|
||||||
deletePath(getCacheDir() + "/nix/git");
|
deletePath(getCacheDir() + "/nix/git");
|
||||||
|
|
||||||
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
|
Path cacheDir = getCacheDir() + "/nix/gitv2/" +
|
||||||
|
hashString(htSHA256, uri).to_string(Base32, false);
|
||||||
|
|
||||||
if (!pathExists(cacheDir)) {
|
if (!pathExists(cacheDir)) {
|
||||||
createDirs(dirOf(cacheDir));
|
createDirs(dirOf(cacheDir));
|
||||||
|
|
@ -122,13 +118,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
|
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
|
||||||
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now;
|
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now;
|
||||||
}
|
}
|
||||||
if (doFetch)
|
if (doFetch) {
|
||||||
{
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
|
fmt("fetching Git repository '%s'", uri));
|
||||||
|
|
||||||
// FIXME: git stderr messes up our progress indicator, so
|
// FIXME: git stderr messes up our progress indicator, so
|
||||||
// we're using --quiet for now. Should process its stderr.
|
// we're using --quiet for now. Should process its stderr.
|
||||||
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
|
runProgram("git", true,
|
||||||
|
{"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri,
|
||||||
|
fmt("%s:%s", *ref, *ref)});
|
||||||
|
|
||||||
struct timeval times[2];
|
struct timeval times[2];
|
||||||
times[0].tv_sec = now;
|
times[0].tv_sec = now;
|
||||||
|
|
@ -146,9 +144,12 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
|
|
||||||
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
|
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
|
||||||
|
|
||||||
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
|
std::string storeLinkName =
|
||||||
|
hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev)
|
||||||
|
.to_string(Base32, false);
|
||||||
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
|
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
|
||||||
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
|
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...",
|
||||||
|
storeLink)); // FIXME: broken
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto json = nlohmann::json::parse(readFile(storeLink));
|
auto json = nlohmann::json::parse(readFile(storeLink));
|
||||||
|
|
@ -177,7 +178,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
|
|
||||||
gitInfo.storePath = store->addToStore(name, tmpDir);
|
gitInfo.storePath = store->addToStore(name, tmpDir);
|
||||||
|
|
||||||
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
|
gitInfo.revCount = std::stoull(runProgram(
|
||||||
|
"git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev}));
|
||||||
|
|
||||||
nlohmann::json json;
|
nlohmann::json json;
|
||||||
json["storePath"] = gitInfo.storePath;
|
json["storePath"] = gitInfo.storePath;
|
||||||
|
|
@ -191,8 +193,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
|
||||||
return gitInfo;
|
return gitInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args,
|
||||||
{
|
Value& v) {
|
||||||
std::string url;
|
std::string url;
|
||||||
std::optional<std::string> ref;
|
std::optional<std::string> ref;
|
||||||
std::string rev;
|
std::string rev;
|
||||||
|
|
@ -202,13 +204,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
state.forceValue(*args[0]);
|
state.forceValue(*args[0]);
|
||||||
|
|
||||||
if (args[0]->type == tAttrs) {
|
if (args[0]->type == tAttrs) {
|
||||||
|
|
||||||
state.forceAttrs(*args[0], pos);
|
state.forceAttrs(*args[0], pos);
|
||||||
|
|
||||||
for (auto& attr : *args[0]->attrs) {
|
for (auto& attr : *args[0]->attrs) {
|
||||||
string n(attr.name);
|
string n(attr.name);
|
||||||
if (n == "url")
|
if (n == "url")
|
||||||
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
|
url =
|
||||||
|
state.coerceToString(*attr.pos, *attr.value, context, false, false);
|
||||||
else if (n == "ref")
|
else if (n == "ref")
|
||||||
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
|
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||||
else if (n == "rev")
|
else if (n == "rev")
|
||||||
|
|
@ -216,7 +218,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
else if (n == "name")
|
else if (n == "name")
|
||||||
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||||
else
|
else
|
||||||
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
|
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s",
|
||||||
|
attr.name, *attr.pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (url.empty())
|
if (url.empty())
|
||||||
|
|
@ -232,10 +235,13 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
auto gitInfo = exportGit(state.store, url, ref, rev, name);
|
auto gitInfo = exportGit(state.store, url, ref, rev, name);
|
||||||
|
|
||||||
state.mkAttrs(v, 8);
|
state.mkAttrs(v, 8);
|
||||||
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
|
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath,
|
||||||
|
PathSet({gitInfo.storePath}));
|
||||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
|
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
|
||||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
|
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
|
||||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
|
gitInfo.shortRev);
|
||||||
|
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
|
||||||
|
gitInfo.revCount);
|
||||||
v.attrs->sort();
|
v.attrs->sort();
|
||||||
|
|
||||||
if (state.allowedPaths)
|
if (state.allowedPaths)
|
||||||
|
|
@ -244,4 +250,4 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
|
|
||||||
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
|
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,17 @@
|
||||||
#include "primops.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
#include "download.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
|
|
||||||
#include <sys/time.h>
|
#include <sys/time.h>
|
||||||
|
|
||||||
#include <regex>
|
|
||||||
|
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
|
#include <regex>
|
||||||
|
#include "download.hh"
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "pathlocks.hh"
|
||||||
|
#include "primops.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
using namespace std::string_literals;
|
using namespace std::string_literals;
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct HgInfo
|
struct HgInfo {
|
||||||
{
|
|
||||||
Path storePath;
|
Path storePath;
|
||||||
std::string branch;
|
std::string branch;
|
||||||
std::string rev;
|
std::string rev;
|
||||||
|
|
@ -25,17 +21,18 @@ struct HgInfo
|
||||||
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
|
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
|
||||||
|
|
||||||
HgInfo exportMercurial(ref<Store> store, const std::string& uri,
|
HgInfo exportMercurial(ref<Store> store, const std::string& uri,
|
||||||
std::string rev, const std::string & name)
|
std::string rev, const std::string& name) {
|
||||||
{
|
|
||||||
if (evalSettings.pureEval && rev == "")
|
if (evalSettings.pureEval && rev == "")
|
||||||
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
|
throw Error(
|
||||||
|
"in pure evaluation mode, 'fetchMercurial' requires a Mercurial "
|
||||||
|
"revision");
|
||||||
|
|
||||||
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
|
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
|
||||||
|
bool clean = runProgram("hg", true,
|
||||||
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
|
{"status", "-R", uri, "--modified", "--added",
|
||||||
|
"--removed"}) == "";
|
||||||
|
|
||||||
if (!clean) {
|
if (!clean) {
|
||||||
|
|
||||||
/* This is an unclean working tree. So copy all tracked
|
/* This is an unclean working tree. So copy all tracked
|
||||||
files. */
|
files. */
|
||||||
|
|
||||||
|
|
@ -46,7 +43,10 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri}));
|
hgInfo.branch = chomp(runProgram("hg", true, {"branch", "-R", uri}));
|
||||||
|
|
||||||
auto files = tokenizeString<std::set<std::string>>(
|
auto files = tokenizeString<std::set<std::string>>(
|
||||||
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
|
runProgram("hg", true,
|
||||||
|
{"status", "-R", uri, "--clean", "--modified", "--added",
|
||||||
|
"--no-status", "--print0"}),
|
||||||
|
"\0"s);
|
||||||
|
|
||||||
PathFilter filter = [&](const Path& p) -> bool {
|
PathFilter filter = [&](const Path& p) -> bool {
|
||||||
assert(hasPrefix(p, uri));
|
assert(hasPrefix(p, uri));
|
||||||
|
|
@ -63,7 +63,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
return files.count(file);
|
return files.count(file);
|
||||||
};
|
};
|
||||||
|
|
||||||
hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
|
hgInfo.storePath =
|
||||||
|
store->addToStore("source", uri, true, htSHA256, filter);
|
||||||
|
|
||||||
return hgInfo;
|
return hgInfo;
|
||||||
}
|
}
|
||||||
|
|
@ -71,39 +72,40 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
|
|
||||||
if (rev == "") rev = "default";
|
if (rev == "") rev = "default";
|
||||||
|
|
||||||
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
|
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(),
|
||||||
|
hashString(htSHA256, uri).to_string(Base32, false));
|
||||||
|
|
||||||
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
|
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir,
|
||||||
|
hashString(htSHA512, rev).to_string(Base32, false));
|
||||||
|
|
||||||
/* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
|
/* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
|
||||||
do so now. */
|
do so now. */
|
||||||
time_t now = time(0);
|
time_t now = time(0);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (stat(stampFile.c_str(), &st) != 0 ||
|
if (stat(stampFile.c_str(), &st) != 0 ||
|
||||||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
|
(uint64_t)st.st_mtime + settings.tarballTtl <= (uint64_t)now) {
|
||||||
{
|
|
||||||
/* Except that if this is a commit hash that we already have,
|
/* Except that if this is a commit hash that we already have,
|
||||||
we don't have to pull again. */
|
we don't have to pull again. */
|
||||||
if (!(std::regex_match(rev, commitHashRegex)
|
if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) &&
|
||||||
&& pathExists(cacheDir)
|
runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev,
|
||||||
&& runProgram(
|
"--template", "1"})
|
||||||
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
|
.killStderr(true))
|
||||||
.killStderr(true)).second == "1"))
|
.second == "1")) {
|
||||||
{
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
|
fmt("fetching Mercurial repository '%s'", uri));
|
||||||
|
|
||||||
if (pathExists(cacheDir)) {
|
if (pathExists(cacheDir)) {
|
||||||
try {
|
try {
|
||||||
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
|
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
|
||||||
}
|
} catch (ExecError& e) {
|
||||||
catch (ExecError & e) {
|
|
||||||
string transJournal = cacheDir + "/.hg/store/journal";
|
string transJournal = cacheDir + "/.hg/store/journal";
|
||||||
/* hg throws "abandoned transaction" error only if this file exists */
|
/* hg throws "abandoned transaction" error only if this file exists */
|
||||||
if (pathExists(transJournal)) {
|
if (pathExists(transJournal)) {
|
||||||
runProgram("hg", true, {"recover", "-R", cacheDir});
|
runProgram("hg", true, {"recover", "-R", cacheDir});
|
||||||
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
|
runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri});
|
||||||
} else {
|
} else {
|
||||||
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
|
throw ExecError(e.status,
|
||||||
|
fmt("'hg pull' %s", statusToString(e.status)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -116,7 +118,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
}
|
}
|
||||||
|
|
||||||
auto tokens = tokenizeString<std::vector<std::string>>(
|
auto tokens = tokenizeString<std::vector<std::string>>(
|
||||||
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
|
runProgram("hg", true,
|
||||||
|
{"log", "-R", cacheDir, "-r", rev, "--template",
|
||||||
|
"{node} {rev} {branch}"}));
|
||||||
assert(tokens.size() == 3);
|
assert(tokens.size() == 3);
|
||||||
|
|
||||||
HgInfo hgInfo;
|
HgInfo hgInfo;
|
||||||
|
|
@ -124,7 +128,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
hgInfo.revCount = std::stoull(tokens[1]);
|
hgInfo.revCount = std::stoull(tokens[1]);
|
||||||
hgInfo.branch = tokens[2];
|
hgInfo.branch = tokens[2];
|
||||||
|
|
||||||
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
|
std::string storeLinkName =
|
||||||
|
hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev)
|
||||||
|
.to_string(Base32, false);
|
||||||
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
|
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -135,7 +141,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
hgInfo.storePath = json["storePath"];
|
hgInfo.storePath = json["storePath"];
|
||||||
|
|
||||||
if (store->isValidPath(hgInfo.storePath)) {
|
if (store->isValidPath(hgInfo.storePath)) {
|
||||||
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
|
printTalkative("using cached Mercurial store path '%s'",
|
||||||
|
hgInfo.storePath);
|
||||||
return hgInfo;
|
return hgInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -165,8 +172,8 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
|
||||||
return hgInfo;
|
return hgInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args,
|
||||||
{
|
Value& v) {
|
||||||
std::string url;
|
std::string url;
|
||||||
std::string rev;
|
std::string rev;
|
||||||
std::string name = "source";
|
std::string name = "source";
|
||||||
|
|
@ -175,19 +182,20 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
|
||||||
state.forceValue(*args[0]);
|
state.forceValue(*args[0]);
|
||||||
|
|
||||||
if (args[0]->type == tAttrs) {
|
if (args[0]->type == tAttrs) {
|
||||||
|
|
||||||
state.forceAttrs(*args[0], pos);
|
state.forceAttrs(*args[0], pos);
|
||||||
|
|
||||||
for (auto& attr : *args[0]->attrs) {
|
for (auto& attr : *args[0]->attrs) {
|
||||||
string n(attr.name);
|
string n(attr.name);
|
||||||
if (n == "url")
|
if (n == "url")
|
||||||
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
|
url =
|
||||||
|
state.coerceToString(*attr.pos, *attr.value, context, false, false);
|
||||||
else if (n == "rev")
|
else if (n == "rev")
|
||||||
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
|
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||||
else if (n == "name")
|
else if (n == "name")
|
||||||
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||||
else
|
else
|
||||||
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
|
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s",
|
||||||
|
attr.name, *attr.pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (url.empty())
|
if (url.empty())
|
||||||
|
|
@ -203,10 +211,12 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
|
||||||
auto hgInfo = exportMercurial(state.store, url, rev, name);
|
auto hgInfo = exportMercurial(state.store, url, rev, name);
|
||||||
|
|
||||||
state.mkAttrs(v, 8);
|
state.mkAttrs(v, 8);
|
||||||
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
|
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath,
|
||||||
|
PathSet({hgInfo.storePath}));
|
||||||
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
|
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
|
||||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
|
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
|
||||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
|
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
|
||||||
|
std::string(hgInfo.rev, 0, 12));
|
||||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
|
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
|
||||||
v.attrs->sort();
|
v.attrs->sort();
|
||||||
|
|
||||||
|
|
@ -216,4 +226,4 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
|
||||||
|
|
||||||
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
|
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
24
third_party/nix/src/libexpr/primops/fromTOML.cc
vendored
24
third_party/nix/src/libexpr/primops/fromTOML.cc
vendored
|
|
@ -1,12 +1,11 @@
|
||||||
#include "primops.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
|
|
||||||
#include "cpptoml/cpptoml.h"
|
#include "cpptoml/cpptoml.h"
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "primops.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args,
|
||||||
{
|
Value& v) {
|
||||||
using namespace cpptoml;
|
using namespace cpptoml;
|
||||||
|
|
||||||
auto toml = state.forceStringNoCtx(*args[0], pos);
|
auto toml = state.forceStringNoCtx(*args[0], pos);
|
||||||
|
|
@ -16,11 +15,12 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
std::function<void(Value&, std::shared_ptr<base>)> visit;
|
std::function<void(Value&, std::shared_ptr<base>)> visit;
|
||||||
|
|
||||||
visit = [&](Value& v, std::shared_ptr<base> t) {
|
visit = [&](Value& v, std::shared_ptr<base> t) {
|
||||||
|
|
||||||
if (auto t2 = t->as_table()) {
|
if (auto t2 = t->as_table()) {
|
||||||
|
|
||||||
size_t size = 0;
|
size_t size = 0;
|
||||||
for (auto & i : *t2) { (void) i; size++; }
|
for (auto& i : *t2) {
|
||||||
|
(void)i;
|
||||||
|
size++;
|
||||||
|
}
|
||||||
|
|
||||||
state.mkAttrs(v, size);
|
state.mkAttrs(v, size);
|
||||||
|
|
||||||
|
|
@ -32,8 +32,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
state.mkList(v2, size2);
|
state.mkList(v2, size2);
|
||||||
for (size_t j = 0; j < size2; ++j)
|
for (size_t j = 0; j < size2; ++j)
|
||||||
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
|
visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
visit(v2, i.second);
|
visit(v2, i.second);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -75,7 +74,8 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
throw EvalError("unsupported value type in TOML");
|
throw EvalError("unsupported value type in TOML");
|
||||||
}
|
}
|
||||||
|
|
||||||
else abort();
|
else
|
||||||
|
abort();
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -87,4 +87,4 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
|
||||||
|
|
||||||
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
|
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
53
third_party/nix/src/libexpr/symbol-table.hh
vendored
53
third_party/nix/src/libexpr/symbol-table.hh
vendored
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
@ -13,8 +12,7 @@ namespace nix {
|
||||||
they can be compared efficiently (using a pointer equality test),
|
they can be compared efficiently (using a pointer equality test),
|
||||||
because the symbol table stores only one copy of each string. */
|
because the symbol table stores only one copy of each string. */
|
||||||
|
|
||||||
class Symbol
|
class Symbol {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
const string* s; // pointer into SymbolTable
|
const string* s; // pointer into SymbolTable
|
||||||
Symbol(const string* s) : s(s){};
|
Symbol(const string* s) : s(s){};
|
||||||
|
|
@ -23,65 +21,40 @@ private:
|
||||||
public:
|
public:
|
||||||
Symbol() : s(0){};
|
Symbol() : s(0){};
|
||||||
|
|
||||||
bool operator == (const Symbol & s2) const
|
bool operator==(const Symbol& s2) const { return s == s2.s; }
|
||||||
{
|
|
||||||
return s == s2.s;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool operator != (const Symbol & s2) const
|
bool operator!=(const Symbol& s2) const { return s != s2.s; }
|
||||||
{
|
|
||||||
return s != s2.s;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool operator < (const Symbol & s2) const
|
bool operator<(const Symbol& s2) const { return s < s2.s; }
|
||||||
{
|
|
||||||
return s < s2.s;
|
|
||||||
}
|
|
||||||
|
|
||||||
operator const string & () const
|
operator const string&() const { return *s; }
|
||||||
{
|
|
||||||
return *s;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool set() const
|
bool set() const { return s; }
|
||||||
{
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool empty() const
|
bool empty() const { return s->empty(); }
|
||||||
{
|
|
||||||
return s->empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
friend std::ostream& operator<<(std::ostream& str, const Symbol& sym);
|
friend std::ostream& operator<<(std::ostream& str, const Symbol& sym);
|
||||||
};
|
};
|
||||||
|
|
||||||
class SymbolTable
|
class SymbolTable {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
typedef std::unordered_set<string> Symbols;
|
typedef std::unordered_set<string> Symbols;
|
||||||
Symbols symbols;
|
Symbols symbols;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
Symbol create(const string & s)
|
Symbol create(const string& s) {
|
||||||
{
|
|
||||||
std::pair<Symbols::iterator, bool> res = symbols.insert(s);
|
std::pair<Symbols::iterator, bool> res = symbols.insert(s);
|
||||||
return Symbol(&*res.first);
|
return Symbol(&*res.first);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t size() const
|
size_t size() const { return symbols.size(); }
|
||||||
{
|
|
||||||
return symbols.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t totalSize() const;
|
size_t totalSize() const;
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
void dump(T callback)
|
void dump(T callback) {
|
||||||
{
|
for (auto& s : symbols) callback(s);
|
||||||
for (auto & s : symbols)
|
|
||||||
callback(s);
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
39
third_party/nix/src/libexpr/value-to-json.cc
vendored
39
third_party/nix/src/libexpr/value-to-json.cc
vendored
|
|
@ -1,23 +1,19 @@
|
||||||
#include "value-to-json.hh"
|
#include "value-to-json.hh"
|
||||||
#include "json.hh"
|
|
||||||
#include "eval-inline.hh"
|
|
||||||
#include "util.hh"
|
|
||||||
|
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
#include <iomanip>
|
#include <iomanip>
|
||||||
|
#include "eval-inline.hh"
|
||||||
|
#include "json.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
void printValueAsJSON(EvalState & state, bool strict,
|
void printValueAsJSON(EvalState& state, bool strict, Value& v,
|
||||||
Value & v, JSONPlaceholder & out, PathSet & context)
|
JSONPlaceholder& out, PathSet& context) {
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
if (strict) state.forceValue(v);
|
if (strict) state.forceValue(v);
|
||||||
|
|
||||||
switch (v.type) {
|
switch (v.type) {
|
||||||
|
|
||||||
case tInt:
|
case tInt:
|
||||||
out.write(v.integer);
|
out.write(v.integer);
|
||||||
break;
|
break;
|
||||||
|
|
@ -40,7 +36,8 @@ void printValueAsJSON(EvalState & state, bool strict,
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case tAttrs: {
|
case tAttrs: {
|
||||||
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
|
auto maybeString =
|
||||||
|
state.tryAttrsToString(noPos, v, context, false, false);
|
||||||
if (maybeString) {
|
if (maybeString) {
|
||||||
out.write(*maybeString);
|
out.write(*maybeString);
|
||||||
break;
|
break;
|
||||||
|
|
@ -49,8 +46,7 @@ void printValueAsJSON(EvalState & state, bool strict,
|
||||||
if (i == v.attrs->end()) {
|
if (i == v.attrs->end()) {
|
||||||
auto obj(out.object());
|
auto obj(out.object());
|
||||||
StringSet names;
|
StringSet names;
|
||||||
for (auto & j : *v.attrs)
|
for (auto& j : *v.attrs) names.insert(j.name);
|
||||||
names.insert(j.name);
|
|
||||||
for (auto& j : names) {
|
for (auto& j : names) {
|
||||||
Attr& a(*v.attrs->find(state.symbols.create(j)));
|
Attr& a(*v.attrs->find(state.symbols.create(j)));
|
||||||
auto placeholder(obj.placeholder(j));
|
auto placeholder(obj.placeholder(j));
|
||||||
|
|
@ -61,11 +57,14 @@ void printValueAsJSON(EvalState & state, bool strict,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case tList1: case tList2: case tListN: {
|
case tList1:
|
||||||
|
case tList2:
|
||||||
|
case tListN: {
|
||||||
auto list(out.list());
|
auto list(out.list());
|
||||||
for (unsigned int n = 0; n < v.listSize(); ++n) {
|
for (unsigned int n = 0; n < v.listSize(); ++n) {
|
||||||
auto placeholder(list.placeholder());
|
auto placeholder(list.placeholder());
|
||||||
printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context);
|
printValueAsJSON(state, strict, *v.listElems()[n], placeholder,
|
||||||
|
context);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -83,18 +82,16 @@ void printValueAsJSON(EvalState & state, bool strict,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void printValueAsJSON(EvalState & state, bool strict,
|
void printValueAsJSON(EvalState& state, bool strict, Value& v,
|
||||||
Value & v, std::ostream & str, PathSet & context)
|
std::ostream& str, PathSet& context) {
|
||||||
{
|
|
||||||
JSONPlaceholder out(str);
|
JSONPlaceholder out(str);
|
||||||
printValueAsJSON(state, strict, v, out, context);
|
printValueAsJSON(state, strict, v, out, context);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict,
|
void ExternalValueBase::printValueAsJSON(EvalState& state, bool strict,
|
||||||
JSONPlaceholder & out, PathSet & context) const
|
JSONPlaceholder& out,
|
||||||
{
|
PathSet& context) const {
|
||||||
throw TypeError(format("cannot convert %1% to JSON") % showType());
|
throw TypeError(format("cannot convert %1% to JSON") % showType());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
17
third_party/nix/src/libexpr/value-to-json.hh
vendored
17
third_party/nix/src/libexpr/value-to-json.hh
vendored
|
|
@ -1,19 +1,18 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "nixexpr.hh"
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include "eval.hh"
|
||||||
|
#include "nixexpr.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class JSONPlaceholder;
|
class JSONPlaceholder;
|
||||||
|
|
||||||
void printValueAsJSON(EvalState & state, bool strict,
|
void printValueAsJSON(EvalState& state, bool strict, Value& v,
|
||||||
Value & v, JSONPlaceholder & out, PathSet & context);
|
JSONPlaceholder& out, PathSet& context);
|
||||||
|
|
||||||
void printValueAsJSON(EvalState & state, bool strict,
|
void printValueAsJSON(EvalState& state, bool strict, Value& v,
|
||||||
Value & v, std::ostream & str, PathSet & context);
|
std::ostream& str, PathSet& context);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
75
third_party/nix/src/libexpr/value-to-xml.cc
vendored
75
third_party/nix/src/libexpr/value-to-xml.cc
vendored
|
|
@ -1,41 +1,33 @@
|
||||||
#include "value-to-xml.hh"
|
#include "value-to-xml.hh"
|
||||||
#include "xml-writer.hh"
|
#include <cstdlib>
|
||||||
#include "eval-inline.hh"
|
#include "eval-inline.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
#include "xml-writer.hh"
|
||||||
#include <cstdlib>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static XMLAttrs singletonAttrs(const string& name, const string& value) {
|
||||||
static XMLAttrs singletonAttrs(const string & name, const string & value)
|
|
||||||
{
|
|
||||||
XMLAttrs attrs;
|
XMLAttrs attrs;
|
||||||
attrs[name] = value;
|
attrs[name] = value;
|
||||||
return attrs;
|
return attrs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void printValueAsXML(EvalState& state, bool strict, bool location,
|
static void printValueAsXML(EvalState& state, bool strict, bool location,
|
||||||
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
|
Value& v, XMLWriter& doc, PathSet& context,
|
||||||
|
PathSet& drvsSeen);
|
||||||
|
|
||||||
|
static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) {
|
||||||
static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
|
|
||||||
{
|
|
||||||
xmlAttrs["path"] = pos.file;
|
xmlAttrs["path"] = pos.file;
|
||||||
xmlAttrs["line"] = (format("%1%") % pos.line).str();
|
xmlAttrs["line"] = (format("%1%") % pos.line).str();
|
||||||
xmlAttrs["column"] = (format("%1%") % pos.column).str();
|
xmlAttrs["column"] = (format("%1%") % pos.column).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void showAttrs(EvalState& state, bool strict, bool location,
|
static void showAttrs(EvalState& state, bool strict, bool location,
|
||||||
Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
|
Bindings& attrs, XMLWriter& doc, PathSet& context,
|
||||||
{
|
PathSet& drvsSeen) {
|
||||||
StringSet names;
|
StringSet names;
|
||||||
|
|
||||||
for (auto & i : attrs)
|
for (auto& i : attrs) names.insert(i.name);
|
||||||
names.insert(i.name);
|
|
||||||
|
|
||||||
for (auto& i : names) {
|
for (auto& i : names) {
|
||||||
Attr& a(*attrs.find(state.symbols.create(i)));
|
Attr& a(*attrs.find(state.symbols.create(i)));
|
||||||
|
|
@ -45,27 +37,26 @@ static void showAttrs(EvalState & state, bool strict, bool location,
|
||||||
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
|
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
|
||||||
|
|
||||||
XMLOpenElement _(doc, "attr", xmlAttrs);
|
XMLOpenElement _(doc, "attr", xmlAttrs);
|
||||||
printValueAsXML(state, strict, location,
|
printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen);
|
||||||
*a.value, doc, context, drvsSeen);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void printValueAsXML(EvalState& state, bool strict, bool location,
|
static void printValueAsXML(EvalState& state, bool strict, bool location,
|
||||||
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
|
Value& v, XMLWriter& doc, PathSet& context,
|
||||||
{
|
PathSet& drvsSeen) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
if (strict) state.forceValue(v);
|
if (strict) state.forceValue(v);
|
||||||
|
|
||||||
switch (v.type) {
|
switch (v.type) {
|
||||||
|
|
||||||
case tInt:
|
case tInt:
|
||||||
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
|
doc.writeEmptyElement(
|
||||||
|
"int", singletonAttrs("value", (format("%1%") % v.integer).str()));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case tBool:
|
case tBool:
|
||||||
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
|
doc.writeEmptyElement(
|
||||||
|
"bool", singletonAttrs("value", v.boolean ? "true" : "false"));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case tString:
|
case tString:
|
||||||
|
|
@ -86,7 +77,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
|
||||||
if (state.isDerivation(v)) {
|
if (state.isDerivation(v)) {
|
||||||
XMLAttrs xmlAttrs;
|
XMLAttrs xmlAttrs;
|
||||||
|
|
||||||
Bindings::iterator a = v.attrs->find(state.symbols.create("derivation"));
|
Bindings::iterator a =
|
||||||
|
v.attrs->find(state.symbols.create("derivation"));
|
||||||
|
|
||||||
Path drvPath;
|
Path drvPath;
|
||||||
a = v.attrs->find(state.sDrvPath);
|
a = v.attrs->find(state.sDrvPath);
|
||||||
|
|
@ -119,10 +111,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case tList1: case tList2: case tListN: {
|
case tList1:
|
||||||
|
case tList2:
|
||||||
|
case tListN: {
|
||||||
XMLOpenElement _(doc, "list");
|
XMLOpenElement _(doc, "list");
|
||||||
for (unsigned int n = 0; n < v.listSize(); ++n)
|
for (unsigned int n = 0; n < v.listSize(); ++n)
|
||||||
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
|
printValueAsXML(state, strict, location, *v.listElems()[n], doc,
|
||||||
|
context, drvsSeen);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -139,17 +134,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
|
||||||
for (auto& i : v.lambda.fun->formals->formals)
|
for (auto& i : v.lambda.fun->formals->formals)
|
||||||
doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
|
doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
|
||||||
} else
|
} else
|
||||||
doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
|
doc.writeEmptyElement("varpat",
|
||||||
|
singletonAttrs("name", v.lambda.fun->arg));
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case tExternal:
|
case tExternal:
|
||||||
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
|
v.external->printValueAsXML(state, strict, location, doc, context,
|
||||||
|
drvsSeen);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case tFloat:
|
case tFloat:
|
||||||
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
|
doc.writeEmptyElement(
|
||||||
|
"float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
|
@ -157,22 +155,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void ExternalValueBase::printValueAsXML(EvalState& state, bool strict,
|
void ExternalValueBase::printValueAsXML(EvalState& state, bool strict,
|
||||||
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
|
bool location, XMLWriter& doc,
|
||||||
{
|
PathSet& context,
|
||||||
|
PathSet& drvsSeen) const {
|
||||||
doc.writeEmptyElement("unevaluated");
|
doc.writeEmptyElement("unevaluated");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
|
||||||
void printValueAsXML(EvalState & state, bool strict, bool location,
|
std::ostream& out, PathSet& context) {
|
||||||
Value & v, std::ostream & out, PathSet & context)
|
|
||||||
{
|
|
||||||
XMLWriter doc(true, out);
|
XMLWriter doc(true, out);
|
||||||
XMLOpenElement root(doc, "expr");
|
XMLOpenElement root(doc, "expr");
|
||||||
PathSet drvsSeen;
|
PathSet drvsSeen;
|
||||||
printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
|
printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
11
third_party/nix/src/libexpr/value-to-xml.hh
vendored
11
third_party/nix/src/libexpr/value-to-xml.hh
vendored
|
|
@ -1,14 +1,13 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "nixexpr.hh"
|
|
||||||
#include "eval.hh"
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include "eval.hh"
|
||||||
|
#include "nixexpr.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
void printValueAsXML(EvalState & state, bool strict, bool location,
|
void printValueAsXML(EvalState& state, bool strict, bool location, Value& v,
|
||||||
Value & v, std::ostream & out, PathSet & context);
|
std::ostream& out, PathSet& context);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
92
third_party/nix/src/libexpr/value.hh
vendored
92
third_party/nix/src/libexpr/value.hh
vendored
|
|
@ -8,7 +8,6 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
tInt = 1,
|
tInt = 1,
|
||||||
tBool,
|
tBool,
|
||||||
|
|
@ -29,7 +28,6 @@ typedef enum {
|
||||||
tFloat
|
tFloat
|
||||||
} ValueType;
|
} ValueType;
|
||||||
|
|
||||||
|
|
||||||
class Bindings;
|
class Bindings;
|
||||||
struct Env;
|
struct Env;
|
||||||
struct Expr;
|
struct Expr;
|
||||||
|
|
@ -42,16 +40,16 @@ class EvalState;
|
||||||
class XMLWriter;
|
class XMLWriter;
|
||||||
class JSONPlaceholder;
|
class JSONPlaceholder;
|
||||||
|
|
||||||
|
|
||||||
typedef int64_t NixInt;
|
typedef int64_t NixInt;
|
||||||
typedef double NixFloat;
|
typedef double NixFloat;
|
||||||
|
|
||||||
/* External values must descend from ExternalValueBase, so that
|
/* External values must descend from ExternalValueBase, so that
|
||||||
* type-agnostic nix functions (e.g. showType) can be implemented
|
* type-agnostic nix functions (e.g. showType) can be implemented
|
||||||
*/
|
*/
|
||||||
class ExternalValueBase
|
class ExternalValueBase {
|
||||||
{
|
friend std::ostream& operator<<(std::ostream& str,
|
||||||
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
|
const ExternalValueBase& v);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
/* Print out the value */
|
/* Print out the value */
|
||||||
virtual std::ostream& print(std::ostream& str) const = 0;
|
virtual std::ostream& print(std::ostream& str) const = 0;
|
||||||
|
|
@ -69,7 +67,8 @@ class ExternalValueBase
|
||||||
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
|
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
|
||||||
* error
|
* error
|
||||||
*/
|
*/
|
||||||
virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
|
virtual string coerceToString(const Pos& pos, PathSet& context, bool copyMore,
|
||||||
|
bool copyToStore) const;
|
||||||
|
|
||||||
/* Compare to another value of the same type. Defaults to uncomparable,
|
/* Compare to another value of the same type. Defaults to uncomparable,
|
||||||
* i.e. always false.
|
* i.e. always false.
|
||||||
|
|
@ -82,21 +81,17 @@ class ExternalValueBase
|
||||||
|
|
||||||
/* Print the value as XML. Defaults to unevaluated */
|
/* Print the value as XML. Defaults to unevaluated */
|
||||||
virtual void printValueAsXML(EvalState& state, bool strict, bool location,
|
virtual void printValueAsXML(EvalState& state, bool strict, bool location,
|
||||||
XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const;
|
XMLWriter& doc, PathSet& context,
|
||||||
|
PathSet& drvsSeen) const;
|
||||||
|
|
||||||
virtual ~ExternalValueBase()
|
virtual ~ExternalValueBase(){};
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v);
|
std::ostream& operator<<(std::ostream& str, const ExternalValueBase& v);
|
||||||
|
|
||||||
|
struct Value {
|
||||||
struct Value
|
|
||||||
{
|
|
||||||
ValueType type;
|
ValueType type;
|
||||||
union
|
union {
|
||||||
{
|
|
||||||
NixInt integer;
|
NixInt integer;
|
||||||
bool boolean;
|
bool boolean;
|
||||||
|
|
||||||
|
|
@ -151,124 +146,95 @@ struct Value
|
||||||
NixFloat fpoint;
|
NixFloat fpoint;
|
||||||
};
|
};
|
||||||
|
|
||||||
bool isList() const
|
bool isList() const {
|
||||||
{
|
|
||||||
return type == tList1 || type == tList2 || type == tListN;
|
return type == tList1 || type == tList2 || type == tListN;
|
||||||
}
|
}
|
||||||
|
|
||||||
Value * * listElems()
|
Value** listElems() {
|
||||||
{
|
|
||||||
return type == tList1 || type == tList2 ? smallList : bigList.elems;
|
return type == tList1 || type == tList2 ? smallList : bigList.elems;
|
||||||
}
|
}
|
||||||
|
|
||||||
const Value * const * listElems() const
|
const Value* const* listElems() const {
|
||||||
{
|
|
||||||
return type == tList1 || type == tList2 ? smallList : bigList.elems;
|
return type == tList1 || type == tList2 ? smallList : bigList.elems;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t listSize() const
|
size_t listSize() const {
|
||||||
{
|
|
||||||
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
|
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* After overwriting an app node, be sure to clear pointers in the
|
/* After overwriting an app node, be sure to clear pointers in the
|
||||||
Value to ensure that the target isn't kept alive unnecessarily. */
|
Value to ensure that the target isn't kept alive unnecessarily. */
|
||||||
static inline void clearValue(Value & v)
|
static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; }
|
||||||
{
|
|
||||||
v.app.left = v.app.right = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
static inline void mkInt(Value& v, NixInt n) {
|
||||||
static inline void mkInt(Value & v, NixInt n)
|
|
||||||
{
|
|
||||||
clearValue(v);
|
clearValue(v);
|
||||||
v.type = tInt;
|
v.type = tInt;
|
||||||
v.integer = n;
|
v.integer = n;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkFloat(Value& v, NixFloat n) {
|
||||||
static inline void mkFloat(Value & v, NixFloat n)
|
|
||||||
{
|
|
||||||
clearValue(v);
|
clearValue(v);
|
||||||
v.type = tFloat;
|
v.type = tFloat;
|
||||||
v.fpoint = n;
|
v.fpoint = n;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkBool(Value& v, bool b) {
|
||||||
static inline void mkBool(Value & v, bool b)
|
|
||||||
{
|
|
||||||
clearValue(v);
|
clearValue(v);
|
||||||
v.type = tBool;
|
v.type = tBool;
|
||||||
v.boolean = b;
|
v.boolean = b;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkNull(Value& v) {
|
||||||
static inline void mkNull(Value & v)
|
|
||||||
{
|
|
||||||
clearValue(v);
|
clearValue(v);
|
||||||
v.type = tNull;
|
v.type = tNull;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkApp(Value& v, Value& left, Value& right) {
|
||||||
static inline void mkApp(Value & v, Value & left, Value & right)
|
|
||||||
{
|
|
||||||
v.type = tApp;
|
v.type = tApp;
|
||||||
v.app.left = &left;
|
v.app.left = &left;
|
||||||
v.app.right = &right;
|
v.app.right = &right;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkPrimOpApp(Value& v, Value& left, Value& right) {
|
||||||
static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
|
|
||||||
{
|
|
||||||
v.type = tPrimOpApp;
|
v.type = tPrimOpApp;
|
||||||
v.app.left = &left;
|
v.app.left = &left;
|
||||||
v.app.right = &right;
|
v.app.right = &right;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkStringNoCopy(Value& v, const char* s) {
|
||||||
static inline void mkStringNoCopy(Value & v, const char * s)
|
|
||||||
{
|
|
||||||
v.type = tString;
|
v.type = tString;
|
||||||
v.string.s = s;
|
v.string.s = s;
|
||||||
v.string.context = 0;
|
v.string.context = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void mkString(Value& v, const Symbol& s) {
|
||||||
static inline void mkString(Value & v, const Symbol & s)
|
|
||||||
{
|
|
||||||
mkStringNoCopy(v, ((const string&)s).c_str());
|
mkStringNoCopy(v, ((const string&)s).c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void mkString(Value& v, const char* s);
|
void mkString(Value& v, const char* s);
|
||||||
|
|
||||||
|
static inline void mkPathNoCopy(Value& v, const char* s) {
|
||||||
static inline void mkPathNoCopy(Value & v, const char * s)
|
|
||||||
{
|
|
||||||
clearValue(v);
|
clearValue(v);
|
||||||
v.type = tPath;
|
v.type = tPath;
|
||||||
v.path = s;
|
v.path = s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void mkPath(Value& v, const char* s);
|
void mkPath(Value& v, const char* s);
|
||||||
|
|
||||||
|
|
||||||
/* Compute the size in bytes of the given value, including all values
|
/* Compute the size in bytes of the given value, including all values
|
||||||
and environments reachable from it. Static expressions (Exprs) are
|
and environments reachable from it. Static expressions (Exprs) are
|
||||||
not included. */
|
not included. */
|
||||||
size_t valueSize(Value& v);
|
size_t valueSize(Value& v);
|
||||||
|
|
||||||
|
|
||||||
#if HAVE_BOEHMGC
|
#if HAVE_BOEHMGC
|
||||||
typedef std::vector<Value*, gc_allocator<Value*> > ValueVector;
|
typedef std::vector<Value*, gc_allocator<Value*> > ValueVector;
|
||||||
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
|
typedef std::map<Symbol, Value*, std::less<Symbol>,
|
||||||
|
gc_allocator<std::pair<const Symbol, Value*> > >
|
||||||
|
ValueMap;
|
||||||
#else
|
#else
|
||||||
typedef std::vector<Value*> ValueVector;
|
typedef std::vector<Value*> ValueVector;
|
||||||
typedef std::map<Symbol, Value*> ValueMap;
|
typedef std::map<Symbol, Value*> ValueMap;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
21
third_party/nix/src/libmain/common-args.cc
vendored
21
third_party/nix/src/libmain/common-args.cc
vendored
|
|
@ -4,8 +4,7 @@
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
MixCommonArgs::MixCommonArgs(const string& programName)
|
MixCommonArgs::MixCommonArgs(const string& programName)
|
||||||
: programName(programName)
|
: programName(programName) {
|
||||||
{
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("verbose")
|
.longName("verbose")
|
||||||
.shortName('v')
|
.shortName('v')
|
||||||
|
|
@ -15,12 +14,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("quiet")
|
.longName("quiet")
|
||||||
.description("decrease verbosity level")
|
.description("decrease verbosity level")
|
||||||
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
|
.handler([]() {
|
||||||
|
verbosity =
|
||||||
|
verbosity > lvlError ? (Verbosity)(verbosity - 1) : lvlError;
|
||||||
|
});
|
||||||
|
|
||||||
mkFlag()
|
mkFlag().longName("debug").description("enable debug output").handler([]() {
|
||||||
.longName("debug")
|
verbosity = lvlDebug;
|
||||||
.description("enable debug output")
|
});
|
||||||
.handler([]() { verbosity = lvlDebug; });
|
|
||||||
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("option")
|
.longName("option")
|
||||||
|
|
@ -40,9 +41,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
|
||||||
.shortName('j')
|
.shortName('j')
|
||||||
.label("jobs")
|
.label("jobs")
|
||||||
.description("maximum number of parallel builds")
|
.description("maximum number of parallel builds")
|
||||||
.handler([=](std::string s) {
|
.handler([=](std::string s) { settings.set("max-jobs", s); });
|
||||||
settings.set("max-jobs", s);
|
|
||||||
});
|
|
||||||
|
|
||||||
std::string cat = "config";
|
std::string cat = "config";
|
||||||
globalConfig.convertToArgs(*this, cat);
|
globalConfig.convertToArgs(*this, cat);
|
||||||
|
|
@ -53,4 +52,4 @@ MixCommonArgs::MixCommonArgs(const string & programName)
|
||||||
hiddenCategories.insert(cat);
|
hiddenCategories.insert(cat);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
22
third_party/nix/src/libmain/common-args.hh
vendored
22
third_party/nix/src/libmain/common-args.hh
vendored
|
|
@ -4,30 +4,24 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct MixCommonArgs : virtual Args
|
struct MixCommonArgs : virtual Args {
|
||||||
{
|
|
||||||
string programName;
|
string programName;
|
||||||
MixCommonArgs(const string& programName);
|
MixCommonArgs(const string& programName);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct MixDryRun : virtual Args
|
struct MixDryRun : virtual Args {
|
||||||
{
|
|
||||||
bool dryRun = false;
|
bool dryRun = false;
|
||||||
|
|
||||||
MixDryRun()
|
MixDryRun() {
|
||||||
{
|
mkFlag(0, "dry-run", "show what this command would do without doing it",
|
||||||
mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
|
&dryRun);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct MixJSON : virtual Args
|
struct MixJSON : virtual Args {
|
||||||
{
|
|
||||||
bool json = false;
|
bool json = false;
|
||||||
|
|
||||||
MixJSON()
|
MixJSON() { mkFlag(0, "json", "produce JSON output", &json); }
|
||||||
{
|
|
||||||
mkFlag(0, "json", "produce JSON output", &json);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
175
third_party/nix/src/libmain/shared.cc
vendored
175
third_party/nix/src/libmain/shared.cc
vendored
|
|
@ -1,30 +1,24 @@
|
||||||
#include "globals.hh"
|
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include <openssl/crypto.h>
|
||||||
#include "util.hh"
|
#include <signal.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <unistd.h>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cctype>
|
#include <cctype>
|
||||||
|
#include <cstdlib>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
#include "globals.hh"
|
||||||
#include <cstdlib>
|
#include "store-api.hh"
|
||||||
#include <sys/time.h>
|
#include "util.hh"
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <signal.h>
|
|
||||||
|
|
||||||
#include <openssl/crypto.h>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
static bool gcWarning = true;
|
static bool gcWarning = true;
|
||||||
|
|
||||||
void printGCWarning()
|
void printGCWarning() {
|
||||||
{
|
|
||||||
if (!gcWarning) return;
|
if (!gcWarning) return;
|
||||||
static bool haveWarned = false;
|
static bool haveWarned = false;
|
||||||
warnOnce(haveWarned,
|
warnOnce(haveWarned,
|
||||||
|
|
@ -32,62 +26,58 @@ void printGCWarning()
|
||||||
"the result might be removed by the garbage collector");
|
"the result might be removed by the garbage collector");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void printMissing(ref<Store> store, const PathSet& paths, Verbosity lvl) {
|
||||||
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
|
|
||||||
{
|
|
||||||
unsigned long long downloadSize, narSize;
|
unsigned long long downloadSize, narSize;
|
||||||
PathSet willBuild, willSubstitute, unknown;
|
PathSet willBuild, willSubstitute, unknown;
|
||||||
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize,
|
||||||
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
|
narSize);
|
||||||
|
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize,
|
||||||
|
lvl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void printMissing(ref<Store> store, const PathSet& willBuild,
|
void printMissing(ref<Store> store, const PathSet& willBuild,
|
||||||
const PathSet& willSubstitute, const PathSet& unknown,
|
const PathSet& willSubstitute, const PathSet& unknown,
|
||||||
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
|
unsigned long long downloadSize, unsigned long long narSize,
|
||||||
{
|
Verbosity lvl) {
|
||||||
if (!willBuild.empty()) {
|
if (!willBuild.empty()) {
|
||||||
printMsg(lvl, "these derivations will be built:");
|
printMsg(lvl, "these derivations will be built:");
|
||||||
Paths sorted = store->topoSortPaths(willBuild);
|
Paths sorted = store->topoSortPaths(willBuild);
|
||||||
reverse(sorted.begin(), sorted.end());
|
reverse(sorted.begin(), sorted.end());
|
||||||
for (auto & i : sorted)
|
for (auto& i : sorted) printMsg(lvl, fmt(" %s", i));
|
||||||
printMsg(lvl, fmt(" %s", i));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!willSubstitute.empty()) {
|
if (!willSubstitute.empty()) {
|
||||||
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
|
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f "
|
||||||
|
"MiB unpacked):",
|
||||||
downloadSize / (1024.0 * 1024.0),
|
downloadSize / (1024.0 * 1024.0),
|
||||||
narSize / (1024.0 * 1024.0)));
|
narSize / (1024.0 * 1024.0)));
|
||||||
for (auto & i : willSubstitute)
|
for (auto& i : willSubstitute) printMsg(lvl, fmt(" %s", i));
|
||||||
printMsg(lvl, fmt(" %s", i));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!unknown.empty()) {
|
if (!unknown.empty()) {
|
||||||
printMsg(lvl, fmt("don't know how to build these paths%s:",
|
printMsg(lvl, fmt("don't know how to build these paths%s:",
|
||||||
(settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
|
(settings.readOnlyMode
|
||||||
for (auto & i : unknown)
|
? " (may be caused by read-only store access)"
|
||||||
printMsg(lvl, fmt(" %s", i));
|
: "")));
|
||||||
|
for (auto& i : unknown) printMsg(lvl, fmt(" %s", i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string getArg(const string& opt, Strings::iterator& i,
|
||||||
string getArg(const string & opt,
|
const Strings::iterator& end) {
|
||||||
Strings::iterator & i, const Strings::iterator & end)
|
|
||||||
{
|
|
||||||
++i;
|
++i;
|
||||||
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
|
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
|
||||||
return *i;
|
return *i;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#if OPENSSL_VERSION_NUMBER < 0x10101000L
|
#if OPENSSL_VERSION_NUMBER < 0x10101000L
|
||||||
/* OpenSSL is not thread-safe by default - it will randomly crash
|
/* OpenSSL is not thread-safe by default - it will randomly crash
|
||||||
unless the user supplies a mutex locking function. So let's do
|
unless the user supplies a mutex locking function. So let's do
|
||||||
that. */
|
that. */
|
||||||
static std::vector<std::mutex> opensslLocks;
|
static std::vector<std::mutex> opensslLocks;
|
||||||
|
|
||||||
static void opensslLockCallback(int mode, int type, const char * file, int line)
|
static void opensslLockCallback(int mode, int type, const char* file,
|
||||||
{
|
int line) {
|
||||||
if (mode & CRYPTO_LOCK)
|
if (mode & CRYPTO_LOCK)
|
||||||
opensslLocks[type].lock();
|
opensslLocks[type].lock();
|
||||||
else
|
else
|
||||||
|
|
@ -95,12 +85,9 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
static void sigHandler(int signo) {}
|
static void sigHandler(int signo) {}
|
||||||
|
|
||||||
|
void initNix() {
|
||||||
void initNix()
|
|
||||||
{
|
|
||||||
/* Turn on buffering for cerr. */
|
/* Turn on buffering for cerr. */
|
||||||
#if HAVE_PUBSETBUF
|
#if HAVE_PUBSETBUF
|
||||||
static char buf[1024];
|
static char buf[1024];
|
||||||
|
|
@ -122,8 +109,7 @@ void initNix()
|
||||||
sigemptyset(&act.sa_mask);
|
sigemptyset(&act.sa_mask);
|
||||||
act.sa_handler = SIG_DFL;
|
act.sa_handler = SIG_DFL;
|
||||||
act.sa_flags = 0;
|
act.sa_flags = 0;
|
||||||
if (sigaction(SIGCHLD, &act, 0))
|
if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD");
|
||||||
throw SysError("resetting SIGCHLD");
|
|
||||||
|
|
||||||
/* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
|
/* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
|
||||||
act.sa_handler = sigHandler;
|
act.sa_handler = sigHandler;
|
||||||
|
|
@ -160,11 +146,11 @@ void initNix()
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LegacyArgs::LegacyArgs(
|
||||||
LegacyArgs::LegacyArgs(const std::string & programName,
|
const std::string& programName,
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
: MixCommonArgs(programName), parseArg(parseArg)
|
parseArg)
|
||||||
{
|
: MixCommonArgs(programName), parseArg(parseArg) {
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.longName("no-build-output")
|
.longName("no-build-output")
|
||||||
.shortName('Q')
|
.shortName('Q')
|
||||||
|
|
@ -189,15 +175,20 @@ LegacyArgs::LegacyArgs(const std::string & programName,
|
||||||
.set(&(bool&)settings.tryFallback, true);
|
.set(&(bool&)settings.tryFallback, true);
|
||||||
|
|
||||||
auto intSettingAlias = [&](char shortName, const std::string& longName,
|
auto intSettingAlias = [&](char shortName, const std::string& longName,
|
||||||
const std::string & description, const std::string & dest) {
|
const std::string& description,
|
||||||
|
const std::string& dest) {
|
||||||
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
|
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
|
||||||
settings.set(dest, std::to_string(n));
|
settings.set(dest, std::to_string(n));
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
|
intSettingAlias(0, "cores",
|
||||||
intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
|
"maximum number of CPU cores to use inside a build", "cores");
|
||||||
intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
|
intSettingAlias(0, "max-silent-time",
|
||||||
|
"number of seconds of silence before a build is killed",
|
||||||
|
"max-silent-time");
|
||||||
|
intSettingAlias(0, "timeout", "number of seconds before a build is killed",
|
||||||
|
"timeout");
|
||||||
|
|
||||||
mkFlag(0, "readonly-mode", "do not write to the Nix store",
|
mkFlag(0, "readonly-mode", "do not write to the Nix store",
|
||||||
&settings.readOnlyMode);
|
&settings.readOnlyMode);
|
||||||
|
|
@ -212,18 +203,14 @@ LegacyArgs::LegacyArgs(const std::string & programName,
|
||||||
.dest(&(std::string&)settings.storeUri);
|
.dest(&(std::string&)settings.storeUri);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) {
|
||||||
bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end)
|
|
||||||
{
|
|
||||||
if (MixCommonArgs::processFlag(pos, end)) return true;
|
if (MixCommonArgs::processFlag(pos, end)) return true;
|
||||||
bool res = parseArg(pos, end);
|
bool res = parseArg(pos, end);
|
||||||
if (res) ++pos;
|
if (res) ++pos;
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool LegacyArgs::processArgs(const Strings& args, bool finish) {
|
||||||
bool LegacyArgs::processArgs(const Strings & args, bool finish)
|
|
||||||
{
|
|
||||||
if (args.empty()) return true;
|
if (args.empty()) return true;
|
||||||
assert(args.size() == 1);
|
assert(args.size() == 1);
|
||||||
Strings ss(args);
|
Strings ss(args);
|
||||||
|
|
@ -233,23 +220,21 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void parseCmdLine(
|
||||||
void parseCmdLine(int argc, char * * argv,
|
int argc, char** argv,
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
{
|
parseArg) {
|
||||||
parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
|
parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void parseCmdLine(
|
||||||
void parseCmdLine(const string & programName, const Strings & args,
|
const string& programName, const Strings& args,
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
{
|
parseArg) {
|
||||||
LegacyArgs(programName, parseArg).parseCmdline(args);
|
LegacyArgs(programName, parseArg).parseCmdline(args);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void printVersion(const string& programName) {
|
||||||
void printVersion(const string & programName)
|
|
||||||
{
|
|
||||||
std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
|
std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
|
||||||
if (verbosity > lvlInfo) {
|
if (verbosity > lvlInfo) {
|
||||||
Strings cfg;
|
Strings cfg;
|
||||||
|
|
@ -260,25 +245,22 @@ void printVersion(const string & programName)
|
||||||
cfg.push_back("signed-caches");
|
cfg.push_back("signed-caches");
|
||||||
#endif
|
#endif
|
||||||
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
|
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
|
||||||
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
|
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf"
|
||||||
|
<< "\n";
|
||||||
std::cout << "Store directory: " << settings.nixStore << "\n";
|
std::cout << "Store directory: " << settings.nixStore << "\n";
|
||||||
std::cout << "State directory: " << settings.nixStateDir << "\n";
|
std::cout << "State directory: " << settings.nixStateDir << "\n";
|
||||||
}
|
}
|
||||||
throw Exit();
|
throw Exit();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void showManPage(const string& name) {
|
||||||
void showManPage(const string & name)
|
|
||||||
{
|
|
||||||
restoreSignals();
|
restoreSignals();
|
||||||
setenv("MANPATH", settings.nixManDir.c_str(), 1);
|
setenv("MANPATH", settings.nixManDir.c_str(), 1);
|
||||||
execlp("man", "man", name.c_str(), nullptr);
|
execlp("man", "man", name.c_str(), nullptr);
|
||||||
throw SysError(format("command 'man %1%' failed") % name.c_str());
|
throw SysError(format("command 'man %1%' failed") % name.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int handleExceptions(const string& programName, std::function<void()> fun) {
|
||||||
int handleExceptions(const string & programName, std::function<void()> fun)
|
|
||||||
{
|
|
||||||
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
|
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
|
||||||
|
|
||||||
string error = ANSI_RED "error:" ANSI_NORMAL " ";
|
string error = ANSI_RED "error:" ANSI_NORMAL " ";
|
||||||
|
|
@ -296,12 +278,12 @@ int handleExceptions(const string & programName, std::function<void()> fun)
|
||||||
} catch (Exit& e) {
|
} catch (Exit& e) {
|
||||||
return e.status;
|
return e.status;
|
||||||
} catch (UsageError& e) {
|
} catch (UsageError& e) {
|
||||||
printError(
|
printError(format(error + "%1%\nTry '%2% --help' for more information.") %
|
||||||
format(error + "%1%\nTry '%2% --help' for more information.")
|
e.what() % programName);
|
||||||
% e.what() % programName);
|
|
||||||
return 1;
|
return 1;
|
||||||
} catch (BaseError& e) {
|
} catch (BaseError& e) {
|
||||||
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
|
printError(format(error + "%1%%2%") %
|
||||||
|
(settings.showTrace ? e.prefix() : "") % e.msg());
|
||||||
if (e.prefix() != "" && !settings.showTrace)
|
if (e.prefix() != "" && !settings.showTrace)
|
||||||
printError("(use '--show-trace' to show detailed location information)");
|
printError("(use '--show-trace' to show detailed location information)");
|
||||||
return e.status;
|
return e.status;
|
||||||
|
|
@ -316,9 +298,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
RunPager::RunPager() {
|
||||||
RunPager::RunPager()
|
|
||||||
{
|
|
||||||
if (!isatty(STDOUT_FILENO)) return;
|
if (!isatty(STDOUT_FILENO)) return;
|
||||||
char* pager = getenv("NIX_PAGER");
|
char* pager = getenv("NIX_PAGER");
|
||||||
if (!pager) pager = getenv("PAGER");
|
if (!pager) pager = getenv("PAGER");
|
||||||
|
|
@ -330,11 +310,9 @@ RunPager::RunPager()
|
||||||
pid = startProcess([&]() {
|
pid = startProcess([&]() {
|
||||||
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
|
if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
|
||||||
throw SysError("dupping stdin");
|
throw SysError("dupping stdin");
|
||||||
if (!getenv("LESS"))
|
if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1);
|
||||||
setenv("LESS", "FRSXMK", 1);
|
|
||||||
restoreSignals();
|
restoreSignals();
|
||||||
if (pager)
|
if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr);
|
||||||
execl("/bin/sh", "sh", "-c", pager, nullptr);
|
|
||||||
execlp("pager", "pager", nullptr);
|
execlp("pager", "pager", nullptr);
|
||||||
execlp("less", "less", nullptr);
|
execlp("less", "less", nullptr);
|
||||||
execlp("more", "more", nullptr);
|
execlp("more", "more", nullptr);
|
||||||
|
|
@ -347,9 +325,7 @@ RunPager::RunPager()
|
||||||
throw SysError("dupping stdout");
|
throw SysError("dupping stdout");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
RunPager::~RunPager() {
|
||||||
RunPager::~RunPager()
|
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
if (pid != -1) {
|
if (pid != -1) {
|
||||||
std::cout.flush();
|
std::cout.flush();
|
||||||
|
|
@ -361,21 +337,16 @@ RunPager::~RunPager()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string showBytes(unsigned long long bytes) {
|
||||||
string showBytes(unsigned long long bytes)
|
|
||||||
{
|
|
||||||
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PrintFreed::~PrintFreed() {
|
||||||
PrintFreed::~PrintFreed()
|
|
||||||
{
|
|
||||||
if (show)
|
if (show)
|
||||||
std::cout << format("%1% store paths deleted, %2% freed\n")
|
std::cout << format("%1% store paths deleted, %2% freed\n") %
|
||||||
% results.paths.size()
|
results.paths.size() % showBytes(results.bytesFreed);
|
||||||
% showBytes(results.bytesFreed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Exit::~Exit() {}
|
Exit::~Exit() {}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
80
third_party/nix/src/libmain/shared.hh
vendored
80
third_party/nix/src/libmain/shared.hh
vendored
|
|
@ -1,18 +1,14 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "util.hh"
|
#include <signal.h>
|
||||||
|
#include <locale>
|
||||||
#include "args.hh"
|
#include "args.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
|
#include "util.hh"
|
||||||
#include <signal.h>
|
|
||||||
|
|
||||||
#include <locale>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class Exit : public std::exception
|
class Exit : public std::exception {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
int status;
|
int status;
|
||||||
Exit() : status(0) {}
|
Exit() : status(0) {}
|
||||||
|
|
@ -25,11 +21,15 @@ int handleExceptions(const string & programName, std::function<void()> fun);
|
||||||
/* Don't forget to call initPlugins() after settings are initialized! */
|
/* Don't forget to call initPlugins() after settings are initialized! */
|
||||||
void initNix();
|
void initNix();
|
||||||
|
|
||||||
void parseCmdLine(int argc, char * * argv,
|
void parseCmdLine(
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
|
int argc, char** argv,
|
||||||
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
|
parseArg);
|
||||||
|
|
||||||
void parseCmdLine(const string & programName, const Strings & args,
|
void parseCmdLine(
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
|
const string& programName, const Strings& args,
|
||||||
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
|
parseArg);
|
||||||
|
|
||||||
void printVersion(const string& programName);
|
void printVersion(const string& programName);
|
||||||
|
|
||||||
|
|
@ -38,18 +38,20 @@ void printGCWarning();
|
||||||
|
|
||||||
class Store;
|
class Store;
|
||||||
|
|
||||||
void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo);
|
void printMissing(ref<Store> store, const PathSet& paths,
|
||||||
|
Verbosity lvl = lvlInfo);
|
||||||
|
|
||||||
void printMissing(ref<Store> store, const PathSet& willBuild,
|
void printMissing(ref<Store> store, const PathSet& willBuild,
|
||||||
const PathSet& willSubstitute, const PathSet& unknown,
|
const PathSet& willSubstitute, const PathSet& unknown,
|
||||||
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
|
unsigned long long downloadSize, unsigned long long narSize,
|
||||||
|
Verbosity lvl = lvlInfo);
|
||||||
|
|
||||||
string getArg(const string & opt,
|
string getArg(const string& opt, Strings::iterator& i,
|
||||||
Strings::iterator & i, const Strings::iterator & end);
|
const Strings::iterator& end);
|
||||||
|
|
||||||
template<class N> N getIntArg(const string & opt,
|
template <class N>
|
||||||
Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
|
N getIntArg(const string& opt, Strings::iterator& i,
|
||||||
{
|
const Strings::iterator& end, bool allowUnit) {
|
||||||
++i;
|
++i;
|
||||||
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
|
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
|
||||||
string s = *i;
|
string s = *i;
|
||||||
|
|
@ -57,11 +59,16 @@ template<class N> N getIntArg(const string & opt,
|
||||||
if (allowUnit && !s.empty()) {
|
if (allowUnit && !s.empty()) {
|
||||||
char u = std::toupper(*s.rbegin());
|
char u = std::toupper(*s.rbegin());
|
||||||
if (std::isalpha(u)) {
|
if (std::isalpha(u)) {
|
||||||
if (u == 'K') multiplier = 1ULL << 10;
|
if (u == 'K')
|
||||||
else if (u == 'M') multiplier = 1ULL << 20;
|
multiplier = 1ULL << 10;
|
||||||
else if (u == 'G') multiplier = 1ULL << 30;
|
else if (u == 'M')
|
||||||
else if (u == 'T') multiplier = 1ULL << 40;
|
multiplier = 1ULL << 20;
|
||||||
else throw UsageError(format("invalid unit specifier '%1%'") % u);
|
else if (u == 'G')
|
||||||
|
multiplier = 1ULL << 30;
|
||||||
|
else if (u == 'T')
|
||||||
|
multiplier = 1ULL << 40;
|
||||||
|
else
|
||||||
|
throw UsageError(format("invalid unit specifier '%1%'") % u);
|
||||||
s.resize(s.size() - 1);
|
s.resize(s.size() - 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -71,27 +78,26 @@ template<class N> N getIntArg(const string & opt,
|
||||||
return n * multiplier;
|
return n * multiplier;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct LegacyArgs : public MixCommonArgs {
|
||||||
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
|
parseArg;
|
||||||
|
|
||||||
struct LegacyArgs : public MixCommonArgs
|
LegacyArgs(
|
||||||
{
|
const std::string& programName,
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;
|
std::function<bool(Strings::iterator& arg, const Strings::iterator& end)>
|
||||||
|
parseArg);
|
||||||
LegacyArgs(const std::string & programName,
|
|
||||||
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
|
|
||||||
|
|
||||||
bool processFlag(Strings::iterator& pos, Strings::iterator end) override;
|
bool processFlag(Strings::iterator& pos, Strings::iterator end) override;
|
||||||
|
|
||||||
bool processArgs(const Strings& args, bool finish) override;
|
bool processArgs(const Strings& args, bool finish) override;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Show the manual page for the specified program. */
|
/* Show the manual page for the specified program. */
|
||||||
void showManPage(const string& name);
|
void showManPage(const string& name);
|
||||||
|
|
||||||
/* The constructor of this class starts a pager if stdout is a
|
/* The constructor of this class starts a pager if stdout is a
|
||||||
terminal and $PAGER is set. Stdout is redirected to the pager. */
|
terminal and $PAGER is set. Stdout is redirected to the pager. */
|
||||||
class RunPager
|
class RunPager {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
RunPager();
|
RunPager();
|
||||||
~RunPager();
|
~RunPager();
|
||||||
|
|
@ -102,15 +108,13 @@ private:
|
||||||
|
|
||||||
extern volatile ::sig_atomic_t blockInt;
|
extern volatile ::sig_atomic_t blockInt;
|
||||||
|
|
||||||
|
|
||||||
/* GC helpers. */
|
/* GC helpers. */
|
||||||
|
|
||||||
string showBytes(unsigned long long bytes);
|
string showBytes(unsigned long long bytes);
|
||||||
|
|
||||||
struct GCResults;
|
struct GCResults;
|
||||||
|
|
||||||
struct PrintFreed
|
struct PrintFreed {
|
||||||
{
|
|
||||||
bool show;
|
bool show;
|
||||||
const GCResults& results;
|
const GCResults& results;
|
||||||
PrintFreed(bool show, const GCResults& results)
|
PrintFreed(bool show, const GCResults& results)
|
||||||
|
|
@ -118,9 +122,7 @@ struct PrintFreed
|
||||||
~PrintFreed();
|
~PrintFreed();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Install a SIGSEGV handler to detect stack overflows. */
|
/* Install a SIGSEGV handler to detect stack overflows. */
|
||||||
void detectStackOverflow();
|
void detectStackOverflow();
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
27
third_party/nix/src/libmain/stack.cc
vendored
27
third_party/nix/src/libmain/stack.cc
vendored
|
|
@ -1,17 +1,13 @@
|
||||||
#include "types.hh"
|
#include <signal.h>
|
||||||
|
#include <unistd.h>
|
||||||
#include <cstring>
|
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
|
#include <cstring>
|
||||||
#include <unistd.h>
|
#include "types.hh"
|
||||||
#include <signal.h>
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) {
|
||||||
static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
|
|
||||||
{
|
|
||||||
/* Detect stack overflows by comparing the faulting address with
|
/* Detect stack overflows by comparing the faulting address with
|
||||||
the stack pointer. Unfortunately, getting the stack pointer is
|
the stack pointer. Unfortunately, getting the stack pointer is
|
||||||
not portable. */
|
not portable. */
|
||||||
|
|
@ -43,9 +39,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
|
||||||
if (sigaction(SIGSEGV, &act, 0)) abort();
|
if (sigaction(SIGSEGV, &act, 0)) abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void detectStackOverflow() {
|
||||||
void detectStackOverflow()
|
|
||||||
{
|
|
||||||
#if defined(SA_SIGINFO) && defined(SA_ONSTACK)
|
#if defined(SA_SIGINFO) && defined(SA_ONSTACK)
|
||||||
/* Install a SIGSEGV handler to detect stack overflows. This
|
/* Install a SIGSEGV handler to detect stack overflows. This
|
||||||
requires an alternative stack, otherwise the signal cannot be
|
requires an alternative stack, otherwise the signal cannot be
|
||||||
|
|
@ -56,16 +50,15 @@ void detectStackOverflow()
|
||||||
stack.ss_sp = stackBuf->data();
|
stack.ss_sp = stackBuf->data();
|
||||||
if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
|
if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
|
||||||
stack.ss_flags = 0;
|
stack.ss_flags = 0;
|
||||||
if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");
|
if (sigaltstack(&stack, 0) == -1)
|
||||||
|
throw SysError("cannot set alternative stack");
|
||||||
|
|
||||||
struct sigaction act;
|
struct sigaction act;
|
||||||
sigfillset(&act.sa_mask);
|
sigfillset(&act.sa_mask);
|
||||||
act.sa_sigaction = sigsegvHandler;
|
act.sa_sigaction = sigsegvHandler;
|
||||||
act.sa_flags = SA_SIGINFO | SA_ONSTACK;
|
act.sa_flags = SA_SIGINFO | SA_ONSTACK;
|
||||||
if (sigaction(SIGSEGV, &act, 0))
|
if (sigaction(SIGSEGV, &act, 0)) throw SysError("resetting SIGSEGV");
|
||||||
throw SysError("resetting SIGSEGV");
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
173
third_party/nix/src/libstore/binary-cache-store.cc
vendored
173
third_party/nix/src/libstore/binary-cache-store.cc
vendored
|
|
@ -1,40 +1,37 @@
|
||||||
#include "archive.hh"
|
|
||||||
#include "binary-cache-store.hh"
|
#include "binary-cache-store.hh"
|
||||||
|
#include <chrono>
|
||||||
|
#include <future>
|
||||||
|
#include "archive.hh"
|
||||||
#include "compression.hh"
|
#include "compression.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
#include "fs-accessor.hh"
|
#include "fs-accessor.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "nar-info.hh"
|
|
||||||
#include "sync.hh"
|
|
||||||
#include "remote-fs-accessor.hh"
|
|
||||||
#include "nar-info-disk-cache.hh"
|
|
||||||
#include "nar-accessor.hh"
|
|
||||||
#include "json.hh"
|
#include "json.hh"
|
||||||
|
#include "nar-accessor.hh"
|
||||||
#include <chrono>
|
#include "nar-info-disk-cache.hh"
|
||||||
|
#include "nar-info.hh"
|
||||||
#include <future>
|
#include "remote-fs-accessor.hh"
|
||||||
|
#include "sync.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
BinaryCacheStore::BinaryCacheStore(const Params & params)
|
BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) {
|
||||||
: Store(params)
|
|
||||||
{
|
|
||||||
if (secretKeyFile != "")
|
if (secretKeyFile != "")
|
||||||
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
|
secretKey =
|
||||||
|
std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
|
||||||
|
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
sink << narVersionMagic1;
|
sink << narVersionMagic1;
|
||||||
narMagic = *sink.s;
|
narMagic = *sink.s;
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::init()
|
void BinaryCacheStore::init() {
|
||||||
{
|
|
||||||
std::string cacheInfoFile = "nix-cache-info";
|
std::string cacheInfoFile = "nix-cache-info";
|
||||||
|
|
||||||
auto cacheInfo = getFile(cacheInfoFile);
|
auto cacheInfo = getFile(cacheInfoFile);
|
||||||
if (!cacheInfo) {
|
if (!cacheInfo) {
|
||||||
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n",
|
||||||
|
"text/x-nix-cache-info");
|
||||||
} else {
|
} else {
|
||||||
for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
|
for (auto& line : tokenizeString<Strings>(*cacheInfo, "\n")) {
|
||||||
size_t colon = line.find(':');
|
size_t colon = line.find(':');
|
||||||
|
|
@ -43,8 +40,9 @@ void BinaryCacheStore::init()
|
||||||
auto value = trim(line.substr(colon + 1, std::string::npos));
|
auto value = trim(line.substr(colon + 1, std::string::npos));
|
||||||
if (name == "StoreDir") {
|
if (name == "StoreDir") {
|
||||||
if (value != storeDir)
|
if (value != storeDir)
|
||||||
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
|
throw Error(format("binary cache '%s' is for Nix stores with prefix "
|
||||||
% getUri() % value % storeDir);
|
"'%s', not '%s'") %
|
||||||
|
getUri() % value % storeDir);
|
||||||
} else if (name == "WantMassQuery") {
|
} else if (name == "WantMassQuery") {
|
||||||
wantMassQuery_ = value == "1";
|
wantMassQuery_ = value == "1";
|
||||||
} else if (name == "Priority") {
|
} else if (name == "Priority") {
|
||||||
|
|
@ -54,19 +52,19 @@ void BinaryCacheStore::init()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::getFile(const std::string & path,
|
void BinaryCacheStore::getFile(
|
||||||
Callback<std::shared_ptr<std::string>> callback) noexcept
|
const std::string& path,
|
||||||
{
|
Callback<std::shared_ptr<std::string>> callback) noexcept {
|
||||||
try {
|
try {
|
||||||
callback(getFile(path));
|
callback(getFile(path));
|
||||||
} catch (...) { callback.rethrow(); }
|
} catch (...) {
|
||||||
|
callback.rethrow();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
void BinaryCacheStore::getFile(const std::string& path, Sink& sink) {
|
||||||
{
|
|
||||||
std::promise<std::shared_ptr<std::string>> promise;
|
std::promise<std::shared_ptr<std::string>> promise;
|
||||||
getFile(path,
|
getFile(path, {[&](std::future<std::shared_ptr<std::string>> result) {
|
||||||
{[&](std::future<std::shared_ptr<std::string>> result) {
|
|
||||||
try {
|
try {
|
||||||
promise.set_value(result.get());
|
promise.set_value(result.get());
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -77,8 +75,8 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
||||||
sink((unsigned char*)data->data(), data->size());
|
sink((unsigned char*)data->data(), data->size());
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
|
std::shared_ptr<std::string> BinaryCacheStore::getFile(
|
||||||
{
|
const std::string& path) {
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
try {
|
try {
|
||||||
getFile(path, sink);
|
getFile(path, sink);
|
||||||
|
|
@ -88,14 +86,12 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
|
||||||
return sink.s;
|
return sink.s;
|
||||||
}
|
}
|
||||||
|
|
||||||
Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
|
Path BinaryCacheStore::narInfoFileFor(const Path& storePath) {
|
||||||
{
|
|
||||||
assertStorePath(storePath);
|
assertStorePath(storePath);
|
||||||
return storePathToHash(storePath) + ".narinfo";
|
return storePathToHash(storePath) + ".narinfo";
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
|
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) {
|
||||||
{
|
|
||||||
auto narInfoFile = narInfoFileFor(narInfo->path);
|
auto narInfoFile = narInfoFileFor(narInfo->path);
|
||||||
|
|
||||||
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
|
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
|
||||||
|
|
@ -108,23 +104,24 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (diskCache)
|
if (diskCache)
|
||||||
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
|
diskCache->upsertNarInfo(getUri(), hashPart,
|
||||||
|
std::shared_ptr<NarInfo>(narInfo));
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
|
void BinaryCacheStore::addToStore(const ValidPathInfo& info,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
|
const ref<std::string>& nar,
|
||||||
{
|
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||||
|
std::shared_ptr<FSAccessor> accessor) {
|
||||||
if (!repair && isValidPath(info.path)) return;
|
if (!repair && isValidPath(info.path)) return;
|
||||||
|
|
||||||
/* Verify that all references are valid. This may do some .narinfo
|
/* Verify that all references are valid. This may do some .narinfo
|
||||||
reads, but typically they'll already be cached. */
|
reads, but typically they'll already be cached. */
|
||||||
for (auto & ref : info.references)
|
for (auto& ref : info.references) try {
|
||||||
try {
|
if (ref != info.path) queryPathInfo(ref);
|
||||||
if (ref != info.path)
|
|
||||||
queryPathInfo(ref);
|
|
||||||
} catch (InvalidPath&) {
|
} catch (InvalidPath&) {
|
||||||
throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
|
throw Error(format("cannot add '%s' to the binary cache because the "
|
||||||
% info.path % ref);
|
"reference '%s' is not valid") %
|
||||||
|
info.path % ref);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
|
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
|
||||||
|
|
@ -135,7 +132,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
narInfo->narHash = hashString(htSHA256, *nar);
|
narInfo->narHash = hashString(htSHA256, *nar);
|
||||||
|
|
||||||
if (info.narHash && info.narHash != narInfo->narHash)
|
if (info.narHash && info.narHash != narInfo->narHash)
|
||||||
throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
|
throw Error(
|
||||||
|
format("refusing to copy corrupted path '%1%' to binary cache") %
|
||||||
|
info.path);
|
||||||
|
|
||||||
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
|
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
|
||||||
|
|
||||||
|
|
@ -150,8 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
|
|
||||||
auto narAccessor = makeNarAccessor(nar);
|
auto narAccessor = makeNarAccessor(nar);
|
||||||
|
|
||||||
if (accessor_)
|
if (accessor_) accessor_->addToCache(info.path, *nar, narAccessor);
|
||||||
accessor_->addToCache(info.path, *nar, narAccessor);
|
|
||||||
|
|
||||||
{
|
{
|
||||||
auto res = jsonRoot.placeholder("root");
|
auto res = jsonRoot.placeholder("root");
|
||||||
|
|
@ -159,12 +157,12 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
|
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(),
|
||||||
|
"application/json");
|
||||||
}
|
}
|
||||||
|
|
||||||
else {
|
else {
|
||||||
if (accessor_)
|
if (accessor_) accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
|
||||||
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Compress the NAR. */
|
/* Compress the NAR. */
|
||||||
|
|
@ -175,18 +173,22 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
narInfo->fileHash = hashString(htSHA256, *narCompressed);
|
narInfo->fileHash = hashString(htSHA256, *narCompressed);
|
||||||
narInfo->fileSize = narCompressed->size();
|
narInfo->fileSize = narCompressed->size();
|
||||||
|
|
||||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
auto duration =
|
||||||
printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
|
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
|
||||||
% narInfo->path % narInfo->narSize
|
.count();
|
||||||
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
|
printMsg(lvlTalkative,
|
||||||
% duration);
|
format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% "
|
||||||
|
"ms) to binary cache") %
|
||||||
|
narInfo->path % narInfo->narSize %
|
||||||
|
((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) %
|
||||||
|
duration);
|
||||||
|
|
||||||
/* Atomically write the NAR file. */
|
/* Atomically write the NAR file. */
|
||||||
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
|
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" +
|
||||||
+ (compression == "xz" ? ".xz" :
|
(compression == "xz" ? ".xz"
|
||||||
compression == "bzip2" ? ".bz2" :
|
: compression == "bzip2"
|
||||||
compression == "br" ? ".br" :
|
? ".bz2"
|
||||||
"");
|
: compression == "br" ? ".br" : "");
|
||||||
if (repair || !fileExists(narInfo->url)) {
|
if (repair || !fileExists(narInfo->url)) {
|
||||||
stats.narWrite++;
|
stats.narWrite++;
|
||||||
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
|
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
|
||||||
|
|
@ -205,16 +207,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
|
||||||
stats.narInfoWrite++;
|
stats.narInfoWrite++;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
|
bool BinaryCacheStore::isValidPathUncached(const Path& storePath) {
|
||||||
{
|
|
||||||
// FIXME: this only checks whether a .narinfo with a matching hash
|
// FIXME: this only checks whether a .narinfo with a matching hash
|
||||||
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
|
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
|
||||||
// though they shouldn't. Not easily fixed.
|
// though they shouldn't. Not easily fixed.
|
||||||
return fileExists(narInfoFileFor(storePath));
|
return fileExists(narInfoFileFor(storePath));
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
|
void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) {
|
||||||
{
|
|
||||||
auto info = queryPathInfo(storePath).cast<const NarInfo>();
|
auto info = queryPathInfo(storePath).cast<const NarInfo>();
|
||||||
|
|
||||||
uint64_t narSize = 0;
|
uint64_t narSize = 0;
|
||||||
|
|
@ -239,20 +239,22 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
|
||||||
stats.narReadBytes += narSize;
|
stats.narReadBytes += narSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
|
void BinaryCacheStore::queryPathInfoUncached(
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
|
const Path& storePath,
|
||||||
{
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
|
||||||
auto uri = getUri();
|
auto uri = getUri();
|
||||||
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
|
auto act = std::make_shared<Activity>(
|
||||||
fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
|
*logger, lvlTalkative, actQueryPathInfo,
|
||||||
|
fmt("querying info about '%s' on '%s'", storePath, uri),
|
||||||
|
Logger::Fields{storePath, uri});
|
||||||
PushActivity pact(act->id);
|
PushActivity pact(act->id);
|
||||||
|
|
||||||
auto narInfoFile = narInfoFileFor(storePath);
|
auto narInfoFile = narInfoFileFor(storePath);
|
||||||
|
|
||||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
||||||
|
|
||||||
getFile(narInfoFile,
|
getFile(
|
||||||
{[=](std::future<std::shared_ptr<std::string>> fut) {
|
narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) {
|
||||||
try {
|
try {
|
||||||
auto data = fut.get();
|
auto data = fut.get();
|
||||||
|
|
||||||
|
|
@ -260,10 +262,12 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
|
||||||
|
|
||||||
stats.narInfoRead++;
|
stats.narInfoRead++;
|
||||||
|
|
||||||
(*callbackPtr)((std::shared_ptr<ValidPathInfo>)
|
(*callbackPtr)(
|
||||||
std::make_shared<NarInfo>(*this, *data, narInfoFile));
|
(std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>(
|
||||||
|
*this, *data, narInfoFile));
|
||||||
|
|
||||||
(void) act; // force Activity into this lambda to ensure it stays alive
|
(void)
|
||||||
|
act; // force Activity into this lambda to ensure it stays alive
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
callbackPtr->rethrow();
|
callbackPtr->rethrow();
|
||||||
}
|
}
|
||||||
|
|
@ -271,8 +275,8 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
|
||||||
}
|
}
|
||||||
|
|
||||||
Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
|
Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
|
||||||
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
bool recursive, HashType hashAlgo,
|
||||||
{
|
PathFilter& filter, RepairFlag repair) {
|
||||||
// FIXME: some cut&paste from LocalStore::addToStore().
|
// FIXME: some cut&paste from LocalStore::addToStore().
|
||||||
|
|
||||||
/* Read the whole path into memory. This is not a very scalable
|
/* Read the whole path into memory. This is not a very scalable
|
||||||
|
|
@ -298,8 +302,8 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
Path BinaryCacheStore::addTextToStore(const string& name, const string& s,
|
Path BinaryCacheStore::addTextToStore(const string& name, const string& s,
|
||||||
const PathSet & references, RepairFlag repair)
|
const PathSet& references,
|
||||||
{
|
RepairFlag repair) {
|
||||||
ValidPathInfo info;
|
ValidPathInfo info;
|
||||||
info.path = computeStorePathForText(name, s, references);
|
info.path = computeStorePathForText(name, s, references);
|
||||||
info.references = references;
|
info.references = references;
|
||||||
|
|
@ -313,13 +317,13 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
|
||||||
return info.path;
|
return info.path;
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
ref<FSAccessor> BinaryCacheStore::getFSAccessor() {
|
||||||
{
|
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()),
|
||||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
|
localNarCache);
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
|
void BinaryCacheStore::addSignatures(const Path& storePath,
|
||||||
{
|
const StringSet& sigs) {
|
||||||
/* Note: this is inherently racy since there is no locking on
|
/* Note: this is inherently racy since there is no locking on
|
||||||
binary caches. In particular, with S3 this unreliable, even
|
binary caches. In particular, with S3 this unreliable, even
|
||||||
when addSignatures() is called sequentially on a path, because
|
when addSignatures() is called sequentially on a path, because
|
||||||
|
|
@ -334,8 +338,7 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
|
||||||
writeNarInfo(narInfo);
|
writeNarInfo(narInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
|
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) {
|
||||||
{
|
|
||||||
Path drvPath;
|
Path drvPath;
|
||||||
|
|
||||||
if (isDerivation(path))
|
if (isDerivation(path))
|
||||||
|
|
@ -358,4 +361,4 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
|
||||||
return getFile(logPath);
|
return getFile(logPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,41 +1,41 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "crypto.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
|
|
||||||
#include "pool.hh"
|
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
#include "crypto.hh"
|
||||||
|
#include "pool.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct NarInfo;
|
struct NarInfo;
|
||||||
|
|
||||||
class BinaryCacheStore : public Store
|
class BinaryCacheStore : public Store {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
const Setting<std::string> compression{
|
||||||
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
this, "xz", "compression",
|
||||||
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
"NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||||
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
const Setting<bool> writeNARListing{
|
||||||
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
|
this, false, "write-nar-listing",
|
||||||
const Setting<bool> parallelCompression{this, false, "parallel-compression",
|
"whether to write a JSON file listing the files in each NAR"};
|
||||||
|
const Setting<Path> secretKeyFile{
|
||||||
|
this, "", "secret-key",
|
||||||
|
"path to secret key used to sign the binary cache"};
|
||||||
|
const Setting<Path> localNarCache{this, "", "local-nar-cache",
|
||||||
|
"path to a local cache of NARs"};
|
||||||
|
const Setting<bool> parallelCompression{
|
||||||
|
this, false, "parallel-compression",
|
||||||
"enable multi-threading compression, available for xz only currently"};
|
"enable multi-threading compression, available for xz only currently"};
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
std::unique_ptr<SecretKey> secretKey;
|
std::unique_ptr<SecretKey> secretKey;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
BinaryCacheStore(const Params& params);
|
BinaryCacheStore(const Params& params);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
virtual bool fileExists(const std::string& path) = 0;
|
virtual bool fileExists(const std::string& path) = 0;
|
||||||
|
|
||||||
virtual void upsertFile(const std::string & path,
|
virtual void upsertFile(const std::string& path, const std::string& data,
|
||||||
const std::string & data,
|
|
||||||
const std::string& mimeType) = 0;
|
const std::string& mimeType) = 0;
|
||||||
|
|
||||||
/* Note: subclasses must implement at least one of the two
|
/* Note: subclasses must implement at least one of the two
|
||||||
|
|
@ -46,22 +46,20 @@ public:
|
||||||
|
|
||||||
/* Fetch the specified file and call the specified callback with
|
/* Fetch the specified file and call the specified callback with
|
||||||
the result. A subclass may implement this asynchronously. */
|
the result. A subclass may implement this asynchronously. */
|
||||||
virtual void getFile(const std::string & path,
|
virtual void getFile(
|
||||||
|
const std::string& path,
|
||||||
Callback<std::shared_ptr<std::string>> callback) noexcept;
|
Callback<std::shared_ptr<std::string>> callback) noexcept;
|
||||||
|
|
||||||
std::shared_ptr<std::string> getFile(const std::string& path);
|
std::shared_ptr<std::string> getFile(const std::string& path);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
bool wantMassQuery_ = false;
|
bool wantMassQuery_ = false;
|
||||||
int priority = 50;
|
int priority = 50;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
virtual void init();
|
virtual void init();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
std::string narMagic;
|
std::string narMagic;
|
||||||
|
|
||||||
std::string narInfoFileFor(const Path& storePath);
|
std::string narInfoFileFor(const Path& storePath);
|
||||||
|
|
@ -69,14 +67,15 @@ private:
|
||||||
void writeNarInfo(ref<NarInfo> narInfo);
|
void writeNarInfo(ref<NarInfo> narInfo);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
bool isValidPathUncached(const Path& path) override;
|
bool isValidPathUncached(const Path& path) override;
|
||||||
|
|
||||||
void queryPathInfoUncached(const Path & path,
|
void queryPathInfoUncached(
|
||||||
|
const Path& path,
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
||||||
|
|
||||||
Path queryPathFromHashPart(const string & hashPart) override
|
Path queryPathFromHashPart(const string& hashPart) override {
|
||||||
{ unsupported("queryPathFromHashPart"); }
|
unsupported("queryPathFromHashPart");
|
||||||
|
}
|
||||||
|
|
||||||
bool wantMassQuery() override { return wantMassQuery_; }
|
bool wantMassQuery() override { return wantMassQuery_; }
|
||||||
|
|
||||||
|
|
@ -84,9 +83,9 @@ public:
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor) override;
|
std::shared_ptr<FSAccessor> accessor) override;
|
||||||
|
|
||||||
Path addToStore(const string & name, const Path & srcPath,
|
Path addToStore(const string& name, const Path& srcPath, bool recursive,
|
||||||
bool recursive, HashType hashAlgo,
|
HashType hashAlgo, PathFilter& filter,
|
||||||
PathFilter & filter, RepairFlag repair) override;
|
RepairFlag repair) override;
|
||||||
|
|
||||||
Path addTextToStore(const string& name, const string& s,
|
Path addTextToStore(const string& name, const string& s,
|
||||||
const PathSet& references, RepairFlag repair) override;
|
const PathSet& references, RepairFlag repair) override;
|
||||||
|
|
@ -94,11 +93,11 @@ public:
|
||||||
void narFromPath(const Path& path, Sink& sink) override;
|
void narFromPath(const Path& path, Sink& sink) override;
|
||||||
|
|
||||||
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
|
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
|
||||||
BuildMode buildMode) override
|
BuildMode buildMode) override {
|
||||||
{ unsupported("buildDerivation"); }
|
unsupported("buildDerivation");
|
||||||
|
}
|
||||||
|
|
||||||
void ensurePath(const Path & path) override
|
void ensurePath(const Path& path) override { unsupported("ensurePath"); }
|
||||||
{ unsupported("ensurePath"); }
|
|
||||||
|
|
||||||
ref<FSAccessor> getFSAccessor() override;
|
ref<FSAccessor> getFSAccessor() override;
|
||||||
|
|
||||||
|
|
@ -107,9 +106,8 @@ public:
|
||||||
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
|
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
|
||||||
|
|
||||||
int getPriority() override { return priority; }
|
int getPriority() override { return priority; }
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
MakeError(NoSuchBinaryCacheFile, Error);
|
MakeError(NoSuchBinaryCacheFile, Error);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
1651
third_party/nix/src/libstore/build.cc
vendored
1651
third_party/nix/src/libstore/build.cc
vendored
File diff suppressed because it is too large
Load diff
2
third_party/nix/src/libstore/builtins.hh
vendored
2
third_party/nix/src/libstore/builtins.hh
vendored
|
|
@ -8,4 +8,4 @@ namespace nix {
|
||||||
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData);
|
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData);
|
||||||
void builtinBuildenv(const BasicDerivation& drv);
|
void builtinBuildenv(const BasicDerivation& drv);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,8 @@
|
||||||
#include "builtins.hh"
|
#include <fcntl.h>
|
||||||
|
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <fcntl.h>
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include "builtins.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
@ -16,23 +15,24 @@ static Priorities priorities;
|
||||||
static unsigned long symlinks;
|
static unsigned long symlinks;
|
||||||
|
|
||||||
/* For each activated package, create symlinks */
|
/* For each activated package, create symlinks */
|
||||||
static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
|
static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
|
||||||
{
|
|
||||||
DirEntries srcFiles;
|
DirEntries srcFiles;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
srcFiles = readDirectory(srcDir);
|
srcFiles = readDirectory(srcDir);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo == ENOTDIR) {
|
if (e.errNo == ENOTDIR) {
|
||||||
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
|
printError(
|
||||||
|
"warning: not including '%s' in the user environment because it's "
|
||||||
|
"not a directory",
|
||||||
|
srcDir);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const auto& ent : srcFiles) {
|
for (const auto& ent : srcFiles) {
|
||||||
if (ent.name[0] == '.')
|
if (ent.name[0] == '.') /* not matched by glob */
|
||||||
/* not matched by glob */
|
|
||||||
continue;
|
continue;
|
||||||
auto srcFile = srcDir + "/" + ent.name;
|
auto srcFile = srcDir + "/" + ent.name;
|
||||||
auto dstFile = dstDir + "/" + ent.name;
|
auto dstFile = dstDir + "/" + ent.name;
|
||||||
|
|
@ -58,8 +58,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
|
||||||
if (hasSuffix(srcFile, "/propagated-build-inputs") ||
|
if (hasSuffix(srcFile, "/propagated-build-inputs") ||
|
||||||
hasSuffix(srcFile, "/nix-support") ||
|
hasSuffix(srcFile, "/nix-support") ||
|
||||||
hasSuffix(srcFile, "/perllocal.pod") ||
|
hasSuffix(srcFile, "/perllocal.pod") ||
|
||||||
hasSuffix(srcFile, "/info/dir") ||
|
hasSuffix(srcFile, "/info/dir") || hasSuffix(srcFile, "/log"))
|
||||||
hasSuffix(srcFile, "/log"))
|
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
else if (S_ISDIR(srcSt.st_mode)) {
|
else if (S_ISDIR(srcSt.st_mode)) {
|
||||||
|
|
@ -72,7 +71,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
|
||||||
} else if (S_ISLNK(dstSt.st_mode)) {
|
} else if (S_ISLNK(dstSt.st_mode)) {
|
||||||
auto target = canonPath(dstFile, true);
|
auto target = canonPath(dstFile, true);
|
||||||
if (!S_ISDIR(lstat(target).st_mode))
|
if (!S_ISDIR(lstat(target).st_mode))
|
||||||
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
|
throw Error("collision between '%1%' and non-directory '%2%'",
|
||||||
|
srcFile, target);
|
||||||
if (unlink(dstFile.c_str()) == -1)
|
if (unlink(dstFile.c_str()) == -1)
|
||||||
throw SysError(format("unlinking '%1%'") % dstFile);
|
throw SysError(format("unlinking '%1%'") % dstFile);
|
||||||
if (mkdir(dstFile.c_str(), 0755) == -1)
|
if (mkdir(dstFile.c_str(), 0755) == -1)
|
||||||
|
|
@ -98,12 +98,13 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
|
||||||
"to change the priority of one of the conflicting packages"
|
"to change the priority of one of the conflicting packages"
|
||||||
" (0 being the highest priority)",
|
" (0 being the highest priority)",
|
||||||
srcFile, readLink(dstFile), priority);
|
srcFile, readLink(dstFile), priority);
|
||||||
if (prevPriority < priority)
|
if (prevPriority < priority) continue;
|
||||||
continue;
|
|
||||||
if (unlink(dstFile.c_str()) == -1)
|
if (unlink(dstFile.c_str()) == -1)
|
||||||
throw SysError(format("unlinking '%1%'") % dstFile);
|
throw SysError(format("unlinking '%1%'") % dstFile);
|
||||||
} else if (S_ISDIR(dstSt.st_mode))
|
} else if (S_ISDIR(dstSt.st_mode))
|
||||||
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
|
throw Error(
|
||||||
|
"collision between non-directory '%1%' and directory '%2%'",
|
||||||
|
srcFile, dstFile);
|
||||||
} else if (errno != ENOENT)
|
} else if (errno != ENOENT)
|
||||||
throw SysError(format("getting status of '%1%'") % dstFile);
|
throw SysError(format("getting status of '%1%'") % dstFile);
|
||||||
}
|
}
|
||||||
|
|
@ -121,17 +122,16 @@ static FileProp postponed = FileProp{};
|
||||||
|
|
||||||
static Path out;
|
static Path out;
|
||||||
|
|
||||||
static void addPkg(const Path & pkgDir, int priority)
|
static void addPkg(const Path& pkgDir, int priority) {
|
||||||
{
|
|
||||||
if (done.count(pkgDir)) return;
|
if (done.count(pkgDir)) return;
|
||||||
done.insert(pkgDir);
|
done.insert(pkgDir);
|
||||||
createLinks(pkgDir, out, priority);
|
createLinks(pkgDir, out, priority);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const auto& p : tokenizeString<std::vector<string>>(
|
for (const auto& p : tokenizeString<std::vector<string>>(
|
||||||
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
|
readFile(pkgDir + "/nix-support/propagated-user-env-packages"),
|
||||||
if (!done.count(p))
|
" \n"))
|
||||||
postponed.insert(p);
|
if (!done.count(p)) postponed.insert(p);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
|
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
|
||||||
}
|
}
|
||||||
|
|
@ -141,13 +141,13 @@ struct Package {
|
||||||
Path path;
|
Path path;
|
||||||
bool active;
|
bool active;
|
||||||
int priority;
|
int priority;
|
||||||
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
|
Package(Path path, bool active, int priority)
|
||||||
|
: path{path}, active{active}, priority{priority} {}
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::vector<Package> Packages;
|
typedef std::vector<Package> Packages;
|
||||||
|
|
||||||
void builtinBuildenv(const BasicDerivation & drv)
|
void builtinBuildenv(const BasicDerivation& drv) {
|
||||||
{
|
|
||||||
auto getAttr = [&](const string& name) {
|
auto getAttr = [&](const string& name) {
|
||||||
auto i = drv.env.find(name);
|
auto i = drv.env.find(name);
|
||||||
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
|
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
|
||||||
|
|
@ -162,12 +162,17 @@ void builtinBuildenv(const BasicDerivation & drv)
|
||||||
Packages pkgs;
|
Packages pkgs;
|
||||||
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
|
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
|
||||||
while (!derivations.empty()) {
|
while (!derivations.empty()) {
|
||||||
/* !!! We're trusting the caller to structure derivations env var correctly */
|
/* !!! We're trusting the caller to structure derivations env var correctly
|
||||||
auto active = derivations.front(); derivations.pop_front();
|
*/
|
||||||
auto priority = stoi(derivations.front()); derivations.pop_front();
|
auto active = derivations.front();
|
||||||
auto outputs = stoi(derivations.front()); derivations.pop_front();
|
derivations.pop_front();
|
||||||
|
auto priority = stoi(derivations.front());
|
||||||
|
derivations.pop_front();
|
||||||
|
auto outputs = stoi(derivations.front());
|
||||||
|
derivations.pop_front();
|
||||||
for (auto n = 0; n < outputs; n++) {
|
for (auto n = 0; n < outputs; n++) {
|
||||||
auto path = derivations.front(); derivations.pop_front();
|
auto path = derivations.front();
|
||||||
|
derivations.pop_front();
|
||||||
pkgs.emplace_back(path, active != "false", priority);
|
pkgs.emplace_back(path, active != "false", priority);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -177,11 +182,11 @@ void builtinBuildenv(const BasicDerivation & drv)
|
||||||
* symlink/unlink steps.
|
* symlink/unlink steps.
|
||||||
*/
|
*/
|
||||||
std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) {
|
std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) {
|
||||||
return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
|
return a.priority < b.priority ||
|
||||||
|
(a.priority == b.priority && a.path < b.path);
|
||||||
});
|
});
|
||||||
for (const auto& pkg : pkgs)
|
for (const auto& pkg : pkgs)
|
||||||
if (pkg.active)
|
if (pkg.active) addPkg(pkg.path, pkg.priority);
|
||||||
addPkg(pkg.path, pkg.priority);
|
|
||||||
|
|
||||||
/* Symlink to the packages that have been "propagated" by packages
|
/* Symlink to the packages that have been "propagated" by packages
|
||||||
* installed by the user (i.e., package X declares that it wants Y
|
* installed by the user (i.e., package X declares that it wants Y
|
||||||
|
|
@ -192,8 +197,7 @@ void builtinBuildenv(const BasicDerivation & drv)
|
||||||
while (!postponed.empty()) {
|
while (!postponed.empty()) {
|
||||||
auto pkgDirs = postponed;
|
auto pkgDirs = postponed;
|
||||||
postponed = FileProp{};
|
postponed = FileProp{};
|
||||||
for (const auto & pkgDir : pkgDirs)
|
for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++);
|
||||||
addPkg(pkgDir, priorityCounter++);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
printError("created %d symlinks in user environment", symlinks);
|
printError("created %d symlinks in user environment", symlinks);
|
||||||
|
|
@ -201,4 +205,4 @@ void builtinBuildenv(const BasicDerivation & drv)
|
||||||
createSymlink(getAttr("manifest"), out + "/manifest.nix");
|
createSymlink(getAttr("manifest"), out + "/manifest.nix");
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,12 @@
|
||||||
|
#include "archive.hh"
|
||||||
#include "builtins.hh"
|
#include "builtins.hh"
|
||||||
|
#include "compression.hh"
|
||||||
#include "download.hh"
|
#include "download.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "archive.hh"
|
|
||||||
#include "compression.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) {
|
||||||
{
|
|
||||||
/* Make the host's netrc data available. Too bad curl requires
|
/* Make the host's netrc data available. Too bad curl requires
|
||||||
this to be stored in a file. It would be nice if we could just
|
this to be stored in a file. It would be nice if we could just
|
||||||
pass a pointer to the data. */
|
pass a pointer to the data. */
|
||||||
|
|
@ -18,7 +17,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
|
|
||||||
auto getAttr = [&](const string& name) {
|
auto getAttr = [&](const string& name) {
|
||||||
auto i = drv.env.find(name);
|
auto i = drv.env.find(name);
|
||||||
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
|
if (i == drv.env.end())
|
||||||
|
throw Error(format("attribute '%s' missing") % name);
|
||||||
return i->second;
|
return i->second;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -31,9 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
auto downloader = makeDownloader();
|
auto downloader = makeDownloader();
|
||||||
|
|
||||||
auto fetch = [&](const std::string& url) {
|
auto fetch = [&](const std::string& url) {
|
||||||
|
|
||||||
auto source = sinkToSource([&](Sink& sink) {
|
auto source = sinkToSource([&](Sink& sink) {
|
||||||
|
|
||||||
/* No need to do TLS verification, because we check the hash of
|
/* No need to do TLS verification, because we check the hash of
|
||||||
the result anyway. */
|
the result anyway. */
|
||||||
DownloadRequest request(url);
|
DownloadRequest request(url);
|
||||||
|
|
@ -60,12 +58,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
|
|
||||||
/* Try the hashed mirrors first. */
|
/* Try the hashed mirrors first. */
|
||||||
if (getAttr("outputHashMode") == "flat")
|
if (getAttr("outputHashMode") == "flat")
|
||||||
for (auto hashedMirror : settings.hashedMirrors.get())
|
for (auto hashedMirror : settings.hashedMirrors.get()) try {
|
||||||
try {
|
|
||||||
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
|
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
|
||||||
auto ht = parseHashType(getAttr("outputHashAlgo"));
|
auto ht = parseHashType(getAttr("outputHashAlgo"));
|
||||||
auto h = Hash(getAttr("outputHash"), ht);
|
auto h = Hash(getAttr("outputHash"), ht);
|
||||||
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
|
fetch(hashedMirror + printHashType(h.type) + "/" +
|
||||||
|
h.to_string(Base16, false));
|
||||||
return;
|
return;
|
||||||
} catch (Error& e) {
|
} catch (Error& e) {
|
||||||
debug(e.what());
|
debug(e.what());
|
||||||
|
|
@ -75,4 +73,4 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
fetch(mainUrl);
|
fetch(mainUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
52
third_party/nix/src/libstore/crypto.cc
vendored
52
third_party/nix/src/libstore/crypto.cc
vendored
|
|
@ -1,6 +1,6 @@
|
||||||
#include "crypto.hh"
|
#include "crypto.hh"
|
||||||
#include "util.hh"
|
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
#include <sodium.h>
|
#include <sodium.h>
|
||||||
|
|
@ -8,30 +8,24 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static std::pair<std::string, std::string> split(const string & s)
|
static std::pair<std::string, std::string> split(const string& s) {
|
||||||
{
|
|
||||||
size_t colon = s.find(':');
|
size_t colon = s.find(':');
|
||||||
if (colon == std::string::npos || colon == 0)
|
if (colon == std::string::npos || colon == 0) return {"", ""};
|
||||||
return {"", ""};
|
|
||||||
return {std::string(s, 0, colon), std::string(s, colon + 1)};
|
return {std::string(s, 0, colon), std::string(s, colon + 1)};
|
||||||
}
|
}
|
||||||
|
|
||||||
Key::Key(const string & s)
|
Key::Key(const string& s) {
|
||||||
{
|
|
||||||
auto ss = split(s);
|
auto ss = split(s);
|
||||||
|
|
||||||
name = ss.first;
|
name = ss.first;
|
||||||
key = ss.second;
|
key = ss.second;
|
||||||
|
|
||||||
if (name == "" || key == "")
|
if (name == "" || key == "") throw Error("secret key is corrupt");
|
||||||
throw Error("secret key is corrupt");
|
|
||||||
|
|
||||||
key = base64Decode(key);
|
key = base64Decode(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
SecretKey::SecretKey(const string & s)
|
SecretKey::SecretKey(const string& s) : Key(s) {
|
||||||
: Key(s)
|
|
||||||
{
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
if (key.size() != crypto_sign_SECRETKEYBYTES)
|
if (key.size() != crypto_sign_SECRETKEYBYTES)
|
||||||
throw Error("secret key is not valid");
|
throw Error("secret key is not valid");
|
||||||
|
|
@ -39,14 +33,14 @@ SecretKey::SecretKey(const string & s)
|
||||||
}
|
}
|
||||||
|
|
||||||
#if !HAVE_SODIUM
|
#if !HAVE_SODIUM
|
||||||
[[noreturn]] static void noSodium()
|
[[noreturn]] static void noSodium() {
|
||||||
{
|
throw Error(
|
||||||
throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
|
"Nix was not compiled with libsodium, required for signed binary cache "
|
||||||
|
"support");
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
std::string SecretKey::signDetached(const std::string & data) const
|
std::string SecretKey::signDetached(const std::string& data) const {
|
||||||
{
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
unsigned char sig[crypto_sign_BYTES];
|
unsigned char sig[crypto_sign_BYTES];
|
||||||
unsigned long long sigLen;
|
unsigned long long sigLen;
|
||||||
|
|
@ -58,8 +52,7 @@ std::string SecretKey::signDetached(const std::string & data) const
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
PublicKey SecretKey::toPublicKey() const
|
PublicKey SecretKey::toPublicKey() const {
|
||||||
{
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
|
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
|
||||||
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data());
|
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data());
|
||||||
|
|
@ -69,9 +62,7 @@ PublicKey SecretKey::toPublicKey() const
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
PublicKey::PublicKey(const string & s)
|
PublicKey::PublicKey(const string& s) : Key(s) {
|
||||||
: Key(s)
|
|
||||||
{
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
if (key.size() != crypto_sign_PUBLICKEYBYTES)
|
if (key.size() != crypto_sign_PUBLICKEYBYTES)
|
||||||
throw Error("public key is not valid");
|
throw Error("public key is not valid");
|
||||||
|
|
@ -79,8 +70,7 @@ PublicKey::PublicKey(const string & s)
|
||||||
}
|
}
|
||||||
|
|
||||||
bool verifyDetached(const std::string& data, const std::string& sig,
|
bool verifyDetached(const std::string& data, const std::string& sig,
|
||||||
const PublicKeys & publicKeys)
|
const PublicKeys& publicKeys) {
|
||||||
{
|
|
||||||
#if HAVE_SODIUM
|
#if HAVE_SODIUM
|
||||||
auto ss = split(sig);
|
auto ss = split(sig);
|
||||||
|
|
||||||
|
|
@ -88,19 +78,17 @@ bool verifyDetached(const std::string & data, const std::string & sig,
|
||||||
if (key == publicKeys.end()) return false;
|
if (key == publicKeys.end()) return false;
|
||||||
|
|
||||||
auto sig2 = base64Decode(ss.second);
|
auto sig2 = base64Decode(ss.second);
|
||||||
if (sig2.size() != crypto_sign_BYTES)
|
if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid");
|
||||||
throw Error("signature is not valid");
|
|
||||||
|
|
||||||
return crypto_sign_verify_detached((unsigned char *) sig2.data(),
|
return crypto_sign_verify_detached(
|
||||||
(unsigned char *) data.data(), data.size(),
|
(unsigned char*)sig2.data(), (unsigned char*)data.data(),
|
||||||
(unsigned char *) key->second.key.data()) == 0;
|
data.size(), (unsigned char*)key->second.key.data()) == 0;
|
||||||
#else
|
#else
|
||||||
noSodium();
|
noSodium();
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
PublicKeys getDefaultPublicKeys()
|
PublicKeys getDefaultPublicKeys() {
|
||||||
{
|
|
||||||
PublicKeys publicKeys;
|
PublicKeys publicKeys;
|
||||||
|
|
||||||
// FIXME: filter duplicates
|
// FIXME: filter duplicates
|
||||||
|
|
@ -123,4 +111,4 @@ PublicKeys getDefaultPublicKeys()
|
||||||
return publicKeys;
|
return publicKeys;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
20
third_party/nix/src/libstore/crypto.hh
vendored
20
third_party/nix/src/libstore/crypto.hh
vendored
|
|
@ -1,13 +1,11 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct Key
|
struct Key {
|
||||||
{
|
|
||||||
std::string name;
|
std::string name;
|
||||||
std::string key;
|
std::string key;
|
||||||
|
|
||||||
|
|
@ -16,14 +14,12 @@ struct Key
|
||||||
Key(const std::string& s);
|
Key(const std::string& s);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
Key(const std::string & name, const std::string & key)
|
Key(const std::string& name, const std::string& key) : name(name), key(key) {}
|
||||||
: name(name), key(key) { }
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct PublicKey;
|
struct PublicKey;
|
||||||
|
|
||||||
struct SecretKey : Key
|
struct SecretKey : Key {
|
||||||
{
|
|
||||||
SecretKey(const std::string& s);
|
SecretKey(const std::string& s);
|
||||||
|
|
||||||
/* Return a detached signature of the given string. */
|
/* Return a detached signature of the given string. */
|
||||||
|
|
@ -32,13 +28,11 @@ struct SecretKey : Key
|
||||||
PublicKey toPublicKey() const;
|
PublicKey toPublicKey() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct PublicKey : Key
|
struct PublicKey : Key {
|
||||||
{
|
|
||||||
PublicKey(const std::string& data);
|
PublicKey(const std::string& data);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
PublicKey(const std::string & name, const std::string & key)
|
PublicKey(const std::string& name, const std::string& key) : Key(name, key) {}
|
||||||
: Key(name, key) { }
|
|
||||||
friend struct SecretKey;
|
friend struct SecretKey;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -51,4 +45,4 @@ bool verifyDetached(const std::string & data, const std::string & sig,
|
||||||
|
|
||||||
PublicKeys getDefaultPublicKeys();
|
PublicKeys getDefaultPublicKeys();
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
265
third_party/nix/src/libstore/derivations.cc
vendored
265
third_party/nix/src/libstore/derivations.cc
vendored
|
|
@ -1,16 +1,14 @@
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
#include "store-api.hh"
|
#include "fs-accessor.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
|
#include "istringstream_nocopy.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "worker-protocol.hh"
|
#include "worker-protocol.hh"
|
||||||
#include "fs-accessor.hh"
|
|
||||||
#include "istringstream_nocopy.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
|
||||||
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
|
|
||||||
{
|
|
||||||
recursive = false;
|
recursive = false;
|
||||||
string algo = hashAlgo;
|
string algo = hashAlgo;
|
||||||
|
|
||||||
|
|
@ -26,29 +24,22 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
|
||||||
hash = Hash(this->hash, hashType);
|
hash = Hash(this->hash, hashType);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path BasicDerivation::findOutput(const string& id) const {
|
||||||
Path BasicDerivation::findOutput(const string & id) const
|
|
||||||
{
|
|
||||||
auto i = outputs.find(id);
|
auto i = outputs.find(id);
|
||||||
if (i == outputs.end())
|
if (i == outputs.end())
|
||||||
throw Error(format("derivation has no output '%1%'") % id);
|
throw Error(format("derivation has no output '%1%'") % id);
|
||||||
return i->second.path;
|
return i->second.path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool BasicDerivation::isBuiltin() const {
|
||||||
bool BasicDerivation::isBuiltin() const
|
|
||||||
{
|
|
||||||
return string(builder, 0, 8) == "builtin:";
|
return string(builder, 0, 8) == "builtin:";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path writeDerivation(ref<Store> store, const Derivation& drv,
|
||||||
Path writeDerivation(ref<Store> store,
|
const string& name, RepairFlag repair) {
|
||||||
const Derivation & drv, const string & name, RepairFlag repair)
|
|
||||||
{
|
|
||||||
PathSet references;
|
PathSet references;
|
||||||
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
||||||
for (auto & i : drv.inputDrvs)
|
for (auto& i : drv.inputDrvs) references.insert(i.first);
|
||||||
references.insert(i.first);
|
|
||||||
/* Note that the outputs of a derivation are *not* references
|
/* Note that the outputs of a derivation are *not* references
|
||||||
(that can be missing (of course) and should not necessarily be
|
(that can be missing (of course) and should not necessarily be
|
||||||
held during a garbage collection). */
|
held during a garbage collection). */
|
||||||
|
|
@ -59,47 +50,43 @@ Path writeDerivation(ref<Store> store,
|
||||||
: store->addTextToStore(suffix, contents, references, repair);
|
: store->addTextToStore(suffix, contents, references, repair);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Read string `s' from stream `str'. */
|
/* Read string `s' from stream `str'. */
|
||||||
static void expect(std::istream & str, const string & s)
|
static void expect(std::istream& str, const string& s) {
|
||||||
{
|
|
||||||
char s2[s.size()];
|
char s2[s.size()];
|
||||||
str.read(s2, s.size());
|
str.read(s2, s.size());
|
||||||
if (string(s2, s.size()) != s)
|
if (string(s2, s.size()) != s)
|
||||||
throw FormatError(format("expected string '%1%'") % s);
|
throw FormatError(format("expected string '%1%'") % s);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Read a C-style string from stream `str'. */
|
/* Read a C-style string from stream `str'. */
|
||||||
static string parseString(std::istream & str)
|
static string parseString(std::istream& str) {
|
||||||
{
|
|
||||||
string res;
|
string res;
|
||||||
expect(str, "\"");
|
expect(str, "\"");
|
||||||
int c;
|
int c;
|
||||||
while ((c = str.get()) != '"')
|
while ((c = str.get()) != '"')
|
||||||
if (c == '\\') {
|
if (c == '\\') {
|
||||||
c = str.get();
|
c = str.get();
|
||||||
if (c == 'n') res += '\n';
|
if (c == 'n')
|
||||||
else if (c == 'r') res += '\r';
|
res += '\n';
|
||||||
else if (c == 't') res += '\t';
|
else if (c == 'r')
|
||||||
else res += c;
|
res += '\r';
|
||||||
}
|
else if (c == 't')
|
||||||
else res += c;
|
res += '\t';
|
||||||
|
else
|
||||||
|
res += c;
|
||||||
|
} else
|
||||||
|
res += c;
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Path parsePath(std::istream& str) {
|
||||||
static Path parsePath(std::istream & str)
|
|
||||||
{
|
|
||||||
string s = parseString(str);
|
string s = parseString(str);
|
||||||
if (s.size() == 0 || s[0] != '/')
|
if (s.size() == 0 || s[0] != '/')
|
||||||
throw FormatError(format("bad path '%1%' in derivation") % s);
|
throw FormatError(format("bad path '%1%' in derivation") % s);
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool endOfList(std::istream& str) {
|
||||||
static bool endOfList(std::istream & str)
|
|
||||||
{
|
|
||||||
if (str.peek() == ',') {
|
if (str.peek() == ',') {
|
||||||
str.get();
|
str.get();
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -111,18 +98,14 @@ static bool endOfList(std::istream & str)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static StringSet parseStrings(std::istream& str, bool arePaths) {
|
||||||
static StringSet parseStrings(std::istream & str, bool arePaths)
|
|
||||||
{
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
while (!endOfList(str))
|
while (!endOfList(str))
|
||||||
res.insert(arePaths ? parsePath(str) : parseString(str));
|
res.insert(arePaths ? parsePath(str) : parseString(str));
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Derivation parseDerivation(const string& s) {
|
||||||
static Derivation parseDerivation(const string & s)
|
|
||||||
{
|
|
||||||
Derivation drv;
|
Derivation drv;
|
||||||
istringstream_nocopy str(s);
|
istringstream_nocopy str(s);
|
||||||
expect(str, "Derive([");
|
expect(str, "Derive([");
|
||||||
|
|
@ -130,10 +113,14 @@ static Derivation parseDerivation(const string & s)
|
||||||
/* Parse the list of outputs. */
|
/* Parse the list of outputs. */
|
||||||
while (!endOfList(str)) {
|
while (!endOfList(str)) {
|
||||||
DerivationOutput out;
|
DerivationOutput out;
|
||||||
expect(str, "("); string id = parseString(str);
|
expect(str, "(");
|
||||||
expect(str, ","); out.path = parsePath(str);
|
string id = parseString(str);
|
||||||
expect(str, ","); out.hashAlgo = parseString(str);
|
expect(str, ",");
|
||||||
expect(str, ","); out.hash = parseString(str);
|
out.path = parsePath(str);
|
||||||
|
expect(str, ",");
|
||||||
|
out.hashAlgo = parseString(str);
|
||||||
|
expect(str, ",");
|
||||||
|
out.hash = parseString(str);
|
||||||
expect(str, ")");
|
expect(str, ")");
|
||||||
drv.outputs[id] = out;
|
drv.outputs[id] = out;
|
||||||
}
|
}
|
||||||
|
|
@ -148,20 +135,24 @@ static Derivation parseDerivation(const string & s)
|
||||||
expect(str, ")");
|
expect(str, ")");
|
||||||
}
|
}
|
||||||
|
|
||||||
expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
|
expect(str, ",[");
|
||||||
expect(str, ","); drv.platform = parseString(str);
|
drv.inputSrcs = parseStrings(str, true);
|
||||||
expect(str, ","); drv.builder = parseString(str);
|
expect(str, ",");
|
||||||
|
drv.platform = parseString(str);
|
||||||
|
expect(str, ",");
|
||||||
|
drv.builder = parseString(str);
|
||||||
|
|
||||||
/* Parse the builder arguments. */
|
/* Parse the builder arguments. */
|
||||||
expect(str, ",[");
|
expect(str, ",[");
|
||||||
while (!endOfList(str))
|
while (!endOfList(str)) drv.args.push_back(parseString(str));
|
||||||
drv.args.push_back(parseString(str));
|
|
||||||
|
|
||||||
/* Parse the environment variables. */
|
/* Parse the environment variables. */
|
||||||
expect(str, ",[");
|
expect(str, ",[");
|
||||||
while (!endOfList(str)) {
|
while (!endOfList(str)) {
|
||||||
expect(str, "("); string name = parseString(str);
|
expect(str, "(");
|
||||||
expect(str, ","); string value = parseString(str);
|
string name = parseString(str);
|
||||||
|
expect(str, ",");
|
||||||
|
string value = parseString(str);
|
||||||
expect(str, ")");
|
expect(str, ")");
|
||||||
drv.env[name] = value;
|
drv.env[name] = value;
|
||||||
}
|
}
|
||||||
|
|
@ -170,94 +161,115 @@ static Derivation parseDerivation(const string & s)
|
||||||
return drv;
|
return drv;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Derivation readDerivation(const Path& drvPath) {
|
||||||
Derivation readDerivation(const Path & drvPath)
|
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
return parseDerivation(readFile(drvPath));
|
return parseDerivation(readFile(drvPath));
|
||||||
} catch (FormatError& e) {
|
} catch (FormatError& e) {
|
||||||
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
|
throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
|
||||||
|
e.msg());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Derivation Store::derivationFromPath(const Path& drvPath) {
|
||||||
Derivation Store::derivationFromPath(const Path & drvPath)
|
|
||||||
{
|
|
||||||
assertStorePath(drvPath);
|
assertStorePath(drvPath);
|
||||||
ensurePath(drvPath);
|
ensurePath(drvPath);
|
||||||
auto accessor = getFSAccessor();
|
auto accessor = getFSAccessor();
|
||||||
try {
|
try {
|
||||||
return parseDerivation(accessor->readFile(drvPath));
|
return parseDerivation(accessor->readFile(drvPath));
|
||||||
} catch (FormatError& e) {
|
} catch (FormatError& e) {
|
||||||
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
|
throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
|
||||||
|
e.msg());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void printString(string& res, const string& s) {
|
||||||
static void printString(string & res, const string & s)
|
|
||||||
{
|
|
||||||
res += '"';
|
res += '"';
|
||||||
for (const char* i = s.c_str(); *i; i++)
|
for (const char* i = s.c_str(); *i; i++)
|
||||||
if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
|
if (*i == '\"' || *i == '\\') {
|
||||||
else if (*i == '\n') res += "\\n";
|
res += "\\";
|
||||||
else if (*i == '\r') res += "\\r";
|
res += *i;
|
||||||
else if (*i == '\t') res += "\\t";
|
} else if (*i == '\n')
|
||||||
else res += *i;
|
res += "\\n";
|
||||||
|
else if (*i == '\r')
|
||||||
|
res += "\\r";
|
||||||
|
else if (*i == '\t')
|
||||||
|
res += "\\t";
|
||||||
|
else
|
||||||
|
res += *i;
|
||||||
res += '"';
|
res += '"';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
template <class ForwardIterator>
|
template <class ForwardIterator>
|
||||||
static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
|
static void printStrings(string& res, ForwardIterator i, ForwardIterator j) {
|
||||||
{
|
|
||||||
res += '[';
|
res += '[';
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (; i != j; ++i) {
|
for (; i != j; ++i) {
|
||||||
if (first) first = false; else res += ',';
|
if (first)
|
||||||
|
first = false;
|
||||||
|
else
|
||||||
|
res += ',';
|
||||||
printString(res, *i);
|
printString(res, *i);
|
||||||
}
|
}
|
||||||
res += ']';
|
res += ']';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string Derivation::unparse() const {
|
||||||
string Derivation::unparse() const
|
|
||||||
{
|
|
||||||
string s;
|
string s;
|
||||||
s.reserve(65536);
|
s.reserve(65536);
|
||||||
s += "Derive([";
|
s += "Derive([";
|
||||||
|
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (auto& i : outputs) {
|
for (auto& i : outputs) {
|
||||||
if (first) first = false; else s += ',';
|
if (first)
|
||||||
s += '('; printString(s, i.first);
|
first = false;
|
||||||
s += ','; printString(s, i.second.path);
|
else
|
||||||
s += ','; printString(s, i.second.hashAlgo);
|
s += ',';
|
||||||
s += ','; printString(s, i.second.hash);
|
s += '(';
|
||||||
|
printString(s, i.first);
|
||||||
|
s += ',';
|
||||||
|
printString(s, i.second.path);
|
||||||
|
s += ',';
|
||||||
|
printString(s, i.second.hashAlgo);
|
||||||
|
s += ',';
|
||||||
|
printString(s, i.second.hash);
|
||||||
s += ')';
|
s += ')';
|
||||||
}
|
}
|
||||||
|
|
||||||
s += "],[";
|
s += "],[";
|
||||||
first = true;
|
first = true;
|
||||||
for (auto& i : inputDrvs) {
|
for (auto& i : inputDrvs) {
|
||||||
if (first) first = false; else s += ',';
|
if (first)
|
||||||
s += '('; printString(s, i.first);
|
first = false;
|
||||||
s += ','; printStrings(s, i.second.begin(), i.second.end());
|
else
|
||||||
|
s += ',';
|
||||||
|
s += '(';
|
||||||
|
printString(s, i.first);
|
||||||
|
s += ',';
|
||||||
|
printStrings(s, i.second.begin(), i.second.end());
|
||||||
s += ')';
|
s += ')';
|
||||||
}
|
}
|
||||||
|
|
||||||
s += "],";
|
s += "],";
|
||||||
printStrings(s, inputSrcs.begin(), inputSrcs.end());
|
printStrings(s, inputSrcs.begin(), inputSrcs.end());
|
||||||
|
|
||||||
s += ','; printString(s, platform);
|
s += ',';
|
||||||
s += ','; printString(s, builder);
|
printString(s, platform);
|
||||||
s += ','; printStrings(s, args.begin(), args.end());
|
s += ',';
|
||||||
|
printString(s, builder);
|
||||||
|
s += ',';
|
||||||
|
printStrings(s, args.begin(), args.end());
|
||||||
|
|
||||||
s += ",[";
|
s += ",[";
|
||||||
first = true;
|
first = true;
|
||||||
for (auto& i : env) {
|
for (auto& i : env) {
|
||||||
if (first) first = false; else s += ',';
|
if (first)
|
||||||
s += '('; printString(s, i.first);
|
first = false;
|
||||||
s += ','; printString(s, i.second);
|
else
|
||||||
|
s += ',';
|
||||||
|
s += '(';
|
||||||
|
printString(s, i.first);
|
||||||
|
s += ',';
|
||||||
|
printString(s, i.second);
|
||||||
s += ')';
|
s += ')';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -266,24 +278,17 @@ string Derivation::unparse() const
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool isDerivation(const string& fileName) {
|
||||||
bool isDerivation(const string & fileName)
|
|
||||||
{
|
|
||||||
return hasSuffix(fileName, drvExtension);
|
return hasSuffix(fileName, drvExtension);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool BasicDerivation::isFixedOutput() const {
|
||||||
bool BasicDerivation::isFixedOutput() const
|
return outputs.size() == 1 && outputs.begin()->first == "out" &&
|
||||||
{
|
|
||||||
return outputs.size() == 1 &&
|
|
||||||
outputs.begin()->first == "out" &&
|
|
||||||
outputs.begin()->second.hash != "";
|
outputs.begin()->second.hash != "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
DrvHashes drvHashes;
|
DrvHashes drvHashes;
|
||||||
|
|
||||||
|
|
||||||
/* Returns the hash of a derivation modulo fixed-output
|
/* Returns the hash of a derivation modulo fixed-output
|
||||||
subderivations. A fixed-output derivation is a derivation with one
|
subderivations. A fixed-output derivation is a derivation with one
|
||||||
output (`out') for which an expected hash and hash algorithm are
|
output (`out') for which an expected hash and hash algorithm are
|
||||||
|
|
@ -304,15 +309,12 @@ DrvHashes drvHashes;
|
||||||
paths have been replaced by the result of a recursive call to this
|
paths have been replaced by the result of a recursive call to this
|
||||||
function, and that for fixed-output derivations we return a hash of
|
function, and that for fixed-output derivations we return a hash of
|
||||||
its output path. */
|
its output path. */
|
||||||
Hash hashDerivationModulo(Store & store, Derivation drv)
|
Hash hashDerivationModulo(Store& store, Derivation drv) {
|
||||||
{
|
|
||||||
/* Return a fixed hash for fixed-output derivations. */
|
/* Return a fixed hash for fixed-output derivations. */
|
||||||
if (drv.isFixedOutput()) {
|
if (drv.isFixedOutput()) {
|
||||||
DerivationOutputs::const_iterator i = drv.outputs.begin();
|
DerivationOutputs::const_iterator i = drv.outputs.begin();
|
||||||
return hashString(htSHA256, "fixed:out:"
|
return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" +
|
||||||
+ i->second.hashAlgo + ":"
|
i->second.hash + ":" + i->second.path);
|
||||||
+ i->second.hash + ":"
|
|
||||||
+ i->second.path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* For other derivations, replace the inputs paths with recursive
|
/* For other derivations, replace the inputs paths with recursive
|
||||||
|
|
@ -333,41 +335,31 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
|
||||||
return hashString(htSHA256, drv.unparse());
|
return hashString(htSHA256, drv.unparse());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s) {
|
||||||
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
|
|
||||||
{
|
|
||||||
size_t n = s.find("!");
|
size_t n = s.find("!");
|
||||||
return n == s.npos
|
return n == s.npos ? DrvPathWithOutputs(s, std::set<string>())
|
||||||
? DrvPathWithOutputs(s, std::set<string>())
|
: DrvPathWithOutputs(string(s, 0, n),
|
||||||
: DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
|
tokenizeString<std::set<string> >(
|
||||||
|
string(s, n + 1), ","));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path makeDrvPathWithOutputs(const Path& drvPath,
|
||||||
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
|
const std::set<string>& outputs) {
|
||||||
{
|
return outputs.empty() ? drvPath
|
||||||
return outputs.empty()
|
|
||||||
? drvPath
|
|
||||||
: drvPath + "!" + concatStringsSep(",", outputs);
|
: drvPath + "!" + concatStringsSep(",", outputs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool wantOutput(const string& output, const std::set<string>& wanted) {
|
||||||
bool wantOutput(const string & output, const std::set<string> & wanted)
|
|
||||||
{
|
|
||||||
return wanted.empty() || wanted.find(output) != wanted.end();
|
return wanted.empty() || wanted.find(output) != wanted.end();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet BasicDerivation::outputPaths() const {
|
||||||
PathSet BasicDerivation::outputPaths() const
|
|
||||||
{
|
|
||||||
PathSet paths;
|
PathSet paths;
|
||||||
for (auto & i : outputs)
|
for (auto& i : outputs) paths.insert(i.second.path);
|
||||||
paths.insert(i.second.path);
|
|
||||||
return paths;
|
return paths;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) {
|
||||||
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
|
|
||||||
{
|
|
||||||
drv.outputs.clear();
|
drv.outputs.clear();
|
||||||
auto nr = readNum<size_t>(in);
|
auto nr = readNum<size_t>(in);
|
||||||
for (size_t n = 0; n < nr; n++) {
|
for (size_t n = 0; n < nr; n++) {
|
||||||
|
|
@ -392,25 +384,20 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
|
||||||
return in;
|
return in;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Sink& operator<<(Sink& out, const BasicDerivation& drv) {
|
||||||
Sink & operator << (Sink & out, const BasicDerivation & drv)
|
|
||||||
{
|
|
||||||
out << drv.outputs.size();
|
out << drv.outputs.size();
|
||||||
for (auto& i : drv.outputs)
|
for (auto& i : drv.outputs)
|
||||||
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
|
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
|
||||||
out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
|
out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
|
||||||
out << drv.env.size();
|
out << drv.env.size();
|
||||||
for (auto & i : drv.env)
|
for (auto& i : drv.env) out << i.first << i.second;
|
||||||
out << i.first << i.second;
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string hashPlaceholder(const std::string& outputName) {
|
||||||
std::string hashPlaceholder(const std::string & outputName)
|
|
||||||
{
|
|
||||||
// FIXME: memoize?
|
// FIXME: memoize?
|
||||||
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
|
return "/" + hashString(htSHA256, "nix-output:" + outputName)
|
||||||
|
.to_string(Base32, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
36
third_party/nix/src/libstore/derivations.hh
vendored
36
third_party/nix/src/libstore/derivations.hh
vendored
|
|
@ -1,31 +1,23 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
#include <map>
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
#include "types.hh"
|
||||||
#include <map>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* Extension of derivations in the Nix store. */
|
/* Extension of derivations in the Nix store. */
|
||||||
const string drvExtension = ".drv";
|
const string drvExtension = ".drv";
|
||||||
|
|
||||||
|
|
||||||
/* Abstract syntax of derivations. */
|
/* Abstract syntax of derivations. */
|
||||||
|
|
||||||
struct DerivationOutput
|
struct DerivationOutput {
|
||||||
{
|
|
||||||
Path path;
|
Path path;
|
||||||
string hashAlgo; /* hash used for expected hash computation */
|
string hashAlgo; /* hash used for expected hash computation */
|
||||||
string hash; /* expected hash, may be null */
|
string hash; /* expected hash, may be null */
|
||||||
DerivationOutput()
|
DerivationOutput() {}
|
||||||
{
|
DerivationOutput(Path path, string hashAlgo, string hash) {
|
||||||
}
|
|
||||||
DerivationOutput(Path path, string hashAlgo, string hash)
|
|
||||||
{
|
|
||||||
this->path = path;
|
this->path = path;
|
||||||
this->hashAlgo = hashAlgo;
|
this->hashAlgo = hashAlgo;
|
||||||
this->hash = hash;
|
this->hash = hash;
|
||||||
|
|
@ -41,8 +33,7 @@ typedef std::map<Path, StringSet> DerivationInputs;
|
||||||
|
|
||||||
typedef std::map<string, string> StringPairs;
|
typedef std::map<string, string> StringPairs;
|
||||||
|
|
||||||
struct BasicDerivation
|
struct BasicDerivation {
|
||||||
{
|
|
||||||
DerivationOutputs outputs; /* keyed on symbolic IDs */
|
DerivationOutputs outputs; /* keyed on symbolic IDs */
|
||||||
PathSet inputSrcs; /* inputs that are sources */
|
PathSet inputSrcs; /* inputs that are sources */
|
||||||
string platform;
|
string platform;
|
||||||
|
|
@ -63,24 +54,20 @@ struct BasicDerivation
|
||||||
|
|
||||||
/* Return the output paths of a derivation. */
|
/* Return the output paths of a derivation. */
|
||||||
PathSet outputPaths() const;
|
PathSet outputPaths() const;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Derivation : BasicDerivation
|
struct Derivation : BasicDerivation {
|
||||||
{
|
|
||||||
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
|
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
|
||||||
|
|
||||||
/* Print a derivation. */
|
/* Print a derivation. */
|
||||||
std::string unparse() const;
|
std::string unparse() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
class Store;
|
class Store;
|
||||||
|
|
||||||
|
|
||||||
/* Write a derivation to the Nix store, and return its path. */
|
/* Write a derivation to the Nix store, and return its path. */
|
||||||
Path writeDerivation(ref<Store> store,
|
Path writeDerivation(ref<Store> store, const Derivation& drv,
|
||||||
const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
|
const string& name, RepairFlag repair = NoRepair);
|
||||||
|
|
||||||
/* Read a derivation from a file. */
|
/* Read a derivation from a file. */
|
||||||
Derivation readDerivation(const Path& drvPath);
|
Derivation readDerivation(const Path& drvPath);
|
||||||
|
|
@ -102,7 +89,8 @@ extern DrvHashes drvHashes; // FIXME: global, not thread-safe
|
||||||
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
|
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
|
||||||
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s);
|
DrvPathWithOutputs parseDrvPathWithOutputs(const string& s);
|
||||||
|
|
||||||
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
|
Path makeDrvPathWithOutputs(const Path& drvPath,
|
||||||
|
const std::set<string>& outputs);
|
||||||
|
|
||||||
bool wantOutput(const string& output, const std::set<string>& wanted);
|
bool wantOutput(const string& output, const std::set<string>& wanted);
|
||||||
|
|
||||||
|
|
@ -114,4 +102,4 @@ Sink & operator << (Sink & out, const BasicDerivation & drv);
|
||||||
|
|
||||||
std::string hashPlaceholder(const std::string& outputName);
|
std::string hashPlaceholder(const std::string& outputName);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
404
third_party/nix/src/libstore/download.cc
vendored
404
third_party/nix/src/libstore/download.cc
vendored
|
|
@ -1,23 +1,21 @@
|
||||||
#include "download.hh"
|
#include "download.hh"
|
||||||
#include "util.hh"
|
#include "archive.hh"
|
||||||
|
#include "compression.hh"
|
||||||
|
#include "finally.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
#include "store-api.hh"
|
|
||||||
#include "archive.hh"
|
|
||||||
#include "s3.hh"
|
|
||||||
#include "compression.hh"
|
|
||||||
#include "pathlocks.hh"
|
#include "pathlocks.hh"
|
||||||
#include "finally.hh"
|
#include "s3.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
#ifdef ENABLE_S3
|
#ifdef ENABLE_S3
|
||||||
#include <aws/core/client/ClientConfiguration.h>
|
#include <aws/core/client/ClientConfiguration.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <fcntl.h>
|
|
||||||
|
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
|
@ -34,31 +32,31 @@ DownloadSettings downloadSettings;
|
||||||
|
|
||||||
static GlobalConfig::Register r1(&downloadSettings);
|
static GlobalConfig::Register r1(&downloadSettings);
|
||||||
|
|
||||||
std::string resolveUri(const std::string & uri)
|
std::string resolveUri(const std::string& uri) {
|
||||||
{
|
|
||||||
if (uri.compare(0, 8, "channel:") == 0)
|
if (uri.compare(0, 8, "channel:") == 0)
|
||||||
return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz";
|
return "https://nixos.org/channels/" + std::string(uri, 8) +
|
||||||
|
"/nixexprs.tar.xz";
|
||||||
else
|
else
|
||||||
return uri;
|
return uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CurlDownloader : public Downloader
|
struct CurlDownloader : public Downloader {
|
||||||
{
|
|
||||||
CURLM* curlm = 0;
|
CURLM* curlm = 0;
|
||||||
|
|
||||||
std::random_device rd;
|
std::random_device rd;
|
||||||
std::mt19937 mt19937;
|
std::mt19937 mt19937;
|
||||||
|
|
||||||
struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
|
struct DownloadItem : public std::enable_shared_from_this<DownloadItem> {
|
||||||
{
|
|
||||||
CurlDownloader& downloader;
|
CurlDownloader& downloader;
|
||||||
DownloadRequest request;
|
DownloadRequest request;
|
||||||
DownloadResult result;
|
DownloadResult result;
|
||||||
Activity act;
|
Activity act;
|
||||||
bool done = false; // whether either the success or failure function has been called
|
bool done = false; // whether either the success or failure function has
|
||||||
|
// been called
|
||||||
Callback<DownloadResult> callback;
|
Callback<DownloadResult> callback;
|
||||||
CURL* req = 0;
|
CURL* req = 0;
|
||||||
bool active = false; // whether the handle has been added to the multi object
|
bool active =
|
||||||
|
false; // whether the handle has been added to the multi object
|
||||||
std::string status;
|
std::string status;
|
||||||
|
|
||||||
unsigned int attempt = 0;
|
unsigned int attempt = 0;
|
||||||
|
|
@ -75,62 +73,62 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
curl_off_t writtenToSink = 0;
|
curl_off_t writtenToSink = 0;
|
||||||
|
|
||||||
DownloadItem(CurlDownloader & downloader,
|
DownloadItem(CurlDownloader& downloader, const DownloadRequest& request,
|
||||||
const DownloadRequest & request,
|
|
||||||
Callback<DownloadResult>&& callback)
|
Callback<DownloadResult>&& callback)
|
||||||
: downloader(downloader)
|
: downloader(downloader),
|
||||||
, request(request)
|
request(request),
|
||||||
, act(*logger, lvlTalkative, actDownload,
|
act(*logger, lvlTalkative, actDownload,
|
||||||
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
|
fmt(request.data ? "uploading '%s'" : "downloading '%s'",
|
||||||
{request.uri}, request.parentAct)
|
request.uri),
|
||||||
, callback(std::move(callback))
|
{request.uri}, request.parentAct),
|
||||||
, finalSink([this](const unsigned char * data, size_t len) {
|
callback(std::move(callback)),
|
||||||
|
finalSink([this](const unsigned char* data, size_t len) {
|
||||||
if (this->request.dataCallback) {
|
if (this->request.dataCallback) {
|
||||||
long httpStatus = 0;
|
long httpStatus = 0;
|
||||||
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
||||||
|
|
||||||
/* Only write data to the sink if this is a
|
/* Only write data to the sink if this is a
|
||||||
successful response. */
|
successful response. */
|
||||||
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
|
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 ||
|
||||||
|
httpStatus == 206) {
|
||||||
writtenToSink += len;
|
writtenToSink += len;
|
||||||
this->request.dataCallback((char*)data, len);
|
this->request.dataCallback((char*)data, len);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
this->result.data->append((char*)data, len);
|
this->result.data->append((char*)data, len);
|
||||||
})
|
}) {
|
||||||
{
|
|
||||||
if (!request.expectedETag.empty())
|
if (!request.expectedETag.empty())
|
||||||
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
requestHeaders = curl_slist_append(
|
||||||
|
requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
||||||
if (!request.mimeType.empty())
|
if (!request.mimeType.empty())
|
||||||
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
requestHeaders = curl_slist_append(
|
||||||
|
requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
~DownloadItem()
|
~DownloadItem() {
|
||||||
{
|
|
||||||
if (req) {
|
if (req) {
|
||||||
if (active)
|
if (active) curl_multi_remove_handle(downloader.curlm, req);
|
||||||
curl_multi_remove_handle(downloader.curlm, req);
|
|
||||||
curl_easy_cleanup(req);
|
curl_easy_cleanup(req);
|
||||||
}
|
}
|
||||||
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
||||||
try {
|
try {
|
||||||
if (!done)
|
if (!done)
|
||||||
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
|
fail(DownloadError(
|
||||||
|
Interrupted,
|
||||||
|
format("download of '%s' was interrupted") % request.uri));
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
ignoreException();
|
ignoreException();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void failEx(std::exception_ptr ex)
|
void failEx(std::exception_ptr ex) {
|
||||||
{
|
|
||||||
assert(!done);
|
assert(!done);
|
||||||
done = true;
|
done = true;
|
||||||
callback.rethrow(ex);
|
callback.rethrow(ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class T>
|
template <class T>
|
||||||
void fail(const T & e)
|
void fail(const T& e) {
|
||||||
{
|
|
||||||
failEx(std::make_exception_ptr(e));
|
failEx(std::make_exception_ptr(e));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -139,8 +137,7 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
std::exception_ptr writeException;
|
std::exception_ptr writeException;
|
||||||
|
|
||||||
size_t writeCallback(void * contents, size_t size, size_t nmemb)
|
size_t writeCallback(void* contents, size_t size, size_t nmemb) {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
size_t realSize = size * nmemb;
|
size_t realSize = size * nmemb;
|
||||||
result.bodySize += realSize;
|
result.bodySize += realSize;
|
||||||
|
|
@ -157,16 +154,16 @@ struct CurlDownloader : public Downloader
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
|
static size_t writeCallbackWrapper(void* contents, size_t size,
|
||||||
{
|
size_t nmemb, void* userp) {
|
||||||
return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb);
|
return ((DownloadItem*)userp)->writeCallback(contents, size, nmemb);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t headerCallback(void * contents, size_t size, size_t nmemb)
|
size_t headerCallback(void* contents, size_t size, size_t nmemb) {
|
||||||
{
|
|
||||||
size_t realSize = size * nmemb;
|
size_t realSize = size * nmemb;
|
||||||
std::string line((char*)contents, realSize);
|
std::string line((char*)contents, realSize);
|
||||||
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
|
printMsg(lvlVomit,
|
||||||
|
format("got header for '%s': %s") % request.uri % trim(line));
|
||||||
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
|
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
|
||||||
result.etag = "";
|
result.etag = "";
|
||||||
auto ss = tokenizeString<vector<string>>(line, " ");
|
auto ss = tokenizeString<vector<string>>(line, " ");
|
||||||
|
|
@ -187,25 +184,26 @@ struct CurlDownloader : public Downloader
|
||||||
down the connection because we already have the
|
down the connection because we already have the
|
||||||
data. */
|
data. */
|
||||||
if (result.etag == request.expectedETag && status == "200") {
|
if (result.etag == request.expectedETag && status == "200") {
|
||||||
debug(format("shutting down on 200 HTTP response with expected ETag"));
|
debug(format(
|
||||||
|
"shutting down on 200 HTTP response with expected ETag"));
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
} else if (name == "content-encoding")
|
} else if (name == "content-encoding")
|
||||||
encoding = trim(string(line, i + 1));
|
encoding = trim(string(line, i + 1));
|
||||||
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
|
else if (name == "accept-ranges" &&
|
||||||
|
toLower(trim(std::string(line, i + 1))) == "bytes")
|
||||||
acceptRanges = true;
|
acceptRanges = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return realSize;
|
return realSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
|
static size_t headerCallbackWrapper(void* contents, size_t size,
|
||||||
{
|
size_t nmemb, void* userp) {
|
||||||
return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb);
|
return ((DownloadItem*)userp)->headerCallback(contents, size, nmemb);
|
||||||
}
|
}
|
||||||
|
|
||||||
int progressCallback(double dltotal, double dlnow)
|
int progressCallback(double dltotal, double dlnow) {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
act.progress(dlnow, dltotal);
|
act.progress(dlnow, dltotal);
|
||||||
} catch (nix::Interrupted&) {
|
} catch (nix::Interrupted&) {
|
||||||
|
|
@ -214,23 +212,22 @@ struct CurlDownloader : public Downloader
|
||||||
return _isInterrupted;
|
return _isInterrupted;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
|
static int progressCallbackWrapper(void* userp, double dltotal,
|
||||||
{
|
double dlnow, double ultotal,
|
||||||
|
double ulnow) {
|
||||||
return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow);
|
return ((DownloadItem*)userp)->progressCallback(dltotal, dlnow);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
|
static int debugCallback(CURL* handle, curl_infotype type, char* data,
|
||||||
{
|
size_t size, void* userptr) {
|
||||||
if (type == CURLINFO_TEXT)
|
if (type == CURLINFO_TEXT)
|
||||||
vomit("curl: %s", chomp(std::string(data, size)));
|
vomit("curl: %s", chomp(std::string(data, size)));
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t readOffset = 0;
|
size_t readOffset = 0;
|
||||||
size_t readCallback(char *buffer, size_t size, size_t nitems)
|
size_t readCallback(char* buffer, size_t size, size_t nitems) {
|
||||||
{
|
if (readOffset == request.data->length()) return 0;
|
||||||
if (readOffset == request.data->length())
|
|
||||||
return 0;
|
|
||||||
auto count = std::min(size * nitems, request.data->length() - readOffset);
|
auto count = std::min(size * nitems, request.data->length() - readOffset);
|
||||||
assert(count);
|
assert(count);
|
||||||
memcpy(buffer, request.data->data() + readOffset, count);
|
memcpy(buffer, request.data->data() + readOffset, count);
|
||||||
|
|
@ -238,20 +235,20 @@ struct CurlDownloader : public Downloader
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
|
static size_t readCallbackWrapper(char* buffer, size_t size, size_t nitems,
|
||||||
{
|
void* userp) {
|
||||||
return ((DownloadItem*)userp)->readCallback(buffer, size, nitems);
|
return ((DownloadItem*)userp)->readCallback(buffer, size, nitems);
|
||||||
}
|
}
|
||||||
|
|
||||||
void init()
|
void init() {
|
||||||
{
|
|
||||||
if (!req) req = curl_easy_init();
|
if (!req) req = curl_easy_init();
|
||||||
|
|
||||||
curl_easy_reset(req);
|
curl_easy_reset(req);
|
||||||
|
|
||||||
if (verbosity >= lvlVomit) {
|
if (verbosity >= lvlVomit) {
|
||||||
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
|
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
|
||||||
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
|
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION,
|
||||||
|
DownloadItem::debugCallback);
|
||||||
}
|
}
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
|
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
|
||||||
|
|
@ -260,7 +257,10 @@ struct CurlDownloader : public Downloader
|
||||||
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
|
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
|
||||||
curl_easy_setopt(req, CURLOPT_USERAGENT,
|
curl_easy_setopt(req, CURLOPT_USERAGENT,
|
||||||
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
|
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
|
||||||
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
|
(downloadSettings.userAgentSuffix != ""
|
||||||
|
? " " + downloadSettings.userAgentSuffix.get()
|
||||||
|
: ""))
|
||||||
|
.c_str());
|
||||||
#if LIBCURL_VERSION_NUM >= 0x072b00
|
#if LIBCURL_VERSION_NUM >= 0x072b00
|
||||||
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -270,9 +270,11 @@ struct CurlDownloader : public Downloader
|
||||||
else
|
else
|
||||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
||||||
#endif
|
#endif
|
||||||
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION,
|
||||||
|
DownloadItem::writeCallbackWrapper);
|
||||||
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
||||||
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION,
|
||||||
|
DownloadItem::headerCallbackWrapper);
|
||||||
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
|
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
|
||||||
|
|
@ -281,14 +283,14 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
|
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
|
||||||
|
|
||||||
if (request.head)
|
if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1);
|
||||||
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
|
|
||||||
|
|
||||||
if (request.data) {
|
if (request.data) {
|
||||||
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
|
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
|
||||||
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
|
||||||
curl_easy_setopt(req, CURLOPT_READDATA, this);
|
curl_easy_setopt(req, CURLOPT_READDATA, this);
|
||||||
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
|
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE,
|
||||||
|
(curl_off_t)request.data->length());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (request.verifyTLS) {
|
if (request.verifyTLS) {
|
||||||
|
|
@ -299,14 +301,17 @@ struct CurlDownloader : public Downloader
|
||||||
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
|
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
|
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT,
|
||||||
|
downloadSettings.connectTimeout.get());
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
|
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME,
|
||||||
|
downloadSettings.stalledDownloadTimeout.get());
|
||||||
|
|
||||||
/* If no file exist in the specified path, curl continues to work
|
/* If no file exist in the specified path, curl continues to work
|
||||||
anyway as if netrc support was disabled. */
|
anyway as if netrc support was disabled. */
|
||||||
curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
|
curl_easy_setopt(req, CURLOPT_NETRC_FILE,
|
||||||
|
settings.netrcFile.get().c_str());
|
||||||
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
|
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
|
||||||
|
|
||||||
if (writtenToSink)
|
if (writtenToSink)
|
||||||
|
|
@ -316,17 +321,17 @@ struct CurlDownloader : public Downloader
|
||||||
result.bodySize = 0;
|
result.bodySize = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void finish(CURLcode code)
|
void finish(CURLcode code) {
|
||||||
{
|
|
||||||
long httpStatus = 0;
|
long httpStatus = 0;
|
||||||
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
||||||
|
|
||||||
char* effectiveUriCStr;
|
char* effectiveUriCStr;
|
||||||
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
|
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
|
||||||
if (effectiveUriCStr)
|
if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr;
|
||||||
result.effectiveUri = effectiveUriCStr;
|
|
||||||
|
|
||||||
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
|
debug(
|
||||||
|
"finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d "
|
||||||
|
"bytes",
|
||||||
request.verb(), request.uri, code, httpStatus, result.bodySize);
|
request.verb(), request.uri, code, httpStatus, result.bodySize);
|
||||||
|
|
||||||
if (decompressionSink) {
|
if (decompressionSink) {
|
||||||
|
|
@ -346,8 +351,10 @@ struct CurlDownloader : public Downloader
|
||||||
failEx(writeException);
|
failEx(writeException);
|
||||||
|
|
||||||
else if (code == CURLE_OK &&
|
else if (code == CURLE_OK &&
|
||||||
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
|
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 ||
|
||||||
{
|
httpStatus == 206 || httpStatus == 304 ||
|
||||||
|
httpStatus == 226 /* FTP */ ||
|
||||||
|
httpStatus == 0 /* other protocol */)) {
|
||||||
result.cached = httpStatus == 304;
|
result.cached = httpStatus == 304;
|
||||||
act.progress(result.bodySize, result.bodySize);
|
act.progress(result.bodySize, result.bodySize);
|
||||||
done = true;
|
done = true;
|
||||||
|
|
@ -358,19 +365,25 @@ struct CurlDownloader : public Downloader
|
||||||
// We treat most errors as transient, but won't retry when hopeless
|
// We treat most errors as transient, but won't retry when hopeless
|
||||||
Error err = Transient;
|
Error err = Transient;
|
||||||
|
|
||||||
if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
|
if (httpStatus == 404 || httpStatus == 410 ||
|
||||||
|
code == CURLE_FILE_COULDNT_READ_FILE) {
|
||||||
// The file is definitely not there
|
// The file is definitely not there
|
||||||
err = NotFound;
|
err = NotFound;
|
||||||
} else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
|
} else if (httpStatus == 401 || httpStatus == 403 ||
|
||||||
|
httpStatus == 407) {
|
||||||
// Don't retry on authentication/authorization failures
|
// Don't retry on authentication/authorization failures
|
||||||
err = Forbidden;
|
err = Forbidden;
|
||||||
} else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
|
} else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 &&
|
||||||
// Most 4xx errors are client errors and are probably not worth retrying:
|
httpStatus != 429) {
|
||||||
|
// Most 4xx errors are client errors and are probably not worth
|
||||||
|
// retrying:
|
||||||
// * 408 means the server timed out waiting for us, so we try again
|
// * 408 means the server timed out waiting for us, so we try again
|
||||||
// * 429 means too many requests, so we retry (with a delay)
|
// * 429 means too many requests, so we retry (with a delay)
|
||||||
err = Misc;
|
err = Misc;
|
||||||
} else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
|
} else if (httpStatus == 501 || httpStatus == 505 ||
|
||||||
// Let's treat most 5xx (server) errors as transient, except for a handful:
|
httpStatus == 511) {
|
||||||
|
// Let's treat most 5xx (server) errors as transient, except for a
|
||||||
|
// handful:
|
||||||
// * 501 not implemented
|
// * 501 not implemented
|
||||||
// * 505 http version not supported
|
// * 505 http version not supported
|
||||||
// * 511 we're behind a captive portal
|
// * 511 we're behind a captive portal
|
||||||
|
|
@ -403,50 +416,57 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
auto exc =
|
auto exc =
|
||||||
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
||||||
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
|
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted",
|
||||||
|
request.verb(), request.uri))
|
||||||
: httpStatus != 0
|
: httpStatus != 0
|
||||||
? DownloadError(err,
|
? DownloadError(
|
||||||
fmt("unable to %s '%s': HTTP error %d",
|
err, fmt("unable to %s '%s': HTTP error %d",
|
||||||
request.verb(), request.uri, httpStatus)
|
request.verb(), request.uri, httpStatus) +
|
||||||
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
|
(code == CURLE_OK
|
||||||
)
|
? ""
|
||||||
: DownloadError(err,
|
: fmt(" (curl error: %s)",
|
||||||
fmt("unable to %s '%s': %s (%d)",
|
curl_easy_strerror(code))))
|
||||||
request.verb(), request.uri, curl_easy_strerror(code), code));
|
: DownloadError(err, fmt("unable to %s '%s': %s (%d)",
|
||||||
|
request.verb(), request.uri,
|
||||||
|
curl_easy_strerror(code), code));
|
||||||
|
|
||||||
/* If this is a transient error, then maybe retry the
|
/* If this is a transient error, then maybe retry the
|
||||||
download after a while. If we're writing to a
|
download after a while. If we're writing to a
|
||||||
sink, we can only retry if the server supports
|
sink, we can only retry if the server supports
|
||||||
ranged requests. */
|
ranged requests. */
|
||||||
if (err == Transient
|
if (err == Transient && attempt < request.tries &&
|
||||||
&& attempt < request.tries
|
(!this->request.dataCallback || writtenToSink == 0 ||
|
||||||
&& (!this->request.dataCallback
|
(acceptRanges && encoding.empty()))) {
|
||||||
|| writtenToSink == 0
|
int ms = request.baseRetryTimeMs *
|
||||||
|| (acceptRanges && encoding.empty())))
|
std::pow(2.0f, attempt - 1 +
|
||||||
{
|
std::uniform_real_distribution<>(
|
||||||
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
|
0.0, 0.5)(downloader.mt19937));
|
||||||
if (writtenToSink)
|
if (writtenToSink)
|
||||||
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
|
warn("%s; retrying from offset %d in %d ms", exc.what(),
|
||||||
|
writtenToSink, ms);
|
||||||
else
|
else
|
||||||
warn("%s; retrying in %d ms", exc.what(), ms);
|
warn("%s; retrying in %d ms", exc.what(), ms);
|
||||||
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
|
embargo =
|
||||||
|
std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
|
||||||
downloader.enqueueItem(shared_from_this());
|
downloader.enqueueItem(shared_from_this());
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
fail(exc);
|
fail(exc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
struct EmbargoComparator {
|
struct EmbargoComparator {
|
||||||
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
|
bool operator()(const std::shared_ptr<DownloadItem>& i1,
|
||||||
|
const std::shared_ptr<DownloadItem>& i2) {
|
||||||
return i1->embargo > i2->embargo;
|
return i1->embargo > i2->embargo;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
bool quit = false;
|
bool quit = false;
|
||||||
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
|
std::priority_queue<std::shared_ptr<DownloadItem>,
|
||||||
|
std::vector<std::shared_ptr<DownloadItem>>,
|
||||||
|
EmbargoComparator>
|
||||||
|
incoming;
|
||||||
};
|
};
|
||||||
|
|
||||||
Sync<State> state_;
|
Sync<State> state_;
|
||||||
|
|
@ -458,9 +478,7 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
std::thread workerThread;
|
std::thread workerThread;
|
||||||
|
|
||||||
CurlDownloader()
|
CurlDownloader() : mt19937(rd()) {
|
||||||
: mt19937(rd())
|
|
||||||
{
|
|
||||||
static std::once_flag globalInit;
|
static std::once_flag globalInit;
|
||||||
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
|
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
|
||||||
|
|
||||||
|
|
@ -480,8 +498,7 @@ struct CurlDownloader : public Downloader
|
||||||
workerThread = std::thread([&]() { workerThreadEntry(); });
|
workerThread = std::thread([&]() { workerThreadEntry(); });
|
||||||
}
|
}
|
||||||
|
|
||||||
~CurlDownloader()
|
~CurlDownloader() {
|
||||||
{
|
|
||||||
stopWorkerThread();
|
stopWorkerThread();
|
||||||
|
|
||||||
workerThread.join();
|
workerThread.join();
|
||||||
|
|
@ -489,8 +506,7 @@ struct CurlDownloader : public Downloader
|
||||||
if (curlm) curl_multi_cleanup(curlm);
|
if (curlm) curl_multi_cleanup(curlm);
|
||||||
}
|
}
|
||||||
|
|
||||||
void stopWorkerThread()
|
void stopWorkerThread() {
|
||||||
{
|
|
||||||
/* Signal the worker thread to exit. */
|
/* Signal the worker thread to exit. */
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
@ -499,12 +515,9 @@ struct CurlDownloader : public Downloader
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ", false);
|
writeFull(wakeupPipe.writeSide.get(), " ", false);
|
||||||
}
|
}
|
||||||
|
|
||||||
void workerThreadMain()
|
void workerThreadMain() {
|
||||||
{
|
|
||||||
/* Cause this thread to be notified on SIGINT. */
|
/* Cause this thread to be notified on SIGINT. */
|
||||||
auto callback = createInterruptCallback([&]() {
|
auto callback = createInterruptCallback([&]() { stopWorkerThread(); });
|
||||||
stopWorkerThread();
|
|
||||||
});
|
|
||||||
|
|
||||||
std::map<CURL*, std::shared_ptr<DownloadItem>> items;
|
std::map<CURL*, std::shared_ptr<DownloadItem>> items;
|
||||||
|
|
||||||
|
|
@ -519,7 +532,9 @@ struct CurlDownloader : public Downloader
|
||||||
int running;
|
int running;
|
||||||
CURLMcode mc = curl_multi_perform(curlm, &running);
|
CURLMcode mc = curl_multi_perform(curlm, &running);
|
||||||
if (mc != CURLM_OK)
|
if (mc != CURLM_OK)
|
||||||
throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
|
throw nix::Error(
|
||||||
|
format("unexpected error from curl_multi_perform(): %s") %
|
||||||
|
curl_multi_strerror(mc));
|
||||||
|
|
||||||
/* Set the promises of any finished requests. */
|
/* Set the promises of any finished requests. */
|
||||||
CURLMsg* msg;
|
CURLMsg* msg;
|
||||||
|
|
@ -544,12 +559,17 @@ struct CurlDownloader : public Downloader
|
||||||
long maxSleepTimeMs = items.empty() ? 10000 : 100;
|
long maxSleepTimeMs = items.empty() ? 10000 : 100;
|
||||||
auto sleepTimeMs =
|
auto sleepTimeMs =
|
||||||
nextWakeup != std::chrono::steady_clock::time_point()
|
nextWakeup != std::chrono::steady_clock::time_point()
|
||||||
? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
|
? std::max(
|
||||||
|
0,
|
||||||
|
(int)std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||||
|
nextWakeup - std::chrono::steady_clock::now())
|
||||||
|
.count())
|
||||||
: maxSleepTimeMs;
|
: maxSleepTimeMs;
|
||||||
vomit("download thread waiting for %d ms", sleepTimeMs);
|
vomit("download thread waiting for %d ms", sleepTimeMs);
|
||||||
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
|
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
|
||||||
if (mc != CURLM_OK)
|
if (mc != CURLM_OK)
|
||||||
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
|
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") %
|
||||||
|
curl_multi_strerror(mc));
|
||||||
|
|
||||||
nextWakeup = std::chrono::steady_clock::time_point();
|
nextWakeup = std::chrono::steady_clock::time_point();
|
||||||
|
|
||||||
|
|
@ -574,8 +594,8 @@ struct CurlDownloader : public Downloader
|
||||||
incoming.push_back(item);
|
incoming.push_back(item);
|
||||||
state->incoming.pop();
|
state->incoming.pop();
|
||||||
} else {
|
} else {
|
||||||
if (nextWakeup == std::chrono::steady_clock::time_point()
|
if (nextWakeup == std::chrono::steady_clock::time_point() ||
|
||||||
|| item->embargo < nextWakeup)
|
item->embargo < nextWakeup)
|
||||||
nextWakeup = item->embargo;
|
nextWakeup = item->embargo;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -595,8 +615,7 @@ struct CurlDownloader : public Downloader
|
||||||
debug("download thread shutting down");
|
debug("download thread shutting down");
|
||||||
}
|
}
|
||||||
|
|
||||||
void workerThreadEntry()
|
void workerThreadEntry() {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
workerThreadMain();
|
workerThreadMain();
|
||||||
} catch (nix::Interrupted& e) {
|
} catch (nix::Interrupted& e) {
|
||||||
|
|
@ -611,30 +630,29 @@ struct CurlDownloader : public Downloader
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void enqueueItem(std::shared_ptr<DownloadItem> item)
|
void enqueueItem(std::shared_ptr<DownloadItem> item) {
|
||||||
{
|
if (item->request.data && !hasPrefix(item->request.uri, "http://") &&
|
||||||
if (item->request.data
|
!hasPrefix(item->request.uri, "https://"))
|
||||||
&& !hasPrefix(item->request.uri, "http://")
|
|
||||||
&& !hasPrefix(item->request.uri, "https://"))
|
|
||||||
throw nix::Error("uploading to '%s' is not supported", item->request.uri);
|
throw nix::Error("uploading to '%s' is not supported", item->request.uri);
|
||||||
|
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (state->quit)
|
if (state->quit)
|
||||||
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
|
throw nix::Error(
|
||||||
|
"cannot enqueue download request because the download thread is "
|
||||||
|
"shutting down");
|
||||||
state->incoming.push(item);
|
state->incoming.push(item);
|
||||||
}
|
}
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
writeFull(wakeupPipe.writeSide.get(), " ");
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef ENABLE_S3
|
#ifdef ENABLE_S3
|
||||||
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
|
std::tuple<std::string, std::string, Store::Params> parseS3Uri(
|
||||||
{
|
std::string uri) {
|
||||||
auto [path, params] = splitUriAndParams(uri);
|
auto [path, params] = splitUriAndParams(uri);
|
||||||
|
|
||||||
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
|
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
|
||||||
if (slash == std::string::npos)
|
if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", path);
|
||||||
throw nix::Error("bad S3 URI '%s'", path);
|
|
||||||
|
|
||||||
std::string bucketName(path, 5, slash - 5);
|
std::string bucketName(path, 5, slash - 5);
|
||||||
std::string key(path, slash + 1);
|
std::string key(path, slash + 1);
|
||||||
|
|
@ -644,8 +662,7 @@ struct CurlDownloader : public Downloader
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void enqueueDownload(const DownloadRequest& request,
|
void enqueueDownload(const DownloadRequest& request,
|
||||||
Callback<DownloadResult> callback) override
|
Callback<DownloadResult> callback) override {
|
||||||
{
|
|
||||||
/* Ugly hack to support s3:// URIs. */
|
/* Ugly hack to support s3:// URIs. */
|
||||||
if (hasPrefix(request.uri, "s3://")) {
|
if (hasPrefix(request.uri, "s3://")) {
|
||||||
// FIXME: do this on a worker thread
|
// FIXME: do this on a worker thread
|
||||||
|
|
@ -664,36 +681,37 @@ struct CurlDownloader : public Downloader
|
||||||
auto s3Res = s3Helper.getObject(bucketName, key);
|
auto s3Res = s3Helper.getObject(bucketName, key);
|
||||||
DownloadResult res;
|
DownloadResult res;
|
||||||
if (!s3Res.data)
|
if (!s3Res.data)
|
||||||
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
throw DownloadError(
|
||||||
|
NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
||||||
res.data = s3Res.data;
|
res.data = s3Res.data;
|
||||||
callback(std::move(res));
|
callback(std::move(res));
|
||||||
#else
|
#else
|
||||||
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
|
throw nix::Error(
|
||||||
|
"cannot download '%s' because Nix is not built with S3 support",
|
||||||
|
request.uri);
|
||||||
#endif
|
#endif
|
||||||
} catch (...) { callback.rethrow(); }
|
} catch (...) {
|
||||||
|
callback.rethrow();
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
|
enqueueItem(
|
||||||
|
std::make_shared<DownloadItem>(*this, request, std::move(callback)));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Downloader> getDownloader()
|
ref<Downloader> getDownloader() {
|
||||||
{
|
|
||||||
static ref<Downloader> downloader = makeDownloader();
|
static ref<Downloader> downloader = makeDownloader();
|
||||||
return downloader;
|
return downloader;
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<Downloader> makeDownloader()
|
ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); }
|
||||||
{
|
|
||||||
return make_ref<CurlDownloader>();
|
|
||||||
}
|
|
||||||
|
|
||||||
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
|
std::future<DownloadResult> Downloader::enqueueDownload(
|
||||||
{
|
const DownloadRequest& request) {
|
||||||
auto promise = std::make_shared<std::promise<DownloadResult>>();
|
auto promise = std::make_shared<std::promise<DownloadResult>>();
|
||||||
enqueueDownload(request,
|
enqueueDownload(request, {[promise](std::future<DownloadResult> fut) {
|
||||||
{[promise](std::future<DownloadResult> fut) {
|
|
||||||
try {
|
try {
|
||||||
promise->set_value(fut.get());
|
promise->set_value(fut.get());
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -703,13 +721,11 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
|
||||||
return promise->get_future();
|
return promise->get_future();
|
||||||
}
|
}
|
||||||
|
|
||||||
DownloadResult Downloader::download(const DownloadRequest & request)
|
DownloadResult Downloader::download(const DownloadRequest& request) {
|
||||||
{
|
|
||||||
return enqueueDownload(request).get();
|
return enqueueDownload(request).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
void Downloader::download(DownloadRequest && request, Sink & sink)
|
void Downloader::download(DownloadRequest&& request, Sink& sink) {
|
||||||
{
|
|
||||||
/* Note: we can't call 'sink' via request.dataCallback, because
|
/* Note: we can't call 'sink' via request.dataCallback, because
|
||||||
that would cause the sink to execute on the downloader
|
that would cause the sink to execute on the downloader
|
||||||
thread. If 'sink' is a coroutine, this will fail. Also, if the
|
thread. If 'sink' is a coroutine, this will fail. Also, if the
|
||||||
|
|
@ -736,7 +752,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
||||||
});
|
});
|
||||||
|
|
||||||
request.dataCallback = [_state](char* buf, size_t len) {
|
request.dataCallback = [_state](char* buf, size_t len) {
|
||||||
|
|
||||||
auto state(_state->lock());
|
auto state(_state->lock());
|
||||||
|
|
||||||
if (state->quit) return;
|
if (state->quit) return;
|
||||||
|
|
@ -757,8 +772,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
||||||
state->avail.notify_one();
|
state->avail.notify_one();
|
||||||
};
|
};
|
||||||
|
|
||||||
enqueueDownload(request,
|
enqueueDownload(request, {[_state](std::future<DownloadResult> fut) {
|
||||||
{[_state](std::future<DownloadResult> fut) {
|
|
||||||
auto state(_state->lock());
|
auto state(_state->lock());
|
||||||
state->quit = true;
|
state->quit = true;
|
||||||
try {
|
try {
|
||||||
|
|
@ -781,7 +795,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
||||||
auto state(_state->lock());
|
auto state(_state->lock());
|
||||||
|
|
||||||
while (state->data.empty()) {
|
while (state->data.empty()) {
|
||||||
|
|
||||||
if (state->quit) {
|
if (state->quit) {
|
||||||
if (state->exc) std::rethrow_exception(state->exc);
|
if (state->exc) std::rethrow_exception(state->exc);
|
||||||
return;
|
return;
|
||||||
|
|
@ -804,8 +817,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
||||||
}
|
}
|
||||||
|
|
||||||
CachedDownloadResult Downloader::downloadCached(
|
CachedDownloadResult Downloader::downloadCached(
|
||||||
ref<Store> store, const CachedDownloadRequest & request)
|
ref<Store> store, const CachedDownloadRequest& request) {
|
||||||
{
|
|
||||||
auto url = resolveUri(request.uri);
|
auto url = resolveUri(request.uri);
|
||||||
|
|
||||||
auto name = request.name;
|
auto name = request.name;
|
||||||
|
|
@ -816,7 +828,8 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
|
|
||||||
Path expectedStorePath;
|
Path expectedStorePath;
|
||||||
if (request.expectedHash) {
|
if (request.expectedHash) {
|
||||||
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
|
expectedStorePath =
|
||||||
|
store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
|
||||||
if (store->isValidPath(expectedStorePath)) {
|
if (store->isValidPath(expectedStorePath)) {
|
||||||
CachedDownloadResult result;
|
CachedDownloadResult result;
|
||||||
result.storePath = expectedStorePath;
|
result.storePath = expectedStorePath;
|
||||||
|
|
@ -828,7 +841,8 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
Path cacheDir = getCacheDir() + "/nix/tarballs";
|
Path cacheDir = getCacheDir() + "/nix/tarballs";
|
||||||
createDirs(cacheDir);
|
createDirs(cacheDir);
|
||||||
|
|
||||||
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
|
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url)
|
||||||
|
.to_string(Base32, false);
|
||||||
|
|
||||||
Path dataFile = cacheDir + "/" + urlHash + ".info";
|
Path dataFile = cacheDir + "/" + urlHash + ".info";
|
||||||
Path fileLink = cacheDir + "/" + urlHash + "-file";
|
Path fileLink = cacheDir + "/" + urlHash + "-file";
|
||||||
|
|
@ -850,7 +864,8 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
|
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
|
||||||
if (ss.size() >= 3 && ss[0] == url) {
|
if (ss.size() >= 3 && ss[0] == url) {
|
||||||
time_t lastChecked;
|
time_t lastChecked;
|
||||||
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
|
if (string2Int(ss[2], lastChecked) &&
|
||||||
|
(uint64_t)lastChecked + request.ttl >= (uint64_t)time(0)) {
|
||||||
skip = true;
|
skip = true;
|
||||||
result.effectiveUri = request.uri;
|
result.effectiveUri = request.uri;
|
||||||
result.etag = ss[1];
|
result.etag = ss[1];
|
||||||
|
|
@ -864,7 +879,6 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!skip) {
|
if (!skip) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
DownloadRequest request2(url);
|
DownloadRequest request2(url);
|
||||||
request2.expectedETag = expectedETag;
|
request2.expectedETag = expectedETag;
|
||||||
|
|
@ -876,7 +890,9 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
ValidPathInfo info;
|
ValidPathInfo info;
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
dumpString(*res.data, sink);
|
dumpString(*res.data, sink);
|
||||||
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
|
Hash hash = hashString(
|
||||||
|
request.expectedHash ? request.expectedHash.type : htSHA256,
|
||||||
|
*res.data);
|
||||||
info.path = store->makeFixedOutputPath(false, hash, name);
|
info.path = store->makeFixedOutputPath(false, hash, name);
|
||||||
info.narHash = hashString(htSHA256, *sink.s);
|
info.narHash = hashString(htSHA256, *sink.s);
|
||||||
info.narSize = sink.s->size();
|
info.narSize = sink.s->size();
|
||||||
|
|
@ -888,7 +904,8 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
assert(!storePath.empty());
|
assert(!storePath.empty());
|
||||||
replaceSymlink(storePath, fileLink);
|
replaceSymlink(storePath, fileLink);
|
||||||
|
|
||||||
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
|
writeFile(dataFile,
|
||||||
|
url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
|
||||||
} catch (DownloadError& e) {
|
} catch (DownloadError& e) {
|
||||||
if (storePath.empty()) throw;
|
if (storePath.empty()) throw;
|
||||||
warn("warning: %s; using cached result", e.msg());
|
warn("warning: %s; using cached result", e.msg());
|
||||||
|
|
@ -898,21 +915,24 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
|
|
||||||
if (request.unpack) {
|
if (request.unpack) {
|
||||||
Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
|
Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
|
||||||
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
|
PathLocks lock2({unpackedLink},
|
||||||
|
fmt("waiting for lock on '%1%'...", unpackedLink));
|
||||||
Path unpackedStorePath;
|
Path unpackedStorePath;
|
||||||
if (pathExists(unpackedLink)) {
|
if (pathExists(unpackedLink)) {
|
||||||
unpackedStorePath = readLink(unpackedLink);
|
unpackedStorePath = readLink(unpackedLink);
|
||||||
store->addTempRoot(unpackedStorePath);
|
store->addTempRoot(unpackedStorePath);
|
||||||
if (!store->isValidPath(unpackedStorePath))
|
if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = "";
|
||||||
unpackedStorePath = "";
|
|
||||||
}
|
}
|
||||||
if (unpackedStorePath.empty()) {
|
if (unpackedStorePath.empty()) {
|
||||||
printInfo(format("unpacking '%1%'...") % url);
|
printInfo(format("unpacking '%1%'...") % url);
|
||||||
Path tmpDir = createTempDir();
|
Path tmpDir = createTempDir();
|
||||||
AutoDelete autoDelete(tmpDir, true);
|
AutoDelete autoDelete(tmpDir, true);
|
||||||
// FIXME: this requires GNU tar for decompression.
|
// FIXME: this requires GNU tar for decompression.
|
||||||
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
|
runProgram("tar", true,
|
||||||
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
|
{"xf", store->toRealPath(storePath), "-C", tmpDir,
|
||||||
|
"--strip-components", "1"});
|
||||||
|
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256,
|
||||||
|
defaultPathFilter, NoRepair);
|
||||||
}
|
}
|
||||||
replaceSymlink(unpackedStorePath, unpackedLink);
|
replaceSymlink(unpackedStorePath, unpackedLink);
|
||||||
storePath = unpackedStorePath;
|
storePath = unpackedStorePath;
|
||||||
|
|
@ -920,11 +940,16 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
|
|
||||||
if (expectedStorePath != "" && storePath != expectedStorePath) {
|
if (expectedStorePath != "" && storePath != expectedStorePath) {
|
||||||
unsigned int statusCode = 102;
|
unsigned int statusCode = 102;
|
||||||
Hash gotHash = request.unpack
|
Hash gotHash =
|
||||||
? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
|
request.unpack
|
||||||
|
? hashPath(request.expectedHash.type, store->toRealPath(storePath))
|
||||||
|
.first
|
||||||
: hashFile(request.expectedHash.type, store->toRealPath(storePath));
|
: hashFile(request.expectedHash.type, store->toRealPath(storePath));
|
||||||
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
|
throw nix::Error(statusCode,
|
||||||
url, request.expectedHash.to_string(), gotHash.to_string());
|
"hash mismatch in file downloaded from '%s':\n wanted: "
|
||||||
|
"%s\n got: %s",
|
||||||
|
url, request.expectedHash.to_string(),
|
||||||
|
gotHash.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
result.storePath = storePath;
|
result.storePath = storePath;
|
||||||
|
|
@ -932,15 +957,14 @@ CachedDownloadResult Downloader::downloadCached(
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool isUri(const string& s) {
|
||||||
bool isUri(const string & s)
|
|
||||||
{
|
|
||||||
if (s.compare(0, 8, "channel:") == 0) return true;
|
if (s.compare(0, 8, "channel:") == 0) return true;
|
||||||
size_t pos = s.find("://");
|
size_t pos = s.find("://");
|
||||||
if (pos == string::npos) return false;
|
if (pos == string::npos) return false;
|
||||||
string scheme(s, 0, pos);
|
string scheme(s, 0, pos);
|
||||||
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
|
return scheme == "http" || scheme == "https" || scheme == "file" ||
|
||||||
|
scheme == "channel" || scheme == "git" || scheme == "s3" ||
|
||||||
|
scheme == "ssh";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
68
third_party/nix/src/libstore/download.hh
vendored
68
third_party/nix/src/libstore/download.hh
vendored
|
|
@ -1,40 +1,45 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "hash.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <future>
|
#include <future>
|
||||||
|
#include <string>
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "hash.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct DownloadSettings : Config
|
struct DownloadSettings : Config {
|
||||||
{
|
|
||||||
Setting<bool> enableHttp2{this, true, "http2",
|
Setting<bool> enableHttp2{this, true, "http2",
|
||||||
"Whether to enable HTTP/2 support."};
|
"Whether to enable HTTP/2 support."};
|
||||||
|
|
||||||
Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
|
Setting<std::string> userAgentSuffix{
|
||||||
|
this, "", "user-agent-suffix",
|
||||||
"String appended to the user agent in HTTP requests."};
|
"String appended to the user agent in HTTP requests."};
|
||||||
|
|
||||||
Setting<size_t> httpConnections{this, 25, "http-connections",
|
Setting<size_t> httpConnections{this,
|
||||||
|
25,
|
||||||
|
"http-connections",
|
||||||
"Number of parallel HTTP connections.",
|
"Number of parallel HTTP connections.",
|
||||||
{"binary-caches-parallel-connections"}};
|
{"binary-caches-parallel-connections"}};
|
||||||
|
|
||||||
Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
|
Setting<unsigned long> connectTimeout{
|
||||||
"Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
|
this, 0, "connect-timeout",
|
||||||
|
"Timeout for connecting to servers during downloads. 0 means use curl's "
|
||||||
|
"builtin default."};
|
||||||
|
|
||||||
Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
|
Setting<unsigned long> stalledDownloadTimeout{
|
||||||
"Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
|
this, 300, "stalled-download-timeout",
|
||||||
|
"Timeout (in seconds) for receiving data from servers during download. "
|
||||||
|
"Nix cancels idle downloads after this timeout's duration."};
|
||||||
|
|
||||||
Setting<unsigned int> tries{this, 5, "download-attempts",
|
Setting<unsigned int> tries{
|
||||||
|
this, 5, "download-attempts",
|
||||||
"How often Nix will attempt to download a file before giving up."};
|
"How often Nix will attempt to download a file before giving up."};
|
||||||
};
|
};
|
||||||
|
|
||||||
extern DownloadSettings downloadSettings;
|
extern DownloadSettings downloadSettings;
|
||||||
|
|
||||||
struct DownloadRequest
|
struct DownloadRequest {
|
||||||
{
|
|
||||||
std::string uri;
|
std::string uri;
|
||||||
std::string expectedETag;
|
std::string expectedETag;
|
||||||
bool verifyTLS = true;
|
bool verifyTLS = true;
|
||||||
|
|
@ -50,14 +55,10 @@ struct DownloadRequest
|
||||||
DownloadRequest(const std::string& uri)
|
DownloadRequest(const std::string& uri)
|
||||||
: uri(uri), parentAct(getCurActivity()) {}
|
: uri(uri), parentAct(getCurActivity()) {}
|
||||||
|
|
||||||
std::string verb()
|
std::string verb() { return data ? "upload" : "download"; }
|
||||||
{
|
|
||||||
return data ? "upload" : "download";
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct DownloadResult
|
struct DownloadResult {
|
||||||
{
|
|
||||||
bool cached = false;
|
bool cached = false;
|
||||||
std::string etag;
|
std::string etag;
|
||||||
std::string effectiveUri;
|
std::string effectiveUri;
|
||||||
|
|
@ -65,20 +66,17 @@ struct DownloadResult
|
||||||
uint64_t bodySize = 0;
|
uint64_t bodySize = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct CachedDownloadRequest
|
struct CachedDownloadRequest {
|
||||||
{
|
|
||||||
std::string uri;
|
std::string uri;
|
||||||
bool unpack = false;
|
bool unpack = false;
|
||||||
std::string name;
|
std::string name;
|
||||||
Hash expectedHash;
|
Hash expectedHash;
|
||||||
unsigned int ttl = settings.tarballTtl;
|
unsigned int ttl = settings.tarballTtl;
|
||||||
|
|
||||||
CachedDownloadRequest(const std::string & uri)
|
CachedDownloadRequest(const std::string& uri) : uri(uri) {}
|
||||||
: uri(uri) { }
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct CachedDownloadResult
|
struct CachedDownloadResult {
|
||||||
{
|
|
||||||
// Note: 'storePath' may be different from 'path' when using a
|
// Note: 'storePath' may be different from 'path' when using a
|
||||||
// chroot store.
|
// chroot store.
|
||||||
Path storePath;
|
Path storePath;
|
||||||
|
|
@ -89,8 +87,7 @@ struct CachedDownloadResult
|
||||||
|
|
||||||
class Store;
|
class Store;
|
||||||
|
|
||||||
struct Downloader
|
struct Downloader {
|
||||||
{
|
|
||||||
virtual ~Downloader() {}
|
virtual ~Downloader() {}
|
||||||
|
|
||||||
/* Enqueue a download request, returning a future to the result of
|
/* Enqueue a download request, returning a future to the result of
|
||||||
|
|
@ -112,7 +109,8 @@ struct Downloader
|
||||||
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
|
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
|
||||||
use the recorded ETag to verify if the server has a more
|
use the recorded ETag to verify if the server has a more
|
||||||
recent version, and if so, download it to the Nix store. */
|
recent version, and if so, download it to the Nix store. */
|
||||||
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
|
CachedDownloadResult downloadCached(ref<Store> store,
|
||||||
|
const CachedDownloadRequest& request);
|
||||||
|
|
||||||
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
||||||
};
|
};
|
||||||
|
|
@ -124,15 +122,13 @@ ref<Downloader> getDownloader();
|
||||||
/* Return a new Downloader object. */
|
/* Return a new Downloader object. */
|
||||||
ref<Downloader> makeDownloader();
|
ref<Downloader> makeDownloader();
|
||||||
|
|
||||||
class DownloadError : public Error
|
class DownloadError : public Error {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
Downloader::Error error;
|
Downloader::Error error;
|
||||||
DownloadError(Downloader::Error error, const FormatOrString& fs)
|
DownloadError(Downloader::Error error, const FormatOrString& fs)
|
||||||
: Error(fs), error(error)
|
: Error(fs), error(error) {}
|
||||||
{ }
|
|
||||||
};
|
};
|
||||||
|
|
||||||
bool isUri(const string& s);
|
bool isUri(const string& s);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
52
third_party/nix/src/libstore/export-import.cc
vendored
52
third_party/nix/src/libstore/export-import.cc
vendored
|
|
@ -1,31 +1,23 @@
|
||||||
#include "store-api.hh"
|
|
||||||
#include "archive.hh"
|
|
||||||
#include "worker-protocol.hh"
|
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include "archive.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
#include "worker-protocol.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct HashAndWriteSink : Sink
|
struct HashAndWriteSink : Sink {
|
||||||
{
|
|
||||||
Sink& writeSink;
|
Sink& writeSink;
|
||||||
HashSink hashSink;
|
HashSink hashSink;
|
||||||
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
|
HashAndWriteSink(Sink& writeSink)
|
||||||
{
|
: writeSink(writeSink), hashSink(htSHA256) {}
|
||||||
}
|
virtual void operator()(const unsigned char* data, size_t len) {
|
||||||
virtual void operator () (const unsigned char * data, size_t len)
|
|
||||||
{
|
|
||||||
writeSink(data, len);
|
writeSink(data, len);
|
||||||
hashSink(data, len);
|
hashSink(data, len);
|
||||||
}
|
}
|
||||||
Hash currentHash()
|
Hash currentHash() { return hashSink.currentHash().first; }
|
||||||
{
|
|
||||||
return hashSink.currentHash().first;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
void Store::exportPaths(const Paths & paths, Sink & sink)
|
void Store::exportPaths(const Paths& paths, Sink& sink) {
|
||||||
{
|
|
||||||
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
|
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
|
||||||
std::reverse(sorted.begin(), sorted.end());
|
std::reverse(sorted.begin(), sorted.end());
|
||||||
|
|
||||||
|
|
@ -42,8 +34,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
|
||||||
sink << 0;
|
sink << 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Store::exportPath(const Path & path, Sink & sink)
|
void Store::exportPath(const Path& path, Sink& sink) {
|
||||||
{
|
|
||||||
auto info = queryPathInfo(path);
|
auto info = queryPathInfo(path);
|
||||||
|
|
||||||
HashAndWriteSink hashAndWriteSink(sink);
|
HashAndWriteSink hashAndWriteSink(sink);
|
||||||
|
|
@ -55,19 +46,22 @@ void Store::exportPath(const Path & path, Sink & sink)
|
||||||
Don't complain if the stored hash is zero (unknown). */
|
Don't complain if the stored hash is zero (unknown). */
|
||||||
Hash hash = hashAndWriteSink.currentHash();
|
Hash hash = hashAndWriteSink.currentHash();
|
||||||
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
|
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
|
||||||
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path
|
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") %
|
||||||
% info->narHash.to_string() % hash.to_string());
|
path % info->narHash.to_string() % hash.to_string());
|
||||||
|
|
||||||
hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
|
hashAndWriteSink << exportMagic << path << info->references << info->deriver
|
||||||
|
<< 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
|
Paths Store::importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
|
||||||
{
|
CheckSigsFlag checkSigs) {
|
||||||
Paths res;
|
Paths res;
|
||||||
while (true) {
|
while (true) {
|
||||||
auto n = readNum<uint64_t>(source);
|
auto n = readNum<uint64_t>(source);
|
||||||
if (n == 0) break;
|
if (n == 0) break;
|
||||||
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
|
if (n != 1)
|
||||||
|
throw Error(
|
||||||
|
"input doesn't look like something created by 'nix-store --export'");
|
||||||
|
|
||||||
/* Extract the NAR from the source. */
|
/* Extract the NAR from the source. */
|
||||||
TeeSink tee(source);
|
TeeSink tee(source);
|
||||||
|
|
@ -81,7 +75,8 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
|
||||||
|
|
||||||
info.path = readStorePath(*this, source);
|
info.path = readStorePath(*this, source);
|
||||||
|
|
||||||
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
|
// Activity act(*logger, lvlInfo, format("importing path '%s'") %
|
||||||
|
// info.path);
|
||||||
|
|
||||||
info.references = readStorePaths<PathSet>(*this, source);
|
info.references = readStorePaths<PathSet>(*this, source);
|
||||||
|
|
||||||
|
|
@ -92,8 +87,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
|
||||||
info.narSize = tee.source.data->size();
|
info.narSize = tee.source.data->size();
|
||||||
|
|
||||||
// Ignore optional legacy signature.
|
// Ignore optional legacy signature.
|
||||||
if (readInt(source) == 1)
|
if (readInt(source) == 1) readString(source);
|
||||||
readString(source);
|
|
||||||
|
|
||||||
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
|
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
|
||||||
|
|
||||||
|
|
@ -103,4 +97,4 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
8
third_party/nix/src/libstore/fs-accessor.hh
vendored
8
third_party/nix/src/libstore/fs-accessor.hh
vendored
|
|
@ -6,13 +6,11 @@ namespace nix {
|
||||||
|
|
||||||
/* An abstract class for accessing a filesystem-like structure, such
|
/* An abstract class for accessing a filesystem-like structure, such
|
||||||
as a (possibly remote) Nix store or the contents of a NAR file. */
|
as a (possibly remote) Nix store or the contents of a NAR file. */
|
||||||
class FSAccessor
|
class FSAccessor {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
enum Type { tMissing, tRegular, tSymlink, tDirectory };
|
enum Type { tMissing, tRegular, tSymlink, tDirectory };
|
||||||
|
|
||||||
struct Stat
|
struct Stat {
|
||||||
{
|
|
||||||
Type type = tMissing;
|
Type type = tMissing;
|
||||||
uint64_t fileSize = 0; // regular files only
|
uint64_t fileSize = 0; // regular files only
|
||||||
bool isExecutable = false; // regular files only
|
bool isExecutable = false; // regular files only
|
||||||
|
|
@ -30,4 +28,4 @@ public:
|
||||||
virtual std::string readLink(const Path& path) = 0;
|
virtual std::string readLink(const Path& path) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
298
third_party/nix/src/libstore/gc.cc
vendored
298
third_party/nix/src/libstore/gc.cc
vendored
|
|
@ -1,42 +1,37 @@
|
||||||
#include "derivations.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "local-store.hh"
|
|
||||||
#include "finally.hh"
|
|
||||||
|
|
||||||
#include <functional>
|
|
||||||
#include <queue>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <regex>
|
|
||||||
#include <random>
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <sys/statvfs.h>
|
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/statvfs.h>
|
||||||
|
#include <sys/types.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <algorithm>
|
||||||
#include <climits>
|
#include <climits>
|
||||||
|
#include <functional>
|
||||||
|
#include <queue>
|
||||||
|
#include <random>
|
||||||
|
#include <regex>
|
||||||
|
#include "derivations.hh"
|
||||||
|
#include "finally.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "local-store.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
static string gcLockName = "gc.lock";
|
static string gcLockName = "gc.lock";
|
||||||
static string gcRootsDir = "gcroots";
|
static string gcRootsDir = "gcroots";
|
||||||
|
|
||||||
|
|
||||||
/* Acquire the global GC lock. This is used to prevent new Nix
|
/* Acquire the global GC lock. This is used to prevent new Nix
|
||||||
processes from starting after the temporary root files have been
|
processes from starting after the temporary root files have been
|
||||||
read. To be precise: when they try to create a new temporary root
|
read. To be precise: when they try to create a new temporary root
|
||||||
file, they will block until the garbage collector has finished /
|
file, they will block until the garbage collector has finished /
|
||||||
yielded the GC lock. */
|
yielded the GC lock. */
|
||||||
AutoCloseFD LocalStore::openGCLock(LockType lockType)
|
AutoCloseFD LocalStore::openGCLock(LockType lockType) {
|
||||||
{
|
Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str();
|
||||||
Path fnGCLock = (format("%1%/%2%")
|
|
||||||
% stateDir % gcLockName).str();
|
|
||||||
|
|
||||||
debug(format("acquiring global GC lock '%1%'") % fnGCLock);
|
debug(format("acquiring global GC lock '%1%'") % fnGCLock);
|
||||||
|
|
||||||
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
|
AutoCloseFD fdGCLock =
|
||||||
|
open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
|
||||||
if (!fdGCLock)
|
if (!fdGCLock)
|
||||||
throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
|
throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
|
||||||
|
|
||||||
|
|
@ -52,69 +47,60 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
|
||||||
return fdGCLock;
|
return fdGCLock;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void makeSymlink(const Path& link, const Path& target) {
|
||||||
static void makeSymlink(const Path & link, const Path & target)
|
|
||||||
{
|
|
||||||
/* Create directories up to `gcRoot'. */
|
/* Create directories up to `gcRoot'. */
|
||||||
createDirs(dirOf(link));
|
createDirs(dirOf(link));
|
||||||
|
|
||||||
/* Create the new symlink. */
|
/* Create the new symlink. */
|
||||||
Path tempLink = (format("%1%.tmp-%2%-%3%")
|
Path tempLink =
|
||||||
% link % getpid() % random()).str();
|
(format("%1%.tmp-%2%-%3%") % link % getpid() % random()).str();
|
||||||
createSymlink(target, tempLink);
|
createSymlink(target, tempLink);
|
||||||
|
|
||||||
/* Atomically replace the old one. */
|
/* Atomically replace the old one. */
|
||||||
if (rename(tempLink.c_str(), link.c_str()) == -1)
|
if (rename(tempLink.c_str(), link.c_str()) == -1)
|
||||||
throw SysError(format("cannot rename '%1%' to '%2%'")
|
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % link);
|
||||||
% tempLink % link);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::syncWithGC() { AutoCloseFD fdGCLock = openGCLock(ltRead); }
|
||||||
|
|
||||||
void LocalStore::syncWithGC()
|
void LocalStore::addIndirectRoot(const Path& path) {
|
||||||
{
|
|
||||||
AutoCloseFD fdGCLock = openGCLock(ltRead);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::addIndirectRoot(const Path & path)
|
|
||||||
{
|
|
||||||
string hash = hashString(htSHA1, path).to_string(Base32, false);
|
string hash = hashString(htSHA1, path).to_string(Base32, false);
|
||||||
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
|
Path realRoot = canonPath(
|
||||||
% stateDir % gcRootsDir % hash).str());
|
(format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str());
|
||||||
makeSymlink(realRoot, path);
|
makeSymlink(realRoot, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot,
|
||||||
Path LocalFSStore::addPermRoot(const Path & _storePath,
|
bool indirect, bool allowOutsideRootsDir) {
|
||||||
const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
|
|
||||||
{
|
|
||||||
Path storePath(canonPath(_storePath));
|
Path storePath(canonPath(_storePath));
|
||||||
Path gcRoot(canonPath(_gcRoot));
|
Path gcRoot(canonPath(_gcRoot));
|
||||||
assertStorePath(storePath);
|
assertStorePath(storePath);
|
||||||
|
|
||||||
if (isInStore(gcRoot))
|
if (isInStore(gcRoot))
|
||||||
throw Error(format(
|
throw Error(format("creating a garbage collector root (%1%) in the Nix "
|
||||||
"creating a garbage collector root (%1%) in the Nix store is forbidden "
|
"store is forbidden "
|
||||||
"(are you running nix-build inside the store?)") % gcRoot);
|
"(are you running nix-build inside the store?)") %
|
||||||
|
gcRoot);
|
||||||
|
|
||||||
if (indirect) {
|
if (indirect) {
|
||||||
/* Don't clobber the link if it already exists and doesn't
|
/* Don't clobber the link if it already exists and doesn't
|
||||||
point to the Nix store. */
|
point to the Nix store. */
|
||||||
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
|
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
|
||||||
throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
|
throw Error(format("cannot create symlink '%1%'; already exists") %
|
||||||
|
gcRoot);
|
||||||
makeSymlink(gcRoot, storePath);
|
makeSymlink(gcRoot, storePath);
|
||||||
addIndirectRoot(gcRoot);
|
addIndirectRoot(gcRoot);
|
||||||
}
|
}
|
||||||
|
|
||||||
else {
|
else {
|
||||||
if (!allowOutsideRootsDir) {
|
if (!allowOutsideRootsDir) {
|
||||||
Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
|
Path rootsDir =
|
||||||
|
canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
|
||||||
|
|
||||||
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
|
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
|
||||||
throw Error(format(
|
throw Error(format("path '%1%' is not a valid garbage collector root; "
|
||||||
"path '%1%' is not a valid garbage collector root; "
|
"it's not in the directory '%2%'") %
|
||||||
"it's not in the directory '%2%'")
|
gcRoot % rootsDir);
|
||||||
% gcRoot % rootsDir);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (baseNameOf(gcRoot) == baseNameOf(storePath))
|
if (baseNameOf(gcRoot) == baseNameOf(storePath))
|
||||||
|
|
@ -132,10 +118,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
|
||||||
Roots roots = findRoots(false);
|
Roots roots = findRoots(false);
|
||||||
if (roots[storePath].count(gcRoot) == 0)
|
if (roots[storePath].count(gcRoot) == 0)
|
||||||
printError(
|
printError(
|
||||||
format(
|
format("warning: '%1%' is not in a directory where the garbage "
|
||||||
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
|
"collector looks for roots; "
|
||||||
"therefore, '%2%' might be removed by the garbage collector")
|
"therefore, '%2%' might be removed by the garbage collector") %
|
||||||
% gcRoot % storePath);
|
gcRoot % storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Grab the global GC root, causing us to block while a GC is in
|
/* Grab the global GC root, causing us to block while a GC is in
|
||||||
|
|
@ -146,14 +132,11 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
|
||||||
return gcRoot;
|
return gcRoot;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::addTempRoot(const Path& path) {
|
||||||
void LocalStore::addTempRoot(const Path & path)
|
|
||||||
{
|
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
||||||
/* Create the temporary roots file for this process. */
|
/* Create the temporary roots file for this process. */
|
||||||
if (!state->fdTempRoots) {
|
if (!state->fdTempRoots) {
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
AutoCloseFD fdGCLock = openGCLock(ltRead);
|
AutoCloseFD fdGCLock = openGCLock(ltRead);
|
||||||
|
|
||||||
|
|
@ -180,7 +163,6 @@ void LocalStore::addTempRoot(const Path & path)
|
||||||
get a lock. (It won't delete the file after we get a
|
get a lock. (It won't delete the file after we get a
|
||||||
lock.) Try again. */
|
lock.) Try again. */
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Upgrade the lock to a write lock. This will cause us to block
|
/* Upgrade the lock to a write lock. This will cause us to block
|
||||||
|
|
@ -196,12 +178,9 @@ void LocalStore::addTempRoot(const Path & path)
|
||||||
lockFile(state->fdTempRoots.get(), ltRead, true);
|
lockFile(state->fdTempRoots.get(), ltRead, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static std::string censored = "{censored}";
|
static std::string censored = "{censored}";
|
||||||
|
|
||||||
|
void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
|
||||||
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
|
|
||||||
{
|
|
||||||
/* Read the `temproots' directory for per-process temporary root
|
/* Read the `temproots' directory for per-process temporary root
|
||||||
files. */
|
files. */
|
||||||
for (auto& i : readDirectory(tempRootsDir)) {
|
for (auto& i : readDirectory(tempRootsDir)) {
|
||||||
|
|
@ -255,21 +234,18 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
|
||||||
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
|
|
||||||
{
|
|
||||||
auto foundRoot = [&](const Path& path, const Path& target) {
|
auto foundRoot = [&](const Path& path, const Path& target) {
|
||||||
Path storePath = toStorePath(target);
|
Path storePath = toStorePath(target);
|
||||||
if (isStorePath(storePath) && isValidPath(storePath))
|
if (isStorePath(storePath) && isValidPath(storePath))
|
||||||
roots[storePath].emplace(path);
|
roots[storePath].emplace(path);
|
||||||
else
|
else
|
||||||
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
|
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path %
|
||||||
|
storePath);
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (type == DT_UNKNOWN) type = getFileType(path);
|
||||||
if (type == DT_UNKNOWN)
|
|
||||||
type = getFileType(path);
|
|
||||||
|
|
||||||
if (type == DT_DIR) {
|
if (type == DT_DIR) {
|
||||||
for (auto& i : readDirectory(path))
|
for (auto& i : readDirectory(path))
|
||||||
|
|
@ -278,15 +254,15 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
|
||||||
|
|
||||||
else if (type == DT_LNK) {
|
else if (type == DT_LNK) {
|
||||||
Path target = readLink(path);
|
Path target = readLink(path);
|
||||||
if (isInStore(target))
|
if (isInStore(target)) foundRoot(path, target);
|
||||||
foundRoot(path, target);
|
|
||||||
|
|
||||||
/* Handle indirect roots. */
|
/* Handle indirect roots. */
|
||||||
else {
|
else {
|
||||||
target = absPath(target, dirOf(path));
|
target = absPath(target, dirOf(path));
|
||||||
if (!pathExists(target)) {
|
if (!pathExists(target)) {
|
||||||
if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
|
if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
|
||||||
printInfo(format("removing stale link from '%1%' to '%2%'") % path % target);
|
printInfo(format("removing stale link from '%1%' to '%2%'") % path %
|
||||||
|
target);
|
||||||
unlink(path.c_str());
|
unlink(path.c_str());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -315,9 +291,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::findRootsNoTemp(Roots& roots, bool censor) {
|
||||||
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
|
|
||||||
{
|
|
||||||
/* Process direct roots in {gcroots,profiles}. */
|
/* Process direct roots in {gcroots,profiles}. */
|
||||||
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
|
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
|
||||||
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
|
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
|
||||||
|
|
@ -328,9 +302,7 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
|
||||||
findRuntimeRoots(roots, censor);
|
findRuntimeRoots(roots, censor);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Roots LocalStore::findRoots(bool censor) {
|
||||||
Roots LocalStore::findRoots(bool censor)
|
|
||||||
{
|
|
||||||
Roots roots;
|
Roots roots;
|
||||||
findRootsNoTemp(roots, censor);
|
findRootsNoTemp(roots, censor);
|
||||||
|
|
||||||
|
|
@ -340,55 +312,49 @@ Roots LocalStore::findRoots(bool censor)
|
||||||
return roots;
|
return roots;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void readProcLink(const string & file, Roots & roots)
|
static void readProcLink(const string& file, Roots& roots) {
|
||||||
{
|
|
||||||
/* 64 is the starting buffer size gnu readlink uses... */
|
/* 64 is the starting buffer size gnu readlink uses... */
|
||||||
auto bufsiz = ssize_t{64};
|
auto bufsiz = ssize_t{64};
|
||||||
try_again:
|
try_again:
|
||||||
char buf[bufsiz];
|
char buf[bufsiz];
|
||||||
auto res = readlink(file.c_str(), buf, bufsiz);
|
auto res = readlink(file.c_str(), buf, bufsiz);
|
||||||
if (res == -1) {
|
if (res == -1) {
|
||||||
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
|
if (errno == ENOENT || errno == EACCES || errno == ESRCH) return;
|
||||||
return;
|
|
||||||
throw SysError("reading symlink");
|
throw SysError("reading symlink");
|
||||||
}
|
}
|
||||||
if (res == bufsiz) {
|
if (res == bufsiz) {
|
||||||
if (SSIZE_MAX / 2 < bufsiz)
|
if (SSIZE_MAX / 2 < bufsiz) throw Error("stupidly long symlink");
|
||||||
throw Error("stupidly long symlink");
|
|
||||||
bufsiz *= 2;
|
bufsiz *= 2;
|
||||||
goto try_again;
|
goto try_again;
|
||||||
}
|
}
|
||||||
if (res > 0 && buf[0] == '/')
|
if (res > 0 && buf[0] == '/')
|
||||||
roots[std::string(static_cast<char *>(buf), res)]
|
roots[std::string(static_cast<char*>(buf), res)].emplace(file);
|
||||||
.emplace(file);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static string quoteRegexChars(const string & raw)
|
static string quoteRegexChars(const string& raw) {
|
||||||
{
|
|
||||||
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
|
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
|
||||||
return std::regex_replace(raw, specialRegex, R"(\$&)");
|
return std::regex_replace(raw, specialRegex, R"(\$&)");
|
||||||
}
|
}
|
||||||
|
|
||||||
static void readFileRoots(const char * path, Roots & roots)
|
static void readFileRoots(const char* path, Roots& roots) {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
roots[readFile(path)].emplace(path);
|
roots[readFile(path)].emplace(path);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo != ENOENT && e.errNo != EACCES)
|
if (e.errNo != ENOENT && e.errNo != EACCES) throw;
|
||||||
throw;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
|
||||||
{
|
|
||||||
Roots unchecked;
|
Roots unchecked;
|
||||||
|
|
||||||
auto procDir = AutoCloseDir{opendir("/proc")};
|
auto procDir = AutoCloseDir{opendir("/proc")};
|
||||||
if (procDir) {
|
if (procDir) {
|
||||||
struct dirent* ent;
|
struct dirent* ent;
|
||||||
auto digitsRegex = std::regex(R"(^\d+$)");
|
auto digitsRegex = std::regex(R"(^\d+$)");
|
||||||
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
|
auto mapRegex =
|
||||||
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
|
std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
|
||||||
|
auto storePathRegex = std::regex(quoteRegexChars(storeDir) +
|
||||||
|
R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
|
||||||
while (errno = 0, ent = readdir(procDir.get())) {
|
while (errno = 0, ent = readdir(procDir.get())) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
if (std::regex_match(ent->d_name, digitsRegex)) {
|
if (std::regex_match(ent->d_name, digitsRegex)) {
|
||||||
|
|
@ -398,8 +364,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||||
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
|
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
|
||||||
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
|
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
|
||||||
if (!fdDir) {
|
if (!fdDir) {
|
||||||
if (errno == ENOENT || errno == EACCES)
|
if (errno == ENOENT || errno == EACCES) continue;
|
||||||
continue;
|
|
||||||
throw SysError(format("opening %1%") % fdStr);
|
throw SysError(format("opening %1%") % fdStr);
|
||||||
}
|
}
|
||||||
struct dirent* fd_ent;
|
struct dirent* fd_ent;
|
||||||
|
|
@ -408,15 +373,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||||
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
|
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
|
||||||
}
|
}
|
||||||
if (errno) {
|
if (errno) {
|
||||||
if (errno == ESRCH)
|
if (errno == ESRCH) continue;
|
||||||
continue;
|
|
||||||
throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
|
throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
|
||||||
}
|
}
|
||||||
fdDir.reset();
|
fdDir.reset();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
|
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
|
||||||
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
|
auto mapLines = tokenizeString<std::vector<string>>(
|
||||||
|
readFile(mapFile, true), "\n");
|
||||||
for (const auto& line : mapLines) {
|
for (const auto& line : mapLines) {
|
||||||
auto match = std::smatch{};
|
auto match = std::smatch{};
|
||||||
if (std::regex_match(line, match, mapRegex))
|
if (std::regex_match(line, match, mapRegex))
|
||||||
|
|
@ -426,28 +391,28 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||||
auto envFile = fmt("/proc/%s/environ", ent->d_name);
|
auto envFile = fmt("/proc/%s/environ", ent->d_name);
|
||||||
auto envString = readFile(envFile, true);
|
auto envString = readFile(envFile, true);
|
||||||
auto env_end = std::sregex_iterator{};
|
auto env_end = std::sregex_iterator{};
|
||||||
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
|
for (auto i = std::sregex_iterator{envString.begin(), envString.end(),
|
||||||
|
storePathRegex};
|
||||||
|
i != env_end; ++i)
|
||||||
unchecked[i->str()].emplace(envFile);
|
unchecked[i->str()].emplace(envFile);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
|
if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue;
|
||||||
continue;
|
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (errno)
|
if (errno) throw SysError("iterating /proc");
|
||||||
throw SysError("iterating /proc");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#if !defined(__linux__)
|
#if !defined(__linux__)
|
||||||
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
|
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test
|
||||||
// See: https://github.com/NixOS/nix/issues/3011
|
// to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we
|
||||||
// Because of this we disable lsof when running the tests.
|
// disable lsof when running the tests.
|
||||||
if (getEnv("_NIX_TEST_NO_LSOF") == "") {
|
if (getEnv("_NIX_TEST_NO_LSOF") == "") {
|
||||||
try {
|
try {
|
||||||
std::regex lsofRegex(R"(^n(/.*)$)");
|
std::regex lsofRegex(R"(^n(/.*)$)");
|
||||||
auto lsofLines =
|
auto lsofLines = tokenizeString<std::vector<string>>(
|
||||||
tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
|
runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n");
|
||||||
for (const auto& line : lsofLines) {
|
for (const auto& line : lsofLines) {
|
||||||
std::smatch match;
|
std::smatch match;
|
||||||
if (std::regex_match(line, match, lsofRegex))
|
if (std::regex_match(line, match, lsofRegex))
|
||||||
|
|
@ -479,12 +444,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
struct GCLimitReached {};
|
struct GCLimitReached {};
|
||||||
|
|
||||||
|
struct LocalStore::GCState {
|
||||||
struct LocalStore::GCState
|
|
||||||
{
|
|
||||||
GCOptions options;
|
GCOptions options;
|
||||||
GCResults& results;
|
GCResults& results;
|
||||||
PathSet roots;
|
PathSet roots;
|
||||||
|
|
@ -499,25 +461,20 @@ struct LocalStore::GCState
|
||||||
GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {}
|
GCState(GCResults& results_) : results(results_), bytesInvalidated(0) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
bool LocalStore::isActiveTempFile(const GCState& state, const Path& path,
|
||||||
bool LocalStore::isActiveTempFile(const GCState & state,
|
const string& suffix) {
|
||||||
const Path & path, const string & suffix)
|
return hasSuffix(path, suffix) &&
|
||||||
{
|
state.tempRoots.find(string(path, 0, path.size() - suffix.size())) !=
|
||||||
return hasSuffix(path, suffix)
|
state.tempRoots.end();
|
||||||
&& state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::deleteGarbage(GCState& state, const Path& path) {
|
||||||
void LocalStore::deleteGarbage(GCState & state, const Path & path)
|
|
||||||
{
|
|
||||||
unsigned long long bytesFreed;
|
unsigned long long bytesFreed;
|
||||||
deletePath(path, bytesFreed);
|
deletePath(path, bytesFreed);
|
||||||
state.results.bytesFreed += bytesFreed;
|
state.results.bytesFreed += bytesFreed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
|
||||||
void LocalStore::deletePathRecursive(GCState & state, const Path & path)
|
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
unsigned long long size = 0;
|
unsigned long long size = 0;
|
||||||
|
|
@ -557,26 +514,29 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
|
||||||
throw SysError(format("making '%1%' writable") % realPath);
|
throw SysError(format("making '%1%' writable") % realPath);
|
||||||
Path tmp = trashDir + "/" + baseNameOf(path);
|
Path tmp = trashDir + "/" + baseNameOf(path);
|
||||||
if (rename(realPath.c_str(), tmp.c_str()))
|
if (rename(realPath.c_str(), tmp.c_str()))
|
||||||
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
|
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath %
|
||||||
|
tmp);
|
||||||
state.bytesInvalidated += size;
|
state.bytesInvalidated += size;
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo == ENOSPC) {
|
if (e.errNo == ENOSPC) {
|
||||||
printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
|
printInfo(format("note: can't create move '%1%': %2%") % realPath %
|
||||||
|
e.msg());
|
||||||
deleteGarbage(state, realPath);
|
deleteGarbage(state, realPath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
deleteGarbage(state, realPath);
|
deleteGarbage(state, realPath);
|
||||||
|
|
||||||
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
|
if (state.results.bytesFreed + state.bytesInvalidated >
|
||||||
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
|
state.options.maxFreed) {
|
||||||
|
printInfo(format("deleted or invalidated more than %1% bytes; stopping") %
|
||||||
|
state.options.maxFreed);
|
||||||
throw GCLimitReached();
|
throw GCLimitReached();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool LocalStore::canReachRoot(GCState& state, PathSet& visited,
|
||||||
bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
|
const Path& path) {
|
||||||
{
|
|
||||||
if (visited.count(path)) return false;
|
if (visited.count(path)) return false;
|
||||||
|
|
||||||
if (state.alive.count(path)) return true;
|
if (state.alive.count(path)) return true;
|
||||||
|
|
@ -611,8 +571,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
|
||||||
are derivers of this path that are not garbage. */
|
are derivers of this path that are not garbage. */
|
||||||
if (state.gcKeepOutputs) {
|
if (state.gcKeepOutputs) {
|
||||||
PathSet derivers = queryValidDerivers(path);
|
PathSet derivers = queryValidDerivers(path);
|
||||||
for (auto & i : derivers)
|
for (auto& i : derivers) incoming.insert(i);
|
||||||
incoming.insert(i);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& i : incoming)
|
for (auto& i : incoming)
|
||||||
|
|
@ -625,15 +584,14 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::tryToDelete(GCState& state, const Path& path) {
|
||||||
void LocalStore::tryToDelete(GCState & state, const Path & path)
|
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
auto realPath = realStoreDir + "/" + baseNameOf(path);
|
auto realPath = realStoreDir + "/" + baseNameOf(path);
|
||||||
if (realPath == linksDir || realPath == trashDir) return;
|
if (realPath == linksDir || realPath == trashDir) return;
|
||||||
|
|
||||||
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
|
// Activity act(*logger, lvlDebug, format("considering whether to delete
|
||||||
|
// '%1%'") % path);
|
||||||
|
|
||||||
if (!isStorePath(path) || !isValidPath(path)) {
|
if (!isStorePath(path) || !isValidPath(path)) {
|
||||||
/* A lock file belonging to a path that we're building right
|
/* A lock file belonging to a path that we're building right
|
||||||
|
|
@ -660,19 +618,16 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
|
||||||
‘nix-store --delete’ doesn't have the unexpected effect of
|
‘nix-store --delete’ doesn't have the unexpected effect of
|
||||||
recursing into derivations and outputs. */
|
recursing into derivations and outputs. */
|
||||||
state.dead.insert(visited.begin(), visited.end());
|
state.dead.insert(visited.begin(), visited.end());
|
||||||
if (state.shouldDelete)
|
if (state.shouldDelete) deletePathRecursive(state, path);
|
||||||
deletePathRecursive(state, path);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Unlink all files in /nix/store/.links that have a link count of 1,
|
/* Unlink all files in /nix/store/.links that have a link count of 1,
|
||||||
which indicates that there are no other links and so they can be
|
which indicates that there are no other links and so they can be
|
||||||
safely deleted. FIXME: race condition with optimisePath(): we
|
safely deleted. FIXME: race condition with optimisePath(): we
|
||||||
might see a link count of 1 just before optimisePath() increases
|
might see a link count of 1 just before optimisePath() increases
|
||||||
the link count. */
|
the link count. */
|
||||||
void LocalStore::removeUnusedLinks(const GCState & state)
|
void LocalStore::removeUnusedLinks(const GCState& state) {
|
||||||
{
|
|
||||||
AutoCloseDir dir(opendir(linksDir.c_str()));
|
AutoCloseDir dir(opendir(linksDir.c_str()));
|
||||||
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
|
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
|
||||||
|
|
||||||
|
|
@ -708,13 +663,11 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
||||||
throw SysError(format("statting '%1%'") % linksDir);
|
throw SysError(format("statting '%1%'") % linksDir);
|
||||||
long long overhead = st.st_blocks * 512ULL;
|
long long overhead = st.st_blocks * 512ULL;
|
||||||
|
|
||||||
printInfo(format("note: currently hard linking saves %.2f MiB")
|
printInfo(format("note: currently hard linking saves %.2f MiB") %
|
||||||
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
|
||||||
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|
||||||
{
|
|
||||||
GCState state(results);
|
GCState state(results);
|
||||||
state.options = options;
|
state.options = options;
|
||||||
state.gcKeepOutputs = settings.gcKeepOutputs;
|
state.gcKeepOutputs = settings.gcKeepOutputs;
|
||||||
|
|
@ -729,10 +682,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
state.gcKeepDerivations = false;
|
state.gcKeepDerivations = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
|
state.shouldDelete = options.action == GCOptions::gcDeleteDead ||
|
||||||
|
options.action == GCOptions::gcDeleteSpecific;
|
||||||
|
|
||||||
if (state.shouldDelete)
|
if (state.shouldDelete) deletePath(reservedPath);
|
||||||
deletePath(reservedPath);
|
|
||||||
|
|
||||||
/* Acquire the global GC root. This prevents
|
/* Acquire the global GC root. This prevents
|
||||||
a) New roots from being added.
|
a) New roots from being added.
|
||||||
|
|
@ -743,8 +696,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
permanent roots cannot increase now. */
|
permanent roots cannot increase now. */
|
||||||
printError(format("finding garbage collector roots..."));
|
printError(format("finding garbage collector roots..."));
|
||||||
Roots rootMap;
|
Roots rootMap;
|
||||||
if (!options.ignoreLiveness)
|
if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true);
|
||||||
findRootsNoTemp(rootMap, true);
|
|
||||||
|
|
||||||
for (auto& i : rootMap) state.roots.insert(i.first);
|
for (auto& i : rootMap) state.roots.insert(i.first);
|
||||||
|
|
||||||
|
|
@ -754,8 +706,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
FDs fds;
|
FDs fds;
|
||||||
Roots tempRoots;
|
Roots tempRoots;
|
||||||
findTempRoots(fds, tempRoots, true);
|
findTempRoots(fds, tempRoots, true);
|
||||||
for (auto & root : tempRoots)
|
for (auto& root : tempRoots) state.tempRoots.insert(root.first);
|
||||||
state.tempRoots.insert(root.first);
|
|
||||||
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
|
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
|
||||||
|
|
||||||
/* After this point the set of roots or temporary roots cannot
|
/* After this point the set of roots or temporary roots cannot
|
||||||
|
|
@ -778,25 +729,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
paths (for gcDeleteSpecific). */
|
paths (for gcDeleteSpecific). */
|
||||||
|
|
||||||
if (options.action == GCOptions::gcDeleteSpecific) {
|
if (options.action == GCOptions::gcDeleteSpecific) {
|
||||||
|
|
||||||
for (auto& i : options.pathsToDelete) {
|
for (auto& i : options.pathsToDelete) {
|
||||||
assertStorePath(i);
|
assertStorePath(i);
|
||||||
tryToDelete(state, i);
|
tryToDelete(state, i);
|
||||||
if (state.dead.find(i) == state.dead.end())
|
if (state.dead.find(i) == state.dead.end())
|
||||||
throw Error(format("cannot delete path '%1%' since it is still alive") % i);
|
throw Error(format("cannot delete path '%1%' since it is still alive") %
|
||||||
|
i);
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if (options.maxFreed > 0) {
|
} else if (options.maxFreed > 0) {
|
||||||
|
|
||||||
if (state.shouldDelete)
|
if (state.shouldDelete)
|
||||||
printError(format("deleting garbage..."));
|
printError(format("deleting garbage..."));
|
||||||
else
|
else
|
||||||
printError(format("determining live/dead paths..."));
|
printError(format("determining live/dead paths..."));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
AutoCloseDir dir(opendir(realStoreDir.c_str()));
|
AutoCloseDir dir(opendir(realStoreDir.c_str()));
|
||||||
if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
|
if (!dir)
|
||||||
|
throw SysError(format("opening directory '%1%'") % realStoreDir);
|
||||||
|
|
||||||
/* Read the store and immediately delete all paths that
|
/* Read the store and immediately delete all paths that
|
||||||
aren't valid. When using --max-freed etc., deleting
|
aren't valid. When using --max-freed etc., deleting
|
||||||
|
|
@ -828,8 +778,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
std::mt19937 gen(1);
|
std::mt19937 gen(1);
|
||||||
std::shuffle(entries_.begin(), entries_.end(), gen);
|
std::shuffle(entries_.begin(), entries_.end(), gen);
|
||||||
|
|
||||||
for (auto & i : entries_)
|
for (auto& i : entries_) tryToDelete(state, i);
|
||||||
tryToDelete(state, i);
|
|
||||||
|
|
||||||
} catch (GCLimitReached& e) {
|
} catch (GCLimitReached& e) {
|
||||||
}
|
}
|
||||||
|
|
@ -854,7 +803,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
deleteGarbage(state, trashDir);
|
deleteGarbage(state, trashDir);
|
||||||
|
|
||||||
/* Clean up the links directory. */
|
/* Clean up the links directory. */
|
||||||
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
|
if (options.action == GCOptions::gcDeleteDead ||
|
||||||
|
options.action == GCOptions::gcDeleteSpecific) {
|
||||||
printError(format("deleting unused links..."));
|
printError(format("deleting unused links..."));
|
||||||
removeUnusedLinks(state);
|
removeUnusedLinks(state);
|
||||||
}
|
}
|
||||||
|
|
@ -863,9 +813,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
// if (options.action == GCOptions::gcDeleteDead) vacuumDB();
|
// if (options.action == GCOptions::gcDeleteDead) vacuumDB();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::autoGC(bool sync) {
|
||||||
void LocalStore::autoGC(bool sync)
|
|
||||||
{
|
|
||||||
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
|
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
|
||||||
|
|
||||||
auto getAvail = [this]() -> uint64_t {
|
auto getAvail = [this]() -> uint64_t {
|
||||||
|
|
@ -892,7 +840,9 @@ void LocalStore::autoGC(bool sync)
|
||||||
|
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return;
|
if (now < state->lastGCCheck +
|
||||||
|
std::chrono::seconds(settings.minFreeCheckInterval))
|
||||||
|
return;
|
||||||
|
|
||||||
auto avail = getAvail();
|
auto avail = getAvail();
|
||||||
|
|
||||||
|
|
@ -908,9 +858,7 @@ void LocalStore::autoGC(bool sync)
|
||||||
future = state->gcFuture = promise.get_future().share();
|
future = state->gcFuture = promise.get_future().share();
|
||||||
|
|
||||||
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
|
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
/* Wake up any threads waiting for the auto-GC to finish. */
|
/* Wake up any threads waiting for the auto-GC to finish. */
|
||||||
Finally wakeup([&]() {
|
Finally wakeup([&]() {
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
@ -935,7 +883,6 @@ void LocalStore::autoGC(bool sync)
|
||||||
// future, but we don't really care.
|
// future, but we don't really care.
|
||||||
ignoreException();
|
ignoreException();
|
||||||
}
|
}
|
||||||
|
|
||||||
}).detach();
|
}).detach();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -944,5 +891,4 @@ void LocalStore::autoGC(bool sync)
|
||||||
if (sync) future.get();
|
if (sync) future.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
123
third_party/nix/src/libstore/globals.cc
vendored
123
third_party/nix/src/libstore/globals.cc
vendored
|
|
@ -1,17 +1,14 @@
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "util.hh"
|
#include <dlfcn.h>
|
||||||
#include "archive.hh"
|
|
||||||
#include "args.hh"
|
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
#include <dlfcn.h>
|
#include "archive.hh"
|
||||||
|
#include "args.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* The default location of the daemon socket, relative to nixStateDir.
|
/* The default location of the daemon socket, relative to nixStateDir.
|
||||||
The socket is in a directory to allow you to control access to the
|
The socket is in a directory to allow you to control access to the
|
||||||
Nix daemon by setting the mode/ownership of the directory
|
Nix daemon by setting the mode/ownership of the directory
|
||||||
|
|
@ -31,23 +28,25 @@ Settings settings;
|
||||||
static GlobalConfig::Register r1(&settings);
|
static GlobalConfig::Register r1(&settings);
|
||||||
|
|
||||||
Settings::Settings()
|
Settings::Settings()
|
||||||
: nixPrefix(NIX_PREFIX)
|
: nixPrefix(NIX_PREFIX),
|
||||||
, nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
|
nixStore(canonPath(
|
||||||
, nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
|
getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))),
|
||||||
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
|
nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))),
|
||||||
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)))
|
nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))),
|
||||||
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
|
nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))),
|
||||||
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
|
nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))),
|
||||||
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
|
nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))),
|
||||||
, nixManDir(canonPath(NIX_MAN_DIR))
|
nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))),
|
||||||
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
|
nixManDir(canonPath(NIX_MAN_DIR)),
|
||||||
{
|
nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) {
|
||||||
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
|
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
|
||||||
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
|
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
|
||||||
|
|
||||||
caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
|
caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
|
||||||
if (caFile == "") {
|
if (caFile == "") {
|
||||||
for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
|
for (auto& fn :
|
||||||
|
{"/etc/ssl/certs/ca-certificates.crt",
|
||||||
|
"/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
|
||||||
if (pathExists(fn)) {
|
if (pathExists(fn)) {
|
||||||
caFile = fn;
|
caFile = fn;
|
||||||
break;
|
break;
|
||||||
|
|
@ -58,8 +57,7 @@ Settings::Settings()
|
||||||
auto s = getEnv("NIX_REMOTE_SYSTEMS");
|
auto s = getEnv("NIX_REMOTE_SYSTEMS");
|
||||||
if (s != "") {
|
if (s != "") {
|
||||||
Strings ss;
|
Strings ss;
|
||||||
for (auto & p : tokenizeString<Strings>(s, ":"))
|
for (auto& p : tokenizeString<Strings>(s, ":")) ss.push_back("@" + p);
|
||||||
ss.push_back("@" + p);
|
|
||||||
builders = concatStringsSep(" ", ss);
|
builders = concatStringsSep(" ", ss);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -67,11 +65,11 @@ Settings::Settings()
|
||||||
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
|
sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
|
allowedImpureHostPrefixes =
|
||||||
|
tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
|
||||||
}
|
}
|
||||||
|
|
||||||
void loadConfFile()
|
void loadConfFile() {
|
||||||
{
|
|
||||||
globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
|
globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
|
||||||
|
|
||||||
/* We only want to send overrides to the daemon, i.e. stuff from
|
/* We only want to send overrides to the daemon, i.e. stuff from
|
||||||
|
|
@ -79,27 +77,25 @@ void loadConfFile()
|
||||||
globalConfig.resetOverriden();
|
globalConfig.resetOverriden();
|
||||||
|
|
||||||
auto dirs = getConfigDirs();
|
auto dirs = getConfigDirs();
|
||||||
// Iterate over them in reverse so that the ones appearing first in the path take priority
|
// Iterate over them in reverse so that the ones appearing first in the path
|
||||||
|
// take priority
|
||||||
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
|
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
|
||||||
globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
|
globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned int Settings::getDefaultCores()
|
unsigned int Settings::getDefaultCores() {
|
||||||
{
|
|
||||||
return std::max(1U, std::thread::hardware_concurrency());
|
return std::max(1U, std::thread::hardware_concurrency());
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet Settings::getDefaultSystemFeatures()
|
StringSet Settings::getDefaultSystemFeatures() {
|
||||||
{
|
|
||||||
/* For backwards compatibility, accept some "features" that are
|
/* For backwards compatibility, accept some "features" that are
|
||||||
used in Nixpkgs to route builds to certain machines but don't
|
used in Nixpkgs to route builds to certain machines but don't
|
||||||
actually require anything special on the machines. */
|
actually require anything special on the machines. */
|
||||||
StringSet features{"nixos-test", "benchmark", "big-parallel"};
|
StringSet features{"nixos-test", "benchmark", "big-parallel"};
|
||||||
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
if (access("/dev/kvm", R_OK | W_OK) == 0)
|
if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm");
|
||||||
features.insert("kvm");
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return features;
|
return features;
|
||||||
|
|
@ -107,29 +103,38 @@ StringSet Settings::getDefaultSystemFeatures()
|
||||||
|
|
||||||
const string nixVersion = PACKAGE_VERSION;
|
const string nixVersion = PACKAGE_VERSION;
|
||||||
|
|
||||||
template<> void BaseSetting<SandboxMode>::set(const std::string & str)
|
template <>
|
||||||
{
|
void BaseSetting<SandboxMode>::set(const std::string& str) {
|
||||||
if (str == "true") value = smEnabled;
|
if (str == "true")
|
||||||
else if (str == "relaxed") value = smRelaxed;
|
value = smEnabled;
|
||||||
else if (str == "false") value = smDisabled;
|
else if (str == "relaxed")
|
||||||
else throw UsageError("option '%s' has invalid value '%s'", name, str);
|
value = smRelaxed;
|
||||||
|
else if (str == "false")
|
||||||
|
value = smDisabled;
|
||||||
|
else
|
||||||
|
throw UsageError("option '%s' has invalid value '%s'", name, str);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<> std::string BaseSetting<SandboxMode>::to_string()
|
template <>
|
||||||
{
|
std::string BaseSetting<SandboxMode>::to_string() {
|
||||||
if (value == smEnabled) return "true";
|
if (value == smEnabled)
|
||||||
else if (value == smRelaxed) return "relaxed";
|
return "true";
|
||||||
else if (value == smDisabled) return "false";
|
else if (value == smRelaxed)
|
||||||
else abort();
|
return "relaxed";
|
||||||
|
else if (value == smDisabled)
|
||||||
|
return "false";
|
||||||
|
else
|
||||||
|
abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
|
template <>
|
||||||
{
|
void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) {
|
||||||
AbstractSetting::toJSON(out);
|
AbstractSetting::toJSON(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
|
template <>
|
||||||
{
|
void BaseSetting<SandboxMode>::convertToArg(Args& args,
|
||||||
|
const std::string& category) {
|
||||||
args.mkFlag()
|
args.mkFlag()
|
||||||
.longName(name)
|
.longName(name)
|
||||||
.description("Enable sandboxing.")
|
.description("Enable sandboxing.")
|
||||||
|
|
@ -147,16 +152,15 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
|
||||||
.category(category);
|
.category(category);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MaxBuildJobsSetting::set(const std::string & str)
|
void MaxBuildJobsSetting::set(const std::string& str) {
|
||||||
{
|
if (str == "auto")
|
||||||
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
|
value = std::max(1U, std::thread::hardware_concurrency());
|
||||||
else if (!string2Int(str, value))
|
else if (!string2Int(str, value))
|
||||||
throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
|
throw UsageError(
|
||||||
|
"configuration setting '%s' should be 'auto' or an integer", name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void initPlugins() {
|
||||||
void initPlugins()
|
|
||||||
{
|
|
||||||
for (const auto& pluginFile : settings.pluginFiles.get()) {
|
for (const auto& pluginFile : settings.pluginFiles.get()) {
|
||||||
Paths pluginFiles;
|
Paths pluginFiles;
|
||||||
try {
|
try {
|
||||||
|
|
@ -164,17 +168,16 @@ void initPlugins()
|
||||||
for (const auto& ent : ents)
|
for (const auto& ent : ents)
|
||||||
pluginFiles.emplace_back(pluginFile + "/" + ent.name);
|
pluginFiles.emplace_back(pluginFile + "/" + ent.name);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo != ENOTDIR)
|
if (e.errNo != ENOTDIR) throw;
|
||||||
throw;
|
|
||||||
pluginFiles.emplace_back(pluginFile);
|
pluginFiles.emplace_back(pluginFile);
|
||||||
}
|
}
|
||||||
for (const auto& file : pluginFiles) {
|
for (const auto& file : pluginFiles) {
|
||||||
/* handle is purposefully leaked as there may be state in the
|
/* handle is purposefully leaked as there may be state in the
|
||||||
DSO needed by the action of the plugin. */
|
DSO needed by the action of the plugin. */
|
||||||
void *handle =
|
void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
|
||||||
dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
|
|
||||||
if (!handle)
|
if (!handle)
|
||||||
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
|
throw Error("could not dynamically open plugin file '%s': %s", file,
|
||||||
|
dlerror());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -184,4 +187,4 @@ void initPlugins()
|
||||||
globalConfig.warnUnknownSettings();
|
globalConfig.warnUnknownSettings();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
312
third_party/nix/src/libstore/globals.hh
vendored
312
third_party/nix/src/libstore/globals.hh
vendored
|
|
@ -1,27 +1,21 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "config.hh"
|
|
||||||
#include "util.hh"
|
|
||||||
|
|
||||||
#include <map>
|
|
||||||
#include <limits>
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
#include <limits>
|
||||||
|
#include <map>
|
||||||
|
#include "config.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
|
typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
|
||||||
|
|
||||||
struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
|
struct MaxBuildJobsSetting : public BaseSetting<unsigned int> {
|
||||||
{
|
MaxBuildJobsSetting(Config* options, unsigned int def,
|
||||||
MaxBuildJobsSetting(Config * options,
|
const std::string& name, const std::string& description,
|
||||||
unsigned int def,
|
|
||||||
const std::string & name,
|
|
||||||
const std::string & description,
|
|
||||||
const std::set<std::string>& aliases = {})
|
const std::set<std::string>& aliases = {})
|
||||||
: BaseSetting<unsigned int>(def, name, description, aliases)
|
: BaseSetting<unsigned int>(def, name, description, aliases) {
|
||||||
{
|
|
||||||
options->addSetting(this);
|
options->addSetting(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -29,13 +23,11 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
|
||||||
};
|
};
|
||||||
|
|
||||||
class Settings : public Config {
|
class Settings : public Config {
|
||||||
|
|
||||||
unsigned int getDefaultCores();
|
unsigned int getDefaultCores();
|
||||||
|
|
||||||
StringSet getDefaultSystemFeatures();
|
StringSet getDefaultSystemFeatures();
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
Settings();
|
Settings();
|
||||||
|
|
||||||
Path nixPrefix;
|
Path nixPrefix;
|
||||||
|
|
@ -69,32 +61,45 @@ public:
|
||||||
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
|
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
|
||||||
"The default Nix store to use."};
|
"The default Nix store to use."};
|
||||||
|
|
||||||
Setting<bool> keepFailed{this, false, "keep-failed",
|
Setting<bool> keepFailed{
|
||||||
|
this, false, "keep-failed",
|
||||||
"Whether to keep temporary directories of failed builds."};
|
"Whether to keep temporary directories of failed builds."};
|
||||||
|
|
||||||
Setting<bool> keepGoing{this, false, "keep-going",
|
Setting<bool> keepGoing{
|
||||||
|
this, false, "keep-going",
|
||||||
"Whether to keep building derivations when another build fails."};
|
"Whether to keep building derivations when another build fails."};
|
||||||
|
|
||||||
Setting<bool> tryFallback{this, false, "fallback",
|
Setting<bool> tryFallback{
|
||||||
|
this,
|
||||||
|
false,
|
||||||
|
"fallback",
|
||||||
"Whether to fall back to building when substitution fails.",
|
"Whether to fall back to building when substitution fails.",
|
||||||
{"build-fallback"}};
|
{"build-fallback"}};
|
||||||
|
|
||||||
/* Whether to show build log output in real time. */
|
/* Whether to show build log output in real time. */
|
||||||
bool verboseBuild = true;
|
bool verboseBuild = true;
|
||||||
|
|
||||||
Setting<size_t> logLines{this, 10, "log-lines",
|
Setting<size_t> logLines{
|
||||||
|
this, 10, "log-lines",
|
||||||
"If verbose-build is false, the number of lines of the tail of "
|
"If verbose-build is false, the number of lines of the tail of "
|
||||||
"the log to show if a build fails."};
|
"the log to show if a build fails."};
|
||||||
|
|
||||||
MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
|
MaxBuildJobsSetting maxBuildJobs{this,
|
||||||
"Maximum number of parallel build jobs. \"auto\" means use number of cores.",
|
1,
|
||||||
|
"max-jobs",
|
||||||
|
"Maximum number of parallel build jobs. "
|
||||||
|
"\"auto\" means use number of cores.",
|
||||||
{"build-max-jobs"}};
|
{"build-max-jobs"}};
|
||||||
|
|
||||||
Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
|
Setting<unsigned int> buildCores{
|
||||||
|
this,
|
||||||
|
getDefaultCores(),
|
||||||
|
"cores",
|
||||||
"Number of CPU cores to utilize in parallel within a build, "
|
"Number of CPU cores to utilize in parallel within a build, "
|
||||||
"i.e. by passing this number to Make via '-j'. 0 means that the "
|
"i.e. by passing this number to Make via '-j'. 0 means that the "
|
||||||
"number of actual CPU cores on the local host ought to be "
|
"number of actual CPU cores on the local host ought to be "
|
||||||
"auto-detected.", {"build-cores"}};
|
"auto-detected.",
|
||||||
|
{"build-cores"}};
|
||||||
|
|
||||||
/* Read-only mode. Don't copy stuff to the store, don't change
|
/* Read-only mode. Don't copy stuff to the store, don't change
|
||||||
the database. */
|
the database. */
|
||||||
|
|
@ -103,28 +108,41 @@ public:
|
||||||
Setting<std::string> thisSystem{this, SYSTEM, "system",
|
Setting<std::string> thisSystem{this, SYSTEM, "system",
|
||||||
"The canonical Nix system name."};
|
"The canonical Nix system name."};
|
||||||
|
|
||||||
Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
|
Setting<time_t> maxSilentTime{
|
||||||
|
this,
|
||||||
|
0,
|
||||||
|
"max-silent-time",
|
||||||
"The maximum time in seconds that a builer can go without "
|
"The maximum time in seconds that a builer can go without "
|
||||||
"producing any output on stdout/stderr before it is killed. "
|
"producing any output on stdout/stderr before it is killed. "
|
||||||
"0 means infinity.",
|
"0 means infinity.",
|
||||||
{"build-max-silent-time"}};
|
{"build-max-silent-time"}};
|
||||||
|
|
||||||
Setting<time_t> buildTimeout{this, 0, "timeout",
|
Setting<time_t> buildTimeout{
|
||||||
|
this,
|
||||||
|
0,
|
||||||
|
"timeout",
|
||||||
"The maximum duration in seconds that a builder can run. "
|
"The maximum duration in seconds that a builder can run. "
|
||||||
"0 means infinity.", {"build-timeout"}};
|
"0 means infinity.",
|
||||||
|
{"build-timeout"}};
|
||||||
|
|
||||||
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
|
PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote",
|
||||||
"The path of the helper program that executes builds to remote machines."};
|
"build-hook",
|
||||||
|
"The path of the helper program that executes builds "
|
||||||
|
"to remote machines."};
|
||||||
|
|
||||||
Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
|
Setting<std::string> builders{this, "@" + nixConfDir + "/machines",
|
||||||
"A semicolon-separated list of build machines, in the format of nix.machines."};
|
"builders",
|
||||||
|
"A semicolon-separated list of build machines, "
|
||||||
|
"in the format of nix.machines."};
|
||||||
|
|
||||||
Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
|
Setting<bool> buildersUseSubstitutes{
|
||||||
|
this, false, "builders-use-substitutes",
|
||||||
"Whether build machines should use their own substitutes for obtaining "
|
"Whether build machines should use their own substitutes for obtaining "
|
||||||
"build dependencies if possible, rather than waiting for this host to "
|
"build dependencies if possible, rather than waiting for this host to "
|
||||||
"upload them."};
|
"upload them."};
|
||||||
|
|
||||||
Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
|
Setting<off_t> reservedSize{
|
||||||
|
this, 8 * 1024 * 1024, "gc-reserved-space",
|
||||||
"Amount of reserved disk space for the garbage collector."};
|
"Amount of reserved disk space for the garbage collector."};
|
||||||
|
|
||||||
Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
|
Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
|
||||||
|
|
@ -133,29 +151,43 @@ public:
|
||||||
Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
|
Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
|
||||||
"Whether SQLite should use WAL mode."};
|
"Whether SQLite should use WAL mode."};
|
||||||
|
|
||||||
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
|
Setting<bool> syncBeforeRegistering{
|
||||||
|
this, false, "sync-before-registering",
|
||||||
"Whether to call sync() before registering a path as valid."};
|
"Whether to call sync() before registering a path as valid."};
|
||||||
|
|
||||||
Setting<bool> useSubstitutes{this, true, "substitute",
|
Setting<bool> useSubstitutes{this,
|
||||||
|
true,
|
||||||
|
"substitute",
|
||||||
"Whether to use substitutes.",
|
"Whether to use substitutes.",
|
||||||
{"build-use-substitutes"}};
|
{"build-use-substitutes"}};
|
||||||
|
|
||||||
Setting<std::string> buildUsersGroup{this, "", "build-users-group",
|
Setting<std::string> buildUsersGroup{
|
||||||
|
this, "", "build-users-group",
|
||||||
"The Unix group that contains the build users."};
|
"The Unix group that contains the build users."};
|
||||||
|
|
||||||
Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
|
Setting<bool> impersonateLinux26{
|
||||||
|
this,
|
||||||
|
false,
|
||||||
|
"impersonate-linux-26",
|
||||||
"Whether to impersonate a Linux 2.6 machine on newer kernels.",
|
"Whether to impersonate a Linux 2.6 machine on newer kernels.",
|
||||||
{"build-impersonate-linux-26"}};
|
{"build-impersonate-linux-26"}};
|
||||||
|
|
||||||
Setting<bool> keepLog{this, true, "keep-build-log",
|
Setting<bool> keepLog{this,
|
||||||
|
true,
|
||||||
|
"keep-build-log",
|
||||||
"Whether to store build logs.",
|
"Whether to store build logs.",
|
||||||
{"build-keep-log"}};
|
{"build-keep-log"}};
|
||||||
|
|
||||||
Setting<bool> compressLog{this, true, "compress-build-log",
|
Setting<bool> compressLog{this,
|
||||||
|
true,
|
||||||
|
"compress-build-log",
|
||||||
"Whether to compress logs.",
|
"Whether to compress logs.",
|
||||||
{"build-compress-log"}};
|
{"build-compress-log"}};
|
||||||
|
|
||||||
Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
|
Setting<unsigned long> maxLogSize{
|
||||||
|
this,
|
||||||
|
0,
|
||||||
|
"max-build-log-size",
|
||||||
"Maximum number of bytes a builder can write to stdout/stderr "
|
"Maximum number of bytes a builder can write to stdout/stderr "
|
||||||
"before being killed (0 means no limit).",
|
"before being killed (0 means no limit).",
|
||||||
{"build-max-log-size"}};
|
{"build-max-log-size"}};
|
||||||
|
|
@ -165,25 +197,37 @@ public:
|
||||||
stderr. Hack to prevent Hydra logs from being polluted. */
|
stderr. Hack to prevent Hydra logs from being polluted. */
|
||||||
bool printRepeatedBuilds = true;
|
bool printRepeatedBuilds = true;
|
||||||
|
|
||||||
Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
|
Setting<unsigned int> pollInterval{
|
||||||
|
this, 5, "build-poll-interval",
|
||||||
"How often (in seconds) to poll for locks."};
|
"How often (in seconds) to poll for locks."};
|
||||||
|
|
||||||
Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
|
Setting<bool> checkRootReachability{
|
||||||
|
this, false, "gc-check-reachability",
|
||||||
"Whether to check if new GC roots can in fact be found by the "
|
"Whether to check if new GC roots can in fact be found by the "
|
||||||
"garbage collector."};
|
"garbage collector."};
|
||||||
|
|
||||||
Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
|
Setting<bool> gcKeepOutputs{
|
||||||
|
this,
|
||||||
|
false,
|
||||||
|
"keep-outputs",
|
||||||
"Whether the garbage collector should keep outputs of live derivations.",
|
"Whether the garbage collector should keep outputs of live derivations.",
|
||||||
{"gc-keep-outputs"}};
|
{"gc-keep-outputs"}};
|
||||||
|
|
||||||
Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
|
Setting<bool> gcKeepDerivations{
|
||||||
|
this,
|
||||||
|
true,
|
||||||
|
"keep-derivations",
|
||||||
"Whether the garbage collector should keep derivers of live paths.",
|
"Whether the garbage collector should keep derivers of live paths.",
|
||||||
{"gc-keep-derivations"}};
|
{"gc-keep-derivations"}};
|
||||||
|
|
||||||
Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
|
Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
|
||||||
"Whether to automatically replace files with identical contents with hard links."};
|
"Whether to automatically replace files with "
|
||||||
|
"identical contents with hard links."};
|
||||||
|
|
||||||
Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
|
Setting<bool> envKeepDerivations{
|
||||||
|
this,
|
||||||
|
false,
|
||||||
|
"keep-env-derivations",
|
||||||
"Whether to add derivations as a dependency of user environments "
|
"Whether to add derivations as a dependency of user environments "
|
||||||
"(to prevent them from being GCed).",
|
"(to prevent them from being GCed).",
|
||||||
{"env-keep-derivations"}};
|
{"env-keep-derivations"}};
|
||||||
|
|
@ -192,170 +236,240 @@ public:
|
||||||
bool lockCPU;
|
bool lockCPU;
|
||||||
|
|
||||||
/* Whether to show a stack trace if Nix evaluation fails. */
|
/* Whether to show a stack trace if Nix evaluation fails. */
|
||||||
Setting<bool> showTrace{this, false, "show-trace",
|
Setting<bool> showTrace{
|
||||||
|
this, false, "show-trace",
|
||||||
"Whether to show a stack trace on evaluation errors."};
|
"Whether to show a stack trace on evaluation errors."};
|
||||||
|
|
||||||
Setting<SandboxMode> sandboxMode{this,
|
Setting<SandboxMode> sandboxMode {
|
||||||
|
this,
|
||||||
#if __linux__
|
#if __linux__
|
||||||
smEnabled
|
smEnabled
|
||||||
#else
|
#else
|
||||||
smDisabled
|
smDisabled
|
||||||
#endif
|
#endif
|
||||||
, "sandbox",
|
,
|
||||||
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
|
"sandbox",
|
||||||
{"build-use-chroot", "build-use-sandbox"}};
|
"Whether to enable sandboxed builds. Can be \"true\", \"false\" or "
|
||||||
|
"\"relaxed\".",
|
||||||
|
{
|
||||||
|
"build-use-chroot", "build-use-sandbox"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
|
Setting<PathSet> sandboxPaths{
|
||||||
|
this,
|
||||||
|
{},
|
||||||
|
"sandbox-paths",
|
||||||
"The paths to make available inside the build sandbox.",
|
"The paths to make available inside the build sandbox.",
|
||||||
{"build-chroot-dirs", "build-sandbox-paths"}};
|
{"build-chroot-dirs", "build-sandbox-paths"}};
|
||||||
|
|
||||||
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
|
Setting<bool> sandboxFallback{
|
||||||
|
this, true, "sandbox-fallback",
|
||||||
"Whether to disable sandboxing when the kernel doesn't allow it."};
|
"Whether to disable sandboxing when the kernel doesn't allow it."};
|
||||||
|
|
||||||
Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
|
Setting<PathSet> extraSandboxPaths{
|
||||||
|
this,
|
||||||
|
{},
|
||||||
|
"extra-sandbox-paths",
|
||||||
"Additional paths to make available inside the build sandbox.",
|
"Additional paths to make available inside the build sandbox.",
|
||||||
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
|
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
|
||||||
|
|
||||||
Setting<size_t> buildRepeat{this, 0, "repeat",
|
Setting<size_t> buildRepeat{
|
||||||
|
this,
|
||||||
|
0,
|
||||||
|
"repeat",
|
||||||
"The number of times to repeat a build in order to verify determinism.",
|
"The number of times to repeat a build in order to verify determinism.",
|
||||||
{"build-repeat"}};
|
{"build-repeat"}};
|
||||||
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
|
Setting<std::string> sandboxShmSize{
|
||||||
|
this, "50%", "sandbox-dev-shm-size",
|
||||||
"The size of /dev/shm in the build sandbox."};
|
"The size of /dev/shm in the build sandbox."};
|
||||||
|
|
||||||
Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
|
Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
|
||||||
"The build directory inside the sandbox."};
|
"The build directory inside the sandbox."};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
|
Setting<PathSet> allowedImpureHostPrefixes{
|
||||||
"Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
|
this,
|
||||||
|
{},
|
||||||
|
"allowed-impure-host-deps",
|
||||||
|
"Which prefixes to allow derivations to ask for access to (primarily for "
|
||||||
|
"Darwin)."};
|
||||||
|
|
||||||
#if __APPLE__
|
#if __APPLE__
|
||||||
Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations",
|
Setting<bool> darwinLogSandboxViolations{
|
||||||
|
this, false, "darwin-log-sandbox-violations",
|
||||||
"Whether to log Darwin sandbox access violations to the system log."};
|
"Whether to log Darwin sandbox access violations to the system log."};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
Setting<bool> runDiffHook{this, false, "run-diff-hook",
|
Setting<bool> runDiffHook{
|
||||||
|
this, false, "run-diff-hook",
|
||||||
"Whether to run the program specified by the diff-hook setting "
|
"Whether to run the program specified by the diff-hook setting "
|
||||||
"repeated builds produce a different result. Typically used to "
|
"repeated builds produce a different result. Typically used to "
|
||||||
"plug in diffoscope."};
|
"plug in diffoscope."};
|
||||||
|
|
||||||
PathSetting diffHook{this, true, "", "diff-hook",
|
PathSetting diffHook{
|
||||||
|
this, true, "", "diff-hook",
|
||||||
"A program that prints out the differences between the two paths "
|
"A program that prints out the differences between the two paths "
|
||||||
"specified on its command line."};
|
"specified on its command line."};
|
||||||
|
|
||||||
Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
|
Setting<bool> enforceDeterminism{
|
||||||
|
this, true, "enforce-determinism",
|
||||||
"Whether to fail if repeated builds produce different output."};
|
"Whether to fail if repeated builds produce different output."};
|
||||||
|
|
||||||
Setting<Strings> trustedPublicKeys{this,
|
Setting<Strings> trustedPublicKeys{
|
||||||
|
this,
|
||||||
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
|
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
|
||||||
"trusted-public-keys",
|
"trusted-public-keys",
|
||||||
"Trusted public keys for secure substitution.",
|
"Trusted public keys for secure substitution.",
|
||||||
{"binary-cache-public-keys"}};
|
{"binary-cache-public-keys"}};
|
||||||
|
|
||||||
Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
|
Setting<Strings> secretKeyFiles{
|
||||||
|
this,
|
||||||
|
{},
|
||||||
|
"secret-key-files",
|
||||||
"Secret keys with which to sign local builds."};
|
"Secret keys with which to sign local builds."};
|
||||||
|
|
||||||
Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
|
Setting<unsigned int> tarballTtl{
|
||||||
|
this, 60 * 60, "tarball-ttl",
|
||||||
"How long downloaded files are considered up-to-date."};
|
"How long downloaded files are considered up-to-date."};
|
||||||
|
|
||||||
Setting<bool> requireSigs{this, true, "require-sigs",
|
Setting<bool> requireSigs{
|
||||||
|
this, true, "require-sigs",
|
||||||
"Whether to check that any non-content-addressed path added to the "
|
"Whether to check that any non-content-addressed path added to the "
|
||||||
"Nix store has a valid signature (that is, one signed using a key "
|
"Nix store has a valid signature (that is, one signed using a key "
|
||||||
"listed in 'trusted-public-keys'."};
|
"listed in 'trusted-public-keys'."};
|
||||||
|
|
||||||
Setting<StringSet> extraPlatforms{this,
|
Setting<StringSet> extraPlatforms{
|
||||||
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
|
this,
|
||||||
|
std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"}
|
||||||
|
: StringSet{},
|
||||||
"extra-platforms",
|
"extra-platforms",
|
||||||
"Additional platforms that can be built on the local system. "
|
"Additional platforms that can be built on the local system. "
|
||||||
"These may be supported natively (e.g. armv7 on some aarch64 CPUs "
|
"These may be supported natively (e.g. armv7 on some aarch64 CPUs "
|
||||||
"or using hacks like qemu-user."};
|
"or using hacks like qemu-user."};
|
||||||
|
|
||||||
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
|
Setting<StringSet> systemFeatures{
|
||||||
"system-features",
|
this, getDefaultSystemFeatures(), "system-features",
|
||||||
"Optional features that this system implements (like \"kvm\")."};
|
"Optional features that this system implements (like \"kvm\")."};
|
||||||
|
|
||||||
Setting<Strings> substituters{this,
|
Setting<Strings> substituters{
|
||||||
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
|
this,
|
||||||
|
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"}
|
||||||
|
: Strings(),
|
||||||
"substituters",
|
"substituters",
|
||||||
"The URIs of substituters (such as https://cache.nixos.org/).",
|
"The URIs of substituters (such as https://cache.nixos.org/).",
|
||||||
{"binary-caches"}};
|
{"binary-caches"}};
|
||||||
|
|
||||||
// FIXME: provide a way to add to option values.
|
// FIXME: provide a way to add to option values.
|
||||||
Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
|
Setting<Strings> extraSubstituters{this,
|
||||||
|
{},
|
||||||
|
"extra-substituters",
|
||||||
"Additional URIs of substituters.",
|
"Additional URIs of substituters.",
|
||||||
{"extra-binary-caches"}};
|
{"extra-binary-caches"}};
|
||||||
|
|
||||||
Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
|
Setting<StringSet> trustedSubstituters{
|
||||||
"Disabled substituters that may be enabled via the substituters option by untrusted users.",
|
this,
|
||||||
|
{},
|
||||||
|
"trusted-substituters",
|
||||||
|
"Disabled substituters that may be enabled via the substituters option "
|
||||||
|
"by untrusted users.",
|
||||||
{"trusted-binary-caches"}};
|
{"trusted-binary-caches"}};
|
||||||
|
|
||||||
Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
|
Setting<Strings> trustedUsers{this,
|
||||||
"Which users or groups are trusted to ask the daemon to do unsafe things."};
|
{"root"},
|
||||||
|
"trusted-users",
|
||||||
|
"Which users or groups are trusted to ask the "
|
||||||
|
"daemon to do unsafe things."};
|
||||||
|
|
||||||
Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
|
Setting<unsigned int> ttlNegativeNarInfoCache{
|
||||||
"The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
|
this, 3600, "narinfo-cache-negative-ttl",
|
||||||
|
"The TTL in seconds for negative lookups in the disk cache i.e binary "
|
||||||
|
"cache lookups that "
|
||||||
"return an invalid path result"};
|
"return an invalid path result"};
|
||||||
|
|
||||||
Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
|
Setting<unsigned int> ttlPositiveNarInfoCache{
|
||||||
"The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
|
this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
|
||||||
|
"The TTL in seconds for positive lookups in the disk cache i.e binary "
|
||||||
|
"cache lookups that "
|
||||||
"return a valid path result."};
|
"return a valid path result."};
|
||||||
|
|
||||||
/* ?Who we trust to use the daemon in safe ways */
|
/* ?Who we trust to use the daemon in safe ways */
|
||||||
Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
|
Setting<Strings> allowedUsers{
|
||||||
|
this,
|
||||||
|
{"*"},
|
||||||
|
"allowed-users",
|
||||||
"Which users or groups are allowed to connect to the daemon."};
|
"Which users or groups are allowed to connect to the daemon."};
|
||||||
|
|
||||||
Setting<bool> printMissing{this, true, "print-missing",
|
Setting<bool> printMissing{
|
||||||
|
this, true, "print-missing",
|
||||||
"Whether to print what paths need to be built or downloaded."};
|
"Whether to print what paths need to be built or downloaded."};
|
||||||
|
|
||||||
Setting<std::string> preBuildHook{this,
|
Setting<std::string> preBuildHook {
|
||||||
|
this,
|
||||||
#if __APPLE__
|
#if __APPLE__
|
||||||
nixLibexecDir + "/nix/resolve-system-dependencies",
|
nixLibexecDir + "/nix/resolve-system-dependencies",
|
||||||
#else
|
#else
|
||||||
"",
|
"",
|
||||||
#endif
|
#endif
|
||||||
"pre-build-hook",
|
"pre-build-hook",
|
||||||
"A program to run just before a build to set derivation-specific build settings."};
|
"A program to run just before a build to set derivation-specific build "
|
||||||
|
"settings."
|
||||||
|
};
|
||||||
|
|
||||||
Setting<std::string> postBuildHook{this, "", "post-build-hook",
|
Setting<std::string> postBuildHook{
|
||||||
|
this, "", "post-build-hook",
|
||||||
"A program to run just after each successful build."};
|
"A program to run just after each successful build."};
|
||||||
|
|
||||||
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
|
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"),
|
||||||
"Path to the netrc file used to obtain usernames/passwords for downloads."};
|
"netrc-file",
|
||||||
|
"Path to the netrc file used to obtain "
|
||||||
|
"usernames/passwords for downloads."};
|
||||||
|
|
||||||
/* Path to the SSL CA file used */
|
/* Path to the SSL CA file used */
|
||||||
Path caFile;
|
Path caFile;
|
||||||
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
Setting<bool> filterSyscalls{this, true, "filter-syscalls",
|
Setting<bool> filterSyscalls{
|
||||||
|
this, true, "filter-syscalls",
|
||||||
"Whether to prevent certain dangerous system calls, such as "
|
"Whether to prevent certain dangerous system calls, such as "
|
||||||
"creation of setuid/setgid files or adding ACLs or extended "
|
"creation of setuid/setgid files or adding ACLs or extended "
|
||||||
"attributes. Only disable this if you're aware of the "
|
"attributes. Only disable this if you're aware of the "
|
||||||
"security implications."};
|
"security implications."};
|
||||||
|
|
||||||
Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
|
Setting<bool> allowNewPrivileges{
|
||||||
|
this, false, "allow-new-privileges",
|
||||||
"Whether builders can acquire new privileges by calling programs with "
|
"Whether builders can acquire new privileges by calling programs with "
|
||||||
"setuid/setgid bits or with file capabilities."};
|
"setuid/setgid bits or with file capabilities."};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
|
Setting<Strings> hashedMirrors{
|
||||||
|
this,
|
||||||
|
{"http://tarballs.nixos.org/"},
|
||||||
|
"hashed-mirrors",
|
||||||
"A list of servers used by builtins.fetchurl to fetch files by hash."};
|
"A list of servers used by builtins.fetchurl to fetch files by hash."};
|
||||||
|
|
||||||
Setting<uint64_t> minFree{this, 0, "min-free",
|
Setting<uint64_t> minFree{this, 0, "min-free",
|
||||||
"Automatically run the garbage collector when free disk space drops below the specified amount."};
|
"Automatically run the garbage collector when free "
|
||||||
|
"disk space drops below the specified amount."};
|
||||||
|
|
||||||
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
|
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(),
|
||||||
"Stop deleting garbage when free disk space is above the specified amount."};
|
"max-free",
|
||||||
|
"Stop deleting garbage when free disk space is "
|
||||||
|
"above the specified amount."};
|
||||||
|
|
||||||
Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
|
Setting<uint64_t> minFreeCheckInterval{
|
||||||
|
this, 5, "min-free-check-interval",
|
||||||
"Number of seconds between checking free disk space."};
|
"Number of seconds between checking free disk space."};
|
||||||
|
|
||||||
Setting<Paths> pluginFiles{this, {}, "plugin-files",
|
Setting<Paths> pluginFiles{
|
||||||
|
this,
|
||||||
|
{},
|
||||||
|
"plugin-files",
|
||||||
"Plugins to dynamically load at nix initialization time."};
|
"Plugins to dynamically load at nix initialization time."};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
// FIXME: don't use a global variable.
|
// FIXME: don't use a global variable.
|
||||||
extern Settings settings;
|
extern Settings settings;
|
||||||
|
|
||||||
|
|
@ -367,4 +481,4 @@ void loadConfFile();
|
||||||
|
|
||||||
extern const string nixVersion;
|
extern const string nixVersion;
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -7,14 +7,11 @@ namespace nix {
|
||||||
|
|
||||||
MakeError(UploadToHTTP, Error);
|
MakeError(UploadToHTTP, Error);
|
||||||
|
|
||||||
class HttpBinaryCacheStore : public BinaryCacheStore
|
class HttpBinaryCacheStore : public BinaryCacheStore {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
Path cacheUri;
|
Path cacheUri;
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
bool enabled = true;
|
bool enabled = true;
|
||||||
std::chrono::steady_clock::time_point disabledUntil;
|
std::chrono::steady_clock::time_point disabledUntil;
|
||||||
};
|
};
|
||||||
|
|
@ -22,25 +19,16 @@ private:
|
||||||
Sync<State> _state;
|
Sync<State> _state;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
HttpBinaryCacheStore(const Params& params, const Path& _cacheUri)
|
||||||
HttpBinaryCacheStore(
|
: BinaryCacheStore(params), cacheUri(_cacheUri) {
|
||||||
const Params & params, const Path & _cacheUri)
|
if (cacheUri.back() == '/') cacheUri.pop_back();
|
||||||
: BinaryCacheStore(params)
|
|
||||||
, cacheUri(_cacheUri)
|
|
||||||
{
|
|
||||||
if (cacheUri.back() == '/')
|
|
||||||
cacheUri.pop_back();
|
|
||||||
|
|
||||||
diskCache = getNarInfoDiskCache();
|
diskCache = getNarInfoDiskCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string getUri() override
|
std::string getUri() override { return cacheUri; }
|
||||||
{
|
|
||||||
return cacheUri;
|
|
||||||
}
|
|
||||||
|
|
||||||
void init() override
|
void init() override {
|
||||||
{
|
|
||||||
// FIXME: do this lazily?
|
// FIXME: do this lazily?
|
||||||
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
|
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -53,20 +41,18 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
void maybeDisable() {
|
||||||
void maybeDisable()
|
|
||||||
{
|
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
if (state->enabled && settings.tryFallback) {
|
if (state->enabled && settings.tryFallback) {
|
||||||
int t = 60;
|
int t = 60;
|
||||||
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
|
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
|
||||||
state->enabled = false;
|
state->enabled = false;
|
||||||
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
|
state->disabledUntil =
|
||||||
|
std::chrono::steady_clock::now() + std::chrono::seconds(t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void checkEnabled()
|
void checkEnabled() {
|
||||||
{
|
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
if (state->enabled) return;
|
if (state->enabled) return;
|
||||||
if (std::chrono::steady_clock::now() > state->disabledUntil) {
|
if (std::chrono::steady_clock::now() > state->disabledUntil) {
|
||||||
|
|
@ -77,8 +63,7 @@ protected:
|
||||||
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
|
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
|
||||||
}
|
}
|
||||||
|
|
||||||
bool fileExists(const std::string & path) override
|
bool fileExists(const std::string& path) override {
|
||||||
{
|
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -96,55 +81,55 @@ protected:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void upsertFile(const std::string & path,
|
void upsertFile(const std::string& path, const std::string& data,
|
||||||
const std::string & data,
|
const std::string& mimeType) override {
|
||||||
const std::string & mimeType) override
|
|
||||||
{
|
|
||||||
auto req = DownloadRequest(cacheUri + "/" + path);
|
auto req = DownloadRequest(cacheUri + "/" + path);
|
||||||
req.data = std::make_shared<string>(data); // FIXME: inefficient
|
req.data = std::make_shared<string>(data); // FIXME: inefficient
|
||||||
req.mimeType = mimeType;
|
req.mimeType = mimeType;
|
||||||
try {
|
try {
|
||||||
getDownloader()->download(req);
|
getDownloader()->download(req);
|
||||||
} catch (DownloadError& e) {
|
} catch (DownloadError& e) {
|
||||||
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
|
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s",
|
||||||
|
cacheUri, e.msg());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
DownloadRequest makeRequest(const std::string & path)
|
DownloadRequest makeRequest(const std::string& path) {
|
||||||
{
|
|
||||||
DownloadRequest request(cacheUri + "/" + path);
|
DownloadRequest request(cacheUri + "/" + path);
|
||||||
return request;
|
return request;
|
||||||
}
|
}
|
||||||
|
|
||||||
void getFile(const std::string & path, Sink & sink) override
|
void getFile(const std::string& path, Sink& sink) override {
|
||||||
{
|
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
auto request(makeRequest(path));
|
auto request(makeRequest(path));
|
||||||
try {
|
try {
|
||||||
getDownloader()->download(std::move(request), sink);
|
getDownloader()->download(std::move(request), sink);
|
||||||
} catch (DownloadError& e) {
|
} catch (DownloadError& e) {
|
||||||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
throw NoSuchBinaryCacheFile(
|
||||||
|
"file '%s' does not exist in binary cache '%s'", path, getUri());
|
||||||
maybeDisable();
|
maybeDisable();
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void getFile(const std::string & path,
|
void getFile(
|
||||||
Callback<std::shared_ptr<std::string>> callback) noexcept override
|
const std::string& path,
|
||||||
{
|
Callback<std::shared_ptr<std::string>> callback) noexcept override {
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
|
|
||||||
auto request(makeRequest(path));
|
auto request(makeRequest(path));
|
||||||
|
|
||||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
auto callbackPtr =
|
||||||
|
std::make_shared<decltype(callback)>(std::move(callback));
|
||||||
|
|
||||||
getDownloader()->enqueueDownload(request,
|
getDownloader()->enqueueDownload(
|
||||||
{[callbackPtr, this](std::future<DownloadResult> result) {
|
request, {[callbackPtr, this](std::future<DownloadResult> result) {
|
||||||
try {
|
try {
|
||||||
(*callbackPtr)(result.get().data);
|
(*callbackPtr)(result.get().data);
|
||||||
} catch (DownloadError& e) {
|
} catch (DownloadError& e) {
|
||||||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
if (e.error == Downloader::NotFound ||
|
||||||
|
e.error == Downloader::Forbidden)
|
||||||
return (*callbackPtr)(std::shared_ptr<std::string>());
|
return (*callbackPtr)(std::shared_ptr<std::string>());
|
||||||
maybeDisable();
|
maybeDisable();
|
||||||
callbackPtr->rethrow();
|
callbackPtr->rethrow();
|
||||||
|
|
@ -153,21 +138,19 @@ protected:
|
||||||
}
|
}
|
||||||
}});
|
}});
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore(
|
||||||
const std::string & uri, const Store::Params & params)
|
[](const std::string& uri,
|
||||||
-> std::shared_ptr<Store>
|
const Store::Params& params) -> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (std::string(uri, 0, 7) != "http://" &&
|
if (std::string(uri, 0, 7) != "http://" &&
|
||||||
std::string(uri, 0, 8) != "https://" &&
|
std::string(uri, 0, 8) != "https://" &&
|
||||||
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
|
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" ||
|
||||||
) return 0;
|
std::string(uri, 0, 7) != "file://"))
|
||||||
|
return 0;
|
||||||
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
|
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
|
||||||
store->init();
|
store->init();
|
||||||
return store;
|
return store;
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
||||||
|
|
|
||||||
205
third_party/nix/src/libstore/legacy-ssh-store.cc
vendored
205
third_party/nix/src/libstore/legacy-ssh-store.cc
vendored
|
|
@ -1,29 +1,34 @@
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
#include "derivations.hh"
|
||||||
#include "pool.hh"
|
#include "pool.hh"
|
||||||
#include "remote-store.hh"
|
#include "remote-store.hh"
|
||||||
#include "serve-protocol.hh"
|
#include "serve-protocol.hh"
|
||||||
|
#include "ssh.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "worker-protocol.hh"
|
#include "worker-protocol.hh"
|
||||||
#include "ssh.hh"
|
|
||||||
#include "derivations.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static std::string uriScheme = "ssh://";
|
static std::string uriScheme = "ssh://";
|
||||||
|
|
||||||
struct LegacySSHStore : public Store
|
struct LegacySSHStore : public Store {
|
||||||
{
|
const Setting<int> maxConnections{
|
||||||
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
|
this, 1, "max-connections",
|
||||||
|
"maximum number of concurrent SSH connections"};
|
||||||
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
|
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
|
||||||
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
|
const Setting<bool> compress{this, false, "compress",
|
||||||
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
|
"whether to compress the connection"};
|
||||||
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
|
const Setting<Path> remoteProgram{
|
||||||
|
this, "nix-store", "remote-program",
|
||||||
|
"path to the nix-store executable on the remote system"};
|
||||||
|
const Setting<std::string> remoteStore{
|
||||||
|
this, "", "remote-store", "URI of the store on the remote system"};
|
||||||
|
|
||||||
// Hack for getting remote build log output.
|
// Hack for getting remote build log output.
|
||||||
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
const Setting<int> logFD{
|
||||||
|
this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||||
|
|
||||||
struct Connection
|
struct Connection {
|
||||||
{
|
|
||||||
std::unique_ptr<SSHMaster::Connection> sshConn;
|
std::unique_ptr<SSHMaster::Connection> sshConn;
|
||||||
FdSink to;
|
FdSink to;
|
||||||
FdSource from;
|
FdSource from;
|
||||||
|
|
@ -38,29 +43,23 @@ struct LegacySSHStore : public Store
|
||||||
SSHMaster master;
|
SSHMaster master;
|
||||||
|
|
||||||
LegacySSHStore(const string& host, const Params& params)
|
LegacySSHStore(const string& host, const Params& params)
|
||||||
: Store(params)
|
: Store(params),
|
||||||
, host(host)
|
host(host),
|
||||||
, connections(make_ref<Pool<Connection>>(
|
connections(make_ref<Pool<Connection>>(
|
||||||
std::max(1, (int)maxConnections),
|
std::max(1, (int)maxConnections),
|
||||||
[this]() { return openConnection(); },
|
[this]() { return openConnection(); },
|
||||||
[](const ref<Connection> & r) { return r->good; }
|
[](const ref<Connection>& r) { return r->good; })),
|
||||||
))
|
master(host, sshKey,
|
||||||
, master(
|
|
||||||
host,
|
|
||||||
sshKey,
|
|
||||||
// Use SSH master only if using more than 1 connection.
|
// Use SSH master only if using more than 1 connection.
|
||||||
connections->capacity() > 1,
|
connections->capacity() > 1, compress, logFD) {}
|
||||||
compress,
|
|
||||||
logFD)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
ref<Connection> openConnection()
|
ref<Connection> openConnection() {
|
||||||
{
|
|
||||||
auto conn = make_ref<Connection>();
|
auto conn = make_ref<Connection>();
|
||||||
conn->sshConn = master.startCommand(
|
conn->sshConn = master.startCommand(
|
||||||
fmt("%s --serve --write", remoteProgram)
|
fmt("%s --serve --write", remoteProgram) +
|
||||||
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
|
(remoteStore.get() == ""
|
||||||
|
? ""
|
||||||
|
: " --store " + shellEscape(remoteStore.get())));
|
||||||
conn->to = FdSink(conn->sshConn->in.get());
|
conn->to = FdSink(conn->sshConn->in.get());
|
||||||
conn->from = FdSource(conn->sshConn->out.get());
|
conn->from = FdSource(conn->sshConn->out.get());
|
||||||
|
|
||||||
|
|
@ -73,7 +72,8 @@ struct LegacySSHStore : public Store
|
||||||
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
|
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
|
||||||
conn->remoteVersion = readInt(conn->from);
|
conn->remoteVersion = readInt(conn->from);
|
||||||
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
|
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
|
||||||
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
|
throw Error("unsupported 'nix-store --serve' protocol version on '%s'",
|
||||||
|
host);
|
||||||
|
|
||||||
} catch (EndOfFile& e) {
|
} catch (EndOfFile& e) {
|
||||||
throw Error("cannot connect to '%1%'", host);
|
throw Error("cannot connect to '%1%'", host);
|
||||||
|
|
@ -82,14 +82,11 @@ struct LegacySSHStore : public Store
|
||||||
return conn;
|
return conn;
|
||||||
};
|
};
|
||||||
|
|
||||||
string getUri() override
|
string getUri() override { return uriScheme + host; }
|
||||||
{
|
|
||||||
return uriScheme + host;
|
|
||||||
}
|
|
||||||
|
|
||||||
void queryPathInfoUncached(const Path & path,
|
void queryPathInfoUncached(
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override
|
const Path& path,
|
||||||
{
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override {
|
||||||
try {
|
try {
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
|
@ -120,30 +117,23 @@ struct LegacySSHStore : public Store
|
||||||
assert(s == "");
|
assert(s == "");
|
||||||
|
|
||||||
callback(std::move(info));
|
callback(std::move(info));
|
||||||
} catch (...) { callback.rethrow(); }
|
} catch (...) {
|
||||||
|
callback.rethrow();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void addToStore(const ValidPathInfo & info, Source & source,
|
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor) override
|
std::shared_ptr<FSAccessor> accessor) override {
|
||||||
{
|
|
||||||
debug("adding path '%s' to remote host '%s'", info.path, host);
|
debug("adding path '%s' to remote host '%s'", info.path, host);
|
||||||
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
|
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
|
||||||
|
conn->to << cmdAddToStoreNar << info.path << info.deriver
|
||||||
conn->to
|
<< info.narHash.to_string(Base16, false) << info.references
|
||||||
<< cmdAddToStoreNar
|
<< info.registrationTime << info.narSize << info.ultimate
|
||||||
<< info.path
|
<< info.sigs << info.ca;
|
||||||
<< info.deriver
|
|
||||||
<< info.narHash.to_string(Base16, false)
|
|
||||||
<< info.references
|
|
||||||
<< info.registrationTime
|
|
||||||
<< info.narSize
|
|
||||||
<< info.ultimate
|
|
||||||
<< info.sigs
|
|
||||||
<< info.ca;
|
|
||||||
try {
|
try {
|
||||||
copyNAR(source, conn->to);
|
copyNAR(source, conn->to);
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -153,33 +143,24 @@ struct LegacySSHStore : public Store
|
||||||
conn->to.flush();
|
conn->to.flush();
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
conn->to << cmdImportPaths << 1;
|
||||||
conn->to
|
|
||||||
<< cmdImportPaths
|
|
||||||
<< 1;
|
|
||||||
try {
|
try {
|
||||||
copyNAR(source, conn->to);
|
copyNAR(source, conn->to);
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
conn->good = false;
|
conn->good = false;
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
conn->to
|
conn->to << exportMagic << info.path << info.references << info.deriver
|
||||||
<< exportMagic
|
<< 0 << 0;
|
||||||
<< info.path
|
|
||||||
<< info.references
|
|
||||||
<< info.deriver
|
|
||||||
<< 0
|
|
||||||
<< 0;
|
|
||||||
conn->to.flush();
|
conn->to.flush();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (readInt(conn->from) != 1)
|
if (readInt(conn->from) != 1)
|
||||||
throw Error("failed to add path '%s' to remote host '%s', info.path, host");
|
throw Error(
|
||||||
|
"failed to add path '%s' to remote host '%s', info.path, host");
|
||||||
}
|
}
|
||||||
|
|
||||||
void narFromPath(const Path & path, Sink & sink) override
|
void narFromPath(const Path& path, Sink& sink) override {
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
conn->to << cmdDumpStorePath << path;
|
conn->to << cmdDumpStorePath << path;
|
||||||
|
|
@ -187,36 +168,31 @@ struct LegacySSHStore : public Store
|
||||||
copyNAR(conn->from, sink);
|
copyNAR(conn->from, sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
Path queryPathFromHashPart(const string & hashPart) override
|
Path queryPathFromHashPart(const string& hashPart) override {
|
||||||
{ unsupported("queryPathFromHashPart"); }
|
unsupported("queryPathFromHashPart");
|
||||||
|
}
|
||||||
|
|
||||||
Path addToStore(const string & name, const Path & srcPath,
|
Path addToStore(const string& name, const Path& srcPath, bool recursive,
|
||||||
bool recursive, HashType hashAlgo,
|
HashType hashAlgo, PathFilter& filter,
|
||||||
PathFilter & filter, RepairFlag repair) override
|
RepairFlag repair) override {
|
||||||
{ unsupported("addToStore"); }
|
unsupported("addToStore");
|
||||||
|
}
|
||||||
|
|
||||||
Path addTextToStore(const string& name, const string& s,
|
Path addTextToStore(const string& name, const string& s,
|
||||||
const PathSet & references, RepairFlag repair) override
|
const PathSet& references, RepairFlag repair) override {
|
||||||
{ unsupported("addTextToStore"); }
|
unsupported("addTextToStore");
|
||||||
|
}
|
||||||
|
|
||||||
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
|
BuildResult buildDerivation(const Path& drvPath, const BasicDerivation& drv,
|
||||||
BuildMode buildMode) override
|
BuildMode buildMode) override {
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
conn->to
|
conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime
|
||||||
<< cmdBuildDerivation
|
|
||||||
<< drvPath
|
|
||||||
<< drv
|
|
||||||
<< settings.maxSilentTime
|
|
||||||
<< settings.buildTimeout;
|
<< settings.buildTimeout;
|
||||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
|
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
|
||||||
conn->to
|
conn->to << settings.maxLogSize;
|
||||||
<< settings.maxLogSize;
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
||||||
conn->to
|
conn->to << settings.buildRepeat << settings.enforceDeterminism;
|
||||||
<< settings.buildRepeat
|
|
||||||
<< settings.enforceDeterminism;
|
|
||||||
|
|
||||||
conn->to.flush();
|
conn->to.flush();
|
||||||
|
|
||||||
|
|
@ -225,29 +201,26 @@ struct LegacySSHStore : public Store
|
||||||
conn->from >> status.errorMsg;
|
conn->from >> status.errorMsg;
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
||||||
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
|
conn->from >> status.timesBuilt >> status.isNonDeterministic >>
|
||||||
|
status.startTime >> status.stopTime;
|
||||||
|
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ensurePath(const Path & path) override
|
void ensurePath(const Path& path) override { unsupported("ensurePath"); }
|
||||||
{ unsupported("ensurePath"); }
|
|
||||||
|
|
||||||
void computeFSClosure(const PathSet & paths,
|
void computeFSClosure(const PathSet& paths, PathSet& out,
|
||||||
PathSet & out, bool flipDirection = false,
|
bool flipDirection = false, bool includeOutputs = false,
|
||||||
bool includeOutputs = false, bool includeDerivers = false) override
|
bool includeDerivers = false) override {
|
||||||
{
|
|
||||||
if (flipDirection || includeDerivers) {
|
if (flipDirection || includeDerivers) {
|
||||||
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
|
Store::computeFSClosure(paths, out, flipDirection, includeOutputs,
|
||||||
|
includeDerivers);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
conn->to
|
conn->to << cmdQueryClosure << includeOutputs << paths;
|
||||||
<< cmdQueryClosure
|
|
||||||
<< includeOutputs
|
|
||||||
<< paths;
|
|
||||||
conn->to.flush();
|
conn->to.flush();
|
||||||
|
|
||||||
auto res = readStorePaths<PathSet>(*this, conn->from);
|
auto res = readStorePaths<PathSet>(*this, conn->from);
|
||||||
|
|
@ -255,39 +228,31 @@ struct LegacySSHStore : public Store
|
||||||
out.insert(res.begin(), res.end());
|
out.insert(res.begin(), res.end());
|
||||||
}
|
}
|
||||||
|
|
||||||
PathSet queryValidPaths(const PathSet & paths,
|
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
|
||||||
SubstituteFlag maybeSubstitute = NoSubstitute) override
|
NoSubstitute) override {
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
|
|
||||||
conn->to
|
conn->to << cmdQueryValidPaths << false // lock
|
||||||
<< cmdQueryValidPaths
|
<< maybeSubstitute << paths;
|
||||||
<< false // lock
|
|
||||||
<< maybeSubstitute
|
|
||||||
<< paths;
|
|
||||||
conn->to.flush();
|
conn->to.flush();
|
||||||
|
|
||||||
return readStorePaths<PathSet>(*this, conn->from);
|
return readStorePaths<PathSet>(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
void connect() override
|
void connect() override { auto conn(connections->get()); }
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned int getProtocol() override
|
unsigned int getProtocol() override {
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
return conn->remoteVersion;
|
return conn->remoteVersion;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore(
|
||||||
const std::string & uri, const Store::Params & params)
|
[](const std::string& uri,
|
||||||
-> std::shared_ptr<Store>
|
const Store::Params& params) -> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||||
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
|
return std::make_shared<LegacySSHStore>(
|
||||||
|
std::string(uri, uriScheme.size()), params);
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -4,70 +4,54 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class LocalBinaryCacheStore : public BinaryCacheStore
|
class LocalBinaryCacheStore : public BinaryCacheStore {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
Path binaryCacheDir;
|
Path binaryCacheDir;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
LocalBinaryCacheStore(const Params& params, const Path& binaryCacheDir)
|
||||||
LocalBinaryCacheStore(
|
: BinaryCacheStore(params), binaryCacheDir(binaryCacheDir) {}
|
||||||
const Params & params, const Path & binaryCacheDir)
|
|
||||||
: BinaryCacheStore(params)
|
|
||||||
, binaryCacheDir(binaryCacheDir)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void init() override;
|
void init() override;
|
||||||
|
|
||||||
std::string getUri() override
|
std::string getUri() override { return "file://" + binaryCacheDir; }
|
||||||
{
|
|
||||||
return "file://" + binaryCacheDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
bool fileExists(const std::string& path) override;
|
bool fileExists(const std::string& path) override;
|
||||||
|
|
||||||
void upsertFile(const std::string & path,
|
void upsertFile(const std::string& path, const std::string& data,
|
||||||
const std::string & data,
|
|
||||||
const std::string& mimeType) override;
|
const std::string& mimeType) override;
|
||||||
|
|
||||||
void getFile(const std::string & path, Sink & sink) override
|
void getFile(const std::string& path, Sink& sink) override {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
readFile(binaryCacheDir + "/" + path, sink);
|
readFile(binaryCacheDir + "/" + path, sink);
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
if (e.errNo == ENOENT)
|
if (e.errNo == ENOENT)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache",
|
||||||
|
path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
PathSet queryAllValidPaths() override
|
PathSet queryAllValidPaths() override {
|
||||||
{
|
|
||||||
PathSet paths;
|
PathSet paths;
|
||||||
|
|
||||||
for (auto& entry : readDirectory(binaryCacheDir)) {
|
for (auto& entry : readDirectory(binaryCacheDir)) {
|
||||||
if (entry.name.size() != 40 ||
|
if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo"))
|
||||||
!hasSuffix(entry.name, ".narinfo"))
|
|
||||||
continue;
|
continue;
|
||||||
paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8));
|
paths.insert(storeDir + "/" +
|
||||||
|
entry.name.substr(0, entry.name.size() - 8));
|
||||||
}
|
}
|
||||||
|
|
||||||
return paths;
|
return paths;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
void LocalBinaryCacheStore::init()
|
void LocalBinaryCacheStore::init() {
|
||||||
{
|
|
||||||
createDirs(binaryCacheDir + "/nar");
|
createDirs(binaryCacheDir + "/nar");
|
||||||
BinaryCacheStore::init();
|
BinaryCacheStore::init();
|
||||||
}
|
}
|
||||||
|
|
||||||
static void atomicWrite(const Path & path, const std::string & s)
|
static void atomicWrite(const Path& path, const std::string& s) {
|
||||||
{
|
|
||||||
Path tmp = path + ".tmp." + std::to_string(getpid());
|
Path tmp = path + ".tmp." + std::to_string(getpid());
|
||||||
AutoDelete del(tmp, false);
|
AutoDelete del(tmp, false);
|
||||||
writeFile(tmp, s);
|
writeFile(tmp, s);
|
||||||
|
|
@ -76,28 +60,26 @@ static void atomicWrite(const Path & path, const std::string & s)
|
||||||
del.cancel();
|
del.cancel();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool LocalBinaryCacheStore::fileExists(const std::string & path)
|
bool LocalBinaryCacheStore::fileExists(const std::string& path) {
|
||||||
{
|
|
||||||
return pathExists(binaryCacheDir + "/" + path);
|
return pathExists(binaryCacheDir + "/" + path);
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalBinaryCacheStore::upsertFile(const std::string& path,
|
void LocalBinaryCacheStore::upsertFile(const std::string& path,
|
||||||
const std::string& data,
|
const std::string& data,
|
||||||
const std::string & mimeType)
|
const std::string& mimeType) {
|
||||||
{
|
|
||||||
atomicWrite(binaryCacheDir + "/" + path, data);
|
atomicWrite(binaryCacheDir + "/" + path, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore(
|
||||||
const std::string & uri, const Store::Params & params)
|
[](const std::string& uri,
|
||||||
-> std::shared_ptr<Store>
|
const Store::Params& params) -> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
|
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
|
||||||
std::string(uri, 0, 7) != "file://")
|
std::string(uri, 0, 7) != "file://")
|
||||||
return 0;
|
return 0;
|
||||||
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
|
auto store =
|
||||||
|
std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
|
||||||
store->init();
|
store->init();
|
||||||
return store;
|
return store;
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
72
third_party/nix/src/libstore/local-fs-store.cc
vendored
72
third_party/nix/src/libstore/local-fs-store.cc
vendored
|
|
@ -1,100 +1,84 @@
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
#include "fs-accessor.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "compression.hh"
|
#include "compression.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
#include "fs-accessor.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
LocalFSStore::LocalFSStore(const Params & params)
|
LocalFSStore::LocalFSStore(const Params& params) : Store(params) {}
|
||||||
: Store(params)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
struct LocalStoreAccessor : public FSAccessor
|
struct LocalStoreAccessor : public FSAccessor {
|
||||||
{
|
|
||||||
ref<LocalFSStore> store;
|
ref<LocalFSStore> store;
|
||||||
|
|
||||||
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {}
|
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) {}
|
||||||
|
|
||||||
Path toRealPath(const Path & path)
|
Path toRealPath(const Path& path) {
|
||||||
{
|
|
||||||
Path storePath = store->toStorePath(path);
|
Path storePath = store->toStorePath(path);
|
||||||
if (!store->isValidPath(storePath))
|
if (!store->isValidPath(storePath))
|
||||||
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
|
throw InvalidPath(format("path '%1%' is not a valid store path") %
|
||||||
|
storePath);
|
||||||
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
|
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
FSAccessor::Stat stat(const Path & path) override
|
FSAccessor::Stat stat(const Path& path) override {
|
||||||
{
|
|
||||||
auto realPath = toRealPath(path);
|
auto realPath = toRealPath(path);
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(realPath.c_str(), &st)) {
|
if (lstat(realPath.c_str(), &st)) {
|
||||||
if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
|
if (errno == ENOENT || errno == ENOTDIR)
|
||||||
|
return {Type::tMissing, 0, false};
|
||||||
throw SysError(format("getting status of '%1%'") % path);
|
throw SysError(format("getting status of '%1%'") % path);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
|
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
|
||||||
throw Error(format("file '%1%' has unsupported type") % path);
|
throw Error(format("file '%1%' has unsupported type") % path);
|
||||||
|
|
||||||
return {
|
return {S_ISREG(st.st_mode)
|
||||||
S_ISREG(st.st_mode) ? Type::tRegular :
|
? Type::tRegular
|
||||||
S_ISLNK(st.st_mode) ? Type::tSymlink :
|
: S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory,
|
||||||
Type::tDirectory,
|
|
||||||
S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0,
|
S_ISREG(st.st_mode) ? (uint64_t)st.st_size : 0,
|
||||||
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
|
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet readDirectory(const Path & path) override
|
StringSet readDirectory(const Path& path) override {
|
||||||
{
|
|
||||||
auto realPath = toRealPath(path);
|
auto realPath = toRealPath(path);
|
||||||
|
|
||||||
auto entries = nix::readDirectory(realPath);
|
auto entries = nix::readDirectory(realPath);
|
||||||
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
for (auto & entry : entries)
|
for (auto& entry : entries) res.insert(entry.name);
|
||||||
res.insert(entry.name);
|
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readFile(const Path & path) override
|
std::string readFile(const Path& path) override {
|
||||||
{
|
|
||||||
return nix::readFile(toRealPath(path));
|
return nix::readFile(toRealPath(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readLink(const Path & path) override
|
std::string readLink(const Path& path) override {
|
||||||
{
|
|
||||||
return nix::readLink(toRealPath(path));
|
return nix::readLink(toRealPath(path));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<FSAccessor> LocalFSStore::getFSAccessor()
|
ref<FSAccessor> LocalFSStore::getFSAccessor() {
|
||||||
{
|
|
||||||
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
|
return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
|
||||||
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
|
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalFSStore::narFromPath(const Path & path, Sink & sink)
|
void LocalFSStore::narFromPath(const Path& path, Sink& sink) {
|
||||||
{
|
if (!isValidPath(path)) throw Error(format("path '%s' is not valid") % path);
|
||||||
if (!isValidPath(path))
|
|
||||||
throw Error(format("path '%s' is not valid") % path);
|
|
||||||
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
|
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
const string LocalFSStore::drvsLogDir = "drvs";
|
const string LocalFSStore::drvsLogDir = "drvs";
|
||||||
|
|
||||||
|
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) {
|
||||||
|
|
||||||
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
|
|
||||||
{
|
|
||||||
auto path(path_);
|
auto path(path_);
|
||||||
|
|
||||||
assertStorePath(path);
|
assertStorePath(path);
|
||||||
|
|
||||||
|
|
||||||
if (!isDerivation(path)) {
|
if (!isDerivation(path)) {
|
||||||
try {
|
try {
|
||||||
path = queryPathInfo(path)->deriver;
|
path = queryPathInfo(path)->deriver;
|
||||||
|
|
@ -107,10 +91,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
|
||||||
string baseName = baseNameOf(path);
|
string baseName = baseNameOf(path);
|
||||||
|
|
||||||
for (int j = 0; j < 2; j++) {
|
for (int j = 0; j < 2; j++) {
|
||||||
|
Path logPath = j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir,
|
||||||
Path logPath =
|
string(baseName, 0, 2), string(baseName, 2))
|
||||||
j == 0
|
|
||||||
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
|
|
||||||
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
|
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
|
||||||
Path logBz2Path = logPath + ".bz2";
|
Path logBz2Path = logPath + ".bz2";
|
||||||
|
|
||||||
|
|
@ -120,12 +102,12 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
|
||||||
else if (pathExists(logBz2Path)) {
|
else if (pathExists(logBz2Path)) {
|
||||||
try {
|
try {
|
||||||
return decompress("bzip2", readFile(logBz2Path));
|
return decompress("bzip2", readFile(logBz2Path));
|
||||||
} catch (Error &) { }
|
} catch (Error&) {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
601
third_party/nix/src/libstore/local-store.cc
vendored
601
third_party/nix/src/libstore/local-store.cc
vendored
File diff suppressed because it is too large
Load diff
83
third_party/nix/src/libstore/local-store.hh
vendored
83
third_party/nix/src/libstore/local-store.hh
vendored
|
|
@ -1,48 +1,37 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "sqlite.hh"
|
|
||||||
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
#include "store-api.hh"
|
|
||||||
#include "sync.hh"
|
|
||||||
#include "util.hh"
|
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <future>
|
#include <future>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
#include "pathlocks.hh"
|
||||||
|
#include "sqlite.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
#include "sync.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
|
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
|
||||||
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
|
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
|
||||||
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
|
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
|
||||||
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
|
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
|
||||||
const int nixSchemaVersion = 10;
|
const int nixSchemaVersion = 10;
|
||||||
|
|
||||||
|
|
||||||
struct Derivation;
|
struct Derivation;
|
||||||
|
|
||||||
|
struct OptimiseStats {
|
||||||
struct OptimiseStats
|
|
||||||
{
|
|
||||||
unsigned long filesLinked = 0;
|
unsigned long filesLinked = 0;
|
||||||
unsigned long long bytesFreed = 0;
|
unsigned long long bytesFreed = 0;
|
||||||
unsigned long long blocksFreed = 0;
|
unsigned long long blocksFreed = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class LocalStore : public LocalFSStore {
|
||||||
class LocalStore : public LocalFSStore
|
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/* Lock file used for upgrading. */
|
/* Lock file used for upgrading. */
|
||||||
AutoCloseFD globalLock;
|
AutoCloseFD globalLock;
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
/* The SQLite database object. */
|
/* The SQLite database object. */
|
||||||
SQLite db;
|
SQLite db;
|
||||||
|
|
||||||
|
|
@ -84,7 +73,6 @@ private:
|
||||||
Sync<State, std::recursive_mutex> _state;
|
Sync<State, std::recursive_mutex> _state;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
PathSetting realStoreDir_;
|
PathSetting realStoreDir_;
|
||||||
|
|
||||||
const Path realStoreDir;
|
const Path realStoreDir;
|
||||||
|
|
@ -97,15 +85,13 @@ public:
|
||||||
const Path fnTempRoots;
|
const Path fnTempRoots;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
Setting<bool> requireSigs{
|
||||||
Setting<bool> requireSigs{(Store*) this,
|
(Store*)this, settings.requireSigs, "require-sigs",
|
||||||
settings.requireSigs,
|
"whether store paths should have a trusted signature on import"};
|
||||||
"require-sigs", "whether store paths should have a trusted signature on import"};
|
|
||||||
|
|
||||||
const PublicKeys& getPublicKeys();
|
const PublicKeys& getPublicKeys();
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// Hack for build-remote.cc.
|
// Hack for build-remote.cc.
|
||||||
PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
|
PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
|
||||||
|
|
||||||
|
|
@ -121,12 +107,13 @@ public:
|
||||||
|
|
||||||
bool isValidPathUncached(const Path& path) override;
|
bool isValidPathUncached(const Path& path) override;
|
||||||
|
|
||||||
PathSet queryValidPaths(const PathSet & paths,
|
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
|
||||||
SubstituteFlag maybeSubstitute = NoSubstitute) override;
|
NoSubstitute) override;
|
||||||
|
|
||||||
PathSet queryAllValidPaths() override;
|
PathSet queryAllValidPaths() override;
|
||||||
|
|
||||||
void queryPathInfoUncached(const Path & path,
|
void queryPathInfoUncached(
|
||||||
|
const Path& path,
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
||||||
|
|
||||||
void queryReferrers(const Path& path, PathSet& referrers) override;
|
void queryReferrers(const Path& path, PathSet& referrers) override;
|
||||||
|
|
@ -144,20 +131,21 @@ public:
|
||||||
void querySubstitutablePathInfos(const PathSet& paths,
|
void querySubstitutablePathInfos(const PathSet& paths,
|
||||||
SubstitutablePathInfos& infos) override;
|
SubstitutablePathInfos& infos) override;
|
||||||
|
|
||||||
void addToStore(const ValidPathInfo & info, Source & source,
|
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor) override;
|
std::shared_ptr<FSAccessor> accessor) override;
|
||||||
|
|
||||||
Path addToStore(const string & name, const Path & srcPath,
|
Path addToStore(const string& name, const Path& srcPath, bool recursive,
|
||||||
bool recursive, HashType hashAlgo,
|
HashType hashAlgo, PathFilter& filter,
|
||||||
PathFilter & filter, RepairFlag repair) override;
|
RepairFlag repair) override;
|
||||||
|
|
||||||
/* Like addToStore(), but the contents of the path are contained
|
/* Like addToStore(), but the contents of the path are contained
|
||||||
in `dump', which is either a NAR serialisation (if recursive ==
|
in `dump', which is either a NAR serialisation (if recursive ==
|
||||||
true) or simply the contents of a regular file (if recursive ==
|
true) or simply the contents of a regular file (if recursive ==
|
||||||
false). */
|
false). */
|
||||||
Path addToStoreFromDump(const string& dump, const string& name,
|
Path addToStoreFromDump(const string& dump, const string& name,
|
||||||
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair);
|
bool recursive = true, HashType hashAlgo = htSHA256,
|
||||||
|
RepairFlag repair = NoRepair);
|
||||||
|
|
||||||
Path addTextToStore(const string& name, const string& s,
|
Path addTextToStore(const string& name, const string& s,
|
||||||
const PathSet& references, RepairFlag repair) override;
|
const PathSet& references, RepairFlag repair) override;
|
||||||
|
|
@ -176,14 +164,12 @@ public:
|
||||||
void syncWithGC() override;
|
void syncWithGC() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
typedef std::shared_ptr<AutoCloseFD> FDPtr;
|
typedef std::shared_ptr<AutoCloseFD> FDPtr;
|
||||||
typedef list<FDPtr> FDs;
|
typedef list<FDPtr> FDs;
|
||||||
|
|
||||||
void findTempRoots(FDs& fds, Roots& roots, bool censor);
|
void findTempRoots(FDs& fds, Roots& roots, bool censor);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
Roots findRoots(bool censor) override;
|
Roots findRoots(bool censor) override;
|
||||||
|
|
||||||
void collectGarbage(const GCOptions& options, GCResults& results) override;
|
void collectGarbage(const GCOptions& options, GCResults& results) override;
|
||||||
|
|
@ -224,7 +210,6 @@ public:
|
||||||
void autoGC(bool sync = true);
|
void autoGC(bool sync = true);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
int getSchema();
|
int getSchema();
|
||||||
|
|
||||||
void openDB(State& state, bool create);
|
void openDB(State& state, bool create);
|
||||||
|
|
@ -233,15 +218,16 @@ private:
|
||||||
|
|
||||||
uint64_t queryValidPathId(State& state, const Path& path);
|
uint64_t queryValidPathId(State& state, const Path& path);
|
||||||
|
|
||||||
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
|
uint64_t addValidPath(State& state, const ValidPathInfo& info,
|
||||||
|
bool checkOutputs = true);
|
||||||
|
|
||||||
void invalidatePath(State& state, const Path& path);
|
void invalidatePath(State& state, const Path& path);
|
||||||
|
|
||||||
/* Delete a path from the Nix store. */
|
/* Delete a path from the Nix store. */
|
||||||
void invalidatePathChecked(const Path& path);
|
void invalidatePathChecked(const Path& path);
|
||||||
|
|
||||||
void verifyPath(const Path & path, const PathSet & store,
|
void verifyPath(const Path& path, const PathSet& store, PathSet& done,
|
||||||
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors);
|
PathSet& validPaths, RepairFlag repair, bool& errors);
|
||||||
|
|
||||||
void updatePathInfo(State& state, const ValidPathInfo& info);
|
void updatePathInfo(State& state, const ValidPathInfo& info);
|
||||||
|
|
||||||
|
|
@ -260,8 +246,8 @@ private:
|
||||||
|
|
||||||
void deletePathRecursive(GCState& state, const Path& path);
|
void deletePathRecursive(GCState& state, const Path& path);
|
||||||
|
|
||||||
bool isActiveTempFile(const GCState & state,
|
bool isActiveTempFile(const GCState& state, const Path& path,
|
||||||
const Path & path, const string & suffix);
|
const string& suffix);
|
||||||
|
|
||||||
AutoCloseFD openGCLock(LockType lockType);
|
AutoCloseFD openGCLock(LockType lockType);
|
||||||
|
|
||||||
|
|
@ -280,8 +266,10 @@ private:
|
||||||
typedef std::unordered_set<ino_t> InodeHash;
|
typedef std::unordered_set<ino_t> InodeHash;
|
||||||
|
|
||||||
InodeHash loadInodeHash();
|
InodeHash loadInodeHash();
|
||||||
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
|
Strings readDirectoryIgnoringInodes(const Path& path,
|
||||||
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
|
const InodeHash& inodeHash);
|
||||||
|
void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path,
|
||||||
|
InodeHash& inodeHash);
|
||||||
|
|
||||||
// Internal versions that are not wrapped in retry_sqlite.
|
// Internal versions that are not wrapped in retry_sqlite.
|
||||||
bool isValidPath_(State& state, const Path& path);
|
bool isValidPath_(State& state, const Path& path);
|
||||||
|
|
@ -299,11 +287,9 @@ private:
|
||||||
friend class SubstitutionGoal;
|
friend class SubstitutionGoal;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
typedef std::pair<dev_t, ino_t> Inode;
|
typedef std::pair<dev_t, ino_t> Inode;
|
||||||
typedef set<Inode> InodesSeen;
|
typedef set<Inode> InodesSeen;
|
||||||
|
|
||||||
|
|
||||||
/* "Fix", or canonicalise, the meta-data of the files in a store path
|
/* "Fix", or canonicalise, the meta-data of the files in a store path
|
||||||
after it has been built. In particular:
|
after it has been built. In particular:
|
||||||
- the last modification date on each file is set to 1 (i.e.,
|
- the last modification date on each file is set to 1 (i.e.,
|
||||||
|
|
@ -312,11 +298,12 @@ typedef set<Inode> InodesSeen;
|
||||||
without execute permission; setuid bits etc. are cleared)
|
without execute permission; setuid bits etc. are cleared)
|
||||||
- the owner and group are set to the Nix user and group, if we're
|
- the owner and group are set to the Nix user and group, if we're
|
||||||
running as root. */
|
running as root. */
|
||||||
void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen);
|
void canonicalisePathMetaData(const Path& path, uid_t fromUid,
|
||||||
|
InodesSeen& inodesSeen);
|
||||||
void canonicalisePathMetaData(const Path& path, uid_t fromUid);
|
void canonicalisePathMetaData(const Path& path, uid_t fromUid);
|
||||||
|
|
||||||
void canonicaliseTimestampAndPermissions(const Path& path);
|
void canonicaliseTimestampAndPermissions(const Path& path);
|
||||||
|
|
||||||
MakeError(PathInUse, Error);
|
MakeError(PathInUse, Error);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
63
third_party/nix/src/libstore/machines.cc
vendored
63
third_party/nix/src/libstore/machines.cc
vendored
|
|
@ -1,27 +1,23 @@
|
||||||
#include "machines.hh"
|
#include "machines.hh"
|
||||||
#include "util.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
Machine::Machine(decltype(storeUri) storeUri,
|
Machine::Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
|
||||||
decltype(systemTypes) systemTypes,
|
decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
|
||||||
decltype(sshKey) sshKey,
|
|
||||||
decltype(maxJobs) maxJobs,
|
|
||||||
decltype(speedFactor) speedFactor,
|
decltype(speedFactor) speedFactor,
|
||||||
decltype(supportedFeatures) supportedFeatures,
|
decltype(supportedFeatures) supportedFeatures,
|
||||||
decltype(mandatoryFeatures) mandatoryFeatures,
|
decltype(mandatoryFeatures) mandatoryFeatures,
|
||||||
decltype(sshPublicHostKey) sshPublicHostKey) :
|
decltype(sshPublicHostKey) sshPublicHostKey)
|
||||||
storeUri(
|
: storeUri(
|
||||||
// Backwards compatibility: if the URI is a hostname,
|
// Backwards compatibility: if the URI is a hostname,
|
||||||
// prepend ssh://.
|
// prepend ssh://.
|
||||||
storeUri.find("://") != std::string::npos
|
storeUri.find("://") != std::string::npos ||
|
||||||
|| hasPrefix(storeUri, "local")
|
hasPrefix(storeUri, "local") ||
|
||||||
|| hasPrefix(storeUri, "remote")
|
hasPrefix(storeUri, "remote") ||
|
||||||
|| hasPrefix(storeUri, "auto")
|
hasPrefix(storeUri, "auto") || hasPrefix(storeUri, "/")
|
||||||
|| hasPrefix(storeUri, "/")
|
|
||||||
? storeUri
|
? storeUri
|
||||||
: "ssh://" + storeUri),
|
: "ssh://" + storeUri),
|
||||||
systemTypes(systemTypes),
|
systemTypes(systemTypes),
|
||||||
|
|
@ -30,8 +26,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|
||||||
speedFactor(std::max(1U, speedFactor)),
|
speedFactor(std::max(1U, speedFactor)),
|
||||||
supportedFeatures(supportedFeatures),
|
supportedFeatures(supportedFeatures),
|
||||||
mandatoryFeatures(mandatoryFeatures),
|
mandatoryFeatures(mandatoryFeatures),
|
||||||
sshPublicHostKey(sshPublicHostKey)
|
sshPublicHostKey(sshPublicHostKey) {}
|
||||||
{}
|
|
||||||
|
|
||||||
bool Machine::allSupported(const std::set<string>& features) const {
|
bool Machine::allSupported(const std::set<string>& features) const {
|
||||||
return std::all_of(features.begin(), features.end(),
|
return std::all_of(features.begin(), features.end(),
|
||||||
|
|
@ -42,14 +37,12 @@ bool Machine::allSupported(const std::set<string> & features) const {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Machine::mandatoryMet(const std::set<string>& features) const {
|
bool Machine::mandatoryMet(const std::set<string>& features) const {
|
||||||
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
|
return std::all_of(
|
||||||
[&](const string & feature) {
|
mandatoryFeatures.begin(), mandatoryFeatures.end(),
|
||||||
return features.count(feature);
|
[&](const string& feature) { return features.count(feature); });
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void parseMachines(const std::string & s, Machines & machines)
|
void parseMachines(const std::string& s, Machines& machines) {
|
||||||
{
|
|
||||||
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
|
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
|
||||||
trim(line);
|
trim(line);
|
||||||
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
|
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
|
||||||
|
|
@ -60,8 +53,7 @@ void parseMachines(const std::string & s, Machines & machines)
|
||||||
try {
|
try {
|
||||||
parseMachines(readFile(file), machines);
|
parseMachines(readFile(file), machines);
|
||||||
} catch (const SysError& e) {
|
} catch (const SysError& e) {
|
||||||
if (e.errNo != ENOENT)
|
if (e.errNo != ENOENT) throw;
|
||||||
throw;
|
|
||||||
debug("cannot find machines file '%s'", file);
|
debug("cannot find machines file '%s'", file);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -69,26 +61,27 @@ void parseMachines(const std::string & s, Machines & machines)
|
||||||
|
|
||||||
auto tokens = tokenizeString<std::vector<string>>(line);
|
auto tokens = tokenizeString<std::vector<string>>(line);
|
||||||
auto sz = tokens.size();
|
auto sz = tokens.size();
|
||||||
if (sz < 1)
|
if (sz < 1) throw FormatError("bad machine specification '%s'", line);
|
||||||
throw FormatError("bad machine specification '%s'", line);
|
|
||||||
|
|
||||||
auto isSet = [&](size_t n) {
|
auto isSet = [&](size_t n) {
|
||||||
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
|
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
|
||||||
};
|
};
|
||||||
|
|
||||||
machines.emplace_back(tokens[0],
|
machines.emplace_back(
|
||||||
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
|
tokens[0],
|
||||||
isSet(2) ? tokens[2] : "",
|
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",")
|
||||||
isSet(3) ? std::stoull(tokens[3]) : 1LL,
|
: std::vector<string>{settings.thisSystem},
|
||||||
|
isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL,
|
||||||
isSet(4) ? std::stoull(tokens[4]) : 1LL,
|
isSet(4) ? std::stoull(tokens[4]) : 1LL,
|
||||||
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
|
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",")
|
||||||
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
|
: std::set<string>{},
|
||||||
|
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",")
|
||||||
|
: std::set<string>{},
|
||||||
isSet(7) ? tokens[7] : "");
|
isSet(7) ? tokens[7] : "");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Machines getMachines()
|
Machines getMachines() {
|
||||||
{
|
|
||||||
static auto machines = [&]() {
|
static auto machines = [&]() {
|
||||||
Machines machines;
|
Machines machines;
|
||||||
parseMachines(settings.builders, machines);
|
parseMachines(settings.builders, machines);
|
||||||
|
|
@ -97,4 +90,4 @@ Machines getMachines()
|
||||||
return machines;
|
return machines;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
9
third_party/nix/src/libstore/machines.hh
vendored
9
third_party/nix/src/libstore/machines.hh
vendored
|
|
@ -5,7 +5,6 @@
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct Machine {
|
struct Machine {
|
||||||
|
|
||||||
const string storeUri;
|
const string storeUri;
|
||||||
const std::vector<string> systemTypes;
|
const std::vector<string> systemTypes;
|
||||||
const string sshKey;
|
const string sshKey;
|
||||||
|
|
@ -20,10 +19,8 @@ struct Machine {
|
||||||
|
|
||||||
bool mandatoryMet(const std::set<string>& features) const;
|
bool mandatoryMet(const std::set<string>& features) const;
|
||||||
|
|
||||||
Machine(decltype(storeUri) storeUri,
|
Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes,
|
||||||
decltype(systemTypes) systemTypes,
|
decltype(sshKey) sshKey, decltype(maxJobs) maxJobs,
|
||||||
decltype(sshKey) sshKey,
|
|
||||||
decltype(maxJobs) maxJobs,
|
|
||||||
decltype(speedFactor) speedFactor,
|
decltype(speedFactor) speedFactor,
|
||||||
decltype(supportedFeatures) supportedFeatures,
|
decltype(supportedFeatures) supportedFeatures,
|
||||||
decltype(mandatoryFeatures) mandatoryFeatures,
|
decltype(mandatoryFeatures) mandatoryFeatures,
|
||||||
|
|
@ -36,4 +33,4 @@ void parseMachines(const std::string & s, Machines & machines);
|
||||||
|
|
||||||
Machines getMachines();
|
Machines getMachines();
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
98
third_party/nix/src/libstore/misc.cc
vendored
98
third_party/nix/src/libstore/misc.cc
vendored
|
|
@ -1,19 +1,16 @@
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
#include "parsed-derivations.hh"
|
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "local-store.hh"
|
#include "local-store.hh"
|
||||||
|
#include "parsed-derivations.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "thread-pool.hh"
|
#include "thread-pool.hh"
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_,
|
||||||
void Store::computeFSClosure(const PathSet & startPaths,
|
bool flipDirection, bool includeOutputs,
|
||||||
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
|
bool includeDerivers) {
|
||||||
{
|
struct State {
|
||||||
struct State
|
|
||||||
{
|
|
||||||
size_t pending;
|
size_t pending;
|
||||||
PathSet& paths;
|
PathSet& paths;
|
||||||
std::exception_ptr exc;
|
std::exception_ptr exc;
|
||||||
|
|
@ -34,23 +31,21 @@ void Store::computeFSClosure(const PathSet & startPaths,
|
||||||
state->pending++;
|
state->pending++;
|
||||||
}
|
}
|
||||||
|
|
||||||
queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
|
queryPathInfo(
|
||||||
|
path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
|
||||||
// FIXME: calls to isValidPath() should be async
|
// FIXME: calls to isValidPath() should be async
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto info = fut.get();
|
auto info = fut.get();
|
||||||
|
|
||||||
if (flipDirection) {
|
if (flipDirection) {
|
||||||
|
|
||||||
PathSet referrers;
|
PathSet referrers;
|
||||||
queryReferrers(path, referrers);
|
queryReferrers(path, referrers);
|
||||||
for (auto& ref : referrers)
|
for (auto& ref : referrers)
|
||||||
if (ref != path)
|
if (ref != path) enqueue(ref);
|
||||||
enqueue(ref);
|
|
||||||
|
|
||||||
if (includeOutputs)
|
if (includeOutputs)
|
||||||
for (auto & i : queryValidDerivers(path))
|
for (auto& i : queryValidDerivers(path)) enqueue(i);
|
||||||
enqueue(i);
|
|
||||||
|
|
||||||
if (includeDerivers && isDerivation(path))
|
if (includeDerivers && isDerivation(path))
|
||||||
for (auto& i : queryDerivationOutputs(path))
|
for (auto& i : queryDerivationOutputs(path))
|
||||||
|
|
@ -58,10 +53,8 @@ void Store::computeFSClosure(const PathSet & startPaths,
|
||||||
enqueue(i);
|
enqueue(i);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
for (auto& ref : info->references)
|
for (auto& ref : info->references)
|
||||||
if (ref != path)
|
if (ref != path) enqueue(ref);
|
||||||
enqueue(ref);
|
|
||||||
|
|
||||||
if (includeOutputs && isDerivation(path))
|
if (includeOutputs && isDerivation(path))
|
||||||
for (auto& i : queryDerivationOutputs(path))
|
for (auto& i : queryDerivationOutputs(path))
|
||||||
|
|
@ -69,7 +62,6 @@ void Store::computeFSClosure(const PathSet & startPaths,
|
||||||
|
|
||||||
if (includeDerivers && isValidPath(info->deriver))
|
if (includeDerivers && isValidPath(info->deriver))
|
||||||
enqueue(info->deriver);
|
enqueue(info->deriver);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|
@ -87,8 +79,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
|
||||||
}});
|
}});
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & startPath : startPaths)
|
for (auto& startPath : startPaths) enqueue(startPath);
|
||||||
enqueue(startPath);
|
|
||||||
|
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
@ -97,41 +88,40 @@ void Store::computeFSClosure(const PathSet & startPaths,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Store::computeFSClosure(const Path& startPath, PathSet& paths_,
|
||||||
void Store::computeFSClosure(const Path & startPath,
|
bool flipDirection, bool includeOutputs,
|
||||||
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
|
bool includeDerivers) {
|
||||||
{
|
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs,
|
||||||
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
|
includeDerivers);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Store::queryMissing(const PathSet& targets, PathSet& willBuild_,
|
||||||
void Store::queryMissing(const PathSet & targets,
|
PathSet& willSubstitute_, PathSet& unknown_,
|
||||||
PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
|
unsigned long long& downloadSize_,
|
||||||
unsigned long long & downloadSize_, unsigned long long & narSize_)
|
unsigned long long& narSize_) {
|
||||||
{
|
Activity act(*logger, lvlDebug, actUnknown,
|
||||||
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
|
"querying info about missing paths");
|
||||||
|
|
||||||
downloadSize_ = narSize_ = 0;
|
downloadSize_ = narSize_ = 0;
|
||||||
|
|
||||||
ThreadPool pool;
|
ThreadPool pool;
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
PathSet done;
|
PathSet done;
|
||||||
PathSet &unknown, &willSubstitute, &willBuild;
|
PathSet &unknown, &willSubstitute, &willBuild;
|
||||||
unsigned long long& downloadSize;
|
unsigned long long& downloadSize;
|
||||||
unsigned long long& narSize;
|
unsigned long long& narSize;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct DrvState
|
struct DrvState {
|
||||||
{
|
|
||||||
size_t left;
|
size_t left;
|
||||||
bool done = false;
|
bool done = false;
|
||||||
PathSet outPaths;
|
PathSet outPaths;
|
||||||
DrvState(size_t left) : left(left) {}
|
DrvState(size_t left) : left(left) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
|
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_,
|
||||||
|
downloadSize_, narSize_});
|
||||||
|
|
||||||
std::function<void(Path)> doPath;
|
std::function<void(Path)> doPath;
|
||||||
|
|
||||||
|
|
@ -142,12 +132,12 @@ void Store::queryMissing(const PathSet & targets,
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& i : drv.inputDrvs)
|
for (auto& i : drv.inputDrvs)
|
||||||
pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
|
pool.enqueue(
|
||||||
|
std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
|
||||||
};
|
};
|
||||||
|
|
||||||
auto checkOutput = [&](
|
auto checkOutput = [&](const Path& drvPath, ref<Derivation> drv,
|
||||||
const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_)
|
const Path& outPath, ref<Sync<DrvState>> drvState_) {
|
||||||
{
|
|
||||||
if (drvState_->lock()->done) return;
|
if (drvState_->lock()->done) return;
|
||||||
|
|
||||||
SubstitutablePathInfos infos;
|
SubstitutablePathInfos infos;
|
||||||
|
|
@ -172,7 +162,6 @@ void Store::queryMissing(const PathSet & targets,
|
||||||
};
|
};
|
||||||
|
|
||||||
doPath = [&](const Path& path) {
|
doPath = [&](const Path& path) {
|
||||||
|
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (state->done.count(path)) return;
|
if (state->done.count(path)) return;
|
||||||
|
|
@ -194,20 +183,19 @@ void Store::queryMissing(const PathSet & targets,
|
||||||
|
|
||||||
PathSet invalid;
|
PathSet invalid;
|
||||||
for (auto& j : drv.outputs)
|
for (auto& j : drv.outputs)
|
||||||
if (wantOutput(j.first, i2.second)
|
if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path))
|
||||||
&& !isValidPath(j.second.path))
|
|
||||||
invalid.insert(j.second.path);
|
invalid.insert(j.second.path);
|
||||||
if (invalid.empty()) return;
|
if (invalid.empty()) return;
|
||||||
|
|
||||||
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||||
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
|
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
|
||||||
for (auto& output : invalid)
|
for (auto& output : invalid)
|
||||||
pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
|
pool.enqueue(std::bind(checkOutput, i2.first,
|
||||||
|
make_ref<Derivation>(drv), output, drvState));
|
||||||
} else
|
} else
|
||||||
mustBuildDrv(i2.first, drv);
|
mustBuildDrv(i2.first, drv);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
if (isValidPath(path)) return;
|
if (isValidPath(path)) return;
|
||||||
|
|
||||||
SubstitutablePathInfos infos;
|
SubstitutablePathInfos infos;
|
||||||
|
|
@ -234,15 +222,12 @@ void Store::queryMissing(const PathSet & targets,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & path : targets)
|
for (auto& path : targets) pool.enqueue(std::bind(doPath, path));
|
||||||
pool.enqueue(std::bind(doPath, path));
|
|
||||||
|
|
||||||
pool.process();
|
pool.process();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Paths Store::topoSortPaths(const PathSet& paths) {
|
||||||
Paths Store::topoSortPaths(const PathSet & paths)
|
|
||||||
{
|
|
||||||
Paths sorted;
|
Paths sorted;
|
||||||
PathSet visited, parents;
|
PathSet visited, parents;
|
||||||
|
|
||||||
|
|
@ -250,7 +235,9 @@ Paths Store::topoSortPaths(const PathSet & paths)
|
||||||
|
|
||||||
dfsVisit = [&](const Path& path, const Path* parent) {
|
dfsVisit = [&](const Path& path, const Path* parent) {
|
||||||
if (parents.find(path) != parents.end())
|
if (parents.find(path) != parents.end())
|
||||||
throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
|
throw BuildError(
|
||||||
|
format("cycle detected in the references of '%1%' from '%2%'") %
|
||||||
|
path % *parent);
|
||||||
|
|
||||||
if (visited.find(path) != visited.end()) return;
|
if (visited.find(path) != visited.end()) return;
|
||||||
visited.insert(path);
|
visited.insert(path);
|
||||||
|
|
@ -265,18 +252,15 @@ Paths Store::topoSortPaths(const PathSet & paths)
|
||||||
for (auto& i : references)
|
for (auto& i : references)
|
||||||
/* Don't traverse into paths that don't exist. That can
|
/* Don't traverse into paths that don't exist. That can
|
||||||
happen due to substitutes for non-existent paths. */
|
happen due to substitutes for non-existent paths. */
|
||||||
if (i != path && paths.find(i) != paths.end())
|
if (i != path && paths.find(i) != paths.end()) dfsVisit(i, &path);
|
||||||
dfsVisit(i, &path);
|
|
||||||
|
|
||||||
sorted.push_front(path);
|
sorted.push_front(path);
|
||||||
parents.erase(path);
|
parents.erase(path);
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & i : paths)
|
for (auto& i : paths) dfsVisit(i, nullptr);
|
||||||
dfsVisit(i, nullptr);
|
|
||||||
|
|
||||||
return sorted;
|
return sorted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
100
third_party/nix/src/libstore/nar-accessor.cc
vendored
100
third_party/nix/src/libstore/nar-accessor.cc
vendored
|
|
@ -1,17 +1,14 @@
|
||||||
#include "nar-accessor.hh"
|
#include "nar-accessor.hh"
|
||||||
|
#include <algorithm>
|
||||||
|
#include <map>
|
||||||
|
#include <nlohmann/json.hpp>
|
||||||
|
#include <stack>
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
#include "json.hh"
|
#include "json.hh"
|
||||||
|
|
||||||
#include <map>
|
|
||||||
#include <stack>
|
|
||||||
#include <algorithm>
|
|
||||||
|
|
||||||
#include <nlohmann/json.hpp>
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct NarMember
|
struct NarMember {
|
||||||
{
|
|
||||||
FSAccessor::Type type = FSAccessor::Type::tMissing;
|
FSAccessor::Type type = FSAccessor::Type::tMissing;
|
||||||
|
|
||||||
bool isExecutable = false;
|
bool isExecutable = false;
|
||||||
|
|
@ -26,16 +23,14 @@ struct NarMember
|
||||||
std::map<std::string, NarMember> children;
|
std::map<std::string, NarMember> children;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct NarAccessor : public FSAccessor
|
struct NarAccessor : public FSAccessor {
|
||||||
{
|
|
||||||
std::shared_ptr<const std::string> nar;
|
std::shared_ptr<const std::string> nar;
|
||||||
|
|
||||||
GetNarBytes getNarBytes;
|
GetNarBytes getNarBytes;
|
||||||
|
|
||||||
NarMember root;
|
NarMember root;
|
||||||
|
|
||||||
struct NarIndexer : ParseSink, StringSource
|
struct NarIndexer : ParseSink, StringSource {
|
||||||
{
|
|
||||||
NarAccessor& acc;
|
NarAccessor& acc;
|
||||||
|
|
||||||
std::stack<NarMember*> parents;
|
std::stack<NarMember*> parents;
|
||||||
|
|
@ -44,8 +39,7 @@ struct NarAccessor : public FSAccessor
|
||||||
bool isExec = false;
|
bool isExec = false;
|
||||||
|
|
||||||
NarIndexer(NarAccessor& acc, const std::string& nar)
|
NarIndexer(NarAccessor& acc, const std::string& nar)
|
||||||
: StringSource(nar), acc(acc)
|
: StringSource(nar), acc(acc) {}
|
||||||
{ }
|
|
||||||
|
|
||||||
void createMember(const Path& path, NarMember member) {
|
void createMember(const Path& path, NarMember member) {
|
||||||
size_t level = std::count(path.begin(), path.end(), '/');
|
size_t level = std::count(path.begin(), path.end(), '/');
|
||||||
|
|
@ -57,36 +51,30 @@ struct NarAccessor : public FSAccessor
|
||||||
} else {
|
} else {
|
||||||
if (parents.top()->type != FSAccessor::Type::tDirectory)
|
if (parents.top()->type != FSAccessor::Type::tDirectory)
|
||||||
throw Error("NAR file missing parent directory of path '%s'", path);
|
throw Error("NAR file missing parent directory of path '%s'", path);
|
||||||
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
|
auto result = parents.top()->children.emplace(baseNameOf(path),
|
||||||
|
std::move(member));
|
||||||
parents.push(&result.first->second);
|
parents.push(&result.first->second);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void createDirectory(const Path & path) override
|
void createDirectory(const Path& path) override {
|
||||||
{
|
|
||||||
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
|
createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
|
||||||
}
|
}
|
||||||
|
|
||||||
void createRegularFile(const Path & path) override
|
void createRegularFile(const Path& path) override {
|
||||||
{
|
|
||||||
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
|
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
|
||||||
}
|
}
|
||||||
|
|
||||||
void isExecutable() override
|
void isExecutable() override { parents.top()->isExecutable = true; }
|
||||||
{
|
|
||||||
parents.top()->isExecutable = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
void preallocateContents(unsigned long long size) override
|
void preallocateContents(unsigned long long size) override {
|
||||||
{
|
|
||||||
currentStart = string(s, pos, 16);
|
currentStart = string(s, pos, 16);
|
||||||
assert(size <= std::numeric_limits<size_t>::max());
|
assert(size <= std::numeric_limits<size_t>::max());
|
||||||
parents.top()->size = (size_t)size;
|
parents.top()->size = (size_t)size;
|
||||||
parents.top()->start = pos;
|
parents.top()->start = pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
void receiveContents(unsigned char * data, unsigned int len) override
|
void receiveContents(unsigned char* data, unsigned int len) override {
|
||||||
{
|
|
||||||
// Sanity check
|
// Sanity check
|
||||||
if (!currentStart.empty()) {
|
if (!currentStart.empty()) {
|
||||||
assert(len < 16 || currentStart == string((char*)data, 16));
|
assert(len < 16 || currentStart == string((char*)data, 16));
|
||||||
|
|
@ -94,22 +82,19 @@ struct NarAccessor : public FSAccessor
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void createSymlink(const Path & path, const string & target) override
|
void createSymlink(const Path& path, const string& target) override {
|
||||||
{
|
|
||||||
createMember(path,
|
createMember(path,
|
||||||
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
NarAccessor(ref<const std::string> nar) : nar(nar)
|
NarAccessor(ref<const std::string> nar) : nar(nar) {
|
||||||
{
|
|
||||||
NarIndexer indexer(*this, *nar);
|
NarIndexer indexer(*this, *nar);
|
||||||
parseDump(indexer, indexer);
|
parseDump(indexer, indexer);
|
||||||
}
|
}
|
||||||
|
|
||||||
NarAccessor(const std::string& listing, GetNarBytes getNarBytes)
|
NarAccessor(const std::string& listing, GetNarBytes getNarBytes)
|
||||||
: getNarBytes(getNarBytes)
|
: getNarBytes(getNarBytes) {
|
||||||
{
|
|
||||||
using json = nlohmann::json;
|
using json = nlohmann::json;
|
||||||
|
|
||||||
std::function<void(NarMember&, json&)> recurse;
|
std::function<void(NarMember&, json&)> recurse;
|
||||||
|
|
@ -131,15 +116,15 @@ struct NarAccessor : public FSAccessor
|
||||||
} else if (type == "symlink") {
|
} else if (type == "symlink") {
|
||||||
member.type = FSAccessor::Type::tSymlink;
|
member.type = FSAccessor::Type::tSymlink;
|
||||||
member.target = v.value("target", "");
|
member.target = v.value("target", "");
|
||||||
} else return;
|
} else
|
||||||
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
json v = json::parse(listing);
|
json v = json::parse(listing);
|
||||||
recurse(root, v);
|
recurse(root, v);
|
||||||
}
|
}
|
||||||
|
|
||||||
NarMember * find(const Path & path)
|
NarMember* find(const Path& path) {
|
||||||
{
|
|
||||||
Path canon = path == "" ? "" : canonPath(path);
|
Path canon = path == "" ? "" : canonPath(path);
|
||||||
NarMember* current = &root;
|
NarMember* current = &root;
|
||||||
auto end = path.end();
|
auto end = path.end();
|
||||||
|
|
@ -171,33 +156,30 @@ struct NarAccessor : public FSAccessor
|
||||||
return *result;
|
return *result;
|
||||||
}
|
}
|
||||||
|
|
||||||
Stat stat(const Path & path) override
|
Stat stat(const Path& path) override {
|
||||||
{
|
|
||||||
auto i = find(path);
|
auto i = find(path);
|
||||||
if (i == nullptr)
|
if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false};
|
||||||
return {FSAccessor::Type::tMissing, 0, false};
|
|
||||||
return {i->type, i->size, i->isExecutable, i->start};
|
return {i->type, i->size, i->isExecutable, i->start};
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet readDirectory(const Path & path) override
|
StringSet readDirectory(const Path& path) override {
|
||||||
{
|
|
||||||
auto i = get(path);
|
auto i = get(path);
|
||||||
|
|
||||||
if (i.type != FSAccessor::Type::tDirectory)
|
if (i.type != FSAccessor::Type::tDirectory)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a directory") % path);
|
throw Error(format("path '%1%' inside NAR file is not a directory") %
|
||||||
|
path);
|
||||||
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
for (auto & child : i.children)
|
for (auto& child : i.children) res.insert(child.first);
|
||||||
res.insert(child.first);
|
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readFile(const Path & path) override
|
std::string readFile(const Path& path) override {
|
||||||
{
|
|
||||||
auto i = get(path);
|
auto i = get(path);
|
||||||
if (i.type != FSAccessor::Type::tRegular)
|
if (i.type != FSAccessor::Type::tRegular)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
|
throw Error(format("path '%1%' inside NAR file is not a regular file") %
|
||||||
|
path);
|
||||||
|
|
||||||
if (getNarBytes) return getNarBytes(i.start, i.size);
|
if (getNarBytes) return getNarBytes(i.start, i.size);
|
||||||
|
|
||||||
|
|
@ -205,8 +187,7 @@ struct NarAccessor : public FSAccessor
|
||||||
return std::string(*nar, i.start, i.size);
|
return std::string(*nar, i.start, i.size);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readLink(const Path & path) override
|
std::string readLink(const Path& path) override {
|
||||||
{
|
|
||||||
auto i = get(path);
|
auto i = get(path);
|
||||||
if (i.type != FSAccessor::Type::tSymlink)
|
if (i.type != FSAccessor::Type::tSymlink)
|
||||||
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
|
throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
|
||||||
|
|
@ -214,20 +195,17 @@ struct NarAccessor : public FSAccessor
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
|
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) {
|
||||||
{
|
|
||||||
return make_ref<NarAccessor>(nar);
|
return make_ref<NarAccessor>(nar);
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
|
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
|
||||||
GetNarBytes getNarBytes)
|
GetNarBytes getNarBytes) {
|
||||||
{
|
|
||||||
return make_ref<NarAccessor>(listing, getNarBytes);
|
return make_ref<NarAccessor>(listing, getNarBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
|
||||||
const Path & path, bool recurse)
|
bool recurse) {
|
||||||
{
|
|
||||||
auto st = accessor->stat(path);
|
auto st = accessor->stat(path);
|
||||||
|
|
||||||
auto obj = res.object();
|
auto obj = res.object();
|
||||||
|
|
@ -236,10 +214,8 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
||||||
case FSAccessor::Type::tRegular:
|
case FSAccessor::Type::tRegular:
|
||||||
obj.attr("type", "regular");
|
obj.attr("type", "regular");
|
||||||
obj.attr("size", st.fileSize);
|
obj.attr("size", st.fileSize);
|
||||||
if (st.isExecutable)
|
if (st.isExecutable) obj.attr("executable", true);
|
||||||
obj.attr("executable", true);
|
if (st.narOffset) obj.attr("narOffset", st.narOffset);
|
||||||
if (st.narOffset)
|
|
||||||
obj.attr("narOffset", st.narOffset);
|
|
||||||
break;
|
break;
|
||||||
case FSAccessor::Type::tDirectory:
|
case FSAccessor::Type::tDirectory:
|
||||||
obj.attr("type", "directory");
|
obj.attr("type", "directory");
|
||||||
|
|
@ -263,4 +239,4 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
10
third_party/nix/src/libstore/nar-accessor.hh
vendored
10
third_party/nix/src/libstore/nar-accessor.hh
vendored
|
|
@ -1,7 +1,6 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
|
|
||||||
#include "fs-accessor.hh"
|
#include "fs-accessor.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
@ -16,15 +15,14 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
|
||||||
inside the NAR. */
|
inside the NAR. */
|
||||||
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
|
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
|
||||||
|
|
||||||
ref<FSAccessor> makeLazyNarAccessor(
|
ref<FSAccessor> makeLazyNarAccessor(const std::string& listing,
|
||||||
const std::string & listing,
|
|
||||||
GetNarBytes getNarBytes);
|
GetNarBytes getNarBytes);
|
||||||
|
|
||||||
class JSONPlaceholder;
|
class JSONPlaceholder;
|
||||||
|
|
||||||
/* Write a JSON representation of the contents of a NAR (except file
|
/* Write a JSON representation of the contents of a NAR (except file
|
||||||
contents). */
|
contents). */
|
||||||
void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
|
void listNar(JSONPlaceholder& res, ref<FSAccessor> accessor, const Path& path,
|
||||||
const Path & path, bool recurse);
|
bool recurse);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
168
third_party/nix/src/libstore/nar-info-disk-cache.cc
vendored
168
third_party/nix/src/libstore/nar-info-disk-cache.cc
vendored
|
|
@ -1,9 +1,8 @@
|
||||||
#include "nar-info-disk-cache.hh"
|
#include "nar-info-disk-cache.hh"
|
||||||
#include "sync.hh"
|
|
||||||
#include "sqlite.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
|
|
||||||
#include <sqlite3.h>
|
#include <sqlite3.h>
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "sqlite.hh"
|
||||||
|
#include "sync.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
@ -45,32 +44,28 @@ create table if not exists LastPurge (
|
||||||
|
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
class NarInfoDiskCacheImpl : public NarInfoDiskCache
|
class NarInfoDiskCacheImpl : public NarInfoDiskCache {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* How often to purge expired entries from the cache. */
|
/* How often to purge expired entries from the cache. */
|
||||||
const int purgeInterval = 24 * 3600;
|
const int purgeInterval = 24 * 3600;
|
||||||
|
|
||||||
struct Cache
|
struct Cache {
|
||||||
{
|
|
||||||
int id;
|
int id;
|
||||||
Path storeDir;
|
Path storeDir;
|
||||||
bool wantMassQuery;
|
bool wantMassQuery;
|
||||||
int priority;
|
int priority;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
SQLite db;
|
SQLite db;
|
||||||
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
|
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR,
|
||||||
|
purgeCache;
|
||||||
std::map<std::string, Cache> caches;
|
std::map<std::string, Cache> caches;
|
||||||
};
|
};
|
||||||
|
|
||||||
Sync<State> _state;
|
Sync<State> _state;
|
||||||
|
|
||||||
NarInfoDiskCacheImpl()
|
NarInfoDiskCacheImpl() {
|
||||||
{
|
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
||||||
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
|
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
|
||||||
|
|
@ -87,21 +82,33 @@ public:
|
||||||
|
|
||||||
state->db.exec(schema);
|
state->db.exec(schema);
|
||||||
|
|
||||||
state->insertCache.create(state->db,
|
state->insertCache.create(
|
||||||
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
|
state->db,
|
||||||
|
"insert or replace into BinaryCaches(url, timestamp, storeDir, "
|
||||||
|
"wantMassQuery, priority) values (?, ?, ?, ?, ?)");
|
||||||
|
|
||||||
state->queryCache.create(state->db,
|
state->queryCache.create(state->db,
|
||||||
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
|
"select id, storeDir, wantMassQuery, priority "
|
||||||
|
"from BinaryCaches where url = ?");
|
||||||
|
|
||||||
state->insertNAR.create(state->db,
|
state->insertNAR.create(
|
||||||
"insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
|
state->db,
|
||||||
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
|
"insert or replace into NARs(cache, hashPart, namePart, url, "
|
||||||
|
"compression, fileHash, fileSize, narHash, "
|
||||||
|
"narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, "
|
||||||
|
"?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
|
||||||
|
|
||||||
state->insertMissingNAR.create(state->db,
|
state->insertMissingNAR.create(
|
||||||
"insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
|
state->db,
|
||||||
|
"insert or replace into NARs(cache, hashPart, timestamp, present) "
|
||||||
|
"values (?, ?, ?, 0)");
|
||||||
|
|
||||||
state->queryNAR.create(state->db,
|
state->queryNAR.create(
|
||||||
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
|
state->db,
|
||||||
|
"select present, namePart, url, compression, fileHash, fileSize, "
|
||||||
|
"narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? "
|
||||||
|
"and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 "
|
||||||
|
"and timestamp > ?))");
|
||||||
|
|
||||||
/* Periodically purge expired entries from the database. */
|
/* Periodically purge expired entries from the database. */
|
||||||
retrySQLite<void>([&]() {
|
retrySQLite<void>([&]() {
|
||||||
|
|
@ -110,46 +117,50 @@ public:
|
||||||
SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
|
SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
|
||||||
auto queryLastPurge_(queryLastPurge.use());
|
auto queryLastPurge_(queryLastPurge.use());
|
||||||
|
|
||||||
if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) {
|
if (!queryLastPurge_.next() ||
|
||||||
|
queryLastPurge_.getInt(0) < now - purgeInterval) {
|
||||||
SQLiteStmt(state->db,
|
SQLiteStmt(state->db,
|
||||||
"delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
|
"delete from NARs where ((present = 0 and timestamp < ?) or "
|
||||||
.use()
|
"(present = 1 and timestamp < ?))")
|
||||||
(now - settings.ttlNegativeNarInfoCache)
|
.use()(now - settings.ttlNegativeNarInfoCache)(
|
||||||
(now - settings.ttlPositiveNarInfoCache)
|
now - settings.ttlPositiveNarInfoCache)
|
||||||
.exec();
|
.exec();
|
||||||
|
|
||||||
debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
|
debug("deleted %d entries from the NAR info disk cache",
|
||||||
|
sqlite3_changes(state->db));
|
||||||
|
|
||||||
SQLiteStmt(state->db,
|
SQLiteStmt(
|
||||||
|
state->db,
|
||||||
"insert or replace into LastPurge(dummy, value) values ('', ?)")
|
"insert or replace into LastPurge(dummy, value) values ('', ?)")
|
||||||
.use()(now).exec();
|
.use()(now)
|
||||||
|
.exec();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
Cache & getCache(State & state, const std::string & uri)
|
Cache& getCache(State& state, const std::string& uri) {
|
||||||
{
|
|
||||||
auto i = state.caches.find(uri);
|
auto i = state.caches.find(uri);
|
||||||
if (i == state.caches.end()) abort();
|
if (i == state.caches.end()) abort();
|
||||||
return i->second;
|
return i->second;
|
||||||
}
|
}
|
||||||
|
|
||||||
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
|
void createCache(const std::string& uri, const Path& storeDir,
|
||||||
{
|
bool wantMassQuery, int priority) override {
|
||||||
retrySQLite<void>([&]() {
|
retrySQLite<void>([&]() {
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
||||||
// FIXME: race
|
// FIXME: race
|
||||||
|
|
||||||
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
|
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)
|
||||||
|
.exec();
|
||||||
assert(sqlite3_changes(state->db) == 1);
|
assert(sqlite3_changes(state->db) == 1);
|
||||||
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
|
state->caches[uri] = Cache{(int)sqlite3_last_insert_rowid(state->db),
|
||||||
|
storeDir, wantMassQuery, priority};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
bool cacheExists(const std::string & uri,
|
bool cacheExists(const std::string& uri, bool& wantMassQuery,
|
||||||
bool & wantMassQuery, int & priority) override
|
int& priority) override {
|
||||||
{
|
|
||||||
return retrySQLite<bool>([&]() {
|
return retrySQLite<bool>([&]() {
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
||||||
|
|
@ -157,8 +168,9 @@ public:
|
||||||
if (i == state->caches.end()) {
|
if (i == state->caches.end()) {
|
||||||
auto queryCache(state->queryCache.use()(uri));
|
auto queryCache(state->queryCache.use()(uri));
|
||||||
if (!queryCache.next()) return false;
|
if (!queryCache.next()) return false;
|
||||||
state->caches.emplace(uri,
|
state->caches.emplace(
|
||||||
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
|
uri, Cache{(int)queryCache.getInt(0), queryCache.getStr(1),
|
||||||
|
queryCache.getInt(2) != 0, (int)queryCache.getInt(3)});
|
||||||
}
|
}
|
||||||
|
|
||||||
auto& cache(getCache(*state, uri));
|
auto& cache(getCache(*state, uri));
|
||||||
|
|
@ -171,8 +183,7 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
|
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
|
||||||
const std::string & uri, const std::string & hashPart) override
|
const std::string& uri, const std::string& hashPart) override {
|
||||||
{
|
|
||||||
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
|
return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
|
||||||
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
|
[&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
@ -181,27 +192,22 @@ public:
|
||||||
|
|
||||||
auto now = time(0);
|
auto now = time(0);
|
||||||
|
|
||||||
auto queryNAR(state->queryNAR.use()
|
auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)(
|
||||||
(cache.id)
|
now - settings.ttlNegativeNarInfoCache)(
|
||||||
(hashPart)
|
now - settings.ttlPositiveNarInfoCache));
|
||||||
(now - settings.ttlNegativeNarInfoCache)
|
|
||||||
(now - settings.ttlPositiveNarInfoCache));
|
|
||||||
|
|
||||||
if (!queryNAR.next())
|
if (!queryNAR.next()) return {oUnknown, 0};
|
||||||
return {oUnknown, 0};
|
|
||||||
|
|
||||||
if (!queryNAR.getInt(0))
|
if (!queryNAR.getInt(0)) return {oInvalid, 0};
|
||||||
return {oInvalid, 0};
|
|
||||||
|
|
||||||
auto narInfo = make_ref<NarInfo>();
|
auto narInfo = make_ref<NarInfo>();
|
||||||
|
|
||||||
auto namePart = queryNAR.getStr(1);
|
auto namePart = queryNAR.getStr(1);
|
||||||
narInfo->path = cache.storeDir + "/" +
|
narInfo->path = cache.storeDir + "/" + hashPart +
|
||||||
hashPart + (namePart.empty() ? "" : "-" + namePart);
|
(namePart.empty() ? "" : "-" + namePart);
|
||||||
narInfo->url = queryNAR.getStr(2);
|
narInfo->url = queryNAR.getStr(2);
|
||||||
narInfo->compression = queryNAR.getStr(3);
|
narInfo->compression = queryNAR.getStr(3);
|
||||||
if (!queryNAR.isNull(4))
|
if (!queryNAR.isNull(4)) narInfo->fileHash = Hash(queryNAR.getStr(4));
|
||||||
narInfo->fileHash = Hash(queryNAR.getStr(4));
|
|
||||||
narInfo->fileSize = queryNAR.getInt(5);
|
narInfo->fileSize = queryNAR.getInt(5);
|
||||||
narInfo->narHash = Hash(queryNAR.getStr(6));
|
narInfo->narHash = Hash(queryNAR.getStr(6));
|
||||||
narInfo->narSize = queryNAR.getInt(7);
|
narInfo->narSize = queryNAR.getInt(7);
|
||||||
|
|
@ -217,51 +223,43 @@ public:
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
void upsertNarInfo(
|
void upsertNarInfo(const std::string& uri, const std::string& hashPart,
|
||||||
const std::string & uri, const std::string & hashPart,
|
std::shared_ptr<ValidPathInfo> info) override {
|
||||||
std::shared_ptr<ValidPathInfo> info) override
|
|
||||||
{
|
|
||||||
retrySQLite<void>([&]() {
|
retrySQLite<void>([&]() {
|
||||||
auto state(_state.lock());
|
auto state(_state.lock());
|
||||||
|
|
||||||
auto& cache(getCache(*state, uri));
|
auto& cache(getCache(*state, uri));
|
||||||
|
|
||||||
if (info) {
|
if (info) {
|
||||||
|
|
||||||
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
|
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
|
||||||
|
|
||||||
assert(hashPart == storePathToHash(info->path));
|
assert(hashPart == storePathToHash(info->path));
|
||||||
|
|
||||||
state->insertNAR.use()
|
state->insertNAR
|
||||||
(cache.id)
|
.use()(cache.id)(hashPart)(storePathToName(info->path))(
|
||||||
(hashPart)
|
narInfo ? narInfo->url : "", narInfo != 0)(
|
||||||
(storePathToName(info->path))
|
narInfo ? narInfo->compression : "", narInfo != 0)(
|
||||||
(narInfo ? narInfo->url : "", narInfo != 0)
|
narInfo && narInfo->fileHash ? narInfo->fileHash.to_string()
|
||||||
(narInfo ? narInfo->compression : "", narInfo != 0)
|
: "",
|
||||||
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
|
narInfo && narInfo->fileHash)(
|
||||||
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
|
narInfo ? narInfo->fileSize : 0,
|
||||||
(info->narHash.to_string())
|
narInfo != 0 && narInfo->fileSize)(info->narHash.to_string())(
|
||||||
(info->narSize)
|
info->narSize)(concatStringsSep(" ", info->shortRefs()))(
|
||||||
(concatStringsSep(" ", info->shortRefs()))
|
info->deriver != "" ? baseNameOf(info->deriver) : "",
|
||||||
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
|
info->deriver !=
|
||||||
(concatStringsSep(" ", info->sigs))
|
"")(concatStringsSep(" ", info->sigs))(info->ca)(time(0))
|
||||||
(info->ca)
|
.exec();
|
||||||
(time(0)).exec();
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
state->insertMissingNAR.use()
|
state->insertMissingNAR.use()(cache.id)(hashPart)(time(0)).exec();
|
||||||
(cache.id)
|
|
||||||
(hashPart)
|
|
||||||
(time(0)).exec();
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<NarInfoDiskCache> getNarInfoDiskCache()
|
ref<NarInfoDiskCache> getNarInfoDiskCache() {
|
||||||
{
|
|
||||||
static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
|
static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
|
||||||
return cache;
|
return cache;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,26 +1,25 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "ref.hh"
|
|
||||||
#include "nar-info.hh"
|
#include "nar-info.hh"
|
||||||
|
#include "ref.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class NarInfoDiskCache
|
class NarInfoDiskCache {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
typedef enum { oValid, oInvalid, oUnknown } Outcome;
|
typedef enum { oValid, oInvalid, oUnknown } Outcome;
|
||||||
|
|
||||||
virtual void createCache(const std::string& uri, const Path& storeDir,
|
virtual void createCache(const std::string& uri, const Path& storeDir,
|
||||||
bool wantMassQuery, int priority) = 0;
|
bool wantMassQuery, int priority) = 0;
|
||||||
|
|
||||||
virtual bool cacheExists(const std::string & uri,
|
virtual bool cacheExists(const std::string& uri, bool& wantMassQuery,
|
||||||
bool & wantMassQuery, int & priority) = 0;
|
int& priority) = 0;
|
||||||
|
|
||||||
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
|
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
|
||||||
const std::string& uri, const std::string& hashPart) = 0;
|
const std::string& uri, const std::string& hashPart) = 0;
|
||||||
|
|
||||||
virtual void upsertNarInfo(
|
virtual void upsertNarInfo(const std::string& uri,
|
||||||
const std::string & uri, const std::string & hashPart,
|
const std::string& hashPart,
|
||||||
std::shared_ptr<ValidPathInfo> info) = 0;
|
std::shared_ptr<ValidPathInfo> info) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -28,4 +27,4 @@ public:
|
||||||
multiple threads. */
|
multiple threads. */
|
||||||
ref<NarInfoDiskCache> getNarInfoDiskCache();
|
ref<NarInfoDiskCache> getNarInfoDiskCache();
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
39
third_party/nix/src/libstore/nar-info.cc
vendored
39
third_party/nix/src/libstore/nar-info.cc
vendored
|
|
@ -1,10 +1,10 @@
|
||||||
#include "globals.hh"
|
|
||||||
#include "nar-info.hh"
|
#include "nar-info.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
|
NarInfo::NarInfo(const Store& store, const std::string& s,
|
||||||
{
|
const std::string& whence) {
|
||||||
auto corrupt = [&]() {
|
auto corrupt = [&]() {
|
||||||
throw Error(format("NAR info file '%1%' is corrupt") % whence);
|
throw Error(format("NAR info file '%1%' is corrupt") % whence);
|
||||||
};
|
};
|
||||||
|
|
@ -20,7 +20,6 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
|
|
||||||
size_t pos = 0;
|
size_t pos = 0;
|
||||||
while (pos < s.size()) {
|
while (pos < s.size()) {
|
||||||
|
|
||||||
size_t colon = s.find(':', pos);
|
size_t colon = s.find(':', pos);
|
||||||
if (colon == std::string::npos) corrupt();
|
if (colon == std::string::npos) corrupt();
|
||||||
|
|
||||||
|
|
@ -34,8 +33,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
if (name == "StorePath") {
|
if (name == "StorePath") {
|
||||||
if (!store.isStorePath(value)) corrupt();
|
if (!store.isStorePath(value)) corrupt();
|
||||||
path = value;
|
path = value;
|
||||||
}
|
} else if (name == "URL")
|
||||||
else if (name == "URL")
|
|
||||||
url = value;
|
url = value;
|
||||||
else if (name == "Compression")
|
else if (name == "Compression")
|
||||||
compression = value;
|
compression = value;
|
||||||
|
|
@ -43,13 +41,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
fileHash = parseHashField(value);
|
fileHash = parseHashField(value);
|
||||||
else if (name == "FileSize") {
|
else if (name == "FileSize") {
|
||||||
if (!string2Int(value, fileSize)) corrupt();
|
if (!string2Int(value, fileSize)) corrupt();
|
||||||
}
|
} else if (name == "NarHash")
|
||||||
else if (name == "NarHash")
|
|
||||||
narHash = parseHashField(value);
|
narHash = parseHashField(value);
|
||||||
else if (name == "NarSize") {
|
else if (name == "NarSize") {
|
||||||
if (!string2Int(value, narSize)) corrupt();
|
if (!string2Int(value, narSize)) corrupt();
|
||||||
}
|
} else if (name == "References") {
|
||||||
else if (name == "References") {
|
|
||||||
auto refs = tokenizeString<Strings>(value, " ");
|
auto refs = tokenizeString<Strings>(value, " ");
|
||||||
if (!references.empty()) corrupt();
|
if (!references.empty()) corrupt();
|
||||||
for (auto& r : refs) {
|
for (auto& r : refs) {
|
||||||
|
|
@ -57,15 +53,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
if (!store.isStorePath(r2)) corrupt();
|
if (!store.isStorePath(r2)) corrupt();
|
||||||
references.insert(r2);
|
references.insert(r2);
|
||||||
}
|
}
|
||||||
}
|
} else if (name == "Deriver") {
|
||||||
else if (name == "Deriver") {
|
|
||||||
if (value != "unknown-deriver") {
|
if (value != "unknown-deriver") {
|
||||||
auto p = store.storeDir + "/" + value;
|
auto p = store.storeDir + "/" + value;
|
||||||
if (!store.isStorePath(p)) corrupt();
|
if (!store.isStorePath(p)) corrupt();
|
||||||
deriver = p;
|
deriver = p;
|
||||||
}
|
}
|
||||||
}
|
} else if (name == "System")
|
||||||
else if (name == "System")
|
|
||||||
system = value;
|
system = value;
|
||||||
else if (name == "Sig")
|
else if (name == "Sig")
|
||||||
sigs.insert(value);
|
sigs.insert(value);
|
||||||
|
|
@ -82,8 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
|
||||||
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
|
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string NarInfo::to_string() const
|
std::string NarInfo::to_string() const {
|
||||||
{
|
|
||||||
std::string res;
|
std::string res;
|
||||||
res += "StorePath: " + path + "\n";
|
res += "StorePath: " + path + "\n";
|
||||||
res += "URL: " + url + "\n";
|
res += "URL: " + url + "\n";
|
||||||
|
|
@ -98,19 +91,15 @@ std::string NarInfo::to_string() const
|
||||||
|
|
||||||
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
|
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
|
||||||
|
|
||||||
if (!deriver.empty())
|
if (!deriver.empty()) res += "Deriver: " + baseNameOf(deriver) + "\n";
|
||||||
res += "Deriver: " + baseNameOf(deriver) + "\n";
|
|
||||||
|
|
||||||
if (!system.empty())
|
if (!system.empty()) res += "System: " + system + "\n";
|
||||||
res += "System: " + system + "\n";
|
|
||||||
|
|
||||||
for (auto sig : sigs)
|
for (auto sig : sigs) res += "Sig: " + sig + "\n";
|
||||||
res += "Sig: " + sig + "\n";
|
|
||||||
|
|
||||||
if (!ca.empty())
|
if (!ca.empty()) res += "CA: " + ca + "\n";
|
||||||
res += "CA: " + ca + "\n";
|
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
7
third_party/nix/src/libstore/nar-info.hh
vendored
7
third_party/nix/src/libstore/nar-info.hh
vendored
|
|
@ -1,13 +1,12 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct NarInfo : ValidPathInfo
|
struct NarInfo : ValidPathInfo {
|
||||||
{
|
|
||||||
std::string url;
|
std::string url;
|
||||||
std::string compression;
|
std::string compression;
|
||||||
Hash fileHash;
|
Hash fileHash;
|
||||||
|
|
@ -21,4 +20,4 @@ struct NarInfo : ValidPathInfo
|
||||||
std::string to_string() const;
|
std::string to_string() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
93
third_party/nix/src/libstore/optimise-store.cc
vendored
93
third_party/nix/src/libstore/optimise-store.cc
vendored
|
|
@ -1,22 +1,18 @@
|
||||||
#include "util.hh"
|
|
||||||
#include "local-store.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
|
|
||||||
#include <cstdlib>
|
|
||||||
#include <cstring>
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <cstring>
|
||||||
#include <regex>
|
#include <regex>
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "local-store.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static void makeWritable(const Path& path) {
|
||||||
static void makeWritable(const Path & path)
|
|
||||||
{
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path '%1%'") % path);
|
throw SysError(format("getting attributes of path '%1%'") % path);
|
||||||
|
|
@ -24,13 +20,10 @@ static void makeWritable(const Path & path)
|
||||||
throw SysError(format("changing writability of '%1%'") % path);
|
throw SysError(format("changing writability of '%1%'") % path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct MakeReadOnly {
|
||||||
struct MakeReadOnly
|
|
||||||
{
|
|
||||||
Path path;
|
Path path;
|
||||||
MakeReadOnly(const Path& path) : path(path) {}
|
MakeReadOnly(const Path& path) : path(path) {}
|
||||||
~MakeReadOnly()
|
~MakeReadOnly() {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
/* This will make the path read-only. */
|
/* This will make the path read-only. */
|
||||||
if (path != "") canonicaliseTimestampAndPermissions(path);
|
if (path != "") canonicaliseTimestampAndPermissions(path);
|
||||||
|
|
@ -40,9 +33,7 @@ struct MakeReadOnly
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
LocalStore::InodeHash LocalStore::loadInodeHash() {
|
||||||
LocalStore::InodeHash LocalStore::loadInodeHash()
|
|
||||||
{
|
|
||||||
debug("loading hash inodes in memory");
|
debug("loading hash inodes in memory");
|
||||||
InodeHash inodeHash;
|
InodeHash inodeHash;
|
||||||
|
|
||||||
|
|
@ -62,9 +53,8 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
|
||||||
return inodeHash;
|
return inodeHash;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
|
||||||
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
|
const InodeHash& inodeHash) {
|
||||||
{
|
|
||||||
Strings names;
|
Strings names;
|
||||||
|
|
||||||
AutoCloseDir dir(opendir(path.c_str()));
|
AutoCloseDir dir(opendir(path.c_str()));
|
||||||
|
|
@ -88,10 +78,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
|
||||||
return names;
|
return names;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
|
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
|
||||||
const Path & path, InodeHash & inodeHash)
|
const Path& path, InodeHash& inodeHash) {
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
|
|
@ -104,8 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
|
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
|
||||||
https://github.com/NixOS/nix/issues/1443 for more discussion. */
|
https://github.com/NixOS/nix/issues/1443 for more discussion. */
|
||||||
|
|
||||||
if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
|
if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) {
|
||||||
{
|
|
||||||
debug(format("'%1%' is not allowed to be linked in macOS") % path);
|
debug(format("'%1%' is not allowed to be linked in macOS") % path);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -113,8 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
if (S_ISDIR(st.st_mode)) {
|
||||||
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
||||||
for (auto & i : names)
|
for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash);
|
||||||
optimisePath_(act, stats, path + "/" + i, inodeHash);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -123,7 +109,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
#if CAN_LINK_SYMLINK
|
#if CAN_LINK_SYMLINK
|
||||||
&& !S_ISLNK(st.st_mode)
|
&& !S_ISLNK(st.st_mode)
|
||||||
#endif
|
#endif
|
||||||
) return;
|
)
|
||||||
|
return;
|
||||||
|
|
||||||
/* Sometimes SNAFUs can cause files in the Nix store to be
|
/* Sometimes SNAFUs can cause files in the Nix store to be
|
||||||
modified, in particular when running programs as root under
|
modified, in particular when running programs as root under
|
||||||
|
|
@ -136,7 +123,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
|
|
||||||
/* This can still happen on top-level files. */
|
/* This can still happen on top-level files. */
|
||||||
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
|
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
|
||||||
debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
|
debug(format("'%1%' is already linked, with %2% other file(s)") % path %
|
||||||
|
(st.st_nlink - 2));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -174,7 +162,8 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
full. When that happens, it's fine to ignore it: we
|
full. When that happens, it's fine to ignore it: we
|
||||||
just effectively disable deduplication of this
|
just effectively disable deduplication of this
|
||||||
file. */
|
file. */
|
||||||
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
|
printInfo("cannot link '%s' to '%s': %s", linkPath, path,
|
||||||
|
strerror(errno));
|
||||||
return;
|
return;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
|
@ -211,8 +200,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
its timestamp back to 0. */
|
its timestamp back to 0. */
|
||||||
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
|
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
|
||||||
|
|
||||||
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
|
Path tempLink =
|
||||||
% realStoreDir % getpid() % random()).str();
|
(format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random())
|
||||||
|
.str();
|
||||||
|
|
||||||
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
||||||
if (errno == EMLINK) {
|
if (errno == EMLINK) {
|
||||||
|
|
@ -245,13 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
stats.bytesFreed += st.st_size;
|
stats.bytesFreed += st.st_size;
|
||||||
stats.blocksFreed += st.st_blocks;
|
stats.blocksFreed += st.st_blocks;
|
||||||
|
|
||||||
if (act)
|
if (act) act->result(resFileLinked, st.st_size, st.st_blocks);
|
||||||
act->result(resFileLinked, st.st_size, st.st_blocks);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LocalStore::optimiseStore(OptimiseStats& stats) {
|
||||||
void LocalStore::optimiseStore(OptimiseStats & stats)
|
|
||||||
{
|
|
||||||
Activity act(*logger, actOptimiseStore);
|
Activity act(*logger, actOptimiseStore);
|
||||||
|
|
||||||
PathSet paths = queryAllValidPaths();
|
PathSet paths = queryAllValidPaths();
|
||||||
|
|
@ -265,7 +252,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
addTempRoot(i);
|
addTempRoot(i);
|
||||||
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
||||||
{
|
{
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i));
|
Activity act(*logger, lvlTalkative, actUnknown,
|
||||||
|
fmt("optimising path '%s'", i));
|
||||||
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
||||||
}
|
}
|
||||||
done++;
|
done++;
|
||||||
|
|
@ -273,30 +261,25 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static string showBytes(unsigned long long bytes)
|
static string showBytes(unsigned long long bytes) {
|
||||||
{
|
|
||||||
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalStore::optimiseStore()
|
void LocalStore::optimiseStore() {
|
||||||
{
|
|
||||||
OptimiseStats stats;
|
OptimiseStats stats;
|
||||||
|
|
||||||
optimiseStore(stats);
|
optimiseStore(stats);
|
||||||
|
|
||||||
printInfo(
|
printInfo(format("%1% freed by hard-linking %2% files") %
|
||||||
format("%1% freed by hard-linking %2% files")
|
showBytes(stats.bytesFreed) % stats.filesLinked);
|
||||||
% showBytes(stats.bytesFreed)
|
|
||||||
% stats.filesLinked);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalStore::optimisePath(const Path & path)
|
void LocalStore::optimisePath(const Path& path) {
|
||||||
{
|
|
||||||
OptimiseStats stats;
|
OptimiseStats stats;
|
||||||
InodeHash inodeHash;
|
InodeHash inodeHash;
|
||||||
|
|
||||||
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
|
if (settings.autoOptimiseStore)
|
||||||
|
optimisePath_(nullptr, stats, path, inodeHash);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -3,28 +3,29 @@
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
|
ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv)
|
||||||
: drvPath(drvPath), drv(drv)
|
: drvPath(drvPath), drv(drv) {
|
||||||
{
|
|
||||||
/* Parse the __json attribute, if any. */
|
/* Parse the __json attribute, if any. */
|
||||||
auto jsonAttr = drv.env.find("__json");
|
auto jsonAttr = drv.env.find("__json");
|
||||||
if (jsonAttr != drv.env.end()) {
|
if (jsonAttr != drv.env.end()) {
|
||||||
try {
|
try {
|
||||||
structuredAttrs = nlohmann::json::parse(jsonAttr->second);
|
structuredAttrs = nlohmann::json::parse(jsonAttr->second);
|
||||||
} catch (std::exception& e) {
|
} catch (std::exception& e) {
|
||||||
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
|
throw Error("cannot process __json attribute of '%s': %s", drvPath,
|
||||||
|
e.what());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
|
std::optional<std::string> ParsedDerivation::getStringAttr(
|
||||||
{
|
const std::string& name) const {
|
||||||
if (structuredAttrs) {
|
if (structuredAttrs) {
|
||||||
auto i = structuredAttrs->find(name);
|
auto i = structuredAttrs->find(name);
|
||||||
if (i == structuredAttrs->end())
|
if (i == structuredAttrs->end())
|
||||||
return {};
|
return {};
|
||||||
else {
|
else {
|
||||||
if (!i->is_string())
|
if (!i->is_string())
|
||||||
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
|
throw Error("attribute '%s' of derivation '%s' must be a string", name,
|
||||||
|
drvPath);
|
||||||
return i->get<std::string>();
|
return i->get<std::string>();
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -36,15 +37,15 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
|
bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const {
|
||||||
{
|
|
||||||
if (structuredAttrs) {
|
if (structuredAttrs) {
|
||||||
auto i = structuredAttrs->find(name);
|
auto i = structuredAttrs->find(name);
|
||||||
if (i == structuredAttrs->end())
|
if (i == structuredAttrs->end())
|
||||||
return def;
|
return def;
|
||||||
else {
|
else {
|
||||||
if (!i->is_boolean())
|
if (!i->is_boolean())
|
||||||
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
|
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name,
|
||||||
|
drvPath);
|
||||||
return i->get<bool>();
|
return i->get<bool>();
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -56,19 +57,23 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
|
std::optional<Strings> ParsedDerivation::getStringsAttr(
|
||||||
{
|
const std::string& name) const {
|
||||||
if (structuredAttrs) {
|
if (structuredAttrs) {
|
||||||
auto i = structuredAttrs->find(name);
|
auto i = structuredAttrs->find(name);
|
||||||
if (i == structuredAttrs->end())
|
if (i == structuredAttrs->end())
|
||||||
return {};
|
return {};
|
||||||
else {
|
else {
|
||||||
if (!i->is_array())
|
if (!i->is_array())
|
||||||
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
|
throw Error(
|
||||||
|
"attribute '%s' of derivation '%s' must be a list of strings", name,
|
||||||
|
drvPath);
|
||||||
Strings res;
|
Strings res;
|
||||||
for (auto j = i->begin(); j != i->end(); ++j) {
|
for (auto j = i->begin(); j != i->end(); ++j) {
|
||||||
if (!j->is_string())
|
if (!j->is_string())
|
||||||
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
|
throw Error(
|
||||||
|
"attribute '%s' of derivation '%s' must be a list of strings",
|
||||||
|
name, drvPath);
|
||||||
res.push_back(j->get<std::string>());
|
res.push_back(j->get<std::string>());
|
||||||
}
|
}
|
||||||
return res;
|
return res;
|
||||||
|
|
@ -82,19 +87,16 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet ParsedDerivation::getRequiredSystemFeatures() const
|
StringSet ParsedDerivation::getRequiredSystemFeatures() const {
|
||||||
{
|
|
||||||
StringSet res;
|
StringSet res;
|
||||||
for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
|
for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
|
||||||
res.insert(i);
|
res.insert(i);
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::canBuildLocally() const
|
bool ParsedDerivation::canBuildLocally() const {
|
||||||
{
|
if (drv.platform != settings.thisSystem.get() &&
|
||||||
if (drv.platform != settings.thisSystem.get()
|
!settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin())
|
||||||
&& !settings.extraPlatforms.get().count(drv.platform)
|
|
||||||
&& !drv.isBuiltin())
|
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
for (auto& feature : getRequiredSystemFeatures())
|
for (auto& feature : getRequiredSystemFeatures())
|
||||||
|
|
@ -103,14 +105,12 @@ bool ParsedDerivation::canBuildLocally() const
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::willBuildLocally() const
|
bool ParsedDerivation::willBuildLocally() const {
|
||||||
{
|
|
||||||
return getBoolAttr("preferLocalBuild") && canBuildLocally();
|
return getBoolAttr("preferLocalBuild") && canBuildLocally();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParsedDerivation::substitutesAllowed() const
|
bool ParsedDerivation::substitutesAllowed() const {
|
||||||
{
|
|
||||||
return getBoolAttr("allowSubstitutes", true);
|
return getBoolAttr("allowSubstitutes", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,17 @@
|
||||||
#include "derivations.hh"
|
|
||||||
|
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
|
#include "derivations.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class ParsedDerivation
|
class ParsedDerivation {
|
||||||
{
|
|
||||||
Path drvPath;
|
Path drvPath;
|
||||||
BasicDerivation& drv;
|
BasicDerivation& drv;
|
||||||
std::optional<nlohmann::json> structuredAttrs;
|
std::optional<nlohmann::json> structuredAttrs;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
ParsedDerivation(const Path& drvPath, BasicDerivation& drv);
|
ParsedDerivation(const Path& drvPath, BasicDerivation& drv);
|
||||||
|
|
||||||
const std::optional<nlohmann::json> & getStructuredAttrs() const
|
const std::optional<nlohmann::json>& getStructuredAttrs() const {
|
||||||
{
|
|
||||||
return structuredAttrs;
|
return structuredAttrs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -34,4 +30,4 @@ public:
|
||||||
bool substitutesAllowed() const;
|
bool substitutesAllowed() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
78
third_party/nix/src/libstore/pathlocks.cc
vendored
78
third_party/nix/src/libstore/pathlocks.cc
vendored
|
|
@ -1,21 +1,16 @@
|
||||||
#include "pathlocks.hh"
|
#include "pathlocks.hh"
|
||||||
#include "util.hh"
|
#include <fcntl.h>
|
||||||
#include "sync.hh"
|
#include <sys/file.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
#include <cerrno>
|
#include <cerrno>
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
|
#include "sync.hh"
|
||||||
#include <fcntl.h>
|
#include "util.hh"
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <sys/file.h>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
AutoCloseFD openLockFile(const Path& path, bool create) {
|
||||||
AutoCloseFD openLockFile(const Path & path, bool create)
|
|
||||||
{
|
|
||||||
AutoCloseFD fd;
|
AutoCloseFD fd;
|
||||||
|
|
||||||
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
|
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
|
||||||
|
|
@ -25,9 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
|
||||||
return fd;
|
return fd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteLockFile(const Path& path, int fd) {
|
||||||
void deleteLockFile(const Path & path, int fd)
|
|
||||||
{
|
|
||||||
/* Get rid of the lock file. Have to be careful not to introduce
|
/* Get rid of the lock file. Have to be careful not to introduce
|
||||||
races. Write a (meaningless) token to the file to indicate to
|
races. Write a (meaningless) token to the file to indicate to
|
||||||
other processes waiting on this lock that the lock is stale
|
other processes waiting on this lock that the lock is stale
|
||||||
|
|
@ -38,14 +31,16 @@ void deleteLockFile(const Path & path, int fd)
|
||||||
file is an optimisation, not a necessity. */
|
file is an optimisation, not a necessity. */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool lockFile(int fd, LockType lockType, bool wait) {
|
||||||
bool lockFile(int fd, LockType lockType, bool wait)
|
|
||||||
{
|
|
||||||
int type;
|
int type;
|
||||||
if (lockType == ltRead) type = LOCK_SH;
|
if (lockType == ltRead)
|
||||||
else if (lockType == ltWrite) type = LOCK_EX;
|
type = LOCK_SH;
|
||||||
else if (lockType == ltNone) type = LOCK_UN;
|
else if (lockType == ltWrite)
|
||||||
else abort();
|
type = LOCK_EX;
|
||||||
|
else if (lockType == ltNone)
|
||||||
|
type = LOCK_UN;
|
||||||
|
else
|
||||||
|
abort();
|
||||||
|
|
||||||
if (wait) {
|
if (wait) {
|
||||||
while (flock(fd, type) != 0) {
|
while (flock(fd, type) != 0) {
|
||||||
|
|
@ -59,31 +54,22 @@ bool lockFile(int fd, LockType lockType, bool wait)
|
||||||
while (flock(fd, type | LOCK_NB) != 0) {
|
while (flock(fd, type | LOCK_NB) != 0) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
if (errno == EWOULDBLOCK) return false;
|
if (errno == EWOULDBLOCK) return false;
|
||||||
if (errno != EINTR)
|
if (errno != EINTR) throw SysError(format("acquiring/releasing lock"));
|
||||||
throw SysError(format("acquiring/releasing lock"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathLocks::PathLocks() : deletePaths(false) {}
|
||||||
PathLocks::PathLocks()
|
|
||||||
: deletePaths(false)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
PathLocks::PathLocks(const PathSet& paths, const string& waitMsg)
|
PathLocks::PathLocks(const PathSet& paths, const string& waitMsg)
|
||||||
: deletePaths(false)
|
: deletePaths(false) {
|
||||||
{
|
|
||||||
lockPaths(paths, waitMsg);
|
lockPaths(paths, waitMsg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
|
||||||
bool PathLocks::lockPaths(const PathSet & paths,
|
bool wait) {
|
||||||
const string & waitMsg, bool wait)
|
|
||||||
{
|
|
||||||
assert(fds.empty());
|
assert(fds.empty());
|
||||||
|
|
||||||
/* Note that `fds' is built incrementally so that the destructor
|
/* Note that `fds' is built incrementally so that the destructor
|
||||||
|
|
@ -101,7 +87,6 @@ bool PathLocks::lockPaths(const PathSet & paths,
|
||||||
AutoCloseFD fd;
|
AutoCloseFD fd;
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
|
|
||||||
/* Open/create the lock file. */
|
/* Open/create the lock file. */
|
||||||
fd = openLockFile(lockPath, true);
|
fd = openLockFile(lockPath, true);
|
||||||
|
|
||||||
|
|
@ -142,9 +127,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathLocks::~PathLocks() {
|
||||||
PathLocks::~PathLocks()
|
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
unlock();
|
unlock();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -152,15 +135,13 @@ PathLocks::~PathLocks()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void PathLocks::unlock() {
|
||||||
void PathLocks::unlock()
|
|
||||||
{
|
|
||||||
for (auto& i : fds) {
|
for (auto& i : fds) {
|
||||||
if (deletePaths) deleteLockFile(i.second, i.first);
|
if (deletePaths) deleteLockFile(i.second, i.first);
|
||||||
|
|
||||||
if (close(i.first) == -1)
|
if (close(i.first) == -1)
|
||||||
printError(
|
printError(format("error (ignored): cannot close lock file on '%1%'") %
|
||||||
format("error (ignored): cannot close lock file on '%1%'") % i.second);
|
i.second);
|
||||||
|
|
||||||
debug(format("lock released on '%1%'") % i.second);
|
debug(format("lock released on '%1%'") % i.second);
|
||||||
}
|
}
|
||||||
|
|
@ -168,11 +149,8 @@ void PathLocks::unlock()
|
||||||
fds.clear();
|
fds.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void PathLocks::setDeletion(bool deletePaths) {
|
||||||
void PathLocks::setDeletion(bool deletePaths)
|
|
||||||
{
|
|
||||||
this->deletePaths = deletePaths;
|
this->deletePaths = deletePaths;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
11
third_party/nix/src/libstore/pathlocks.hh
vendored
11
third_party/nix/src/libstore/pathlocks.hh
vendored
|
|
@ -16,8 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
|
||||||
|
|
||||||
bool lockFile(int fd, LockType lockType, bool wait);
|
bool lockFile(int fd, LockType lockType, bool wait);
|
||||||
|
|
||||||
class PathLocks
|
class PathLocks {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
typedef std::pair<int, Path> FDPair;
|
typedef std::pair<int, Path> FDPair;
|
||||||
list<FDPair> fds;
|
list<FDPair> fds;
|
||||||
|
|
@ -25,14 +24,12 @@ private:
|
||||||
|
|
||||||
public:
|
public:
|
||||||
PathLocks();
|
PathLocks();
|
||||||
PathLocks(const PathSet & paths,
|
PathLocks(const PathSet& paths, const string& waitMsg = "");
|
||||||
const string & waitMsg = "");
|
bool lockPaths(const PathSet& _paths, const string& waitMsg = "",
|
||||||
bool lockPaths(const PathSet & _paths,
|
|
||||||
const string & waitMsg = "",
|
|
||||||
bool wait = true);
|
bool wait = true);
|
||||||
~PathLocks();
|
~PathLocks();
|
||||||
void unlock();
|
void unlock();
|
||||||
void setDeletion(bool deletePaths);
|
void setDeletion(bool deletePaths);
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
99
third_party/nix/src/libstore/profiles.cc
vendored
99
third_party/nix/src/libstore/profiles.cc
vendored
|
|
@ -1,27 +1,21 @@
|
||||||
#include "profiles.hh"
|
#include "profiles.hh"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <unistd.h>
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <errno.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static bool cmpGensByNumber(const Generation& a, const Generation& b) {
|
||||||
static bool cmpGensByNumber(const Generation & a, const Generation & b)
|
|
||||||
{
|
|
||||||
return a.number < b.number;
|
return a.number < b.number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Parse a generation name of the format
|
/* Parse a generation name of the format
|
||||||
`<profilename>-<number>-link'. */
|
`<profilename>-<number>-link'. */
|
||||||
static int parseName(const string & profileName, const string & name)
|
static int parseName(const string& profileName, const string& name) {
|
||||||
{
|
|
||||||
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
|
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
|
||||||
string s = string(name, profileName.size() + 1);
|
string s = string(name, profileName.size() + 1);
|
||||||
string::size_type p = s.find("-link");
|
string::size_type p = s.find("-link");
|
||||||
|
|
@ -33,10 +27,7 @@ static int parseName(const string & profileName, const string & name)
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Generations findGenerations(Path profile, int& curGen) {
|
||||||
|
|
||||||
Generations findGenerations(Path profile, int & curGen)
|
|
||||||
{
|
|
||||||
Generations gens;
|
Generations gens;
|
||||||
|
|
||||||
Path profileDir = dirOf(profile);
|
Path profileDir = dirOf(profile);
|
||||||
|
|
@ -58,24 +49,17 @@ Generations findGenerations(Path profile, int & curGen)
|
||||||
|
|
||||||
gens.sort(cmpGensByNumber);
|
gens.sort(cmpGensByNumber);
|
||||||
|
|
||||||
curGen = pathExists(profile)
|
curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1;
|
||||||
? parseName(profileName, readLink(profile))
|
|
||||||
: -1;
|
|
||||||
|
|
||||||
return gens;
|
return gens;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void makeName(const Path& profile, unsigned int num, Path& outLink) {
|
||||||
static void makeName(const Path & profile, unsigned int num,
|
|
||||||
Path & outLink)
|
|
||||||
{
|
|
||||||
Path prefix = (format("%1%-%2%") % profile % num).str();
|
Path prefix = (format("%1%-%2%") % profile % num).str();
|
||||||
outLink = prefix + "-link";
|
outLink = prefix + "-link";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) {
|
||||||
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
|
|
||||||
{
|
|
||||||
/* The new generation number should be higher than old the
|
/* The new generation number should be higher than old the
|
||||||
previous ones. */
|
previous ones. */
|
||||||
int dummy;
|
int dummy;
|
||||||
|
|
@ -113,24 +97,19 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
|
||||||
return generation;
|
return generation;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void removeFile(const Path& path) {
|
||||||
static void removeFile(const Path & path)
|
|
||||||
{
|
|
||||||
if (remove(path.c_str()) == -1)
|
if (remove(path.c_str()) == -1)
|
||||||
throw SysError(format("cannot unlink '%1%'") % path);
|
throw SysError(format("cannot unlink '%1%'") % path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteGeneration(const Path& profile, unsigned int gen) {
|
||||||
void deleteGeneration(const Path & profile, unsigned int gen)
|
|
||||||
{
|
|
||||||
Path generation;
|
Path generation;
|
||||||
makeName(profile, gen, generation);
|
makeName(profile, gen, generation);
|
||||||
removeFile(generation);
|
removeFile(generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void deleteGeneration2(const Path& profile, unsigned int gen,
|
||||||
static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
|
bool dryRun) {
|
||||||
{
|
|
||||||
if (dryRun)
|
if (dryRun)
|
||||||
printInfo(format("would remove generation %1%") % gen);
|
printInfo(format("would remove generation %1%") % gen);
|
||||||
else {
|
else {
|
||||||
|
|
@ -139,9 +118,9 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteGenerations(const Path& profile,
|
||||||
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
|
const std::set<unsigned int>& gensToDelete,
|
||||||
{
|
bool dryRun) {
|
||||||
PathLocks lock;
|
PathLocks lock;
|
||||||
lockProfile(lock, profile);
|
lockProfile(lock, profile);
|
||||||
|
|
||||||
|
|
@ -149,7 +128,8 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
|
||||||
Generations gens = findGenerations(profile, curGen);
|
Generations gens = findGenerations(profile, curGen);
|
||||||
|
|
||||||
if (gensToDelete.find(curGen) != gensToDelete.end())
|
if (gensToDelete.find(curGen) != gensToDelete.end())
|
||||||
throw Error(format("cannot delete current generation of profile %1%'") % profile);
|
throw Error(format("cannot delete current generation of profile %1%'") %
|
||||||
|
profile);
|
||||||
|
|
||||||
for (auto& i : gens) {
|
for (auto& i : gens) {
|
||||||
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
|
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
|
||||||
|
|
@ -157,8 +137,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
|
void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) {
|
||||||
{
|
|
||||||
PathLocks lock;
|
PathLocks lock;
|
||||||
lockProfile(lock, profile);
|
lockProfile(lock, profile);
|
||||||
|
|
||||||
|
|
@ -181,8 +160,7 @@ void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void deleteOldGenerations(const Path & profile, bool dryRun)
|
void deleteOldGenerations(const Path& profile, bool dryRun) {
|
||||||
{
|
|
||||||
PathLocks lock;
|
PathLocks lock;
|
||||||
lockProfile(lock, profile);
|
lockProfile(lock, profile);
|
||||||
|
|
||||||
|
|
@ -190,13 +168,10 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
|
||||||
Generations gens = findGenerations(profile, curGen);
|
Generations gens = findGenerations(profile, curGen);
|
||||||
|
|
||||||
for (auto& i : gens)
|
for (auto& i : gens)
|
||||||
if (i.number != curGen)
|
if (i.number != curGen) deleteGeneration2(profile, i.number, dryRun);
|
||||||
deleteGeneration2(profile, i.number, dryRun);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) {
|
||||||
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
|
|
||||||
{
|
|
||||||
PathLocks lock;
|
PathLocks lock;
|
||||||
lockProfile(lock, profile);
|
lockProfile(lock, profile);
|
||||||
|
|
||||||
|
|
@ -207,8 +182,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
|
||||||
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
|
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
|
||||||
if (canDelete) {
|
if (canDelete) {
|
||||||
assert(i->creationTime < t);
|
assert(i->creationTime < t);
|
||||||
if (i->number != curGen)
|
if (i->number != curGen) deleteGeneration2(profile, i->number, dryRun);
|
||||||
deleteGeneration2(profile, i->number, dryRun);
|
|
||||||
} else if (i->creationTime < t) {
|
} else if (i->creationTime < t) {
|
||||||
/* We may now start deleting generations, but we don't
|
/* We may now start deleting generations, but we don't
|
||||||
delete this generation yet, because this generation was
|
delete this generation yet, because this generation was
|
||||||
|
|
@ -218,9 +192,8 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
|
||||||
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
|
bool dryRun) {
|
||||||
{
|
|
||||||
time_t curTime = time(0);
|
time_t curTime = time(0);
|
||||||
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
|
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
|
||||||
int days;
|
int days;
|
||||||
|
|
@ -233,27 +206,21 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
|
||||||
deleteGenerationsOlderThan(profile, oldTime, dryRun);
|
deleteGenerationsOlderThan(profile, oldTime, dryRun);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void switchLink(Path link, Path target) {
|
||||||
void switchLink(Path link, Path target)
|
|
||||||
{
|
|
||||||
/* Hacky. */
|
/* Hacky. */
|
||||||
if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
|
if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
|
||||||
|
|
||||||
replaceSymlink(target, link);
|
replaceSymlink(target, link);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void lockProfile(PathLocks& lock, const Path& profile) {
|
||||||
void lockProfile(PathLocks & lock, const Path & profile)
|
lock.lockPaths({profile},
|
||||||
{
|
(format("waiting for lock on profile '%1%'") % profile).str());
|
||||||
lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());
|
|
||||||
lock.setDeletion(true);
|
lock.setDeletion(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string optimisticLockProfile(const Path& profile) {
|
||||||
string optimisticLockProfile(const Path & profile)
|
|
||||||
{
|
|
||||||
return pathExists(profile) ? readLink(profile) : "";
|
return pathExists(profile) ? readLink(profile) : "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
32
third_party/nix/src/libstore/profiles.hh
vendored
32
third_party/nix/src/libstore/profiles.hh
vendored
|
|
@ -1,32 +1,21 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "pathlocks.hh"
|
|
||||||
|
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include "pathlocks.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
struct Generation {
|
||||||
struct Generation
|
|
||||||
{
|
|
||||||
int number;
|
int number;
|
||||||
Path path;
|
Path path;
|
||||||
time_t creationTime;
|
time_t creationTime;
|
||||||
Generation()
|
Generation() { number = -1; }
|
||||||
{
|
operator bool() const { return number != -1; }
|
||||||
number = -1;
|
|
||||||
}
|
|
||||||
operator bool() const
|
|
||||||
{
|
|
||||||
return number != -1;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef list<Generation> Generations;
|
typedef list<Generation> Generations;
|
||||||
|
|
||||||
|
|
||||||
/* Returns the list of currently present generations for the specified
|
/* Returns the list of currently present generations for the specified
|
||||||
profile, sorted by generation number. */
|
profile, sorted by generation number. */
|
||||||
Generations findGenerations(Path profile, int& curGen);
|
Generations findGenerations(Path profile, int& curGen);
|
||||||
|
|
@ -37,15 +26,18 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
|
||||||
|
|
||||||
void deleteGeneration(const Path& profile, unsigned int gen);
|
void deleteGeneration(const Path& profile, unsigned int gen);
|
||||||
|
|
||||||
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
|
void deleteGenerations(const Path& profile,
|
||||||
|
const std::set<unsigned int>& gensToDelete, bool dryRun);
|
||||||
|
|
||||||
void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
|
void deleteGenerationsGreaterThan(const Path& profile, const int max,
|
||||||
|
bool dryRun);
|
||||||
|
|
||||||
void deleteOldGenerations(const Path& profile, bool dryRun);
|
void deleteOldGenerations(const Path& profile, bool dryRun);
|
||||||
|
|
||||||
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun);
|
void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun);
|
||||||
|
|
||||||
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun);
|
void deleteGenerationsOlderThan(const Path& profile, const string& timeSpec,
|
||||||
|
bool dryRun);
|
||||||
|
|
||||||
void switchLink(Path link, Path target);
|
void switchLink(Path link, Path target);
|
||||||
|
|
||||||
|
|
@ -64,4 +56,4 @@ void lockProfile(PathLocks & lock, const Path & profile);
|
||||||
rebuilt. */
|
rebuilt. */
|
||||||
string optimisticLockProfile(const Path& profile);
|
string optimisticLockProfile(const Path& profile);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
46
third_party/nix/src/libstore/references.cc
vendored
46
third_party/nix/src/libstore/references.cc
vendored
|
|
@ -1,21 +1,16 @@
|
||||||
#include "references.hh"
|
#include "references.hh"
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <map>
|
||||||
|
#include "archive.hh"
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "archive.hh"
|
|
||||||
|
|
||||||
#include <map>
|
|
||||||
#include <cstdlib>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
static unsigned int refLength = 32; /* characters */
|
static unsigned int refLength = 32; /* characters */
|
||||||
|
|
||||||
|
static void search(const unsigned char* s, size_t len, StringSet& hashes,
|
||||||
static void search(const unsigned char * s, size_t len,
|
StringSet& seen) {
|
||||||
StringSet & hashes, StringSet & seen)
|
|
||||||
{
|
|
||||||
static bool initialised = false;
|
static bool initialised = false;
|
||||||
static bool isBase32[256];
|
static bool isBase32[256];
|
||||||
if (!initialised) {
|
if (!initialised) {
|
||||||
|
|
@ -37,8 +32,7 @@ static void search(const unsigned char * s, size_t len,
|
||||||
if (!match) continue;
|
if (!match) continue;
|
||||||
string ref((const char*)s + i, refLength);
|
string ref((const char*)s + i, refLength);
|
||||||
if (hashes.find(ref) != hashes.end()) {
|
if (hashes.find(ref) != hashes.end()) {
|
||||||
debug(format("found reference to '%1%' at offset '%2%'")
|
debug(format("found reference to '%1%' at offset '%2%'") % ref % i);
|
||||||
% ref % i);
|
|
||||||
seen.insert(ref);
|
seen.insert(ref);
|
||||||
hashes.erase(ref);
|
hashes.erase(ref);
|
||||||
}
|
}
|
||||||
|
|
@ -46,9 +40,7 @@ static void search(const unsigned char * s, size_t len,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct RefScanSink : Sink {
|
||||||
struct RefScanSink : Sink
|
|
||||||
{
|
|
||||||
HashSink hashSink;
|
HashSink hashSink;
|
||||||
StringSet hashes;
|
StringSet hashes;
|
||||||
StringSet seen;
|
StringSet seen;
|
||||||
|
|
@ -60,29 +52,27 @@ struct RefScanSink : Sink
|
||||||
void operator()(const unsigned char* data, size_t len);
|
void operator()(const unsigned char* data, size_t len);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
void RefScanSink::operator()(const unsigned char* data, size_t len) {
|
||||||
void RefScanSink::operator () (const unsigned char * data, size_t len)
|
|
||||||
{
|
|
||||||
hashSink(data, len);
|
hashSink(data, len);
|
||||||
|
|
||||||
/* It's possible that a reference spans the previous and current
|
/* It's possible that a reference spans the previous and current
|
||||||
fragment, so search in the concatenation of the tail of the
|
fragment, so search in the concatenation of the tail of the
|
||||||
previous fragment and the start of the current fragment. */
|
previous fragment and the start of the current fragment. */
|
||||||
string s = tail + string((const char *) data, len > refLength ? refLength : len);
|
string s =
|
||||||
|
tail + string((const char*)data, len > refLength ? refLength : len);
|
||||||
search((const unsigned char*)s.data(), s.size(), hashes, seen);
|
search((const unsigned char*)s.data(), s.size(), hashes, seen);
|
||||||
|
|
||||||
search(data, len, hashes, seen);
|
search(data, len, hashes, seen);
|
||||||
|
|
||||||
size_t tailLen = len <= refLength ? len : refLength;
|
size_t tailLen = len <= refLength ? len : refLength;
|
||||||
tail =
|
tail = string(tail, tail.size() < refLength - tailLen
|
||||||
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
|
? 0
|
||||||
|
: tail.size() - (refLength - tailLen)) +
|
||||||
string((const char*)data + len - tailLen, tailLen);
|
string((const char*)data + len - tailLen, tailLen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet scanForReferences(const string& path, const PathSet& refs,
|
||||||
PathSet scanForReferences(const string & path,
|
HashResult& hash) {
|
||||||
const PathSet & refs, HashResult & hash)
|
|
||||||
{
|
|
||||||
RefScanSink sink;
|
RefScanSink sink;
|
||||||
std::map<string, Path> backMap;
|
std::map<string, Path> backMap;
|
||||||
|
|
||||||
|
|
@ -92,8 +82,7 @@ PathSet scanForReferences(const string & path,
|
||||||
for (auto& i : refs) {
|
for (auto& i : refs) {
|
||||||
string baseName = baseNameOf(i);
|
string baseName = baseNameOf(i);
|
||||||
string::size_type pos = baseName.find('-');
|
string::size_type pos = baseName.find('-');
|
||||||
if (pos == string::npos)
|
if (pos == string::npos) throw Error(format("bad reference '%1%'") % i);
|
||||||
throw Error(format("bad reference '%1%'") % i);
|
|
||||||
string s = string(baseName, 0, pos);
|
string s = string(baseName, 0, pos);
|
||||||
assert(s.size() == refLength);
|
assert(s.size() == refLength);
|
||||||
assert(backMap.find(s) == backMap.end());
|
assert(backMap.find(s) == backMap.end());
|
||||||
|
|
@ -118,5 +107,4 @@ PathSet scanForReferences(const string & path,
|
||||||
return found;
|
return found;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
2
third_party/nix/src/libstore/references.hh
vendored
2
third_party/nix/src/libstore/references.hh
vendored
|
|
@ -1,7 +1,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "hash.hh"
|
#include "hash.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,30 +1,25 @@
|
||||||
#include "remote-fs-accessor.hh"
|
#include "remote-fs-accessor.hh"
|
||||||
#include "nar-accessor.hh"
|
|
||||||
#include "json.hh"
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include "json.hh"
|
||||||
|
#include "nar-accessor.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir)
|
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path& cacheDir)
|
||||||
: store(store)
|
: store(store), cacheDir(cacheDir) {
|
||||||
, cacheDir(cacheDir)
|
if (cacheDir != "") createDirs(cacheDir);
|
||||||
{
|
|
||||||
if (cacheDir != "")
|
|
||||||
createDirs(cacheDir);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
|
Path RemoteFSAccessor::makeCacheFile(const Path& storePath,
|
||||||
{
|
const std::string& ext) {
|
||||||
assert(cacheDir != "");
|
assert(cacheDir != "");
|
||||||
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
|
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
|
||||||
}
|
}
|
||||||
|
|
||||||
void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar,
|
void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar,
|
||||||
ref<FSAccessor> narAccessor)
|
ref<FSAccessor> narAccessor) {
|
||||||
{
|
|
||||||
nars.emplace(storePath, narAccessor);
|
nars.emplace(storePath, narAccessor);
|
||||||
|
|
||||||
if (cacheDir != "") {
|
if (cacheDir != "") {
|
||||||
|
|
@ -43,15 +38,15 @@ void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & na
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) {
|
||||||
{
|
|
||||||
auto path = canonPath(path_);
|
auto path = canonPath(path_);
|
||||||
|
|
||||||
auto storePath = store->toStorePath(path);
|
auto storePath = store->toStorePath(path);
|
||||||
std::string restPath = std::string(path, storePath.size());
|
std::string restPath = std::string(path, storePath.size());
|
||||||
|
|
||||||
if (!store->isValidPath(storePath))
|
if (!store->isValidPath(storePath))
|
||||||
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
|
throw InvalidPath(format("path '%1%' is not a valid store path") %
|
||||||
|
storePath);
|
||||||
|
|
||||||
auto i = nars.find(storePath);
|
auto i = nars.find(storePath);
|
||||||
if (i != nars.end()) return {i->second, restPath};
|
if (i != nars.end()) return {i->second, restPath};
|
||||||
|
|
@ -60,17 +55,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
std::string listing;
|
std::string listing;
|
||||||
Path cacheFile;
|
Path cacheFile;
|
||||||
|
|
||||||
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
|
if (cacheDir != "" &&
|
||||||
|
pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
|
||||||
try {
|
try {
|
||||||
listing = nix::readFile(makeCacheFile(storePath, "ls"));
|
listing = nix::readFile(makeCacheFile(storePath, "ls"));
|
||||||
|
|
||||||
auto narAccessor = makeLazyNarAccessor(listing,
|
auto narAccessor = makeLazyNarAccessor(
|
||||||
[cacheFile](uint64_t offset, uint64_t length) {
|
listing, [cacheFile](uint64_t offset, uint64_t length) {
|
||||||
|
|
||||||
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
|
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
|
||||||
if (!fd)
|
if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile);
|
||||||
throw SysError("opening NAR cache file '%s'", cacheFile);
|
|
||||||
|
|
||||||
if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset)
|
if (lseek(fd.get(), offset, SEEK_SET) != (off_t)offset)
|
||||||
throw SysError("seeking in '%s'", cacheFile);
|
throw SysError("seeking in '%s'", cacheFile);
|
||||||
|
|
@ -84,7 +77,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
nars.emplace(storePath, narAccessor);
|
nars.emplace(storePath, narAccessor);
|
||||||
return {narAccessor, restPath};
|
return {narAccessor, restPath};
|
||||||
|
|
||||||
} catch (SysError &) { }
|
} catch (SysError&) {
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
*sink.s = nix::readFile(cacheFile);
|
*sink.s = nix::readFile(cacheFile);
|
||||||
|
|
@ -93,7 +87,8 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
nars.emplace(storePath, narAccessor);
|
nars.emplace(storePath, narAccessor);
|
||||||
return {narAccessor, restPath};
|
return {narAccessor, restPath};
|
||||||
|
|
||||||
} catch (SysError &) { }
|
} catch (SysError&) {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
store->narFromPath(storePath, sink);
|
store->narFromPath(storePath, sink);
|
||||||
|
|
@ -102,28 +97,24 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||||
return {narAccessor, restPath};
|
return {narAccessor, restPath};
|
||||||
}
|
}
|
||||||
|
|
||||||
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
|
FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) {
|
||||||
{
|
|
||||||
auto res = fetch(path);
|
auto res = fetch(path);
|
||||||
return res.first->stat(res.second);
|
return res.first->stat(res.second);
|
||||||
}
|
}
|
||||||
|
|
||||||
StringSet RemoteFSAccessor::readDirectory(const Path & path)
|
StringSet RemoteFSAccessor::readDirectory(const Path& path) {
|
||||||
{
|
|
||||||
auto res = fetch(path);
|
auto res = fetch(path);
|
||||||
return res.first->readDirectory(res.second);
|
return res.first->readDirectory(res.second);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string RemoteFSAccessor::readFile(const Path & path)
|
std::string RemoteFSAccessor::readFile(const Path& path) {
|
||||||
{
|
|
||||||
auto res = fetch(path);
|
auto res = fetch(path);
|
||||||
return res.first->readFile(res.second);
|
return res.first->readFile(res.second);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string RemoteFSAccessor::readLink(const Path & path)
|
std::string RemoteFSAccessor::readLink(const Path& path) {
|
||||||
{
|
|
||||||
auto res = fetch(path);
|
auto res = fetch(path);
|
||||||
return res.first->readLink(res.second);
|
return res.first->readLink(res.second);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,7 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class RemoteFSAccessor : public FSAccessor
|
class RemoteFSAccessor : public FSAccessor {
|
||||||
{
|
|
||||||
ref<Store> store;
|
ref<Store> store;
|
||||||
|
|
||||||
std::map<Path, ref<FSAccessor>> nars;
|
std::map<Path, ref<FSAccessor>> nars;
|
||||||
|
|
@ -24,7 +23,6 @@ class RemoteFSAccessor : public FSAccessor
|
||||||
ref<FSAccessor> narAccessor);
|
ref<FSAccessor> narAccessor);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
RemoteFSAccessor(ref<Store> store,
|
RemoteFSAccessor(ref<Store> store,
|
||||||
const /* FIXME: use std::optional */ Path& cacheDir = "");
|
const /* FIXME: use std::optional */ Path& cacheDir = "");
|
||||||
|
|
||||||
|
|
@ -37,4 +35,4 @@ public:
|
||||||
std::string readLink(const Path& path) override;
|
std::string readLink(const Path& path) override;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
380
third_party/nix/src/libstore/remote-store.cc
vendored
380
third_party/nix/src/libstore/remote-store.cc
vendored
|
|
@ -1,37 +1,32 @@
|
||||||
#include "serialise.hh"
|
|
||||||
#include "util.hh"
|
|
||||||
#include "remote-store.hh"
|
#include "remote-store.hh"
|
||||||
#include "worker-protocol.hh"
|
|
||||||
#include "archive.hh"
|
|
||||||
#include "affinity.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "derivations.hh"
|
|
||||||
#include "pool.hh"
|
|
||||||
#include "finally.hh"
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <sys/socket.h>
|
|
||||||
#include <sys/un.h>
|
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <sys/un.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
#include "affinity.hh"
|
||||||
|
#include "archive.hh"
|
||||||
|
#include "derivations.hh"
|
||||||
|
#include "finally.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "pool.hh"
|
||||||
|
#include "serialise.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
#include "worker-protocol.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
Path readStorePath(Store& store, Source& from) {
|
||||||
Path readStorePath(Store & store, Source & from)
|
|
||||||
{
|
|
||||||
Path path = readString(from);
|
Path path = readString(from);
|
||||||
store.assertStorePath(path);
|
store.assertStorePath(path);
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <class T>
|
||||||
template<class T> T readStorePaths(Store & store, Source & from)
|
T readStorePaths(Store& store, Source& from) {
|
||||||
{
|
|
||||||
T paths = readStrings<T>(from);
|
T paths = readStrings<T>(from);
|
||||||
for (auto& i : paths) store.assertStorePath(i);
|
for (auto& i : paths) store.assertStorePath(i);
|
||||||
return paths;
|
return paths;
|
||||||
|
|
@ -40,28 +35,24 @@ template<class T> T readStorePaths(Store & store, Source & from)
|
||||||
template PathSet readStorePaths(Store& store, Source& from);
|
template PathSet readStorePaths(Store& store, Source& from);
|
||||||
template Paths readStorePaths(Store& store, Source& from);
|
template Paths readStorePaths(Store& store, Source& from);
|
||||||
|
|
||||||
/* TODO: Separate these store impls into different files, give them better names */
|
/* TODO: Separate these store impls into different files, give them better names
|
||||||
|
*/
|
||||||
RemoteStore::RemoteStore(const Params& params)
|
RemoteStore::RemoteStore(const Params& params)
|
||||||
: Store(params)
|
: Store(params),
|
||||||
, connections(make_ref<Pool<Connection>>(
|
connections(make_ref<Pool<Connection>>(
|
||||||
std::max(1, (int)maxConnections),
|
std::max(1, (int)maxConnections),
|
||||||
[this]() { return openConnectionWrapper(); },
|
[this]() { return openConnectionWrapper(); },
|
||||||
[this](const ref<Connection>& r) {
|
[this](const ref<Connection>& r) {
|
||||||
return
|
return r->to.good() && r->from.good() &&
|
||||||
r->to.good()
|
std::chrono::duration_cast<std::chrono::seconds>(
|
||||||
&& r->from.good()
|
std::chrono::steady_clock::now() - r->startTime)
|
||||||
&& std::chrono::duration_cast<std::chrono::seconds>(
|
.count() < maxConnectionAge;
|
||||||
std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
|
})) {}
|
||||||
}
|
|
||||||
))
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() {
|
||||||
ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
|
||||||
{
|
|
||||||
if (failed)
|
if (failed)
|
||||||
throw Error("opening a connection to remote store '%s' previously failed", getUri());
|
throw Error("opening a connection to remote store '%s' previously failed",
|
||||||
|
getUri());
|
||||||
try {
|
try {
|
||||||
return openConnection();
|
return openConnection();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -70,26 +61,16 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
UDSRemoteStore::UDSRemoteStore(const Params& params)
|
UDSRemoteStore::UDSRemoteStore(const Params& params)
|
||||||
: Store(params)
|
: Store(params), LocalFSStore(params), RemoteStore(params) {}
|
||||||
, LocalFSStore(params)
|
|
||||||
, RemoteStore(params)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params)
|
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params& params)
|
||||||
: Store(params)
|
: Store(params),
|
||||||
, LocalFSStore(params)
|
LocalFSStore(params),
|
||||||
, RemoteStore(params)
|
RemoteStore(params),
|
||||||
, path(socket_path)
|
path(socket_path) {}
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
std::string UDSRemoteStore::getUri() {
|
||||||
std::string UDSRemoteStore::getUri()
|
|
||||||
{
|
|
||||||
if (path) {
|
if (path) {
|
||||||
return std::string("unix://") + *path;
|
return std::string("unix://") + *path;
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -97,19 +78,18 @@ std::string UDSRemoteStore::getUri()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ref<RemoteStore::Connection> UDSRemoteStore::openConnection() {
|
||||||
ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
|
||||||
{
|
|
||||||
auto conn = make_ref<Connection>();
|
auto conn = make_ref<Connection>();
|
||||||
|
|
||||||
/* Connect to a daemon that does the privileged work for us. */
|
/* Connect to a daemon that does the privileged work for us. */
|
||||||
conn->fd = socket(PF_UNIX, SOCK_STREAM
|
conn->fd = socket(PF_UNIX,
|
||||||
|
SOCK_STREAM
|
||||||
#ifdef SOCK_CLOEXEC
|
#ifdef SOCK_CLOEXEC
|
||||||
| SOCK_CLOEXEC
|
| SOCK_CLOEXEC
|
||||||
#endif
|
#endif
|
||||||
, 0);
|
,
|
||||||
if (!conn->fd)
|
0);
|
||||||
throw SysError("cannot create Unix domain socket");
|
if (!conn->fd) throw SysError("cannot create Unix domain socket");
|
||||||
closeOnExec(conn->fd.get());
|
closeOnExec(conn->fd.get());
|
||||||
|
|
||||||
string socketPath = path ? *path : settings.nixDaemonSocketFile;
|
string socketPath = path ? *path : settings.nixDaemonSocketFile;
|
||||||
|
|
@ -133,9 +113,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
||||||
return conn;
|
return conn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::initConnection(Connection& conn) {
|
||||||
void RemoteStore::initConnection(Connection & conn)
|
|
||||||
{
|
|
||||||
/* Send the magic greeting, check for the reply. */
|
/* Send the magic greeting, check for the reply. */
|
||||||
try {
|
try {
|
||||||
conn.to << WORKER_MAGIC_1;
|
conn.to << WORKER_MAGIC_1;
|
||||||
|
|
@ -144,7 +122,8 @@ void RemoteStore::initConnection(Connection & conn)
|
||||||
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
|
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
|
||||||
|
|
||||||
conn.from >> conn.daemonVersion;
|
conn.from >> conn.daemonVersion;
|
||||||
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) !=
|
||||||
|
GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
||||||
throw Error("Nix daemon protocol version not supported");
|
throw Error("Nix daemon protocol version not supported");
|
||||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
|
||||||
throw Error("the Nix daemon version is too old");
|
throw Error("the Nix daemon version is too old");
|
||||||
|
|
@ -158,35 +137,26 @@ void RemoteStore::initConnection(Connection & conn)
|
||||||
conn.to << 0;
|
conn.to << 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false;
|
||||||
conn.to << false;
|
|
||||||
|
|
||||||
auto ex = conn.processStderr();
|
auto ex = conn.processStderr();
|
||||||
if (ex) std::rethrow_exception(ex);
|
if (ex) std::rethrow_exception(ex);
|
||||||
}
|
} catch (Error& e) {
|
||||||
catch (Error & e) {
|
throw Error("cannot open connection to remote store '%s': %s", getUri(),
|
||||||
throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
|
e.what());
|
||||||
}
|
}
|
||||||
|
|
||||||
setOptions(conn);
|
setOptions(conn);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::setOptions(Connection& conn) {
|
||||||
void RemoteStore::setOptions(Connection & conn)
|
conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing
|
||||||
{
|
<< settings.tryFallback << verbosity << settings.maxBuildJobs
|
||||||
conn.to << wopSetOptions
|
<< settings.maxSilentTime << true
|
||||||
<< settings.keepFailed
|
|
||||||
<< settings.keepGoing
|
|
||||||
<< settings.tryFallback
|
|
||||||
<< verbosity
|
|
||||||
<< settings.maxBuildJobs
|
|
||||||
<< settings.maxSilentTime
|
|
||||||
<< true
|
|
||||||
<< (settings.verboseBuild ? lvlError : lvlVomit)
|
<< (settings.verboseBuild ? lvlError : lvlVomit)
|
||||||
<< 0 // obsolete log type
|
<< 0 // obsolete log type
|
||||||
<< 0 /* obsolete print build trace */
|
<< 0 /* obsolete print build trace */
|
||||||
<< settings.buildCores
|
<< settings.buildCores << settings.useSubstitutes;
|
||||||
<< settings.useSubstitutes;
|
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
|
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
|
||||||
std::map<std::string, Config::SettingInfo> overrides;
|
std::map<std::string, Config::SettingInfo> overrides;
|
||||||
|
|
@ -200,35 +170,28 @@ void RemoteStore::setOptions(Connection & conn)
|
||||||
overrides.erase(settings.useSubstitutes.name);
|
overrides.erase(settings.useSubstitutes.name);
|
||||||
overrides.erase(settings.showTrace.name);
|
overrides.erase(settings.showTrace.name);
|
||||||
conn.to << overrides.size();
|
conn.to << overrides.size();
|
||||||
for (auto & i : overrides)
|
for (auto& i : overrides) conn.to << i.first << i.second.value;
|
||||||
conn.to << i.first << i.second.value;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto ex = conn.processStderr();
|
auto ex = conn.processStderr();
|
||||||
if (ex) std::rethrow_exception(ex);
|
if (ex) std::rethrow_exception(ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
|
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
|
||||||
the connection as bad (causing it to be closed) if a non-daemon
|
the connection as bad (causing it to be closed) if a non-daemon
|
||||||
exception is thrown before the handle is closed. Such an exception
|
exception is thrown before the handle is closed. Such an exception
|
||||||
causes a deviation from the expected protocol and therefore a
|
causes a deviation from the expected protocol and therefore a
|
||||||
desynchronization between the client and daemon. */
|
desynchronization between the client and daemon. */
|
||||||
struct ConnectionHandle
|
struct ConnectionHandle {
|
||||||
{
|
|
||||||
Pool<RemoteStore::Connection>::Handle handle;
|
Pool<RemoteStore::Connection>::Handle handle;
|
||||||
bool daemonException = false;
|
bool daemonException = false;
|
||||||
|
|
||||||
ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle)
|
ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle)
|
||||||
: handle(std::move(handle))
|
: handle(std::move(handle)) {}
|
||||||
{ }
|
|
||||||
|
|
||||||
ConnectionHandle(ConnectionHandle && h)
|
ConnectionHandle(ConnectionHandle&& h) : handle(std::move(h.handle)) {}
|
||||||
: handle(std::move(h.handle))
|
|
||||||
{ }
|
|
||||||
|
|
||||||
~ConnectionHandle()
|
~ConnectionHandle() {
|
||||||
{
|
|
||||||
if (!daemonException && std::uncaught_exceptions()) {
|
if (!daemonException && std::uncaught_exceptions()) {
|
||||||
handle.markBad();
|
handle.markBad();
|
||||||
debug("closing daemon connection because of an exception");
|
debug("closing daemon connection because of an exception");
|
||||||
|
|
@ -237,8 +200,7 @@ struct ConnectionHandle
|
||||||
|
|
||||||
RemoteStore::Connection* operator->() { return &*handle; }
|
RemoteStore::Connection* operator->() { return &*handle; }
|
||||||
|
|
||||||
void processStderr(Sink * sink = 0, Source * source = 0)
|
void processStderr(Sink* sink = 0, Source* source = 0) {
|
||||||
{
|
|
||||||
auto ex = handle->processStderr(sink, source);
|
auto ex = handle->processStderr(sink, source);
|
||||||
if (ex) {
|
if (ex) {
|
||||||
daemonException = true;
|
daemonException = true;
|
||||||
|
|
@ -247,24 +209,19 @@ struct ConnectionHandle
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
ConnectionHandle RemoteStore::getConnection() {
|
||||||
ConnectionHandle RemoteStore::getConnection()
|
|
||||||
{
|
|
||||||
return ConnectionHandle(connections->get());
|
return ConnectionHandle(connections->get());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool RemoteStore::isValidPathUncached(const Path& path) {
|
||||||
bool RemoteStore::isValidPathUncached(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopIsValidPath << path;
|
conn->to << wopIsValidPath << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readInt(conn->from);
|
return readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryValidPaths(const PathSet& paths,
|
||||||
PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
|
SubstituteFlag maybeSubstitute) {
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
||||||
PathSet res;
|
PathSet res;
|
||||||
|
|
@ -278,18 +235,14 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryAllValidPaths() {
|
||||||
PathSet RemoteStore::queryAllValidPaths()
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryAllValidPaths;
|
conn->to << wopQueryAllValidPaths;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readStorePaths<PathSet>(*this, conn->from);
|
return readStorePaths<PathSet>(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::querySubstitutablePaths(const PathSet& paths) {
|
||||||
PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
||||||
PathSet res;
|
PathSet res;
|
||||||
|
|
@ -306,16 +259,13 @@ PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void RemoteStore::querySubstitutablePathInfos(const PathSet& paths,
|
void RemoteStore::querySubstitutablePathInfos(const PathSet& paths,
|
||||||
SubstitutablePathInfos & infos)
|
SubstitutablePathInfos& infos) {
|
||||||
{
|
|
||||||
if (paths.empty()) return;
|
if (paths.empty()) return;
|
||||||
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
|
||||||
|
|
||||||
for (auto& i : paths) {
|
for (auto& i : paths) {
|
||||||
SubstitutablePathInfo info;
|
SubstitutablePathInfo info;
|
||||||
conn->to << wopQuerySubstitutablePathInfo << i;
|
conn->to << wopQuerySubstitutablePathInfo << i;
|
||||||
|
|
@ -331,7 +281,6 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
conn->to << wopQuerySubstitutablePathInfos << paths;
|
conn->to << wopQuerySubstitutablePathInfos << paths;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
size_t count = readNum<size_t>(conn->from);
|
size_t count = readNum<size_t>(conn->from);
|
||||||
|
|
@ -344,14 +293,12 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
|
||||||
info.downloadSize = readLongLong(conn->from);
|
info.downloadSize = readLongLong(conn->from);
|
||||||
info.narSize = readLongLong(conn->from);
|
info.narSize = readLongLong(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::queryPathInfoUncached(
|
||||||
void RemoteStore::queryPathInfoUncached(const Path & path,
|
const Path& path,
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
std::shared_ptr<ValidPathInfo> info;
|
std::shared_ptr<ValidPathInfo> info;
|
||||||
{
|
{
|
||||||
|
|
@ -366,7 +313,8 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
|
||||||
bool valid; conn->from >> valid;
|
bool valid;
|
||||||
|
conn->from >> valid;
|
||||||
if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
|
if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
|
||||||
}
|
}
|
||||||
info = std::make_shared<ValidPathInfo>();
|
info = std::make_shared<ValidPathInfo>();
|
||||||
|
|
@ -383,13 +331,12 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
callback(std::move(info));
|
callback(std::move(info));
|
||||||
} catch (...) { callback.rethrow(); }
|
} catch (...) {
|
||||||
|
callback.rethrow();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::queryReferrers(const Path& path, PathSet& referrers) {
|
||||||
void RemoteStore::queryReferrers(const Path & path,
|
|
||||||
PathSet & referrers)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryReferrers << path;
|
conn->to << wopQueryReferrers << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
|
@ -397,36 +344,28 @@ void RemoteStore::queryReferrers(const Path & path,
|
||||||
referrers.insert(referrers2.begin(), referrers2.end());
|
referrers.insert(referrers2.begin(), referrers2.end());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryValidDerivers(const Path& path) {
|
||||||
PathSet RemoteStore::queryValidDerivers(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryValidDerivers << path;
|
conn->to << wopQueryValidDerivers << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readStorePaths<PathSet>(*this, conn->from);
|
return readStorePaths<PathSet>(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryDerivationOutputs(const Path& path) {
|
||||||
PathSet RemoteStore::queryDerivationOutputs(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryDerivationOutputs << path;
|
conn->to << wopQueryDerivationOutputs << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readStorePaths<PathSet>(*this, conn->from);
|
return readStorePaths<PathSet>(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryDerivationOutputNames(const Path& path) {
|
||||||
PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryDerivationOutputNames << path;
|
conn->to << wopQueryDerivationOutputNames << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readStrings<PathSet>(conn->from);
|
return readStrings<PathSet>(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path RemoteStore::queryPathFromHashPart(const string& hashPart) {
|
||||||
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopQueryPathFromHashPart << hashPart;
|
conn->to << wopQueryPathFromHashPart << hashPart;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
|
@ -435,10 +374,9 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void RemoteStore::addToStore(const ValidPathInfo& info, Source& source,
|
void RemoteStore::addToStore(const ValidPathInfo& info, Source& source,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
|
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||||
{
|
std::shared_ptr<FSAccessor> accessor) {
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
|
||||||
|
|
@ -448,11 +386,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
sink << 1 // == path follows
|
sink << 1 // == path follows
|
||||||
;
|
;
|
||||||
copyNAR(source, sink);
|
copyNAR(source, sink);
|
||||||
sink
|
sink << exportMagic << info.path << info.references << info.deriver
|
||||||
<< exportMagic
|
|
||||||
<< info.path
|
|
||||||
<< info.references
|
|
||||||
<< info.deriver
|
|
||||||
<< 0 // == no legacy signature
|
<< 0 // == no legacy signature
|
||||||
<< 0 // == no path follows
|
<< 0 // == no path follows
|
||||||
;
|
;
|
||||||
|
|
@ -465,31 +399,32 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
}
|
}
|
||||||
|
|
||||||
else {
|
else {
|
||||||
conn->to << wopAddToStoreNar
|
conn->to << wopAddToStoreNar << info.path << info.deriver
|
||||||
<< info.path << info.deriver << info.narHash.to_string(Base16, false)
|
<< info.narHash.to_string(Base16, false) << info.references
|
||||||
<< info.references << info.registrationTime << info.narSize
|
<< info.registrationTime << info.narSize << info.ultimate
|
||||||
<< info.ultimate << info.sigs << info.ca
|
<< info.sigs << info.ca << repair << !checkSigs;
|
||||||
<< repair << !checkSigs;
|
|
||||||
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
|
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
|
||||||
if (!tunnel) copyNAR(source, conn->to);
|
if (!tunnel) copyNAR(source, conn->to);
|
||||||
conn.processStderr(0, tunnel ? &source : nullptr);
|
conn.processStderr(0, tunnel ? &source : nullptr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Path RemoteStore::addToStore(const string& name, const Path& _srcPath,
|
Path RemoteStore::addToStore(const string& name, const Path& _srcPath,
|
||||||
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
bool recursive, HashType hashAlgo,
|
||||||
{
|
PathFilter& filter, RepairFlag repair) {
|
||||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
if (repair)
|
||||||
|
throw Error(
|
||||||
|
"repairing is not supported when building through the Nix daemon");
|
||||||
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
|
|
||||||
Path srcPath(absPath(_srcPath));
|
Path srcPath(absPath(_srcPath));
|
||||||
|
|
||||||
conn->to << wopAddToStore << name
|
conn->to << wopAddToStore << name
|
||||||
<< ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
|
<< ((hashAlgo == htSHA256 && recursive)
|
||||||
<< (recursive ? 1 : 0)
|
? 0
|
||||||
<< printHashType(hashAlgo);
|
: 1) /* backwards compatibility hack */
|
||||||
|
<< (recursive ? 1 : 0) << printHashType(hashAlgo);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
conn->to.written = 0;
|
conn->to.written = 0;
|
||||||
|
|
@ -504,21 +439,21 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
|
||||||
} catch (SysError& e) {
|
} catch (SysError& e) {
|
||||||
/* Daemon closed while we were sending the path. Probably OOM
|
/* Daemon closed while we were sending the path. Probably OOM
|
||||||
or I/O error. */
|
or I/O error. */
|
||||||
if (e.errNo == EPIPE)
|
if (e.errNo == EPIPE) try {
|
||||||
try {
|
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
} catch (EndOfFile & e) { }
|
} catch (EndOfFile& e) {
|
||||||
|
}
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|
||||||
return readStorePath(*this, conn->from);
|
return readStorePath(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Path RemoteStore::addTextToStore(const string& name, const string& s,
|
Path RemoteStore::addTextToStore(const string& name, const string& s,
|
||||||
const PathSet & references, RepairFlag repair)
|
const PathSet& references, RepairFlag repair) {
|
||||||
{
|
if (repair)
|
||||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
throw Error(
|
||||||
|
"repairing is not supported when building through the Nix daemon");
|
||||||
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopAddTextToStore << name << s << references;
|
conn->to << wopAddTextToStore << name << s << references;
|
||||||
|
|
@ -527,9 +462,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
|
||||||
return readStorePath(*this, conn->from);
|
return readStorePath(*this, conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::buildPaths(const PathSet& drvPaths, BuildMode buildMode) {
|
||||||
void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopBuildPaths;
|
conn->to << wopBuildPaths;
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
|
||||||
|
|
@ -540,23 +473,23 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
|
||||||
/* Old daemons did not take a 'buildMode' parameter, so we
|
/* Old daemons did not take a 'buildMode' parameter, so we
|
||||||
need to validate it here on the client side. */
|
need to validate it here on the client side. */
|
||||||
if (buildMode != bmNormal)
|
if (buildMode != bmNormal)
|
||||||
throw Error("repairing or checking is not supported when building through the Nix daemon");
|
throw Error(
|
||||||
|
"repairing or checking is not supported when building through the "
|
||||||
|
"Nix daemon");
|
||||||
} else {
|
} else {
|
||||||
/* For backwards compatibility with old daemons, strip output
|
/* For backwards compatibility with old daemons, strip output
|
||||||
identifiers. */
|
identifiers. */
|
||||||
PathSet drvPaths2;
|
PathSet drvPaths2;
|
||||||
for (auto & i : drvPaths)
|
for (auto& i : drvPaths) drvPaths2.insert(string(i, 0, i.find('!')));
|
||||||
drvPaths2.insert(string(i, 0, i.find('!')));
|
|
||||||
conn->to << drvPaths2;
|
conn->to << drvPaths2;
|
||||||
}
|
}
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
BuildResult RemoteStore::buildDerivation(const Path& drvPath,
|
||||||
BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
|
const BasicDerivation& drv,
|
||||||
BuildMode buildMode)
|
BuildMode buildMode) {
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopBuildDerivation << drvPath << drv << buildMode;
|
conn->to << wopBuildDerivation << drvPath << drv << buildMode;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
|
@ -567,45 +500,35 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::ensurePath(const Path& path) {
|
||||||
void RemoteStore::ensurePath(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopEnsurePath << path;
|
conn->to << wopEnsurePath << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::addTempRoot(const Path& path) {
|
||||||
void RemoteStore::addTempRoot(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopAddTempRoot << path;
|
conn->to << wopAddTempRoot << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::addIndirectRoot(const Path& path) {
|
||||||
void RemoteStore::addIndirectRoot(const Path & path)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopAddIndirectRoot << path;
|
conn->to << wopAddIndirectRoot << path;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::syncWithGC() {
|
||||||
void RemoteStore::syncWithGC()
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopSyncWithGC;
|
conn->to << wopSyncWithGC;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Roots RemoteStore::findRoots(bool censor) {
|
||||||
Roots RemoteStore::findRoots(bool censor)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopFindRoots;
|
conn->to << wopFindRoots;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
|
@ -619,13 +542,11 @@ Roots RemoteStore::findRoots(bool censor)
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::collectGarbage(const GCOptions& options, GCResults& results) {
|
||||||
void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
|
|
||||||
conn->to
|
conn->to << wopCollectGarbage << options.action << options.pathsToDelete
|
||||||
<< wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
|
<< options.ignoreLiveness
|
||||||
<< options.maxFreed
|
<< options.maxFreed
|
||||||
/* removed options */
|
/* removed options */
|
||||||
<< 0 << 0 << 0;
|
<< 0 << 0 << 0;
|
||||||
|
|
@ -642,38 +563,31 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::optimiseStore() {
|
||||||
void RemoteStore::optimiseStore()
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopOptimiseStore;
|
conn->to << wopOptimiseStore;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) {
|
||||||
bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopVerifyStore << checkContents << repair;
|
conn->to << wopVerifyStore << checkContents << repair;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return readInt(conn->from);
|
return readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::addSignatures(const Path& storePath, const StringSet& sigs) {
|
||||||
void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
|
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << wopAddSignatures << storePath << sigs;
|
conn->to << wopAddSignatures << storePath << sigs;
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
readInt(conn->from);
|
readInt(conn->from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::queryMissing(const PathSet& targets, PathSet& willBuild,
|
||||||
void RemoteStore::queryMissing(const PathSet & targets,
|
PathSet& willSubstitute, PathSet& unknown,
|
||||||
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
|
unsigned long long& downloadSize,
|
||||||
unsigned long long & downloadSize, unsigned long long & narSize)
|
unsigned long long& narSize) {
|
||||||
{
|
|
||||||
{
|
{
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
|
||||||
|
|
@ -690,32 +604,20 @@ void RemoteStore::queryMissing(const PathSet & targets,
|
||||||
}
|
}
|
||||||
|
|
||||||
fallback:
|
fallback:
|
||||||
return Store::queryMissing(targets, willBuild, willSubstitute,
|
return Store::queryMissing(targets, willBuild, willSubstitute, unknown,
|
||||||
unknown, downloadSize, narSize);
|
downloadSize, narSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::connect() { auto conn(getConnection()); }
|
||||||
|
|
||||||
void RemoteStore::connect()
|
unsigned int RemoteStore::getProtocol() {
|
||||||
{
|
|
||||||
auto conn(getConnection());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
unsigned int RemoteStore::getProtocol()
|
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
return conn->daemonVersion;
|
return conn->daemonVersion;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void RemoteStore::flushBadConnections() { connections->flushBad(); }
|
||||||
|
|
||||||
void RemoteStore::flushBadConnections()
|
RemoteStore::Connection::~Connection() {
|
||||||
{
|
|
||||||
connections->flushBad();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
RemoteStore::Connection::~Connection()
|
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
to.flush();
|
to.flush();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -723,9 +625,7 @@ RemoteStore::Connection::~Connection()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Logger::Fields readFields(Source& from) {
|
||||||
static Logger::Fields readFields(Source & from)
|
|
||||||
{
|
|
||||||
Logger::Fields fields;
|
Logger::Fields fields;
|
||||||
size_t size = readInt(from);
|
size_t size = readInt(from);
|
||||||
for (size_t n = 0; n < size; n++) {
|
for (size_t n = 0; n < size; n++) {
|
||||||
|
|
@ -740,13 +640,11 @@ static Logger::Fields readFields(Source & from)
|
||||||
return fields;
|
return fields;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink,
|
||||||
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
|
Source* source) {
|
||||||
{
|
|
||||||
to.flush();
|
to.flush();
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
||||||
auto msg = readNum<uint64_t>(from);
|
auto msg = readNum<uint64_t>(from);
|
||||||
|
|
||||||
if (msg == STDERR_WRITE) {
|
if (msg == STDERR_WRITE) {
|
||||||
|
|
@ -806,12 +704,12 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
||||||
|
|
||||||
static std::string uriScheme = "unix://";
|
static std::string uriScheme = "unix://";
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore(
|
||||||
const std::string & uri, const Store::Params & params)
|
[](const std::string& uri,
|
||||||
-> std::shared_ptr<Store>
|
const Store::Params& params) -> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||||
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
|
return std::make_shared<UDSRemoteStore>(
|
||||||
|
std::string(uri, uriScheme.size()), params);
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
57
third_party/nix/src/libstore/remote-store.hh
vendored
57
third_party/nix/src/libstore/remote-store.hh
vendored
|
|
@ -2,31 +2,28 @@
|
||||||
|
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
class Pipe;
|
class Pipe;
|
||||||
class Pid;
|
class Pid;
|
||||||
struct FdSink;
|
struct FdSink;
|
||||||
struct FdSource;
|
struct FdSource;
|
||||||
template<typename T> class Pool;
|
template <typename T>
|
||||||
|
class Pool;
|
||||||
struct ConnectionHandle;
|
struct ConnectionHandle;
|
||||||
|
|
||||||
|
|
||||||
/* FIXME: RemoteStore is a misnomer - should be something like
|
/* FIXME: RemoteStore is a misnomer - should be something like
|
||||||
DaemonStore. */
|
DaemonStore. */
|
||||||
class RemoteStore : public virtual Store
|
class RemoteStore : public virtual Store {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
const Setting<int> maxConnections{
|
||||||
|
(Store*)this, 1, "max-connections",
|
||||||
|
"maximum number of concurrent connections to the Nix daemon"};
|
||||||
|
|
||||||
const Setting<int> maxConnections{(Store*) this, 1,
|
const Setting<unsigned int> maxConnectionAge{
|
||||||
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
|
(Store*)this, std::numeric_limits<unsigned int>::max(),
|
||||||
|
|
||||||
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
|
|
||||||
"max-connection-age", "number of seconds to reuse a connection"};
|
"max-connection-age", "number of seconds to reuse a connection"};
|
||||||
|
|
||||||
virtual bool sameMachine() = 0;
|
virtual bool sameMachine() = 0;
|
||||||
|
|
@ -37,12 +34,13 @@ public:
|
||||||
|
|
||||||
bool isValidPathUncached(const Path& path) override;
|
bool isValidPathUncached(const Path& path) override;
|
||||||
|
|
||||||
PathSet queryValidPaths(const PathSet & paths,
|
PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute =
|
||||||
SubstituteFlag maybeSubstitute = NoSubstitute) override;
|
NoSubstitute) override;
|
||||||
|
|
||||||
PathSet queryAllValidPaths() override;
|
PathSet queryAllValidPaths() override;
|
||||||
|
|
||||||
void queryPathInfoUncached(const Path & path,
|
void queryPathInfoUncached(
|
||||||
|
const Path& path,
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
|
||||||
|
|
||||||
void queryReferrers(const Path& path, PathSet& referrers) override;
|
void queryReferrers(const Path& path, PathSet& referrers) override;
|
||||||
|
|
@ -60,13 +58,14 @@ public:
|
||||||
void querySubstitutablePathInfos(const PathSet& paths,
|
void querySubstitutablePathInfos(const PathSet& paths,
|
||||||
SubstitutablePathInfos& infos) override;
|
SubstitutablePathInfos& infos) override;
|
||||||
|
|
||||||
void addToStore(const ValidPathInfo & info, Source & nar,
|
void addToStore(const ValidPathInfo& info, Source& nar, RepairFlag repair,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor) override;
|
std::shared_ptr<FSAccessor> accessor) override;
|
||||||
|
|
||||||
Path addToStore(const string& name, const Path& srcPath,
|
Path addToStore(const string& name, const Path& srcPath,
|
||||||
bool recursive = true, HashType hashAlgo = htSHA256,
|
bool recursive = true, HashType hashAlgo = htSHA256,
|
||||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
|
PathFilter& filter = defaultPathFilter,
|
||||||
|
RepairFlag repair = NoRepair) override;
|
||||||
|
|
||||||
Path addTextToStore(const string& name, const string& s,
|
Path addTextToStore(const string& name, const string& s,
|
||||||
const PathSet& references, RepairFlag repair) override;
|
const PathSet& references, RepairFlag repair) override;
|
||||||
|
|
@ -94,9 +93,10 @@ public:
|
||||||
|
|
||||||
void addSignatures(const Path& storePath, const StringSet& sigs) override;
|
void addSignatures(const Path& storePath, const StringSet& sigs) override;
|
||||||
|
|
||||||
void queryMissing(const PathSet & targets,
|
void queryMissing(const PathSet& targets, PathSet& willBuild,
|
||||||
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
|
PathSet& willSubstitute, PathSet& unknown,
|
||||||
unsigned long long & downloadSize, unsigned long long & narSize) override;
|
unsigned long long& downloadSize,
|
||||||
|
unsigned long long& narSize) override;
|
||||||
|
|
||||||
void connect() override;
|
void connect() override;
|
||||||
|
|
||||||
|
|
@ -105,9 +105,7 @@ public:
|
||||||
void flushBadConnections();
|
void flushBadConnections();
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
struct Connection {
|
||||||
struct Connection
|
|
||||||
{
|
|
||||||
AutoCloseFD fd;
|
AutoCloseFD fd;
|
||||||
FdSink to;
|
FdSink to;
|
||||||
FdSource from;
|
FdSource from;
|
||||||
|
|
@ -134,28 +132,21 @@ protected:
|
||||||
friend struct ConnectionHandle;
|
friend struct ConnectionHandle;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
std::atomic_bool failed{false};
|
std::atomic_bool failed{false};
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore
|
class UDSRemoteStore : public LocalFSStore, public RemoteStore {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
UDSRemoteStore(const Params& params);
|
UDSRemoteStore(const Params& params);
|
||||||
UDSRemoteStore(std::string path, const Params& params);
|
UDSRemoteStore(std::string path, const Params& params);
|
||||||
|
|
||||||
std::string getUri() override;
|
std::string getUri() override;
|
||||||
|
|
||||||
bool sameMachine()
|
bool sameMachine() { return true; }
|
||||||
{ return true; }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
ref<RemoteStore::Connection> openConnection() override;
|
ref<RemoteStore::Connection> openConnection() override;
|
||||||
std::optional<std::string> path;
|
std::optional<std::string> path;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,6 @@
|
||||||
#if ENABLE_S3
|
#if ENABLE_S3
|
||||||
|
|
||||||
#include "s3.hh"
|
|
||||||
#include "s3-binary-cache-store.hh"
|
#include "s3-binary-cache-store.hh"
|
||||||
#include "nar-info.hh"
|
|
||||||
#include "nar-info-disk-cache.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "compression.hh"
|
|
||||||
#include "download.hh"
|
|
||||||
#include "istringstream_nocopy.hh"
|
|
||||||
|
|
||||||
#include <aws/core/Aws.h>
|
#include <aws/core/Aws.h>
|
||||||
#include <aws/core/VersionConfig.h>
|
#include <aws/core/VersionConfig.h>
|
||||||
#include <aws/core/auth/AWSCredentialsProvider.h>
|
#include <aws/core/auth/AWSCredentialsProvider.h>
|
||||||
|
|
@ -24,13 +16,19 @@
|
||||||
#include <aws/s3/model/ListObjectsRequest.h>
|
#include <aws/s3/model/ListObjectsRequest.h>
|
||||||
#include <aws/s3/model/PutObjectRequest.h>
|
#include <aws/s3/model/PutObjectRequest.h>
|
||||||
#include <aws/transfer/TransferManager.h>
|
#include <aws/transfer/TransferManager.h>
|
||||||
|
#include "compression.hh"
|
||||||
|
#include "download.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "istringstream_nocopy.hh"
|
||||||
|
#include "nar-info-disk-cache.hh"
|
||||||
|
#include "nar-info.hh"
|
||||||
|
#include "s3.hh"
|
||||||
|
|
||||||
using namespace Aws::Transfer;
|
using namespace Aws::Transfer;
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct S3Error : public Error
|
struct S3Error : public Error {
|
||||||
{
|
|
||||||
Aws::S3::S3Errors err;
|
Aws::S3::S3Errors err;
|
||||||
S3Error(Aws::S3::S3Errors err, const FormatOrString& fs)
|
S3Error(Aws::S3::S3Errors err, const FormatOrString& fs)
|
||||||
: Error(fs), err(err){};
|
: Error(fs), err(err){};
|
||||||
|
|
@ -39,27 +37,22 @@ struct S3Error : public Error
|
||||||
/* Helper: given an Outcome<R, E>, return R in case of success, or
|
/* Helper: given an Outcome<R, E>, return R in case of success, or
|
||||||
throw an exception in case of an error. */
|
throw an exception in case of an error. */
|
||||||
template <typename R, typename E>
|
template <typename R, typename E>
|
||||||
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
|
R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) {
|
||||||
{
|
|
||||||
if (!outcome.IsSuccess())
|
if (!outcome.IsSuccess())
|
||||||
throw S3Error(
|
throw S3Error(outcome.GetError().GetErrorType(),
|
||||||
outcome.GetError().GetErrorType(),
|
|
||||||
fs.s + ": " + outcome.GetError().GetMessage());
|
fs.s + ": " + outcome.GetError().GetMessage());
|
||||||
return outcome.GetResultWithOwnership();
|
return outcome.GetResultWithOwnership();
|
||||||
}
|
}
|
||||||
|
|
||||||
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
|
class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem {
|
||||||
{
|
|
||||||
using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
|
using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
|
||||||
|
|
||||||
void ProcessFormattedStatement(Aws::String && statement) override
|
void ProcessFormattedStatement(Aws::String&& statement) override {
|
||||||
{
|
|
||||||
debug("AWS: %s", chomp(statement));
|
debug("AWS: %s", chomp(statement));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
static void initAWS()
|
static void initAWS() {
|
||||||
{
|
|
||||||
static std::once_flag flag;
|
static std::once_flag flag;
|
||||||
std::call_once(flag, []() {
|
std::call_once(flag, []() {
|
||||||
Aws::SDKOptions options;
|
Aws::SDKOptions options;
|
||||||
|
|
@ -70,8 +63,7 @@ static void initAWS()
|
||||||
|
|
||||||
if (verbosity >= lvlDebug) {
|
if (verbosity >= lvlDebug) {
|
||||||
options.loggingOptions.logLevel =
|
options.loggingOptions.logLevel =
|
||||||
verbosity == lvlDebug
|
verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug
|
||||||
? Aws::Utils::Logging::LogLevel::Debug
|
|
||||||
: Aws::Utils::Logging::LogLevel::Trace;
|
: Aws::Utils::Logging::LogLevel::Trace;
|
||||||
options.loggingOptions.logger_create_fn = [options]() {
|
options.loggingOptions.logger_create_fn = [options]() {
|
||||||
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
|
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
|
||||||
|
|
@ -82,14 +74,18 @@ static void initAWS()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
|
S3Helper::S3Helper(const string& profile, const string& region,
|
||||||
: config(makeConfig(region, scheme, endpoint))
|
const string& scheme, const string& endpoint)
|
||||||
, client(make_ref<Aws::S3::S3Client>(
|
: config(makeConfig(region, scheme, endpoint)),
|
||||||
|
client(make_ref<Aws::S3::S3Client>(
|
||||||
profile == ""
|
profile == ""
|
||||||
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
||||||
std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
|
std::make_shared<
|
||||||
|
Aws::Auth::DefaultAWSCredentialsProviderChain>())
|
||||||
: std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
: std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
|
||||||
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
|
std::make_shared<
|
||||||
|
Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(
|
||||||
|
profile.c_str())),
|
||||||
*config,
|
*config,
|
||||||
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
|
// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
|
||||||
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
|
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
|
||||||
|
|
@ -97,25 +93,25 @@ S3Helper::S3Helper(const string & profile, const string & region, const string &
|
||||||
#else
|
#else
|
||||||
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
||||||
#endif
|
#endif
|
||||||
endpoint.empty()))
|
endpoint.empty())) {
|
||||||
{
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Log AWS retries. */
|
/* Log AWS retries. */
|
||||||
class RetryStrategy : public Aws::Client::DefaultRetryStrategy
|
class RetryStrategy : public Aws::Client::DefaultRetryStrategy {
|
||||||
{
|
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
|
||||||
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
|
long attemptedRetries) const override {
|
||||||
{
|
auto retry =
|
||||||
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
|
Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
|
||||||
if (retry)
|
if (retry)
|
||||||
printError("AWS error '%s' (%s), will retry in %d ms",
|
printError("AWS error '%s' (%s), will retry in %d ms",
|
||||||
error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
|
error.GetExceptionName(), error.GetMessage(),
|
||||||
|
CalculateDelayBeforeNextRetry(error, attemptedRetries));
|
||||||
return retry;
|
return retry;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
|
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
|
||||||
{
|
const string& region, const string& scheme, const string& endpoint) {
|
||||||
initAWS();
|
initAWS();
|
||||||
auto res = make_ref<Aws::Client::ClientConfiguration>();
|
auto res = make_ref<Aws::Client::ClientConfiguration>();
|
||||||
res->region = region;
|
res->region = region;
|
||||||
|
|
@ -132,30 +128,26 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
S3Helper::DownloadResult S3Helper::getObject(
|
S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName,
|
||||||
const std::string & bucketName, const std::string & key)
|
const std::string& key) {
|
||||||
{
|
|
||||||
debug("fetching 's3://%s/%s'...", bucketName, key);
|
debug("fetching 's3://%s/%s'...", bucketName, key);
|
||||||
|
|
||||||
auto request =
|
auto request =
|
||||||
Aws::S3::Model::GetObjectRequest()
|
Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key);
|
||||||
.WithBucket(bucketName)
|
|
||||||
.WithKey(key);
|
|
||||||
|
|
||||||
request.SetResponseStreamFactory([&]() {
|
request.SetResponseStreamFactory(
|
||||||
return Aws::New<std::stringstream>("STRINGSTREAM");
|
[&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); });
|
||||||
});
|
|
||||||
|
|
||||||
DownloadResult res;
|
DownloadResult res;
|
||||||
|
|
||||||
auto now1 = std::chrono::steady_clock::now();
|
auto now1 = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
auto result = checkAws(fmt("AWS error fetching '%s'", key),
|
auto result = checkAws(fmt("AWS error fetching '%s'", key),
|
||||||
client->GetObject(request));
|
client->GetObject(request));
|
||||||
|
|
||||||
res.data = decompress(result.GetContentEncoding(),
|
res.data =
|
||||||
|
decompress(result.GetContentEncoding(),
|
||||||
dynamic_cast<std::stringstream&>(result.GetBody()).str());
|
dynamic_cast<std::stringstream&>(result.GetBody()).str());
|
||||||
|
|
||||||
} catch (S3Error& e) {
|
} catch (S3Error& e) {
|
||||||
|
|
@ -164,24 +156,35 @@ S3Helper::DownloadResult S3Helper::getObject(
|
||||||
|
|
||||||
auto now2 = std::chrono::steady_clock::now();
|
auto now2 = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
res.durationMs =
|
||||||
|
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
|
||||||
|
.count();
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore {
|
||||||
{
|
const Setting<std::string> profile{
|
||||||
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
|
this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||||
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
const Setting<std::string> region{
|
||||||
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
|
this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||||
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
const Setting<std::string> scheme{
|
||||||
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
|
this, "", "scheme",
|
||||||
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
|
"The scheme to use for S3 requests, https by default."};
|
||||||
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
|
const Setting<std::string> endpoint{
|
||||||
const Setting<bool> multipartUpload{
|
this, "", "endpoint",
|
||||||
this, false, "multipart-upload", "whether to use multi-part uploads"};
|
"An optional override of the endpoint to use when talking to S3."};
|
||||||
|
const Setting<std::string> narinfoCompression{
|
||||||
|
this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||||
|
const Setting<std::string> lsCompression{this, "", "ls-compression",
|
||||||
|
"compression method for .ls files"};
|
||||||
|
const Setting<std::string> logCompression{
|
||||||
|
this, "", "log-compression", "compression method for log/* files"};
|
||||||
|
const Setting<bool> multipartUpload{this, false, "multipart-upload",
|
||||||
|
"whether to use multi-part uploads"};
|
||||||
const Setting<uint64_t> bufferSize{
|
const Setting<uint64_t> bufferSize{
|
||||||
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
|
this, 5 * 1024 * 1024, "buffer-size",
|
||||||
|
"size (in bytes) of each part in multi-part uploads"};
|
||||||
|
|
||||||
std::string bucketName;
|
std::string bucketName;
|
||||||
|
|
||||||
|
|
@ -189,41 +192,30 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
|
|
||||||
S3Helper s3Helper;
|
S3Helper s3Helper;
|
||||||
|
|
||||||
S3BinaryCacheStoreImpl(
|
S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName)
|
||||||
const Params & params, const std::string & bucketName)
|
: S3BinaryCacheStore(params),
|
||||||
: S3BinaryCacheStore(params)
|
bucketName(bucketName),
|
||||||
, bucketName(bucketName)
|
s3Helper(profile, region, scheme, endpoint) {
|
||||||
, s3Helper(profile, region, scheme, endpoint)
|
|
||||||
{
|
|
||||||
diskCache = getNarInfoDiskCache();
|
diskCache = getNarInfoDiskCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string getUri() override
|
std::string getUri() override { return "s3://" + bucketName; }
|
||||||
{
|
|
||||||
return "s3://" + bucketName;
|
|
||||||
}
|
|
||||||
|
|
||||||
void init() override
|
void init() override {
|
||||||
{
|
|
||||||
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
|
if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
|
||||||
|
|
||||||
BinaryCacheStore::init();
|
BinaryCacheStore::init();
|
||||||
|
|
||||||
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
|
diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const Stats & getS3Stats() override
|
const Stats& getS3Stats() override { return stats; }
|
||||||
{
|
|
||||||
return stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* This is a specialisation of isValidPath() that optimistically
|
/* This is a specialisation of isValidPath() that optimistically
|
||||||
fetches the .narinfo file, rather than first checking for its
|
fetches the .narinfo file, rather than first checking for its
|
||||||
existence via a HEAD request. Since .narinfos are small, doing
|
existence via a HEAD request. Since .narinfos are small, doing
|
||||||
a GET is unlikely to be slower than HEAD. */
|
a GET is unlikely to be slower than HEAD. */
|
||||||
bool isValidPathUncached(const Path & storePath) override
|
bool isValidPathUncached(const Path& storePath) override {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
queryPathInfo(storePath);
|
queryPathInfo(storePath);
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -232,23 +224,22 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool fileExists(const std::string & path) override
|
bool fileExists(const std::string& path) override {
|
||||||
{
|
|
||||||
stats.head++;
|
stats.head++;
|
||||||
|
|
||||||
auto res = s3Helper.client->HeadObject(
|
auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest()
|
||||||
Aws::S3::Model::HeadObjectRequest()
|
|
||||||
.WithBucket(bucketName)
|
.WithBucket(bucketName)
|
||||||
.WithKey(path));
|
.WithKey(path));
|
||||||
|
|
||||||
if (!res.IsSuccess()) {
|
if (!res.IsSuccess()) {
|
||||||
auto& error = res.GetError();
|
auto& error = res.GetError();
|
||||||
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND ||
|
||||||
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
|
error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
|
||||||
// If bucket listing is disabled, 404s turn into 403s
|
// If bucket listing is disabled, 404s turn into 403s
|
||||||
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
|
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
|
||||||
return false;
|
return false;
|
||||||
throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
|
throw Error(format("AWS error fetching '%s': %s") % path %
|
||||||
|
error.GetMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -259,17 +250,17 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
|
|
||||||
void uploadFile(const std::string& path, const std::string& data,
|
void uploadFile(const std::string& path, const std::string& data,
|
||||||
const std::string& mimeType,
|
const std::string& mimeType,
|
||||||
const std::string & contentEncoding)
|
const std::string& contentEncoding) {
|
||||||
{
|
|
||||||
auto stream = std::make_shared<istringstream_nocopy>(data);
|
auto stream = std::make_shared<istringstream_nocopy>(data);
|
||||||
|
|
||||||
auto maxThreads = std::thread::hardware_concurrency();
|
auto maxThreads = std::thread::hardware_concurrency();
|
||||||
|
|
||||||
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
|
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
|
||||||
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
|
executor =
|
||||||
|
std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(
|
||||||
|
maxThreads);
|
||||||
|
|
||||||
std::call_once(transferManagerCreated, [&]()
|
std::call_once(transferManagerCreated, [&]() {
|
||||||
{
|
|
||||||
if (multipartUpload) {
|
if (multipartUpload) {
|
||||||
TransferManagerConfiguration transferConfig(executor.get());
|
TransferManagerConfiguration transferConfig(executor.get());
|
||||||
|
|
||||||
|
|
@ -278,9 +269,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
|
|
||||||
transferConfig.uploadProgressCallback =
|
transferConfig.uploadProgressCallback =
|
||||||
[](const TransferManager* transferManager,
|
[](const TransferManager* transferManager,
|
||||||
const std::shared_ptr<const TransferHandle>
|
const std::shared_ptr<const TransferHandle>& transferHandle) {
|
||||||
&transferHandle)
|
|
||||||
{
|
|
||||||
// FIXME: find a way to properly abort the multipart upload.
|
// FIXME: find a way to properly abort the multipart upload.
|
||||||
// checkInterrupt();
|
// checkInterrupt();
|
||||||
debug("upload progress ('%s'): '%d' of '%d' bytes",
|
debug("upload progress ('%s'): '%d' of '%d' bytes",
|
||||||
|
|
@ -296,37 +285,35 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
auto now1 = std::chrono::steady_clock::now();
|
auto now1 = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
if (transferManager) {
|
if (transferManager) {
|
||||||
|
|
||||||
if (contentEncoding != "")
|
if (contentEncoding != "")
|
||||||
throw Error("setting a content encoding is not supported with S3 multi-part uploads");
|
throw Error(
|
||||||
|
"setting a content encoding is not supported with S3 multi-part "
|
||||||
|
"uploads");
|
||||||
|
|
||||||
std::shared_ptr<TransferHandle> transferHandle =
|
std::shared_ptr<TransferHandle> transferHandle =
|
||||||
transferManager->UploadFile(
|
transferManager->UploadFile(stream, bucketName, path, mimeType,
|
||||||
stream, bucketName, path, mimeType,
|
|
||||||
Aws::Map<Aws::String, Aws::String>(),
|
Aws::Map<Aws::String, Aws::String>(),
|
||||||
nullptr /*, contentEncoding */);
|
nullptr /*, contentEncoding */);
|
||||||
|
|
||||||
transferHandle->WaitUntilFinished();
|
transferHandle->WaitUntilFinished();
|
||||||
|
|
||||||
if (transferHandle->GetStatus() == TransferStatus::FAILED)
|
if (transferHandle->GetStatus() == TransferStatus::FAILED)
|
||||||
throw Error("AWS error: failed to upload 's3://%s/%s': %s",
|
throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName,
|
||||||
bucketName, path, transferHandle->GetLastError().GetMessage());
|
path, transferHandle->GetLastError().GetMessage());
|
||||||
|
|
||||||
if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
|
if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
|
||||||
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
|
throw Error(
|
||||||
|
"AWS error: transfer status of 's3://%s/%s' in unexpected state",
|
||||||
bucketName, path);
|
bucketName, path);
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
|
auto request = Aws::S3::Model::PutObjectRequest()
|
||||||
auto request =
|
|
||||||
Aws::S3::Model::PutObjectRequest()
|
|
||||||
.WithBucket(bucketName)
|
.WithBucket(bucketName)
|
||||||
.WithKey(path);
|
.WithKey(path);
|
||||||
|
|
||||||
request.SetContentType(mimeType);
|
request.SetContentType(mimeType);
|
||||||
|
|
||||||
if (contentEncoding != "")
|
if (contentEncoding != "") request.SetContentEncoding(contentEncoding);
|
||||||
request.SetContentEncoding(contentEncoding);
|
|
||||||
|
|
||||||
auto stream = std::make_shared<istringstream_nocopy>(data);
|
auto stream = std::make_shared<istringstream_nocopy>(data);
|
||||||
|
|
||||||
|
|
@ -351,20 +338,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
}
|
}
|
||||||
|
|
||||||
void upsertFile(const std::string& path, const std::string& data,
|
void upsertFile(const std::string& path, const std::string& data,
|
||||||
const std::string & mimeType) override
|
const std::string& mimeType) override {
|
||||||
{
|
|
||||||
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
|
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
|
||||||
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
|
uploadFile(path, *compress(narinfoCompression, data), mimeType,
|
||||||
|
narinfoCompression);
|
||||||
else if (lsCompression != "" && hasSuffix(path, ".ls"))
|
else if (lsCompression != "" && hasSuffix(path, ".ls"))
|
||||||
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
|
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
|
||||||
else if (logCompression != "" && hasPrefix(path, "log/"))
|
else if (logCompression != "" && hasPrefix(path, "log/"))
|
||||||
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
|
uploadFile(path, *compress(logCompression, data), mimeType,
|
||||||
|
logCompression);
|
||||||
else
|
else
|
||||||
uploadFile(path, data, mimeType, "");
|
uploadFile(path, data, mimeType, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
void getFile(const std::string & path, Sink & sink) override
|
void getFile(const std::string& path, Sink& sink) override {
|
||||||
{
|
|
||||||
stats.get++;
|
stats.get++;
|
||||||
|
|
||||||
// FIXME: stream output to sink.
|
// FIXME: stream output to sink.
|
||||||
|
|
@ -374,33 +361,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
stats.getTimeMs += res.durationMs;
|
stats.getTimeMs += res.durationMs;
|
||||||
|
|
||||||
if (res.data) {
|
if (res.data) {
|
||||||
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
|
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName,
|
||||||
bucketName, path, res.data->size(), res.durationMs);
|
path, res.data->size(), res.durationMs);
|
||||||
|
|
||||||
sink((unsigned char*)res.data->data(), res.data->size());
|
sink((unsigned char*)res.data->data(), res.data->size());
|
||||||
} else
|
} else
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
throw NoSuchBinaryCacheFile(
|
||||||
|
"file '%s' does not exist in binary cache '%s'", path, getUri());
|
||||||
}
|
}
|
||||||
|
|
||||||
PathSet queryAllValidPaths() override
|
PathSet queryAllValidPaths() override {
|
||||||
{
|
|
||||||
PathSet paths;
|
PathSet paths;
|
||||||
std::string marker;
|
std::string marker;
|
||||||
|
|
||||||
do {
|
do {
|
||||||
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);
|
debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName %
|
||||||
|
marker);
|
||||||
|
|
||||||
auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
|
auto res = checkAws(
|
||||||
s3Helper.client->ListObjects(
|
format("AWS error listing bucket '%s'") % bucketName,
|
||||||
Aws::S3::Model::ListObjectsRequest()
|
s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest()
|
||||||
.WithBucket(bucketName)
|
.WithBucket(bucketName)
|
||||||
.WithDelimiter("/")
|
.WithDelimiter("/")
|
||||||
.WithMarker(marker)));
|
.WithMarker(marker)));
|
||||||
|
|
||||||
auto& contents = res.GetContents();
|
auto& contents = res.GetContents();
|
||||||
|
|
||||||
debug(format("got %d keys, next marker '%s'")
|
debug(format("got %d keys, next marker '%s'") % contents.size() %
|
||||||
% contents.size() % res.GetNextMarker());
|
res.GetNextMarker());
|
||||||
|
|
||||||
for (auto object : contents) {
|
for (auto object : contents) {
|
||||||
auto& key = object.GetKey();
|
auto& key = object.GetKey();
|
||||||
|
|
@ -413,19 +401,18 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
|
|
||||||
return paths;
|
return paths;
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore(
|
||||||
const std::string & uri, const Store::Params & params)
|
[](const std::string& uri,
|
||||||
-> std::shared_ptr<Store>
|
const Store::Params& params) -> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (std::string(uri, 0, 5) != "s3://") return 0;
|
if (std::string(uri, 0, 5) != "s3://") return 0;
|
||||||
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
|
auto store =
|
||||||
|
std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
|
||||||
store->init();
|
store->init();
|
||||||
return store;
|
return store;
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
||||||
|
|
@ -1,23 +1,16 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "binary-cache-store.hh"
|
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
#include "binary-cache-store.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class S3BinaryCacheStore : public BinaryCacheStore
|
class S3BinaryCacheStore : public BinaryCacheStore {
|
||||||
{
|
|
||||||
protected:
|
protected:
|
||||||
|
S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {}
|
||||||
S3BinaryCacheStore(const Params & params)
|
|
||||||
: BinaryCacheStore(params)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
struct Stats {
|
||||||
struct Stats
|
|
||||||
{
|
|
||||||
std::atomic<uint64_t> put{0};
|
std::atomic<uint64_t> put{0};
|
||||||
std::atomic<uint64_t> putBytes{0};
|
std::atomic<uint64_t> putBytes{0};
|
||||||
std::atomic<uint64_t> putTimeMs{0};
|
std::atomic<uint64_t> putTimeMs{0};
|
||||||
|
|
@ -30,4 +23,4 @@ public:
|
||||||
virtual const Stats& getS3Stats() = 0;
|
virtual const Stats& getS3Stats() = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
31
third_party/nix/src/libstore/s3.hh
vendored
31
third_party/nix/src/libstore/s3.hh
vendored
|
|
@ -4,30 +4,39 @@
|
||||||
|
|
||||||
#include "ref.hh"
|
#include "ref.hh"
|
||||||
|
|
||||||
namespace Aws { namespace Client { class ClientConfiguration; } }
|
namespace Aws {
|
||||||
namespace Aws { namespace S3 { class S3Client; } }
|
namespace Client {
|
||||||
|
class ClientConfiguration;
|
||||||
|
}
|
||||||
|
} // namespace Aws
|
||||||
|
namespace Aws {
|
||||||
|
namespace S3 {
|
||||||
|
class S3Client;
|
||||||
|
}
|
||||||
|
} // namespace Aws
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct S3Helper
|
struct S3Helper {
|
||||||
{
|
|
||||||
ref<Aws::Client::ClientConfiguration> config;
|
ref<Aws::Client::ClientConfiguration> config;
|
||||||
ref<Aws::S3::S3Client> client;
|
ref<Aws::S3::S3Client> client;
|
||||||
|
|
||||||
S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
|
S3Helper(const std::string& profile, const std::string& region,
|
||||||
|
const std::string& scheme, const std::string& endpoint);
|
||||||
|
|
||||||
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
|
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region,
|
||||||
|
const std::string& scheme,
|
||||||
|
const std::string& endpoint);
|
||||||
|
|
||||||
struct DownloadResult
|
struct DownloadResult {
|
||||||
{
|
|
||||||
std::shared_ptr<std::string> data;
|
std::shared_ptr<std::string> data;
|
||||||
unsigned int durationMs;
|
unsigned int durationMs;
|
||||||
};
|
};
|
||||||
|
|
||||||
DownloadResult getObject(
|
DownloadResult getObject(const std::string& bucketName,
|
||||||
const std::string & bucketName, const std::string & key);
|
const std::string& key);
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
||||||
|
|
@ -21,4 +21,4 @@ typedef enum {
|
||||||
cmdAddToStoreNar = 9,
|
cmdAddToStoreNar = 9,
|
||||||
} ServeCommand;
|
} ServeCommand;
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
84
third_party/nix/src/libstore/sqlite.cc
vendored
84
third_party/nix/src/libstore/sqlite.cc
vendored
|
|
@ -1,14 +1,11 @@
|
||||||
#include "sqlite.hh"
|
#include "sqlite.hh"
|
||||||
#include "util.hh"
|
|
||||||
|
|
||||||
#include <sqlite3.h>
|
#include <sqlite3.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs)
|
[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) {
|
||||||
{
|
|
||||||
int err = sqlite3_errcode(db);
|
int err = sqlite3_errcode(db);
|
||||||
int exterr = sqlite3_extended_errcode(db);
|
int exterr = sqlite3_extended_errcode(db);
|
||||||
|
|
||||||
|
|
@ -20,20 +17,18 @@ namespace nix {
|
||||||
err == SQLITE_PROTOCOL
|
err == SQLITE_PROTOCOL
|
||||||
? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
|
? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
|
||||||
: fmt("SQLite database '%s' is busy", path));
|
: fmt("SQLite database '%s' is busy", path));
|
||||||
}
|
} else
|
||||||
else
|
|
||||||
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
|
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLite::SQLite(const Path & path)
|
SQLite::SQLite(const Path& path) {
|
||||||
{
|
|
||||||
if (sqlite3_open_v2(path.c_str(), &db,
|
if (sqlite3_open_v2(path.c_str(), &db,
|
||||||
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
|
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
|
||||||
|
0) != SQLITE_OK)
|
||||||
throw Error(format("cannot open SQLite database '%s'") % path);
|
throw Error(format("cannot open SQLite database '%s'") % path);
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLite::~SQLite()
|
SQLite::~SQLite() {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
if (db && sqlite3_close(db) != SQLITE_OK)
|
if (db && sqlite3_close(db) != SQLITE_OK)
|
||||||
throwSQLiteError(db, "closing database");
|
throwSQLiteError(db, "closing database");
|
||||||
|
|
@ -42,16 +37,14 @@ SQLite::~SQLite()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLite::exec(const std::string & stmt)
|
void SQLite::exec(const std::string& stmt) {
|
||||||
{
|
|
||||||
retrySQLite<void>([&]() {
|
retrySQLite<void>([&]() {
|
||||||
if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
|
if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
|
||||||
throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
|
throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLiteStmt::create(sqlite3 * db, const string & sql)
|
void SQLiteStmt::create(sqlite3* db, const string& sql) {
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
assert(!stmt);
|
assert(!stmt);
|
||||||
if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
|
if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
|
||||||
|
|
@ -60,8 +53,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & sql)
|
||||||
this->sql = sql;
|
this->sql = sql;
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteStmt::~SQLiteStmt()
|
SQLiteStmt::~SQLiteStmt() {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
|
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
|
||||||
throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
|
throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
|
||||||
|
|
@ -70,32 +62,27 @@ SQLiteStmt::~SQLiteStmt()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteStmt::Use::Use(SQLiteStmt & stmt)
|
SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) {
|
||||||
: stmt(stmt)
|
|
||||||
{
|
|
||||||
assert(stmt.stmt);
|
assert(stmt.stmt);
|
||||||
/* Note: sqlite3_reset() returns the error code for the most
|
/* Note: sqlite3_reset() returns the error code for the most
|
||||||
recent call to sqlite3_step(). So ignore it. */
|
recent call to sqlite3_step(). So ignore it. */
|
||||||
sqlite3_reset(stmt);
|
sqlite3_reset(stmt);
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteStmt::Use::~Use()
|
SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); }
|
||||||
{
|
|
||||||
sqlite3_reset(stmt);
|
|
||||||
}
|
|
||||||
|
|
||||||
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
|
SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value,
|
||||||
{
|
bool notNull) {
|
||||||
if (notNull) {
|
if (notNull) {
|
||||||
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
|
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1,
|
||||||
|
SQLITE_TRANSIENT) != SQLITE_OK)
|
||||||
throwSQLiteError(stmt.db, "binding argument");
|
throwSQLiteError(stmt.db, "binding argument");
|
||||||
} else
|
} else
|
||||||
bind();
|
bind();
|
||||||
return *this;
|
return *this;
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
|
SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) {
|
||||||
{
|
|
||||||
if (notNull) {
|
if (notNull) {
|
||||||
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
|
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
|
||||||
throwSQLiteError(stmt.db, "binding argument");
|
throwSQLiteError(stmt.db, "binding argument");
|
||||||
|
|
@ -104,69 +91,57 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
|
||||||
return *this;
|
return *this;
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteStmt::Use & SQLiteStmt::Use::bind()
|
SQLiteStmt::Use& SQLiteStmt::Use::bind() {
|
||||||
{
|
|
||||||
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
|
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
|
||||||
throwSQLiteError(stmt.db, "binding argument");
|
throwSQLiteError(stmt.db, "binding argument");
|
||||||
return *this;
|
return *this;
|
||||||
}
|
}
|
||||||
|
|
||||||
int SQLiteStmt::Use::step()
|
int SQLiteStmt::Use::step() { return sqlite3_step(stmt); }
|
||||||
{
|
|
||||||
return sqlite3_step(stmt);
|
|
||||||
}
|
|
||||||
|
|
||||||
void SQLiteStmt::Use::exec()
|
void SQLiteStmt::Use::exec() {
|
||||||
{
|
|
||||||
int r = step();
|
int r = step();
|
||||||
assert(r != SQLITE_ROW);
|
assert(r != SQLITE_ROW);
|
||||||
if (r != SQLITE_DONE)
|
if (r != SQLITE_DONE)
|
||||||
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
|
throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SQLiteStmt::Use::next()
|
bool SQLiteStmt::Use::next() {
|
||||||
{
|
|
||||||
int r = step();
|
int r = step();
|
||||||
if (r != SQLITE_DONE && r != SQLITE_ROW)
|
if (r != SQLITE_DONE && r != SQLITE_ROW)
|
||||||
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
|
throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
|
||||||
return r == SQLITE_ROW;
|
return r == SQLITE_ROW;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string SQLiteStmt::Use::getStr(int col)
|
std::string SQLiteStmt::Use::getStr(int col) {
|
||||||
{
|
|
||||||
auto s = (const char*)sqlite3_column_text(stmt, col);
|
auto s = (const char*)sqlite3_column_text(stmt, col);
|
||||||
assert(s);
|
assert(s);
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
int64_t SQLiteStmt::Use::getInt(int col)
|
int64_t SQLiteStmt::Use::getInt(int col) {
|
||||||
{
|
|
||||||
// FIXME: detect nulls?
|
// FIXME: detect nulls?
|
||||||
return sqlite3_column_int64(stmt, col);
|
return sqlite3_column_int64(stmt, col);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool SQLiteStmt::Use::isNull(int col)
|
bool SQLiteStmt::Use::isNull(int col) {
|
||||||
{
|
|
||||||
return sqlite3_column_type(stmt, col) == SQLITE_NULL;
|
return sqlite3_column_type(stmt, col) == SQLITE_NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteTxn::SQLiteTxn(sqlite3 * db)
|
SQLiteTxn::SQLiteTxn(sqlite3* db) {
|
||||||
{
|
|
||||||
this->db = db;
|
this->db = db;
|
||||||
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
|
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
|
||||||
throwSQLiteError(db, "starting transaction");
|
throwSQLiteError(db, "starting transaction");
|
||||||
active = true;
|
active = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLiteTxn::commit()
|
void SQLiteTxn::commit() {
|
||||||
{
|
|
||||||
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
|
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
|
||||||
throwSQLiteError(db, "committing transaction");
|
throwSQLiteError(db, "committing transaction");
|
||||||
active = false;
|
active = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
SQLiteTxn::~SQLiteTxn()
|
SQLiteTxn::~SQLiteTxn() {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
|
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
|
||||||
throwSQLiteError(db, "aborting transaction");
|
throwSQLiteError(db, "aborting transaction");
|
||||||
|
|
@ -175,8 +150,7 @@ SQLiteTxn::~SQLiteTxn()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void handleSQLiteBusy(const SQLiteBusy & e)
|
void handleSQLiteBusy(const SQLiteBusy& e) {
|
||||||
{
|
|
||||||
static std::atomic<time_t> lastWarned{0};
|
static std::atomic<time_t> lastWarned{0};
|
||||||
|
|
||||||
time_t now = time(0);
|
time_t now = time(0);
|
||||||
|
|
@ -195,4 +169,4 @@ void handleSQLiteBusy(const SQLiteBusy & e)
|
||||||
nanosleep(&t, 0);
|
nanosleep(&t, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
32
third_party/nix/src/libstore/sqlite.hh
vendored
32
third_party/nix/src/libstore/sqlite.hh
vendored
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
|
|
||||||
class sqlite3;
|
class sqlite3;
|
||||||
|
|
@ -11,14 +10,17 @@ class sqlite3_stmt;
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
/* RAII wrapper to close a SQLite database automatically. */
|
/* RAII wrapper to close a SQLite database automatically. */
|
||||||
struct SQLite
|
struct SQLite {
|
||||||
{
|
|
||||||
sqlite3* db = 0;
|
sqlite3* db = 0;
|
||||||
SQLite() {}
|
SQLite() {}
|
||||||
SQLite(const Path& path);
|
SQLite(const Path& path);
|
||||||
SQLite(const SQLite& from) = delete;
|
SQLite(const SQLite& from) = delete;
|
||||||
SQLite& operator=(const SQLite& from) = delete;
|
SQLite& operator=(const SQLite& from) = delete;
|
||||||
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
|
SQLite& operator=(SQLite&& from) {
|
||||||
|
db = from.db;
|
||||||
|
from.db = 0;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
~SQLite();
|
~SQLite();
|
||||||
operator sqlite3*() { return db; }
|
operator sqlite3*() { return db; }
|
||||||
|
|
||||||
|
|
@ -26,8 +28,7 @@ struct SQLite
|
||||||
};
|
};
|
||||||
|
|
||||||
/* RAII wrapper to create and destroy SQLite prepared statements. */
|
/* RAII wrapper to create and destroy SQLite prepared statements. */
|
||||||
struct SQLiteStmt
|
struct SQLiteStmt {
|
||||||
{
|
|
||||||
sqlite3* db = 0;
|
sqlite3* db = 0;
|
||||||
sqlite3_stmt* stmt = 0;
|
sqlite3_stmt* stmt = 0;
|
||||||
std::string sql;
|
std::string sql;
|
||||||
|
|
@ -38,16 +39,15 @@ struct SQLiteStmt
|
||||||
operator sqlite3_stmt*() { return stmt; }
|
operator sqlite3_stmt*() { return stmt; }
|
||||||
|
|
||||||
/* Helper for binding / executing statements. */
|
/* Helper for binding / executing statements. */
|
||||||
class Use
|
class Use {
|
||||||
{
|
|
||||||
friend struct SQLiteStmt;
|
friend struct SQLiteStmt;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
SQLiteStmt& stmt;
|
SQLiteStmt& stmt;
|
||||||
unsigned int curArg = 1;
|
unsigned int curArg = 1;
|
||||||
Use(SQLiteStmt& stmt);
|
Use(SQLiteStmt& stmt);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
~Use();
|
~Use();
|
||||||
|
|
||||||
/* Bind the next parameter. */
|
/* Bind the next parameter. */
|
||||||
|
|
@ -69,16 +69,12 @@ struct SQLiteStmt
|
||||||
bool isNull(int col);
|
bool isNull(int col);
|
||||||
};
|
};
|
||||||
|
|
||||||
Use use()
|
Use use() { return Use(*this); }
|
||||||
{
|
|
||||||
return Use(*this);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/* RAII helper that ensures transactions are aborted unless explicitly
|
/* RAII helper that ensures transactions are aborted unless explicitly
|
||||||
committed. */
|
committed. */
|
||||||
struct SQLiteTxn
|
struct SQLiteTxn {
|
||||||
{
|
|
||||||
bool active = false;
|
bool active = false;
|
||||||
sqlite3* db;
|
sqlite3* db;
|
||||||
|
|
||||||
|
|
@ -89,7 +85,6 @@ struct SQLiteTxn
|
||||||
~SQLiteTxn();
|
~SQLiteTxn();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
MakeError(SQLiteError, Error);
|
MakeError(SQLiteError, Error);
|
||||||
MakeError(SQLiteBusy, SQLiteError);
|
MakeError(SQLiteBusy, SQLiteError);
|
||||||
|
|
||||||
|
|
@ -100,8 +95,7 @@ void handleSQLiteBusy(const SQLiteBusy & e);
|
||||||
/* Convenience function for retrying a SQLite transaction when the
|
/* Convenience function for retrying a SQLite transaction when the
|
||||||
database is busy. */
|
database is busy. */
|
||||||
template <typename T>
|
template <typename T>
|
||||||
T retrySQLite(std::function<T()> fun)
|
T retrySQLite(std::function<T()> fun) {
|
||||||
{
|
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
return fun();
|
return fun();
|
||||||
|
|
@ -111,4 +105,4 @@ T retrySQLite(std::function<T()> fun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
66
third_party/nix/src/libstore/ssh-store.cc
vendored
66
third_party/nix/src/libstore/ssh-store.cc
vendored
|
|
@ -1,51 +1,40 @@
|
||||||
#include "store-api.hh"
|
|
||||||
#include "remote-store.hh"
|
|
||||||
#include "remote-fs-accessor.hh"
|
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
#include "worker-protocol.hh"
|
|
||||||
#include "pool.hh"
|
#include "pool.hh"
|
||||||
|
#include "remote-fs-accessor.hh"
|
||||||
|
#include "remote-store.hh"
|
||||||
#include "ssh.hh"
|
#include "ssh.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
#include "worker-protocol.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static std::string uriScheme = "ssh-ng://";
|
static std::string uriScheme = "ssh-ng://";
|
||||||
|
|
||||||
class SSHStore : public RemoteStore
|
class SSHStore : public RemoteStore {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
const Setting<Path> sshKey{(Store*)this, "", "ssh-key",
|
||||||
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
|
"path to an SSH private key"};
|
||||||
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
|
const Setting<bool> compress{(Store*)this, false, "compress",
|
||||||
|
"whether to compress the connection"};
|
||||||
|
|
||||||
SSHStore(const std::string& host, const Params& params)
|
SSHStore(const std::string& host, const Params& params)
|
||||||
: Store(params)
|
: Store(params),
|
||||||
, RemoteStore(params)
|
RemoteStore(params),
|
||||||
, host(host)
|
host(host),
|
||||||
, master(
|
master(host, sshKey,
|
||||||
host,
|
|
||||||
sshKey,
|
|
||||||
// Use SSH master only if using more than 1 connection.
|
// Use SSH master only if using more than 1 connection.
|
||||||
connections->capacity() > 1,
|
connections->capacity() > 1, compress) {}
|
||||||
compress)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string getUri() override
|
std::string getUri() override { return uriScheme + host; }
|
||||||
{
|
|
||||||
return uriScheme + host;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool sameMachine()
|
bool sameMachine() { return false; }
|
||||||
{ return false; }
|
|
||||||
|
|
||||||
void narFromPath(const Path& path, Sink& sink) override;
|
void narFromPath(const Path& path, Sink& sink) override;
|
||||||
|
|
||||||
ref<FSAccessor> getFSAccessor() override;
|
ref<FSAccessor> getFSAccessor() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
struct Connection : RemoteStore::Connection {
|
||||||
struct Connection : RemoteStore::Connection
|
|
||||||
{
|
|
||||||
std::unique_ptr<SSHMaster::Connection> sshConn;
|
std::unique_ptr<SSHMaster::Connection> sshConn;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -55,8 +44,7 @@ private:
|
||||||
|
|
||||||
SSHMaster master;
|
SSHMaster master;
|
||||||
|
|
||||||
void setOptions(RemoteStore::Connection & conn) override
|
void setOptions(RemoteStore::Connection& conn) override{
|
||||||
{
|
|
||||||
/* TODO Add a way to explicitly ask for some options to be
|
/* TODO Add a way to explicitly ask for some options to be
|
||||||
forwarded. One option: A way to query the daemon for its
|
forwarded. One option: A way to query the daemon for its
|
||||||
settings, and then a series of params to SSHStore like
|
settings, and then a series of params to SSHStore like
|
||||||
|
|
@ -66,21 +54,18 @@ private:
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
void SSHStore::narFromPath(const Path & path, Sink & sink)
|
void SSHStore::narFromPath(const Path& path, Sink& sink) {
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
auto conn(connections->get());
|
||||||
conn->to << wopNarFromPath << path;
|
conn->to << wopNarFromPath << path;
|
||||||
conn->processStderr();
|
conn->processStderr();
|
||||||
copyNAR(conn->from, sink);
|
copyNAR(conn->from, sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<FSAccessor> SSHStore::getFSAccessor()
|
ref<FSAccessor> SSHStore::getFSAccessor() {
|
||||||
{
|
|
||||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<RemoteStore::Connection> SSHStore::openConnection()
|
ref<RemoteStore::Connection> SSHStore::openConnection() {
|
||||||
{
|
|
||||||
auto conn = make_ref<Connection>();
|
auto conn = make_ref<Connection>();
|
||||||
conn->sshConn = master.startCommand("nix-daemon --stdio");
|
conn->sshConn = master.startCommand("nix-daemon --stdio");
|
||||||
conn->to = FdSink(conn->sshConn->in.get());
|
conn->to = FdSink(conn->sshConn->in.get());
|
||||||
|
|
@ -89,12 +74,11 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
|
||||||
return conn;
|
return conn;
|
||||||
}
|
}
|
||||||
|
|
||||||
static RegisterStoreImplementation regStore([](
|
static RegisterStoreImplementation regStore([](const std::string& uri,
|
||||||
const std::string & uri, const Store::Params & params)
|
const Store::Params& params)
|
||||||
-> std::shared_ptr<Store>
|
-> std::shared_ptr<Store> {
|
||||||
{
|
|
||||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||||
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
|
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
72
third_party/nix/src/libstore/ssh.cc
vendored
72
third_party/nix/src/libstore/ssh.cc
vendored
|
|
@ -2,30 +2,27 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
|
SSHMaster::SSHMaster(const std::string& host, const std::string& keyFile,
|
||||||
: host(host)
|
bool useMaster, bool compress, int logFD)
|
||||||
, fakeSSH(host == "localhost")
|
: host(host),
|
||||||
, keyFile(keyFile)
|
fakeSSH(host == "localhost"),
|
||||||
, useMaster(useMaster && !fakeSSH)
|
keyFile(keyFile),
|
||||||
, compress(compress)
|
useMaster(useMaster && !fakeSSH),
|
||||||
, logFD(logFD)
|
compress(compress),
|
||||||
{
|
logFD(logFD) {
|
||||||
if (host == "" || hasPrefix(host, "-"))
|
if (host == "" || hasPrefix(host, "-"))
|
||||||
throw Error("invalid SSH host name '%s'", host);
|
throw Error("invalid SSH host name '%s'", host);
|
||||||
}
|
}
|
||||||
|
|
||||||
void SSHMaster::addCommonSSHOpts(Strings & args)
|
void SSHMaster::addCommonSSHOpts(Strings& args) {
|
||||||
{
|
|
||||||
for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
|
for (auto& i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
|
||||||
args.push_back(i);
|
args.push_back(i);
|
||||||
if (!keyFile.empty())
|
if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile});
|
||||||
args.insert(args.end(), {"-i", keyFile});
|
if (compress) args.push_back("-C");
|
||||||
if (compress)
|
|
||||||
args.push_back("-C");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
|
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
|
||||||
{
|
const std::string& command) {
|
||||||
Path socketPath = startMaster();
|
Path socketPath = startMaster();
|
||||||
|
|
||||||
Pipe in, out;
|
Pipe in, out;
|
||||||
|
|
@ -36,7 +33,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
|
||||||
ProcessOptions options;
|
ProcessOptions options;
|
||||||
options.dieWithParent = false;
|
options.dieWithParent = false;
|
||||||
|
|
||||||
conn->sshPid = startProcess([&]() {
|
conn->sshPid = startProcess(
|
||||||
|
[&]() {
|
||||||
restoreSignals();
|
restoreSignals();
|
||||||
|
|
||||||
close(in.writeSide.get());
|
close(in.writeSide.get());
|
||||||
|
|
@ -56,10 +54,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
|
||||||
} else {
|
} else {
|
||||||
args = {"ssh", host.c_str(), "-x", "-a"};
|
args = {"ssh", host.c_str(), "-x", "-a"};
|
||||||
addCommonSSHOpts(args);
|
addCommonSSHOpts(args);
|
||||||
if (socketPath != "")
|
if (socketPath != "") args.insert(args.end(), {"-S", socketPath});
|
||||||
args.insert(args.end(), {"-S", socketPath});
|
if (verbosity >= lvlChatty) args.push_back("-v");
|
||||||
if (verbosity >= lvlChatty)
|
|
||||||
args.push_back("-v");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
args.push_back(command);
|
args.push_back(command);
|
||||||
|
|
@ -67,8 +63,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
|
||||||
|
|
||||||
// could not exec ssh/bash
|
// could not exec ssh/bash
|
||||||
throw SysError("unable to execute '%s'", args.front());
|
throw SysError("unable to execute '%s'", args.front());
|
||||||
}, options);
|
},
|
||||||
|
options);
|
||||||
|
|
||||||
in.readSide = -1;
|
in.readSide = -1;
|
||||||
out.writeSide = -1;
|
out.writeSide = -1;
|
||||||
|
|
@ -79,15 +75,15 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
|
||||||
return conn;
|
return conn;
|
||||||
}
|
}
|
||||||
|
|
||||||
Path SSHMaster::startMaster()
|
Path SSHMaster::startMaster() {
|
||||||
{
|
|
||||||
if (!useMaster) return "";
|
if (!useMaster) return "";
|
||||||
|
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
if (state->sshMaster != -1) return state->socketPath;
|
if (state->sshMaster != -1) return state->socketPath;
|
||||||
|
|
||||||
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
|
state->tmpDir =
|
||||||
|
std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
|
||||||
|
|
||||||
state->socketPath = (Path)*state->tmpDir + "/ssh.sock";
|
state->socketPath = (Path)*state->tmpDir + "/ssh.sock";
|
||||||
|
|
||||||
|
|
@ -97,7 +93,8 @@ Path SSHMaster::startMaster()
|
||||||
ProcessOptions options;
|
ProcessOptions options;
|
||||||
options.dieWithParent = false;
|
options.dieWithParent = false;
|
||||||
|
|
||||||
state->sshMaster = startProcess([&]() {
|
state->sshMaster = startProcess(
|
||||||
|
[&]() {
|
||||||
restoreSignals();
|
restoreSignals();
|
||||||
|
|
||||||
close(out.readSide.get());
|
close(out.readSide.get());
|
||||||
|
|
@ -105,25 +102,26 @@ Path SSHMaster::startMaster()
|
||||||
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
|
if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
|
||||||
throw SysError("duping over stdout");
|
throw SysError("duping over stdout");
|
||||||
|
|
||||||
Strings args =
|
Strings args = {"ssh", host.c_str(),
|
||||||
{ "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath
|
"-M", "-N",
|
||||||
, "-o", "LocalCommand=echo started"
|
"-S", state->socketPath,
|
||||||
, "-o", "PermitLocalCommand=yes"
|
"-o", "LocalCommand=echo started",
|
||||||
};
|
"-o", "PermitLocalCommand=yes"};
|
||||||
if (verbosity >= lvlChatty)
|
if (verbosity >= lvlChatty) args.push_back("-v");
|
||||||
args.push_back("-v");
|
|
||||||
addCommonSSHOpts(args);
|
addCommonSSHOpts(args);
|
||||||
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
|
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
|
||||||
|
|
||||||
throw SysError("unable to execute '%s'", args.front());
|
throw SysError("unable to execute '%s'", args.front());
|
||||||
}, options);
|
},
|
||||||
|
options);
|
||||||
|
|
||||||
out.writeSide = -1;
|
out.writeSide = -1;
|
||||||
|
|
||||||
std::string reply;
|
std::string reply;
|
||||||
try {
|
try {
|
||||||
reply = readLine(out.readSide.get());
|
reply = readLine(out.readSide.get());
|
||||||
} catch (EndOfFile & e) { }
|
} catch (EndOfFile& e) {
|
||||||
|
}
|
||||||
|
|
||||||
if (reply != "started")
|
if (reply != "started")
|
||||||
throw Error("failed to start SSH master connection to '%s'", host);
|
throw Error("failed to start SSH master connection to '%s'", host);
|
||||||
|
|
@ -131,4 +129,4 @@ Path SSHMaster::startMaster()
|
||||||
return state->socketPath;
|
return state->socketPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
18
third_party/nix/src/libstore/ssh.hh
vendored
18
third_party/nix/src/libstore/ssh.hh
vendored
|
|
@ -1,14 +1,12 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "util.hh"
|
|
||||||
#include "sync.hh"
|
#include "sync.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
class SSHMaster
|
class SSHMaster {
|
||||||
{
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
const std::string host;
|
const std::string host;
|
||||||
bool fakeSSH;
|
bool fakeSSH;
|
||||||
const std::string keyFile;
|
const std::string keyFile;
|
||||||
|
|
@ -16,8 +14,7 @@ private:
|
||||||
const bool compress;
|
const bool compress;
|
||||||
const int logFD;
|
const int logFD;
|
||||||
|
|
||||||
struct State
|
struct State {
|
||||||
{
|
|
||||||
Pid sshMaster;
|
Pid sshMaster;
|
||||||
std::unique_ptr<AutoDelete> tmpDir;
|
std::unique_ptr<AutoDelete> tmpDir;
|
||||||
Path socketPath;
|
Path socketPath;
|
||||||
|
|
@ -28,11 +25,10 @@ private:
|
||||||
void addCommonSSHOpts(Strings& args);
|
void addCommonSSHOpts(Strings& args);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
SSHMaster(const std::string& host, const std::string& keyFile, bool useMaster,
|
||||||
|
bool compress, int logFD = -1);
|
||||||
|
|
||||||
SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
|
struct Connection {
|
||||||
|
|
||||||
struct Connection
|
|
||||||
{
|
|
||||||
Pid sshPid;
|
Pid sshPid;
|
||||||
AutoCloseFD out, in;
|
AutoCloseFD out, in;
|
||||||
};
|
};
|
||||||
|
|
@ -42,4 +38,4 @@ public:
|
||||||
Path startMaster();
|
Path startMaster();
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
427
third_party/nix/src/libstore/store-api.cc
vendored
427
third_party/nix/src/libstore/store-api.cc
vendored
|
|
@ -1,41 +1,31 @@
|
||||||
#include "crypto.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "util.hh"
|
#include <future>
|
||||||
|
#include "crypto.hh"
|
||||||
|
#include "derivations.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "json.hh"
|
||||||
#include "nar-info-disk-cache.hh"
|
#include "nar-info-disk-cache.hh"
|
||||||
#include "thread-pool.hh"
|
#include "thread-pool.hh"
|
||||||
#include "json.hh"
|
#include "util.hh"
|
||||||
#include "derivations.hh"
|
|
||||||
|
|
||||||
#include <future>
|
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
bool Store::isInStore(const Path& path) const {
|
||||||
bool Store::isInStore(const Path & path) const
|
|
||||||
{
|
|
||||||
return isInDir(path, storeDir);
|
return isInDir(path, storeDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool Store::isStorePath(const Path& path) const {
|
||||||
bool Store::isStorePath(const Path & path) const
|
return isInStore(path) &&
|
||||||
{
|
path.size() >= storeDir.size() + 1 + storePathHashLen &&
|
||||||
return isInStore(path)
|
path.find('/', storeDir.size() + 1) == Path::npos;
|
||||||
&& path.size() >= storeDir.size() + 1 + storePathHashLen
|
|
||||||
&& path.find('/', storeDir.size() + 1) == Path::npos;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Store::assertStorePath(const Path& path) const {
|
||||||
void Store::assertStorePath(const Path & path) const
|
|
||||||
{
|
|
||||||
if (!isStorePath(path))
|
if (!isStorePath(path))
|
||||||
throw Error(format("path '%1%' is not in the Nix store") % path);
|
throw Error(format("path '%1%' is not in the Nix store") % path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path Store::toStorePath(const Path& path) const {
|
||||||
Path Store::toStorePath(const Path & path) const
|
|
||||||
{
|
|
||||||
if (!isInStore(path))
|
if (!isInStore(path))
|
||||||
throw Error(format("path '%1%' is not in the Nix store") % path);
|
throw Error(format("path '%1%' is not in the Nix store") % path);
|
||||||
Path::size_type slash = path.find('/', storeDir.size() + 1);
|
Path::size_type slash = path.find('/', storeDir.size() + 1);
|
||||||
|
|
@ -45,9 +35,7 @@ Path Store::toStorePath(const Path & path) const
|
||||||
return Path(path, 0, slash);
|
return Path(path, 0, slash);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path Store::followLinksToStore(const Path& _path) const {
|
||||||
Path Store::followLinksToStore(const Path & _path) const
|
|
||||||
{
|
|
||||||
Path path = absPath(_path);
|
Path path = absPath(_path);
|
||||||
while (!isInStore(path)) {
|
while (!isInStore(path)) {
|
||||||
if (!isLink(path)) break;
|
if (!isLink(path)) break;
|
||||||
|
|
@ -59,39 +47,36 @@ Path Store::followLinksToStore(const Path & _path) const
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path Store::followLinksToStorePath(const Path& path) const {
|
||||||
Path Store::followLinksToStorePath(const Path & path) const
|
|
||||||
{
|
|
||||||
return toStorePath(followLinksToStore(path));
|
return toStorePath(followLinksToStore(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string storePathToName(const Path& path) {
|
||||||
string storePathToName(const Path & path)
|
|
||||||
{
|
|
||||||
auto base = baseNameOf(path);
|
auto base = baseNameOf(path);
|
||||||
assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
|
assert(base.size() == storePathHashLen ||
|
||||||
return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
|
(base.size() > storePathHashLen && base[storePathHashLen] == '-'));
|
||||||
|
return base.size() == storePathHashLen ? ""
|
||||||
|
: string(base, storePathHashLen + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string storePathToHash(const Path& path) {
|
||||||
string storePathToHash(const Path & path)
|
|
||||||
{
|
|
||||||
auto base = baseNameOf(path);
|
auto base = baseNameOf(path);
|
||||||
assert(base.size() >= storePathHashLen);
|
assert(base.size() >= storePathHashLen);
|
||||||
return string(base, 0, storePathHashLen);
|
return string(base, 0, storePathHashLen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void checkStoreName(const string& name) {
|
||||||
void checkStoreName(const string & name)
|
|
||||||
{
|
|
||||||
string validChars = "+-._?=";
|
string validChars = "+-._?=";
|
||||||
|
|
||||||
auto baseError = format("The path name '%2%' is invalid: %3%. "
|
auto baseError =
|
||||||
|
format(
|
||||||
|
"The path name '%2%' is invalid: %3%. "
|
||||||
"Path names are alphanumeric and can include the symbols %1% "
|
"Path names are alphanumeric and can include the symbols %1% "
|
||||||
"and must not begin with a period. "
|
"and must not begin with a period. "
|
||||||
"Note: If '%2%' is a source file and you cannot rename it on "
|
"Note: If '%2%' is a source file and you cannot rename it on "
|
||||||
"disk, builtins.path { name = ... } can be used to give it an "
|
"disk, builtins.path { name = ... } can be used to give it an "
|
||||||
"alternative name.") % validChars % name;
|
"alternative name.") %
|
||||||
|
validChars % name;
|
||||||
|
|
||||||
/* Disallow names starting with a dot for possible security
|
/* Disallow names starting with a dot for possible security
|
||||||
reasons (e.g., "." and ".."). */
|
reasons (e.g., "." and ".."). */
|
||||||
|
|
@ -102,16 +87,12 @@ void checkStoreName(const string & name)
|
||||||
if (name.length() > 211)
|
if (name.length() > 211)
|
||||||
throw Error(baseError % "name must be less than 212 characters");
|
throw Error(baseError % "name must be less than 212 characters");
|
||||||
for (auto& i : name)
|
for (auto& i : name)
|
||||||
if (!((i >= 'A' && i <= 'Z') ||
|
if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') ||
|
||||||
(i >= 'a' && i <= 'z') ||
|
(i >= '0' && i <= '9') || validChars.find(i) != string::npos)) {
|
||||||
(i >= '0' && i <= '9') ||
|
|
||||||
validChars.find(i) != string::npos))
|
|
||||||
{
|
|
||||||
throw Error(baseError % (format("the '%1%' character is invalid") % i));
|
throw Error(baseError % (format("the '%1%' character is invalid") % i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Store paths have the following form:
|
/* Store paths have the following form:
|
||||||
|
|
||||||
<store>/<h>-<name>
|
<store>/<h>-<name>
|
||||||
|
|
@ -182,44 +163,38 @@ void checkStoreName(const string & name)
|
||||||
"source:".
|
"source:".
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
Path Store::makeStorePath(const string& type, const Hash& hash,
|
||||||
Path Store::makeStorePath(const string & type,
|
const string& name) const {
|
||||||
const Hash & hash, const string & name) const
|
|
||||||
{
|
|
||||||
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
||||||
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
|
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
|
||||||
|
|
||||||
checkStoreName(name);
|
checkStoreName(name);
|
||||||
|
|
||||||
return storeDir + "/"
|
return storeDir + "/" +
|
||||||
+ compressHash(hashString(htSHA256, s), 20).to_string(Base32, false)
|
compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) +
|
||||||
+ "-" + name;
|
"-" + name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path Store::makeOutputPath(const string& id, const Hash& hash,
|
||||||
Path Store::makeOutputPath(const string & id,
|
const string& name) const {
|
||||||
const Hash & hash, const string & name) const
|
|
||||||
{
|
|
||||||
return makeStorePath("output:" + id, hash,
|
return makeStorePath("output:" + id, hash,
|
||||||
name + (id == "out" ? "" : "-" + id));
|
name + (id == "out" ? "" : "-" + id));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path Store::makeFixedOutputPath(bool recursive, const Hash& hash,
|
||||||
Path Store::makeFixedOutputPath(bool recursive,
|
const string& name) const {
|
||||||
const Hash & hash, const string & name) const
|
|
||||||
{
|
|
||||||
return hash.type == htSHA256 && recursive
|
return hash.type == htSHA256 && recursive
|
||||||
? makeStorePath("source", hash, name)
|
? makeStorePath("source", hash, name)
|
||||||
: makeStorePath("output:out", hashString(htSHA256,
|
: makeStorePath(
|
||||||
|
"output:out",
|
||||||
|
hashString(htSHA256,
|
||||||
"fixed:out:" + (recursive ? (string) "r:" : "") +
|
"fixed:out:" + (recursive ? (string) "r:" : "") +
|
||||||
hash.to_string(Base16) + ":"),
|
hash.to_string(Base16) + ":"),
|
||||||
name);
|
name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Path Store::makeTextPath(const string& name, const Hash& hash,
|
Path Store::makeTextPath(const string& name, const Hash& hash,
|
||||||
const PathSet & references) const
|
const PathSet& references) const {
|
||||||
{
|
|
||||||
assert(hash.type == htSHA256);
|
assert(hash.type == htSHA256);
|
||||||
/* Stuff the references (if any) into the type. This is a bit
|
/* Stuff the references (if any) into the type. This is a bit
|
||||||
hacky, but we can't put them in `s' since that would be
|
hacky, but we can't put them in `s' since that would be
|
||||||
|
|
@ -232,38 +207,28 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
|
||||||
return makeStorePath(type, hash, name);
|
return makeStorePath(type, hash, name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::pair<Path, Hash> Store::computeStorePathForPath(const string& name,
|
std::pair<Path, Hash> Store::computeStorePathForPath(const string& name,
|
||||||
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
|
const Path& srcPath,
|
||||||
{
|
bool recursive,
|
||||||
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
|
HashType hashAlgo,
|
||||||
|
PathFilter& filter) const {
|
||||||
|
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first
|
||||||
|
: hashFile(hashAlgo, srcPath);
|
||||||
Path dstPath = makeFixedOutputPath(recursive, h, name);
|
Path dstPath = makeFixedOutputPath(recursive, h, name);
|
||||||
return std::pair<Path, Hash>(dstPath, h);
|
return std::pair<Path, Hash>(dstPath, h);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Path Store::computeStorePathForText(const string& name, const string& s,
|
Path Store::computeStorePathForText(const string& name, const string& s,
|
||||||
const PathSet & references) const
|
const PathSet& references) const {
|
||||||
{
|
|
||||||
return makeTextPath(name, hashString(htSHA256, s), references);
|
return makeTextPath(name, hashString(htSHA256, s), references);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Store::Store(const Params& params)
|
Store::Store(const Params& params)
|
||||||
: Config(params)
|
: Config(params), state({(size_t)pathInfoCacheSize}) {}
|
||||||
, state({(size_t) pathInfoCacheSize})
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
|
std::string Store::getUri() { return ""; }
|
||||||
|
|
||||||
std::string Store::getUri()
|
bool Store::isValidPath(const Path& storePath) {
|
||||||
{
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
bool Store::isValidPath(const Path & storePath)
|
|
||||||
{
|
|
||||||
assertStorePath(storePath);
|
assertStorePath(storePath);
|
||||||
|
|
||||||
auto hashPart = storePathToHash(storePath);
|
auto hashPart = storePathToHash(storePath);
|
||||||
|
|
@ -282,8 +247,8 @@ bool Store::isValidPath(const Path & storePath)
|
||||||
if (res.first != NarInfoDiskCache::oUnknown) {
|
if (res.first != NarInfoDiskCache::oUnknown) {
|
||||||
stats.narInfoReadAverted++;
|
stats.narInfoReadAverted++;
|
||||||
auto state_(state.lock());
|
auto state_(state.lock());
|
||||||
state_->pathInfoCache.upsert(hashPart,
|
state_->pathInfoCache.upsert(
|
||||||
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
|
hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
|
||||||
return res.first == NarInfoDiskCache::oValid;
|
return res.first == NarInfoDiskCache::oValid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -297,11 +262,9 @@ bool Store::isValidPath(const Path & storePath)
|
||||||
return valid;
|
return valid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Default implementation for stores that only implement
|
/* Default implementation for stores that only implement
|
||||||
queryPathInfoUncached(). */
|
queryPathInfoUncached(). */
|
||||||
bool Store::isValidPathUncached(const Path & path)
|
bool Store::isValidPathUncached(const Path& path) {
|
||||||
{
|
|
||||||
try {
|
try {
|
||||||
queryPathInfo(path);
|
queryPathInfo(path);
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -310,13 +273,10 @@ bool Store::isValidPathUncached(const Path & path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) {
|
||||||
ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
|
|
||||||
{
|
|
||||||
std::promise<ref<ValidPathInfo>> promise;
|
std::promise<ref<ValidPathInfo>> promise;
|
||||||
|
|
||||||
queryPathInfo(storePath,
|
queryPathInfo(storePath, {[&](std::future<ref<ValidPathInfo>> result) {
|
||||||
{[&](std::future<ref<ValidPathInfo>> result) {
|
|
||||||
try {
|
try {
|
||||||
promise.set_value(result.get());
|
promise.set_value(result.get());
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
|
|
@ -327,10 +287,8 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
|
||||||
return promise.get_future().get();
|
return promise.get_future().get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Store::queryPathInfo(const Path& storePath,
|
void Store::queryPathInfo(const Path& storePath,
|
||||||
Callback<ref<ValidPathInfo>> callback) noexcept
|
Callback<ref<ValidPathInfo>> callback) noexcept {
|
||||||
{
|
|
||||||
std::string hashPart;
|
std::string hashPart;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -354,51 +312,53 @@ void Store::queryPathInfo(const Path & storePath,
|
||||||
stats.narInfoReadAverted++;
|
stats.narInfoReadAverted++;
|
||||||
{
|
{
|
||||||
auto state_(state.lock());
|
auto state_(state.lock());
|
||||||
state_->pathInfoCache.upsert(hashPart,
|
state_->pathInfoCache.upsert(
|
||||||
|
hashPart,
|
||||||
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
|
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
|
||||||
if (res.first == NarInfoDiskCache::oInvalid ||
|
if (res.first == NarInfoDiskCache::oInvalid ||
|
||||||
(res.second->path != storePath && storePathToName(storePath) != ""))
|
(res.second->path != storePath &&
|
||||||
|
storePathToName(storePath) != ""))
|
||||||
throw InvalidPath(format("path '%s' is not valid") % storePath);
|
throw InvalidPath(format("path '%s' is not valid") % storePath);
|
||||||
}
|
}
|
||||||
return callback(ref<ValidPathInfo>(res.second));
|
return callback(ref<ValidPathInfo>(res.second));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (...) { return callback.rethrow(); }
|
} catch (...) {
|
||||||
|
return callback.rethrow();
|
||||||
|
}
|
||||||
|
|
||||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
||||||
|
|
||||||
queryPathInfoUncached(storePath,
|
queryPathInfoUncached(
|
||||||
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) {
|
storePath, {[this, storePath, hashPart, callbackPtr](
|
||||||
|
std::future<std::shared_ptr<ValidPathInfo>> fut) {
|
||||||
try {
|
try {
|
||||||
auto info = fut.get();
|
auto info = fut.get();
|
||||||
|
|
||||||
if (diskCache)
|
if (diskCache) diskCache->upsertNarInfo(getUri(), hashPart, info);
|
||||||
diskCache->upsertNarInfo(getUri(), hashPart, info);
|
|
||||||
|
|
||||||
{
|
{
|
||||||
auto state_(state.lock());
|
auto state_(state.lock());
|
||||||
state_->pathInfoCache.upsert(hashPart, info);
|
state_->pathInfoCache.upsert(hashPart, info);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!info
|
if (!info ||
|
||||||
|| (info->path != storePath && storePathToName(storePath) != ""))
|
(info->path != storePath && storePathToName(storePath) != "")) {
|
||||||
{
|
|
||||||
stats.narInfoMissing++;
|
stats.narInfoMissing++;
|
||||||
throw InvalidPath("path '%s' is not valid", storePath);
|
throw InvalidPath("path '%s' is not valid", storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
(*callbackPtr)(ref<ValidPathInfo>(info));
|
(*callbackPtr)(ref<ValidPathInfo>(info));
|
||||||
} catch (...) { callbackPtr->rethrow(); }
|
} catch (...) {
|
||||||
|
callbackPtr->rethrow();
|
||||||
|
}
|
||||||
}});
|
}});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PathSet Store::queryValidPaths(const PathSet& paths,
|
||||||
PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
|
SubstituteFlag maybeSubstitute) {
|
||||||
{
|
struct State {
|
||||||
struct State
|
|
||||||
{
|
|
||||||
size_t left;
|
size_t left;
|
||||||
PathSet valid;
|
PathSet valid;
|
||||||
std::exception_ptr exc;
|
std::exception_ptr exc;
|
||||||
|
|
@ -411,7 +371,8 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
|
||||||
|
|
||||||
auto doQuery = [&](const Path& path) {
|
auto doQuery = [&](const Path& path) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
|
queryPathInfo(
|
||||||
|
path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
try {
|
try {
|
||||||
auto info = fut.get();
|
auto info = fut.get();
|
||||||
|
|
@ -421,13 +382,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
|
||||||
state->exc = std::current_exception();
|
state->exc = std::current_exception();
|
||||||
}
|
}
|
||||||
assert(state->left);
|
assert(state->left);
|
||||||
if (!--state->left)
|
if (!--state->left) wakeup.notify_one();
|
||||||
wakeup.notify_one();
|
|
||||||
}});
|
}});
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & path : paths)
|
for (auto& path : paths) pool.enqueue(std::bind(doQuery, path));
|
||||||
pool.enqueue(std::bind(doQuery, path));
|
|
||||||
|
|
||||||
pool.process();
|
pool.process();
|
||||||
|
|
||||||
|
|
@ -441,13 +400,11 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Return a string accepted by decodeValidPathInfo() that
|
/* Return a string accepted by decodeValidPathInfo() that
|
||||||
registers the specified paths as valid. Note: it's the
|
registers the specified paths as valid. Note: it's the
|
||||||
responsibility of the caller to provide a closure. */
|
responsibility of the caller to provide a closure. */
|
||||||
string Store::makeValidityRegistration(const PathSet & paths,
|
string Store::makeValidityRegistration(const PathSet& paths, bool showDerivers,
|
||||||
bool showDerivers, bool showHash)
|
bool showHash) {
|
||||||
{
|
|
||||||
string s = "";
|
string s = "";
|
||||||
|
|
||||||
for (auto& i : paths) {
|
for (auto& i : paths) {
|
||||||
|
|
@ -465,17 +422,15 @@ string Store::makeValidityRegistration(const PathSet & paths,
|
||||||
|
|
||||||
s += (format("%1%\n") % info->references.size()).str();
|
s += (format("%1%\n") % info->references.size()).str();
|
||||||
|
|
||||||
for (auto & j : info->references)
|
for (auto& j : info->references) s += j + "\n";
|
||||||
s += j + "\n";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths,
|
void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths,
|
||||||
bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
|
bool includeImpureInfo, bool showClosureSize,
|
||||||
{
|
AllowInvalidFlag allowInvalid) {
|
||||||
auto jsonList = jsonOut.list();
|
auto jsonList = jsonOut.list();
|
||||||
|
|
||||||
for (auto storePath : storePaths) {
|
for (auto storePath : storePaths) {
|
||||||
|
|
@ -486,18 +441,15 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
|
||||||
auto info = queryPathInfo(storePath);
|
auto info = queryPathInfo(storePath);
|
||||||
storePath = info->path;
|
storePath = info->path;
|
||||||
|
|
||||||
jsonPath
|
jsonPath.attr("narHash", info->narHash.to_string())
|
||||||
.attr("narHash", info->narHash.to_string())
|
|
||||||
.attr("narSize", info->narSize);
|
.attr("narSize", info->narSize);
|
||||||
|
|
||||||
{
|
{
|
||||||
auto jsonRefs = jsonPath.list("references");
|
auto jsonRefs = jsonPath.list("references");
|
||||||
for (auto & ref : info->references)
|
for (auto& ref : info->references) jsonRefs.elem(ref);
|
||||||
jsonRefs.elem(ref);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (info->ca != "")
|
if (info->ca != "") jsonPath.attr("ca", info->ca);
|
||||||
jsonPath.attr("ca", info->ca);
|
|
||||||
|
|
||||||
std::pair<uint64_t, uint64_t> closureSizes;
|
std::pair<uint64_t, uint64_t> closureSizes;
|
||||||
|
|
||||||
|
|
@ -507,28 +459,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
|
||||||
}
|
}
|
||||||
|
|
||||||
if (includeImpureInfo) {
|
if (includeImpureInfo) {
|
||||||
|
if (info->deriver != "") jsonPath.attr("deriver", info->deriver);
|
||||||
if (info->deriver != "")
|
|
||||||
jsonPath.attr("deriver", info->deriver);
|
|
||||||
|
|
||||||
if (info->registrationTime)
|
if (info->registrationTime)
|
||||||
jsonPath.attr("registrationTime", info->registrationTime);
|
jsonPath.attr("registrationTime", info->registrationTime);
|
||||||
|
|
||||||
if (info->ultimate)
|
if (info->ultimate) jsonPath.attr("ultimate", info->ultimate);
|
||||||
jsonPath.attr("ultimate", info->ultimate);
|
|
||||||
|
|
||||||
if (!info->sigs.empty()) {
|
if (!info->sigs.empty()) {
|
||||||
auto jsonSigs = jsonPath.list("signatures");
|
auto jsonSigs = jsonPath.list("signatures");
|
||||||
for (auto & sig : info->sigs)
|
for (auto& sig : info->sigs) jsonSigs.elem(sig);
|
||||||
jsonSigs.elem(sig);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
||||||
std::shared_ptr<const ValidPathInfo>(info));
|
std::shared_ptr<const ValidPathInfo>(info));
|
||||||
|
|
||||||
if (narInfo) {
|
if (narInfo) {
|
||||||
if (!narInfo->url.empty())
|
if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url);
|
||||||
jsonPath.attr("url", narInfo->url);
|
|
||||||
if (narInfo->fileHash)
|
if (narInfo->fileHash)
|
||||||
jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
|
jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
|
||||||
if (narInfo->fileSize)
|
if (narInfo->fileSize)
|
||||||
|
|
@ -544,9 +491,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) {
|
||||||
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
|
|
||||||
{
|
|
||||||
uint64_t totalNarSize = 0, totalDownloadSize = 0;
|
uint64_t totalNarSize = 0, totalDownloadSize = 0;
|
||||||
PathSet closure;
|
PathSet closure;
|
||||||
computeFSClosure(storePath, closure, false, false);
|
computeFSClosure(storePath, closure, false, false);
|
||||||
|
|
@ -555,15 +500,12 @@ std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
|
||||||
totalNarSize += info->narSize;
|
totalNarSize += info->narSize;
|
||||||
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
||||||
std::shared_ptr<const ValidPathInfo>(info));
|
std::shared_ptr<const ValidPathInfo>(info));
|
||||||
if (narInfo)
|
if (narInfo) totalDownloadSize += narInfo->fileSize;
|
||||||
totalDownloadSize += narInfo->fileSize;
|
|
||||||
}
|
}
|
||||||
return {totalNarSize, totalDownloadSize};
|
return {totalNarSize, totalDownloadSize};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const Store::Stats& Store::getStats() {
|
||||||
const Store::Stats & Store::getStats()
|
|
||||||
{
|
|
||||||
{
|
{
|
||||||
auto state_(state.lock());
|
auto state_(state.lock());
|
||||||
stats.pathInfoCacheSize = state_->pathInfoCache.size();
|
stats.pathInfoCacheSize = state_->pathInfoCache.size();
|
||||||
|
|
@ -571,21 +513,16 @@ const Store::Stats & Store::getStats()
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Store::buildPaths(const PathSet& paths, BuildMode buildMode) {
|
||||||
void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
|
|
||||||
{
|
|
||||||
for (auto& path : paths)
|
for (auto& path : paths)
|
||||||
if (isDerivation(path))
|
if (isDerivation(path)) unsupported("buildPaths");
|
||||||
unsupported("buildPaths");
|
|
||||||
|
|
||||||
if (queryValidPaths(paths).size() != paths.size())
|
if (queryValidPaths(paths).size() != paths.size()) unsupported("buildPaths");
|
||||||
unsupported("buildPaths");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
|
const Path& storePath, RepairFlag repair,
|
||||||
{
|
CheckSigsFlag checkSigs) {
|
||||||
auto srcUri = srcStore->getUri();
|
auto srcUri = srcStore->getUri();
|
||||||
auto dstUri = dstStore->getUri();
|
auto dstUri = dstStore->getUri();
|
||||||
|
|
||||||
|
|
@ -594,7 +531,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
? fmt("copying path '%s' to '%s'", storePath, dstUri)
|
? fmt("copying path '%s' to '%s'", storePath, dstUri)
|
||||||
: dstUri == "local" || dstUri == "daemon"
|
: dstUri == "local" || dstUri == "daemon"
|
||||||
? fmt("copying path '%s' from '%s'", storePath, srcUri)
|
? fmt("copying path '%s' from '%s'", storePath, srcUri)
|
||||||
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
|
: fmt("copying path '%s' from '%s' to '%s'", storePath,
|
||||||
|
srcUri, dstUri),
|
||||||
{storePath, srcUri, dstUri});
|
{storePath, srcUri, dstUri});
|
||||||
PushActivity pact(act.id);
|
PushActivity pact(act.id);
|
||||||
|
|
||||||
|
|
@ -622,24 +560,26 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
info = info2;
|
info = info2;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto source = sinkToSource([&](Sink & sink) {
|
auto source = sinkToSource(
|
||||||
|
[&](Sink& sink) {
|
||||||
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
|
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
|
||||||
sink(data, len);
|
sink(data, len);
|
||||||
total += len;
|
total += len;
|
||||||
act.progress(total, info->narSize);
|
act.progress(total, info->narSize);
|
||||||
});
|
});
|
||||||
srcStore->narFromPath({storePath}, wrapperSink);
|
srcStore->narFromPath({storePath}, wrapperSink);
|
||||||
}, [&]() {
|
},
|
||||||
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
|
[&]() {
|
||||||
|
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete",
|
||||||
|
storePath, srcStore->getUri());
|
||||||
});
|
});
|
||||||
|
|
||||||
dstStore->addToStore(*info, *source, repair, checkSigs);
|
dstStore->addToStore(*info, *source, repair, checkSigs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
|
const PathSet& storePaths, RepairFlag repair,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
|
CheckSigsFlag checkSigs, SubstituteFlag substitute) {
|
||||||
{
|
|
||||||
PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
|
PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
|
||||||
|
|
||||||
PathSet missing;
|
PathSet missing;
|
||||||
|
|
@ -648,7 +588,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
|
||||||
|
|
||||||
if (missing.empty()) return;
|
if (missing.empty()) return;
|
||||||
|
|
||||||
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
|
Activity act(*logger, lvlInfo, actCopyPaths,
|
||||||
|
fmt("copying %d paths", missing.size()));
|
||||||
|
|
||||||
std::atomic<size_t> nrDone{0};
|
std::atomic<size_t> nrDone{0};
|
||||||
std::atomic<size_t> nrFailed{0};
|
std::atomic<size_t> nrFailed{0};
|
||||||
|
|
@ -661,8 +602,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
|
||||||
|
|
||||||
ThreadPool pool;
|
ThreadPool pool;
|
||||||
|
|
||||||
processGraph<Path>(pool,
|
processGraph<Path>(
|
||||||
PathSet(missing.begin(), missing.end()),
|
pool, PathSet(missing.begin(), missing.end()),
|
||||||
|
|
||||||
[&](const Path& storePath) {
|
[&](const Path& storePath) {
|
||||||
if (dstStore->isValidPath(storePath)) {
|
if (dstStore->isValidPath(storePath)) {
|
||||||
|
|
@ -689,9 +630,9 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
|
||||||
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
|
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
|
||||||
} catch (Error& e) {
|
} catch (Error& e) {
|
||||||
nrFailed++;
|
nrFailed++;
|
||||||
if (!settings.keepGoing)
|
if (!settings.keepGoing) throw e;
|
||||||
throw e;
|
logger->log(lvlError,
|
||||||
logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
|
format("could not copy %s: %s") % storePath % e.what());
|
||||||
showProgress();
|
showProgress();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -702,22 +643,21 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
|
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
|
const PathSet& storePaths, RepairFlag repair,
|
||||||
SubstituteFlag substitute)
|
CheckSigsFlag checkSigs, SubstituteFlag substitute) {
|
||||||
{
|
|
||||||
PathSet closure;
|
PathSet closure;
|
||||||
srcStore->computeFSClosure({storePaths}, closure);
|
srcStore->computeFSClosure({storePaths}, closure);
|
||||||
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) {
|
||||||
ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
|
|
||||||
{
|
|
||||||
ValidPathInfo info;
|
ValidPathInfo info;
|
||||||
getline(str, info.path);
|
getline(str, info.path);
|
||||||
if (str.eof()) { info.path = ""; return info; }
|
if (str.eof()) {
|
||||||
|
info.path = "";
|
||||||
|
return info;
|
||||||
|
}
|
||||||
if (hashGiven) {
|
if (hashGiven) {
|
||||||
string s;
|
string s;
|
||||||
getline(str, s);
|
getline(str, s);
|
||||||
|
|
@ -726,7 +666,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
|
||||||
if (!string2Int(s, info.narSize)) throw Error("number expected");
|
if (!string2Int(s, info.narSize)) throw Error("number expected");
|
||||||
}
|
}
|
||||||
getline(str, info.deriver);
|
getline(str, info.deriver);
|
||||||
string s; int n;
|
string s;
|
||||||
|
int n;
|
||||||
getline(str, s);
|
getline(str, s);
|
||||||
if (!string2Int(s, n)) throw Error("number expected");
|
if (!string2Int(s, n)) throw Error("number expected");
|
||||||
while (n--) {
|
while (n--) {
|
||||||
|
|
@ -737,9 +678,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string showPaths(const PathSet& paths) {
|
||||||
string showPaths(const PathSet & paths)
|
|
||||||
{
|
|
||||||
string s;
|
string s;
|
||||||
for (auto& i : paths) {
|
for (auto& i : paths) {
|
||||||
if (s.size() != 0) s += ", ";
|
if (s.size() != 0) s += ", ";
|
||||||
|
|
@ -748,30 +687,24 @@ string showPaths(const PathSet & paths)
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string ValidPathInfo::fingerprint() const {
|
||||||
std::string ValidPathInfo::fingerprint() const
|
|
||||||
{
|
|
||||||
if (narSize == 0 || !narHash)
|
if (narSize == 0 || !narHash)
|
||||||
throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known")
|
throw Error(format("cannot calculate fingerprint of path '%s' because its "
|
||||||
% path);
|
"size/hash is not known") %
|
||||||
return
|
path);
|
||||||
"1;" + path + ";"
|
return "1;" + path + ";" + narHash.to_string(Base32) + ";" +
|
||||||
+ narHash.to_string(Base32) + ";"
|
std::to_string(narSize) + ";" + concatStringsSep(",", references);
|
||||||
+ std::to_string(narSize) + ";"
|
|
||||||
+ concatStringsSep(",", references);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ValidPathInfo::sign(const SecretKey& secretKey) {
|
||||||
void ValidPathInfo::sign(const SecretKey & secretKey)
|
|
||||||
{
|
|
||||||
sigs.insert(secretKey.signDetached(fingerprint()));
|
sigs.insert(secretKey.signDetached(fingerprint()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool ValidPathInfo::isContentAddressed(const Store& store) const {
|
||||||
bool ValidPathInfo::isContentAddressed(const Store & store) const
|
|
||||||
{
|
|
||||||
auto warn = [&]() {
|
auto warn = [&]() {
|
||||||
printError(format("warning: path '%s' claims to be content-addressed but isn't") % path);
|
printError(
|
||||||
|
format("warning: path '%s' claims to be content-addressed but isn't") %
|
||||||
|
path);
|
||||||
};
|
};
|
||||||
|
|
||||||
if (hasPrefix(ca, "text:")) {
|
if (hasPrefix(ca, "text:")) {
|
||||||
|
|
@ -786,7 +719,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
|
||||||
bool recursive = ca.compare(6, 2, "r:") == 0;
|
bool recursive = ca.compare(6, 2, "r:") == 0;
|
||||||
Hash hash(std::string(ca, recursive ? 8 : 6));
|
Hash hash(std::string(ca, recursive ? 8 : 6));
|
||||||
if (references.empty() &&
|
if (references.empty() &&
|
||||||
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
|
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) ==
|
||||||
|
path)
|
||||||
return true;
|
return true;
|
||||||
else
|
else
|
||||||
warn();
|
warn();
|
||||||
|
|
@ -795,70 +729,58 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ValidPathInfo::checkSignatures(const Store& store,
|
||||||
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
|
const PublicKeys& publicKeys) const {
|
||||||
{
|
|
||||||
if (isContentAddressed(store)) return maxSigs;
|
if (isContentAddressed(store)) return maxSigs;
|
||||||
|
|
||||||
size_t good = 0;
|
size_t good = 0;
|
||||||
for (auto& sig : sigs)
|
for (auto& sig : sigs)
|
||||||
if (checkSignature(publicKeys, sig))
|
if (checkSignature(publicKeys, sig)) good++;
|
||||||
good++;
|
|
||||||
return good;
|
return good;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys,
|
||||||
bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
|
const std::string& sig) const {
|
||||||
{
|
|
||||||
return verifyDetached(fingerprint(), sig, publicKeys);
|
return verifyDetached(fingerprint(), sig, publicKeys);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Strings ValidPathInfo::shortRefs() const {
|
||||||
Strings ValidPathInfo::shortRefs() const
|
|
||||||
{
|
|
||||||
Strings refs;
|
Strings refs;
|
||||||
for (auto & r : references)
|
for (auto& r : references) refs.push_back(baseNameOf(r));
|
||||||
refs.push_back(baseNameOf(r));
|
|
||||||
return refs;
|
return refs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string makeFixedOutputCA(bool recursive, const Hash& hash) {
|
||||||
std::string makeFixedOutputCA(bool recursive, const Hash & hash)
|
|
||||||
{
|
|
||||||
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
|
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void Store::addToStore(const ValidPathInfo& info, Source& narSource,
|
void Store::addToStore(const ValidPathInfo& info, Source& narSource,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor)
|
std::shared_ptr<FSAccessor> accessor) {
|
||||||
{
|
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs,
|
||||||
addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
|
accessor);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar,
|
void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar,
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor)
|
std::shared_ptr<FSAccessor> accessor) {
|
||||||
{
|
|
||||||
StringSource source(*nar);
|
StringSource source(*nar);
|
||||||
addToStore(info, source, repair, checkSigs, accessor);
|
addToStore(info, source, repair, checkSigs, accessor);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
||||||
|
|
||||||
#include "local-store.hh"
|
#include "local-store.hh"
|
||||||
#include "remote-store.hh"
|
#include "remote-store.hh"
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
RegisterStoreImplementation::Implementations*
|
||||||
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
|
RegisterStoreImplementation::implementations = 0;
|
||||||
|
|
||||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
|
std::pair<std::string, Store::Params> splitUriAndParams(
|
||||||
{
|
const std::string& uri_) {
|
||||||
auto uri(uri_);
|
auto uri(uri_);
|
||||||
Store::Params params;
|
Store::Params params;
|
||||||
auto q = uri.find('?');
|
auto q = uri.find('?');
|
||||||
|
|
@ -890,8 +812,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
||||||
}
|
}
|
||||||
|
|
||||||
ref<Store> openStore(const std::string& uri_,
|
ref<Store> openStore(const std::string& uri_,
|
||||||
const Store::Params & extraParams)
|
const Store::Params& extraParams) {
|
||||||
{
|
|
||||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||||
auto params = extraParams;
|
auto params = extraParams;
|
||||||
params.insert(uriParams.begin(), uriParams.end());
|
params.insert(uriParams.begin(), uriParams.end());
|
||||||
|
|
@ -907,9 +828,7 @@ ref<Store> openStore(const std::string & uri_,
|
||||||
throw Error("don't know how to open Nix store '%s'", uri);
|
throw Error("don't know how to open Nix store '%s'", uri);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
StoreType getStoreType(const std::string& uri, const std::string& stateDir) {
|
||||||
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
|
||||||
{
|
|
||||||
if (uri == "daemon") {
|
if (uri == "daemon") {
|
||||||
return tDaemon;
|
return tDaemon;
|
||||||
} else if (uri == "local" || hasPrefix(uri, "/")) {
|
} else if (uri == "local" || hasPrefix(uri, "/")) {
|
||||||
|
|
@ -926,18 +845,15 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static RegisterStoreImplementation regStore([](const std::string& uri,
|
||||||
static RegisterStoreImplementation regStore([](
|
const Store::Params& params)
|
||||||
const std::string & uri, const Store::Params & params)
|
-> std::shared_ptr<Store> {
|
||||||
-> std::shared_ptr<Store>
|
|
||||||
{
|
|
||||||
switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
|
switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
|
||||||
case tDaemon:
|
case tDaemon:
|
||||||
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
||||||
case tLocal: {
|
case tLocal: {
|
||||||
Store::Params params2 = params;
|
Store::Params params2 = params;
|
||||||
if (hasPrefix(uri, "/"))
|
if (hasPrefix(uri, "/")) params2["root"] = uri;
|
||||||
params2["root"] = uri;
|
|
||||||
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
|
@ -945,9 +861,7 @@ static RegisterStoreImplementation regStore([](
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
std::list<ref<Store>> getDefaultSubstituters() {
|
||||||
std::list<ref<Store>> getDefaultSubstituters()
|
|
||||||
{
|
|
||||||
static auto stores([]() {
|
static auto stores([]() {
|
||||||
std::list<ref<Store>> stores;
|
std::list<ref<Store>> stores;
|
||||||
|
|
||||||
|
|
@ -963,11 +877,9 @@ std::list<ref<Store>> getDefaultSubstituters()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto uri : settings.substituters.get())
|
for (auto uri : settings.substituters.get()) addStore(uri);
|
||||||
addStore(uri);
|
|
||||||
|
|
||||||
for (auto uri : settings.extraSubstituters.get())
|
for (auto uri : settings.extraSubstituters.get()) addStore(uri);
|
||||||
addStore(uri);
|
|
||||||
|
|
||||||
stores.sort([](ref<Store>& a, ref<Store>& b) {
|
stores.sort([](ref<Store>& a, ref<Store>& b) {
|
||||||
return a->getPriority() < b->getPriority();
|
return a->getPriority() < b->getPriority();
|
||||||
|
|
@ -979,5 +891,4 @@ std::list<ref<Store>> getDefaultSubstituters()
|
||||||
return stores;
|
return stores;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
324
third_party/nix/src/libstore/store-api.hh
vendored
324
third_party/nix/src/libstore/store-api.hh
vendored
|
|
@ -1,32 +1,26 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "hash.hh"
|
|
||||||
#include "serialise.hh"
|
|
||||||
#include "crypto.hh"
|
|
||||||
#include "lru-cache.hh"
|
|
||||||
#include "sync.hh"
|
|
||||||
#include "globals.hh"
|
|
||||||
#include "config.hh"
|
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <unordered_map>
|
|
||||||
#include <unordered_set>
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <unordered_set>
|
||||||
|
#include "config.hh"
|
||||||
|
#include "crypto.hh"
|
||||||
|
#include "globals.hh"
|
||||||
|
#include "hash.hh"
|
||||||
|
#include "lru-cache.hh"
|
||||||
|
#include "serialise.hh"
|
||||||
|
#include "sync.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
MakeError(SubstError, Error)
|
MakeError(SubstError, Error)
|
||||||
MakeError(BuildError, Error) /* denotes a permanent build failure */
|
MakeError(BuildError, Error) /* denotes a permanent build failure */
|
||||||
MakeError(InvalidPath, Error)
|
MakeError(InvalidPath, Error) MakeError(Unsupported, Error)
|
||||||
MakeError(Unsupported, Error)
|
MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error)
|
||||||
MakeError(SubstituteGone, Error)
|
|
||||||
MakeError(SubstituterDisabled, Error)
|
|
||||||
|
|
||||||
|
|
||||||
struct BasicDerivation;
|
struct BasicDerivation;
|
||||||
struct Derivation;
|
struct Derivation;
|
||||||
|
|
@ -35,25 +29,20 @@ class NarInfoDiskCache;
|
||||||
class Store;
|
class Store;
|
||||||
class JSONPlaceholder;
|
class JSONPlaceholder;
|
||||||
|
|
||||||
|
|
||||||
enum RepairFlag : bool { NoRepair = false, Repair = true };
|
enum RepairFlag : bool { NoRepair = false, Repair = true };
|
||||||
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
|
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
|
||||||
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
|
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
|
||||||
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
|
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
|
||||||
|
|
||||||
|
|
||||||
/* Size of the hash part of store paths, in base-32 characters. */
|
/* Size of the hash part of store paths, in base-32 characters. */
|
||||||
const size_t storePathHashLen = 32; // i.e. 160 bits
|
const size_t storePathHashLen = 32; // i.e. 160 bits
|
||||||
|
|
||||||
/* Magic header of exportPath() output (obsolete). */
|
/* Magic header of exportPath() output (obsolete). */
|
||||||
const uint32_t exportMagic = 0x4558494e;
|
const uint32_t exportMagic = 0x4558494e;
|
||||||
|
|
||||||
|
|
||||||
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
|
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
|
||||||
|
|
||||||
|
struct GCOptions {
|
||||||
struct GCOptions
|
|
||||||
{
|
|
||||||
/* Garbage collector operation:
|
/* Garbage collector operation:
|
||||||
|
|
||||||
- `gcReturnLive': return the set of paths reachable from
|
- `gcReturnLive': return the set of paths reachable from
|
||||||
|
|
@ -89,9 +78,7 @@ struct GCOptions
|
||||||
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
|
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct GCResults {
|
||||||
struct GCResults
|
|
||||||
{
|
|
||||||
/* Depending on the action, the GC roots, or the paths that would
|
/* Depending on the action, the GC roots, or the paths that would
|
||||||
be or have been deleted. */
|
be or have been deleted. */
|
||||||
PathSet paths;
|
PathSet paths;
|
||||||
|
|
@ -101,9 +88,7 @@ struct GCResults
|
||||||
unsigned long long bytesFreed = 0;
|
unsigned long long bytesFreed = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct SubstitutablePathInfo {
|
||||||
struct SubstitutablePathInfo
|
|
||||||
{
|
|
||||||
Path deriver;
|
Path deriver;
|
||||||
PathSet references;
|
PathSet references;
|
||||||
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
|
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
|
||||||
|
|
@ -112,9 +97,7 @@ struct SubstitutablePathInfo
|
||||||
|
|
||||||
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
|
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
|
||||||
|
|
||||||
|
struct ValidPathInfo {
|
||||||
struct ValidPathInfo
|
|
||||||
{
|
|
||||||
Path path;
|
Path path;
|
||||||
Path deriver;
|
Path deriver;
|
||||||
Hash narHash;
|
Hash narHash;
|
||||||
|
|
@ -157,12 +140,8 @@ struct ValidPathInfo
|
||||||
*/
|
*/
|
||||||
std::string ca;
|
std::string ca;
|
||||||
|
|
||||||
bool operator == (const ValidPathInfo & i) const
|
bool operator==(const ValidPathInfo& i) const {
|
||||||
{
|
return path == i.path && narHash == i.narHash && references == i.references;
|
||||||
return
|
|
||||||
path == i.path
|
|
||||||
&& narHash == i.narHash
|
|
||||||
&& references == i.references;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Return a fingerprint of the store path to be used in binary
|
/* Return a fingerprint of the store path to be used in binary
|
||||||
|
|
@ -183,10 +162,12 @@ struct ValidPathInfo
|
||||||
/* Return the number of signatures on this .narinfo that were
|
/* Return the number of signatures on this .narinfo that were
|
||||||
produced by one of the specified keys, or maxSigs if the path
|
produced by one of the specified keys, or maxSigs if the path
|
||||||
is content-addressed. */
|
is content-addressed. */
|
||||||
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
|
size_t checkSignatures(const Store& store,
|
||||||
|
const PublicKeys& publicKeys) const;
|
||||||
|
|
||||||
/* Verify a single signature. */
|
/* Verify a single signature. */
|
||||||
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
|
bool checkSignature(const PublicKeys& publicKeys,
|
||||||
|
const std::string& sig) const;
|
||||||
|
|
||||||
Strings shortRefs() const;
|
Strings shortRefs() const;
|
||||||
|
|
||||||
|
|
@ -195,12 +176,9 @@ struct ValidPathInfo
|
||||||
|
|
||||||
typedef list<ValidPathInfo> ValidPathInfos;
|
typedef list<ValidPathInfo> ValidPathInfos;
|
||||||
|
|
||||||
|
|
||||||
enum BuildMode { bmNormal, bmRepair, bmCheck };
|
enum BuildMode { bmNormal, bmRepair, bmCheck };
|
||||||
|
|
||||||
|
struct BuildResult {
|
||||||
struct BuildResult
|
|
||||||
{
|
|
||||||
/* Note: don't remove status codes, and only add new status codes
|
/* Note: don't remove status codes, and only add new status codes
|
||||||
at the end of the list, to prevent client/server
|
at the end of the list, to prevent client/server
|
||||||
incompatibilities in the nix-store --serve protocol. */
|
incompatibilities in the nix-store --serve protocol. */
|
||||||
|
|
@ -239,25 +217,25 @@ struct BuildResult
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class Store : public std::enable_shared_from_this<Store>, public Config {
|
||||||
class Store : public std::enable_shared_from_this<Store>, public Config
|
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
typedef std::map<std::string, std::string> Params;
|
typedef std::map<std::string, std::string> Params;
|
||||||
|
|
||||||
const PathSetting storeDir_{this, false, settings.nixStore,
|
const PathSetting storeDir_{this, false, settings.nixStore, "store",
|
||||||
"store", "path to the Nix store"};
|
"path to the Nix store"};
|
||||||
const Path storeDir = storeDir_;
|
const Path storeDir = storeDir_;
|
||||||
|
|
||||||
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
|
const Setting<int> pathInfoCacheSize{
|
||||||
|
this, 65536, "path-info-cache-size",
|
||||||
|
"size of the in-memory store path information cache"};
|
||||||
|
|
||||||
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
|
const Setting<bool> isTrusted{
|
||||||
|
this, false, "trusted",
|
||||||
|
"whether paths from this store can be used as substitutes even when they "
|
||||||
|
"lack trusted signatures"};
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
struct State {
|
||||||
struct State
|
|
||||||
{
|
|
||||||
LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
|
LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -268,7 +246,6 @@ protected:
|
||||||
Store(const Params& params);
|
Store(const Params& params);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
virtual ~Store() {}
|
virtual ~Store() {}
|
||||||
|
|
||||||
virtual std::string getUri() = 0;
|
virtual std::string getUri() = 0;
|
||||||
|
|
@ -296,14 +273,14 @@ public:
|
||||||
Path followLinksToStorePath(const Path& path) const;
|
Path followLinksToStorePath(const Path& path) const;
|
||||||
|
|
||||||
/* Constructs a unique store path name. */
|
/* Constructs a unique store path name. */
|
||||||
Path makeStorePath(const string & type,
|
Path makeStorePath(const string& type, const Hash& hash,
|
||||||
const Hash & hash, const string & name) const;
|
const string& name) const;
|
||||||
|
|
||||||
Path makeOutputPath(const string & id,
|
Path makeOutputPath(const string& id, const Hash& hash,
|
||||||
const Hash & hash, const string & name) const;
|
const string& name) const;
|
||||||
|
|
||||||
Path makeFixedOutputPath(bool recursive,
|
Path makeFixedOutputPath(bool recursive, const Hash& hash,
|
||||||
const Hash & hash, const string & name) const;
|
const string& name) const;
|
||||||
|
|
||||||
Path makeTextPath(const string& name, const Hash& hash,
|
Path makeTextPath(const string& name, const Hash& hash,
|
||||||
const PathSet& references) const;
|
const PathSet& references) const;
|
||||||
|
|
@ -311,9 +288,10 @@ public:
|
||||||
/* This is the preparatory part of addToStore(); it computes the
|
/* This is the preparatory part of addToStore(); it computes the
|
||||||
store path to which srcPath is to be copied. Returns the store
|
store path to which srcPath is to be copied. Returns the store
|
||||||
path and the cryptographic hash of the contents of srcPath. */
|
path and the cryptographic hash of the contents of srcPath. */
|
||||||
std::pair<Path, Hash> computeStorePathForPath(const string & name,
|
std::pair<Path, Hash> computeStorePathForPath(
|
||||||
const Path & srcPath, bool recursive = true,
|
const string& name, const Path& srcPath, bool recursive = true,
|
||||||
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
|
HashType hashAlgo = htSHA256,
|
||||||
|
PathFilter& filter = defaultPathFilter) const;
|
||||||
|
|
||||||
/* Preparatory part of addTextToStore().
|
/* Preparatory part of addTextToStore().
|
||||||
|
|
||||||
|
|
@ -336,23 +314,20 @@ public:
|
||||||
bool isValidPath(const Path& path);
|
bool isValidPath(const Path& path);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
virtual bool isValidPathUncached(const Path& path);
|
virtual bool isValidPathUncached(const Path& path);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* Query which of the given paths is valid. Optionally, try to
|
/* Query which of the given paths is valid. Optionally, try to
|
||||||
substitute missing paths. */
|
substitute missing paths. */
|
||||||
virtual PathSet queryValidPaths(const PathSet & paths,
|
virtual PathSet queryValidPaths(
|
||||||
SubstituteFlag maybeSubstitute = NoSubstitute);
|
const PathSet& paths, SubstituteFlag maybeSubstitute = NoSubstitute);
|
||||||
|
|
||||||
/* Query the set of all valid paths. Note that for some store
|
/* Query the set of all valid paths. Note that for some store
|
||||||
backends, the name part of store paths may be omitted
|
backends, the name part of store paths may be omitted
|
||||||
(i.e. you'll get /nix/store/<hash> rather than
|
(i.e. you'll get /nix/store/<hash> rather than
|
||||||
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
|
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
|
||||||
full store path. */
|
full store path. */
|
||||||
virtual PathSet queryAllValidPaths()
|
virtual PathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); }
|
||||||
{ unsupported("queryAllValidPaths"); }
|
|
||||||
|
|
||||||
/* Query information about a valid path. It is permitted to omit
|
/* Query information about a valid path. It is permitted to omit
|
||||||
the name part of the store path. */
|
the name part of the store path. */
|
||||||
|
|
@ -363,16 +338,16 @@ public:
|
||||||
Callback<ref<ValidPathInfo>> callback) noexcept;
|
Callback<ref<ValidPathInfo>> callback) noexcept;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
virtual void queryPathInfoUncached(
|
||||||
virtual void queryPathInfoUncached(const Path & path,
|
const Path& path,
|
||||||
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
|
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* Queries the set of incoming FS references for a store path.
|
/* Queries the set of incoming FS references for a store path.
|
||||||
The result is not cleared. */
|
The result is not cleared. */
|
||||||
virtual void queryReferrers(const Path & path, PathSet & referrers)
|
virtual void queryReferrers(const Path& path, PathSet& referrers) {
|
||||||
{ unsupported("queryReferrers"); }
|
unsupported("queryReferrers");
|
||||||
|
}
|
||||||
|
|
||||||
/* Return all currently valid derivations that have `path' as an
|
/* Return all currently valid derivations that have `path' as an
|
||||||
output. (Note that the result of `queryDeriver()' is the
|
output. (Note that the result of `queryDeriver()' is the
|
||||||
|
|
@ -381,12 +356,14 @@ public:
|
||||||
virtual PathSet queryValidDerivers(const Path& path) { return {}; };
|
virtual PathSet queryValidDerivers(const Path& path) { return {}; };
|
||||||
|
|
||||||
/* Query the outputs of the derivation denoted by `path'. */
|
/* Query the outputs of the derivation denoted by `path'. */
|
||||||
virtual PathSet queryDerivationOutputs(const Path & path)
|
virtual PathSet queryDerivationOutputs(const Path& path) {
|
||||||
{ unsupported("queryDerivationOutputs"); }
|
unsupported("queryDerivationOutputs");
|
||||||
|
}
|
||||||
|
|
||||||
/* Query the output names of the derivation denoted by `path'. */
|
/* Query the output names of the derivation denoted by `path'. */
|
||||||
virtual StringSet queryDerivationOutputNames(const Path & path)
|
virtual StringSet queryDerivationOutputNames(const Path& path) {
|
||||||
{ unsupported("queryDerivationOutputNames"); }
|
unsupported("queryDerivationOutputNames");
|
||||||
|
}
|
||||||
|
|
||||||
/* Query the full store path given the hash part of a valid store
|
/* Query the full store path given the hash part of a valid store
|
||||||
path, or "" if the path doesn't exist. */
|
path, or "" if the path doesn't exist. */
|
||||||
|
|
@ -399,18 +376,23 @@ public:
|
||||||
sizes) of a set of paths. If a path does not have substitute
|
sizes) of a set of paths. If a path does not have substitute
|
||||||
info, it's omitted from the resulting ‘infos’ map. */
|
info, it's omitted from the resulting ‘infos’ map. */
|
||||||
virtual void querySubstitutablePathInfos(const PathSet& paths,
|
virtual void querySubstitutablePathInfos(const PathSet& paths,
|
||||||
SubstitutablePathInfos & infos) { return; };
|
SubstitutablePathInfos& infos) {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
virtual bool wantMassQuery() { return false; }
|
virtual bool wantMassQuery() { return false; }
|
||||||
|
|
||||||
/* Import a path into the store. */
|
/* Import a path into the store. */
|
||||||
virtual void addToStore(const ValidPathInfo& info, Source& narSource,
|
virtual void addToStore(const ValidPathInfo& info, Source& narSource,
|
||||||
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
|
RepairFlag repair = NoRepair,
|
||||||
|
CheckSigsFlag checkSigs = CheckSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor = 0);
|
std::shared_ptr<FSAccessor> accessor = 0);
|
||||||
|
|
||||||
// FIXME: remove
|
// FIXME: remove
|
||||||
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
|
virtual void addToStore(const ValidPathInfo& info,
|
||||||
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
|
const ref<std::string>& nar,
|
||||||
|
RepairFlag repair = NoRepair,
|
||||||
|
CheckSigsFlag checkSigs = CheckSigs,
|
||||||
std::shared_ptr<FSAccessor> accessor = 0);
|
std::shared_ptr<FSAccessor> accessor = 0);
|
||||||
|
|
||||||
/* Copy the contents of a path to the store and register the
|
/* Copy the contents of a path to the store and register the
|
||||||
|
|
@ -419,12 +401,14 @@ public:
|
||||||
libutil/archive.hh). */
|
libutil/archive.hh). */
|
||||||
virtual Path addToStore(const string& name, const Path& srcPath,
|
virtual Path addToStore(const string& name, const Path& srcPath,
|
||||||
bool recursive = true, HashType hashAlgo = htSHA256,
|
bool recursive = true, HashType hashAlgo = htSHA256,
|
||||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
|
PathFilter& filter = defaultPathFilter,
|
||||||
|
RepairFlag repair = NoRepair) = 0;
|
||||||
|
|
||||||
/* Like addToStore, but the contents written to the output path is
|
/* Like addToStore, but the contents written to the output path is
|
||||||
a regular file containing the given string. */
|
a regular file containing the given string. */
|
||||||
virtual Path addTextToStore(const string& name, const string& s,
|
virtual Path addTextToStore(const string& name, const string& s,
|
||||||
const PathSet & references, RepairFlag repair = NoRepair) = 0;
|
const PathSet& references,
|
||||||
|
RepairFlag repair = NoRepair) = 0;
|
||||||
|
|
||||||
/* Write a NAR dump of a store path. */
|
/* Write a NAR dump of a store path. */
|
||||||
virtual void narFromPath(const Path& path, Sink& sink) = 0;
|
virtual void narFromPath(const Path& path, Sink& sink) = 0;
|
||||||
|
|
@ -442,7 +426,8 @@ public:
|
||||||
/* Build a single non-materialized derivation (i.e. not from an
|
/* Build a single non-materialized derivation (i.e. not from an
|
||||||
on-disk .drv file). Note that ‘drvPath’ is only used for
|
on-disk .drv file). Note that ‘drvPath’ is only used for
|
||||||
informational purposes. */
|
informational purposes. */
|
||||||
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
|
virtual BuildResult buildDerivation(const Path& drvPath,
|
||||||
|
const BasicDerivation& drv,
|
||||||
BuildMode buildMode = bmNormal) = 0;
|
BuildMode buildMode = bmNormal) = 0;
|
||||||
|
|
||||||
/* Ensure that a path is valid. If it is not currently valid, it
|
/* Ensure that a path is valid. If it is not currently valid, it
|
||||||
|
|
@ -452,16 +437,16 @@ public:
|
||||||
|
|
||||||
/* Add a store path as a temporary root of the garbage collector.
|
/* Add a store path as a temporary root of the garbage collector.
|
||||||
The root disappears as soon as we exit. */
|
The root disappears as soon as we exit. */
|
||||||
virtual void addTempRoot(const Path & path)
|
virtual void addTempRoot(const Path& path) { unsupported("addTempRoot"); }
|
||||||
{ unsupported("addTempRoot"); }
|
|
||||||
|
|
||||||
/* Add an indirect root, which is merely a symlink to `path' from
|
/* Add an indirect root, which is merely a symlink to `path' from
|
||||||
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
|
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
|
||||||
to be a symlink to a store path. The garbage collector will
|
to be a symlink to a store path. The garbage collector will
|
||||||
automatically remove the indirect root when it finds that
|
automatically remove the indirect root when it finds that
|
||||||
`path' has disappeared. */
|
`path' has disappeared. */
|
||||||
virtual void addIndirectRoot(const Path & path)
|
virtual void addIndirectRoot(const Path& path) {
|
||||||
{ unsupported("addIndirectRoot"); }
|
unsupported("addIndirectRoot");
|
||||||
|
}
|
||||||
|
|
||||||
/* Acquire the global GC lock, then immediately release it. This
|
/* Acquire the global GC lock, then immediately release it. This
|
||||||
function must be called after registering a new permanent root,
|
function must be called after registering a new permanent root,
|
||||||
|
|
@ -488,18 +473,18 @@ public:
|
||||||
outside of the Nix store that point to `storePath'. If
|
outside of the Nix store that point to `storePath'. If
|
||||||
'censor' is true, privacy-sensitive information about roots
|
'censor' is true, privacy-sensitive information about roots
|
||||||
found in /proc is censored. */
|
found in /proc is censored. */
|
||||||
virtual Roots findRoots(bool censor)
|
virtual Roots findRoots(bool censor) { unsupported("findRoots"); }
|
||||||
{ unsupported("findRoots"); }
|
|
||||||
|
|
||||||
/* Perform a garbage collection. */
|
/* Perform a garbage collection. */
|
||||||
virtual void collectGarbage(const GCOptions & options, GCResults & results)
|
virtual void collectGarbage(const GCOptions& options, GCResults& results) {
|
||||||
{ unsupported("collectGarbage"); }
|
unsupported("collectGarbage");
|
||||||
|
}
|
||||||
|
|
||||||
/* Return a string representing information about the path that
|
/* Return a string representing information about the path that
|
||||||
can be loaded into the database using `nix-store --load-db' or
|
can be loaded into the database using `nix-store --load-db' or
|
||||||
`nix-store --register-validity'. */
|
`nix-store --register-validity'. */
|
||||||
string makeValidityRegistration(const PathSet & paths,
|
string makeValidityRegistration(const PathSet& paths, bool showDerivers,
|
||||||
bool showDerivers, bool showHash);
|
bool showHash);
|
||||||
|
|
||||||
/* Write a JSON representation of store path metadata, such as the
|
/* Write a JSON representation of store path metadata, such as the
|
||||||
hash and the references. If ‘includeImpureInfo’ is true,
|
hash and the references. If ‘includeImpureInfo’ is true,
|
||||||
|
|
@ -521,16 +506,18 @@ public:
|
||||||
|
|
||||||
/* Check the integrity of the Nix store. Returns true if errors
|
/* Check the integrity of the Nix store. Returns true if errors
|
||||||
remain. */
|
remain. */
|
||||||
virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; };
|
virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
/* Return an object to access files in the Nix store. */
|
/* Return an object to access files in the Nix store. */
|
||||||
virtual ref<FSAccessor> getFSAccessor()
|
virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); }
|
||||||
{ unsupported("getFSAccessor"); }
|
|
||||||
|
|
||||||
/* Add signatures to the specified store path. The signatures are
|
/* Add signatures to the specified store path. The signatures are
|
||||||
not verified. */
|
not verified. */
|
||||||
virtual void addSignatures(const Path & storePath, const StringSet & sigs)
|
virtual void addSignatures(const Path& storePath, const StringSet& sigs) {
|
||||||
{ unsupported("addSignatures"); }
|
unsupported("addSignatures");
|
||||||
|
}
|
||||||
|
|
||||||
/* Utility functions. */
|
/* Utility functions. */
|
||||||
|
|
||||||
|
|
@ -545,20 +532,22 @@ public:
|
||||||
`storePath' is returned; that is, the closures under the
|
`storePath' is returned; that is, the closures under the
|
||||||
`referrers' relation instead of the `references' relation is
|
`referrers' relation instead of the `references' relation is
|
||||||
returned. */
|
returned. */
|
||||||
virtual void computeFSClosure(const PathSet & paths,
|
virtual void computeFSClosure(const PathSet& paths, PathSet& out,
|
||||||
PathSet & out, bool flipDirection = false,
|
bool flipDirection = false,
|
||||||
bool includeOutputs = false, bool includeDerivers = false);
|
bool includeOutputs = false,
|
||||||
|
bool includeDerivers = false);
|
||||||
|
|
||||||
void computeFSClosure(const Path & path,
|
void computeFSClosure(const Path& path, PathSet& out,
|
||||||
PathSet & out, bool flipDirection = false,
|
bool flipDirection = false, bool includeOutputs = false,
|
||||||
bool includeOutputs = false, bool includeDerivers = false);
|
bool includeDerivers = false);
|
||||||
|
|
||||||
/* Given a set of paths that are to be built, return the set of
|
/* Given a set of paths that are to be built, return the set of
|
||||||
derivations that will be built, and the set of output paths
|
derivations that will be built, and the set of output paths
|
||||||
that will be substituted. */
|
that will be substituted. */
|
||||||
virtual void queryMissing(const PathSet & targets,
|
virtual void queryMissing(const PathSet& targets, PathSet& willBuild,
|
||||||
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
|
PathSet& willSubstitute, PathSet& unknown,
|
||||||
unsigned long long & downloadSize, unsigned long long & narSize);
|
unsigned long long& downloadSize,
|
||||||
|
unsigned long long& narSize);
|
||||||
|
|
||||||
/* Sort a set of paths topologically under the references
|
/* Sort a set of paths topologically under the references
|
||||||
relation. If p refers to q, then p precedes q in this list. */
|
relation. If p refers to q, then p precedes q in this list. */
|
||||||
|
|
@ -577,8 +566,7 @@ public:
|
||||||
Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
|
Paths importPaths(Source& source, std::shared_ptr<FSAccessor> accessor,
|
||||||
CheckSigsFlag checkSigs = CheckSigs);
|
CheckSigsFlag checkSigs = CheckSigs);
|
||||||
|
|
||||||
struct Stats
|
struct Stats {
|
||||||
{
|
|
||||||
std::atomic<uint64_t> narInfoRead{0};
|
std::atomic<uint64_t> narInfoRead{0};
|
||||||
std::atomic<uint64_t> narInfoReadAverted{0};
|
std::atomic<uint64_t> narInfoReadAverted{0};
|
||||||
std::atomic<uint64_t> narInfoMissing{0};
|
std::atomic<uint64_t> narInfoMissing{0};
|
||||||
|
|
@ -598,67 +586,55 @@ public:
|
||||||
|
|
||||||
/* Return the build log of the specified store path, if available,
|
/* Return the build log of the specified store path, if available,
|
||||||
or null otherwise. */
|
or null otherwise. */
|
||||||
virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
|
virtual std::shared_ptr<std::string> getBuildLog(const Path& path) {
|
||||||
{ return nullptr; }
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
/* Hack to allow long-running processes like hydra-queue-runner to
|
/* Hack to allow long-running processes like hydra-queue-runner to
|
||||||
occasionally flush their path info cache. */
|
occasionally flush their path info cache. */
|
||||||
void clearPathInfoCache()
|
void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); }
|
||||||
{
|
|
||||||
state.lock()->pathInfoCache.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Establish a connection to the store, for store types that have
|
/* Establish a connection to the store, for store types that have
|
||||||
a notion of connection. Otherwise this is a no-op. */
|
a notion of connection. Otherwise this is a no-op. */
|
||||||
virtual void connect(){};
|
virtual void connect(){};
|
||||||
|
|
||||||
/* Get the protocol version of this store or it's connection. */
|
/* Get the protocol version of this store or it's connection. */
|
||||||
virtual unsigned int getProtocol()
|
virtual unsigned int getProtocol() { return 0; };
|
||||||
{
|
|
||||||
return 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Get the priority of the store, used to order substituters. In
|
/* Get the priority of the store, used to order substituters. In
|
||||||
particular, binary caches can specify a priority field in their
|
particular, binary caches can specify a priority field in their
|
||||||
"nix-cache-info" file. Lower value means higher priority. */
|
"nix-cache-info" file. Lower value means higher priority. */
|
||||||
virtual int getPriority() { return 0; }
|
virtual int getPriority() { return 0; }
|
||||||
|
|
||||||
virtual Path toRealPath(const Path & storePath)
|
virtual Path toRealPath(const Path& storePath) { return storePath; }
|
||||||
{
|
|
||||||
return storePath;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void createUser(const std::string & userName, uid_t userId)
|
virtual void createUser(const std::string& userName, uid_t userId) {}
|
||||||
{ }
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
Stats stats;
|
Stats stats;
|
||||||
|
|
||||||
/* Unsupported methods. */
|
/* Unsupported methods. */
|
||||||
[[noreturn]] void unsupported(const std::string & op)
|
[[noreturn]] void unsupported(const std::string& op) {
|
||||||
{
|
throw Unsupported("operation '%s' is not supported by store '%s'", op,
|
||||||
throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri());
|
getUri());
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class LocalFSStore : public virtual Store {
|
||||||
class LocalFSStore : public virtual Store
|
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
// FIXME: the (Store*) cast works around a bug in gcc that causes
|
// FIXME: the (Store*) cast works around a bug in gcc that causes
|
||||||
// it to emit the call to the Option constructor. Clang works fine
|
// it to emit the call to the Option constructor. Clang works fine
|
||||||
// either way.
|
// either way.
|
||||||
const PathSetting rootDir{(Store*) this, true, "",
|
const PathSetting rootDir{(Store*)this, true, "", "root",
|
||||||
"root", "directory prefixed to all other paths"};
|
"directory prefixed to all other paths"};
|
||||||
const PathSetting stateDir{(Store*) this, false,
|
const PathSetting stateDir{
|
||||||
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
|
(Store*)this, false,
|
||||||
"state", "directory where Nix will store state"};
|
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state",
|
||||||
const PathSetting logDir{(Store*) this, false,
|
"directory where Nix will store state"};
|
||||||
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
|
const PathSetting logDir{
|
||||||
"log", "directory where Nix will store state"};
|
(Store*)this, false,
|
||||||
|
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log",
|
||||||
|
"directory where Nix will store state"};
|
||||||
|
|
||||||
const static string drvsLogDir;
|
const static string drvsLogDir;
|
||||||
|
|
||||||
|
|
@ -668,21 +644,20 @@ public:
|
||||||
ref<FSAccessor> getFSAccessor() override;
|
ref<FSAccessor> getFSAccessor() override;
|
||||||
|
|
||||||
/* Register a permanent GC root. */
|
/* Register a permanent GC root. */
|
||||||
Path addPermRoot(const Path & storePath,
|
Path addPermRoot(const Path& storePath, const Path& gcRoot, bool indirect,
|
||||||
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
|
bool allowOutsideRootsDir = false);
|
||||||
|
|
||||||
virtual Path getRealStoreDir() { return storeDir; }
|
virtual Path getRealStoreDir() { return storeDir; }
|
||||||
|
|
||||||
Path toRealPath(const Path & storePath) override
|
Path toRealPath(const Path& storePath) override {
|
||||||
{
|
|
||||||
assert(isInStore(storePath));
|
assert(isInStore(storePath));
|
||||||
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
|
return getRealStoreDir() + "/" +
|
||||||
|
std::string(storePath, storeDir.size() + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
|
std::shared_ptr<std::string> getBuildLog(const Path& path) override;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Extract the name part of the given store path. */
|
/* Extract the name part of the given store path. */
|
||||||
string storePathToName(const Path& path);
|
string storePathToName(const Path& path);
|
||||||
|
|
||||||
|
|
@ -694,37 +669,32 @@ string storePathToHash(const Path & path);
|
||||||
a dot. */
|
a dot. */
|
||||||
void checkStoreName(const string& name);
|
void checkStoreName(const string& name);
|
||||||
|
|
||||||
|
|
||||||
/* Copy a path from one store to another. */
|
/* Copy a path from one store to another. */
|
||||||
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
|
const Path& storePath, RepairFlag repair = NoRepair,
|
||||||
|
CheckSigsFlag checkSigs = CheckSigs);
|
||||||
|
|
||||||
/* Copy store paths from one store to another. The paths may be copied
|
/* Copy store paths from one store to another. The paths may be copied
|
||||||
in parallel. They are copied in a topologically sorted order
|
in parallel. They are copied in a topologically sorted order
|
||||||
(i.e. if A is a reference of B, then A is copied before B), but
|
(i.e. if A is a reference of B, then A is copied before B), but
|
||||||
the set of store paths is not automatically closed; use
|
the set of store paths is not automatically closed; use
|
||||||
copyClosure() for that. */
|
copyClosure() for that. */
|
||||||
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
|
void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
RepairFlag repair = NoRepair,
|
const PathSet& storePaths, RepairFlag repair = NoRepair,
|
||||||
CheckSigsFlag checkSigs = CheckSigs,
|
CheckSigsFlag checkSigs = CheckSigs,
|
||||||
SubstituteFlag substitute = NoSubstitute);
|
SubstituteFlag substitute = NoSubstitute);
|
||||||
|
|
||||||
|
|
||||||
/* Copy the closure of the specified paths from one store to another. */
|
/* Copy the closure of the specified paths from one store to another. */
|
||||||
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
|
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
|
||||||
const PathSet & storePaths,
|
const PathSet& storePaths, RepairFlag repair = NoRepair,
|
||||||
RepairFlag repair = NoRepair,
|
|
||||||
CheckSigsFlag checkSigs = CheckSigs,
|
CheckSigsFlag checkSigs = CheckSigs,
|
||||||
SubstituteFlag substitute = NoSubstitute);
|
SubstituteFlag substitute = NoSubstitute);
|
||||||
|
|
||||||
|
|
||||||
/* Remove the temporary roots file for this process. Any temporary
|
/* Remove the temporary roots file for this process. Any temporary
|
||||||
root becomes garbage after this point unless it has been registered
|
root becomes garbage after this point unless it has been registered
|
||||||
as a (permanent) root. */
|
as a (permanent) root. */
|
||||||
void removeTempRoots();
|
void removeTempRoots();
|
||||||
|
|
||||||
|
|
||||||
/* Return a Store object to access the Nix store denoted by
|
/* Return a Store object to access the Nix store denoted by
|
||||||
‘uri’ (slight misnomer...). Supported values are:
|
‘uri’ (slight misnomer...). Supported values are:
|
||||||
|
|
||||||
|
|
@ -757,13 +727,7 @@ void removeTempRoots();
|
||||||
ref<Store> openStore(const std::string& uri = settings.storeUri.get(),
|
ref<Store> openStore(const std::string& uri = settings.storeUri.get(),
|
||||||
const Store::Params& extraParams = Store::Params());
|
const Store::Params& extraParams = Store::Params());
|
||||||
|
|
||||||
|
enum StoreType { tDaemon, tLocal, tOther };
|
||||||
enum StoreType {
|
|
||||||
tDaemon,
|
|
||||||
tLocal,
|
|
||||||
tOther
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
StoreType getStoreType(const std::string& uri = settings.storeUri.get(),
|
StoreType getStoreType(const std::string& uri = settings.storeUri.get(),
|
||||||
const std::string& stateDir = settings.nixStateDir);
|
const std::string& stateDir = settings.nixStateDir);
|
||||||
|
|
@ -772,40 +736,32 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
|
||||||
‘substituters’ option and various legacy options. */
|
‘substituters’ option and various legacy options. */
|
||||||
std::list<ref<Store>> getDefaultSubstituters();
|
std::list<ref<Store>> getDefaultSubstituters();
|
||||||
|
|
||||||
|
|
||||||
/* Store implementation registration. */
|
/* Store implementation registration. */
|
||||||
typedef std::function<std::shared_ptr<Store>(
|
typedef std::function<std::shared_ptr<Store>(const std::string& uri,
|
||||||
const std::string & uri, const Store::Params & params)> OpenStore;
|
const Store::Params& params)>
|
||||||
|
OpenStore;
|
||||||
|
|
||||||
struct RegisterStoreImplementation
|
struct RegisterStoreImplementation {
|
||||||
{
|
|
||||||
typedef std::vector<OpenStore> Implementations;
|
typedef std::vector<OpenStore> Implementations;
|
||||||
static Implementations* implementations;
|
static Implementations* implementations;
|
||||||
|
|
||||||
RegisterStoreImplementation(OpenStore fun)
|
RegisterStoreImplementation(OpenStore fun) {
|
||||||
{
|
|
||||||
if (!implementations) implementations = new Implementations;
|
if (!implementations) implementations = new Implementations;
|
||||||
implementations->push_back(fun);
|
implementations->push_back(fun);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* Display a set of paths in human-readable form (i.e., between quotes
|
/* Display a set of paths in human-readable form (i.e., between quotes
|
||||||
and separated by commas). */
|
and separated by commas). */
|
||||||
string showPaths(const PathSet& paths);
|
string showPaths(const PathSet& paths);
|
||||||
|
|
||||||
|
ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven = false);
|
||||||
ValidPathInfo decodeValidPathInfo(std::istream & str,
|
|
||||||
bool hashGiven = false);
|
|
||||||
|
|
||||||
|
|
||||||
/* Compute the content-addressability assertion (ValidPathInfo::ca)
|
/* Compute the content-addressability assertion (ValidPathInfo::ca)
|
||||||
for paths created by makeFixedOutputPath() / addToStore(). */
|
for paths created by makeFixedOutputPath() / addToStore(). */
|
||||||
std::string makeFixedOutputCA(bool recursive, const Hash& hash);
|
std::string makeFixedOutputCA(bool recursive, const Hash& hash);
|
||||||
|
|
||||||
|
|
||||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri);
|
std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
10
third_party/nix/src/libstore/worker-protocol.hh
vendored
10
third_party/nix/src/libstore/worker-protocol.hh
vendored
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
#define WORKER_MAGIC_1 0x6e697863
|
#define WORKER_MAGIC_1 0x6e697863
|
||||||
#define WORKER_MAGIC_2 0x6478696f
|
#define WORKER_MAGIC_2 0x6478696f
|
||||||
|
|
||||||
|
|
@ -10,7 +9,6 @@ namespace nix {
|
||||||
#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
|
#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00)
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff)
|
||||||
|
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
wopIsValidPath = 1,
|
wopIsValidPath = 1,
|
||||||
wopHasSubstitutes = 3,
|
wopHasSubstitutes = 3,
|
||||||
|
|
@ -51,7 +49,6 @@ typedef enum {
|
||||||
wopQueryMissing = 40,
|
wopQueryMissing = 40,
|
||||||
} WorkerOp;
|
} WorkerOp;
|
||||||
|
|
||||||
|
|
||||||
#define STDERR_NEXT 0x6f6c6d67
|
#define STDERR_NEXT 0x6f6c6d67
|
||||||
#define STDERR_READ 0x64617461 // data needed from source
|
#define STDERR_READ 0x64617461 // data needed from source
|
||||||
#define STDERR_WRITE 0x64617416 // data for sink
|
#define STDERR_WRITE 0x64617416 // data for sink
|
||||||
|
|
@ -61,9 +58,8 @@ typedef enum {
|
||||||
#define STDERR_STOP_ACTIVITY 0x53544f50
|
#define STDERR_STOP_ACTIVITY 0x53544f50
|
||||||
#define STDERR_RESULT 0x52534c54
|
#define STDERR_RESULT 0x52534c54
|
||||||
|
|
||||||
|
|
||||||
Path readStorePath(Store& store, Source& from);
|
Path readStorePath(Store& store, Source& from);
|
||||||
template<class T> T readStorePaths(Store & store, Source & from);
|
template <class T>
|
||||||
|
T readStorePaths(Store& store, Source& from);
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
18
third_party/nix/src/libutil/affinity.cc
vendored
18
third_party/nix/src/libutil/affinity.cc
vendored
|
|
@ -1,6 +1,6 @@
|
||||||
|
#include "affinity.hh"
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "affinity.hh"
|
|
||||||
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
#include <sched.h>
|
#include <sched.h>
|
||||||
|
|
@ -8,15 +8,12 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
static bool didSaveAffinity = false;
|
static bool didSaveAffinity = false;
|
||||||
static cpu_set_t savedAffinity;
|
static cpu_set_t savedAffinity;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
void setAffinityTo(int cpu) {
|
||||||
void setAffinityTo(int cpu)
|
|
||||||
{
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
|
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
|
||||||
didSaveAffinity = true;
|
didSaveAffinity = true;
|
||||||
|
|
@ -29,9 +26,7 @@ void setAffinityTo(int cpu)
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int lockToCurrentCPU() {
|
||||||
int lockToCurrentCPU()
|
|
||||||
{
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
int cpu = sched_getcpu();
|
int cpu = sched_getcpu();
|
||||||
if (cpu != -1) setAffinityTo(cpu);
|
if (cpu != -1) setAffinityTo(cpu);
|
||||||
|
|
@ -41,9 +36,7 @@ int lockToCurrentCPU()
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void restoreAffinity() {
|
||||||
void restoreAffinity()
|
|
||||||
{
|
|
||||||
#if __linux__
|
#if __linux__
|
||||||
if (!didSaveAffinity) return;
|
if (!didSaveAffinity) return;
|
||||||
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
|
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
|
||||||
|
|
@ -51,5 +44,4 @@ void restoreAffinity()
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
2
third_party/nix/src/libutil/affinity.hh
vendored
2
third_party/nix/src/libutil/affinity.hh
vendored
|
|
@ -6,4 +6,4 @@ void setAffinityTo(int cpu);
|
||||||
int lockToCurrentCPU();
|
int lockToCurrentCPU();
|
||||||
void restoreAffinity();
|
void restoreAffinity();
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
158
third_party/nix/src/libutil/archive.cc
vendored
158
third_party/nix/src/libutil/archive.cc
vendored
|
|
@ -1,32 +1,31 @@
|
||||||
#include <cerrno>
|
#include "archive.hh"
|
||||||
#include <algorithm>
|
|
||||||
#include <vector>
|
|
||||||
#include <map>
|
|
||||||
|
|
||||||
#include <strings.h> // for strcasecmp
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <strings.h> // for strcasecmp
|
||||||
#include "archive.hh"
|
#include <sys/stat.h>
|
||||||
#include "util.hh"
|
#include <sys/types.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cerrno>
|
||||||
|
#include <map>
|
||||||
|
#include <vector>
|
||||||
#include "config.hh"
|
#include "config.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct ArchiveSettings : Config
|
struct ArchiveSettings : Config {
|
||||||
{
|
Setting<bool> useCaseHack {
|
||||||
Setting<bool> useCaseHack{this,
|
this,
|
||||||
#if __APPLE__
|
#if __APPLE__
|
||||||
true,
|
true,
|
||||||
#else
|
#else
|
||||||
false,
|
false,
|
||||||
#endif
|
#endif
|
||||||
"use-case-hack",
|
"use-case-hack",
|
||||||
"Whether to enable a Darwin-specific hack for dealing with file name collisions."};
|
"Whether to enable a Darwin-specific hack for dealing with file name "
|
||||||
|
"collisions."
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
static ArchiveSettings archiveSettings;
|
static ArchiveSettings archiveSettings;
|
||||||
|
|
@ -39,10 +38,7 @@ static string caseHackSuffix = "~nix~case~hack~";
|
||||||
|
|
||||||
PathFilter defaultPathFilter = [](const Path&) { return true; };
|
PathFilter defaultPathFilter = [](const Path&) { return true; };
|
||||||
|
|
||||||
|
static void dumpContents(const Path& path, size_t size, Sink& sink) {
|
||||||
static void dumpContents(const Path & path, size_t size,
|
|
||||||
Sink & sink)
|
|
||||||
{
|
|
||||||
sink << "contents" << size;
|
sink << "contents" << size;
|
||||||
|
|
||||||
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
||||||
|
|
@ -61,9 +57,7 @@ static void dumpContents(const Path & path, size_t size,
|
||||||
writePadding(size, sink);
|
writePadding(size, sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void dump(const Path& path, Sink& sink, PathFilter& filter) {
|
||||||
static void dump(const Path & path, Sink & sink, PathFilter & filter)
|
|
||||||
{
|
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
|
|
@ -73,14 +67,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
|
||||||
sink << "(";
|
sink << "(";
|
||||||
|
|
||||||
if (S_ISREG(st.st_mode)) {
|
if (S_ISREG(st.st_mode)) {
|
||||||
sink << "type" << "regular";
|
sink << "type"
|
||||||
|
<< "regular";
|
||||||
if (st.st_mode & S_IXUSR)
|
if (st.st_mode & S_IXUSR)
|
||||||
sink << "executable" << "";
|
sink << "executable"
|
||||||
|
<< "";
|
||||||
dumpContents(path, (size_t)st.st_size, sink);
|
dumpContents(path, (size_t)st.st_size, sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (S_ISDIR(st.st_mode)) {
|
else if (S_ISDIR(st.st_mode)) {
|
||||||
sink << "type" << "directory";
|
sink << "type"
|
||||||
|
<< "directory";
|
||||||
|
|
||||||
/* If we're on a case-insensitive system like macOS, undo
|
/* If we're on a case-insensitive system like macOS, undo
|
||||||
the case hack applied by restorePath(). */
|
the case hack applied by restorePath(). */
|
||||||
|
|
@ -90,52 +87,54 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
|
||||||
string name(i.name);
|
string name(i.name);
|
||||||
size_t pos = i.name.find(caseHackSuffix);
|
size_t pos = i.name.find(caseHackSuffix);
|
||||||
if (pos != string::npos) {
|
if (pos != string::npos) {
|
||||||
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name));
|
debug(format("removing case hack suffix from '%1%'") %
|
||||||
|
(path + "/" + i.name));
|
||||||
name.erase(pos);
|
name.erase(pos);
|
||||||
}
|
}
|
||||||
if (unhacked.find(name) != unhacked.end())
|
if (unhacked.find(name) != unhacked.end())
|
||||||
throw Error(format("file name collision in between '%1%' and '%2%'")
|
throw Error(format("file name collision in between '%1%' and '%2%'") %
|
||||||
% (path + "/" + unhacked[name]) % (path + "/" + i.name));
|
(path + "/" + unhacked[name]) % (path + "/" + i.name));
|
||||||
unhacked[name] = i.name;
|
unhacked[name] = i.name;
|
||||||
} else
|
} else
|
||||||
unhacked[i.name] = i.name;
|
unhacked[i.name] = i.name;
|
||||||
|
|
||||||
for (auto& i : unhacked)
|
for (auto& i : unhacked)
|
||||||
if (filter(path + "/" + i.first)) {
|
if (filter(path + "/" + i.first)) {
|
||||||
sink << "entry" << "(" << "name" << i.first << "node";
|
sink << "entry"
|
||||||
|
<< "("
|
||||||
|
<< "name" << i.first << "node";
|
||||||
dump(path + "/" + i.second, sink, filter);
|
dump(path + "/" + i.second, sink, filter);
|
||||||
sink << ")";
|
sink << ")";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (S_ISLNK(st.st_mode))
|
else if (S_ISLNK(st.st_mode))
|
||||||
sink << "type" << "symlink" << "target" << readLink(path);
|
sink << "type"
|
||||||
|
<< "symlink"
|
||||||
|
<< "target" << readLink(path);
|
||||||
|
|
||||||
else throw Error(format("file '%1%' has an unsupported type") % path);
|
else
|
||||||
|
throw Error(format("file '%1%' has an unsupported type") % path);
|
||||||
|
|
||||||
sink << ")";
|
sink << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void dumpPath(const Path& path, Sink& sink, PathFilter& filter) {
|
||||||
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
|
|
||||||
{
|
|
||||||
sink << narVersionMagic1;
|
sink << narVersionMagic1;
|
||||||
dump(path, sink, filter);
|
dump(path, sink, filter);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void dumpString(const std::string& s, Sink& sink) {
|
||||||
void dumpString(const std::string & s, Sink & sink)
|
sink << narVersionMagic1 << "("
|
||||||
{
|
<< "type"
|
||||||
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
|
<< "regular"
|
||||||
|
<< "contents" << s << ")";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static SerialisationError badArchive(string s) {
|
||||||
static SerialisationError badArchive(string s)
|
|
||||||
{
|
|
||||||
return SerialisationError("bad archive: " + s);
|
return SerialisationError("bad archive: " + s);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#if 0
|
#if 0
|
||||||
static void skipGeneric(Source & source)
|
static void skipGeneric(Source & source)
|
||||||
{
|
{
|
||||||
|
|
@ -146,9 +145,7 @@ static void skipGeneric(Source & source)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
static void parseContents(ParseSink& sink, Source& source, const Path& path) {
|
||||||
static void parseContents(ParseSink & sink, Source & source, const Path & path)
|
|
||||||
{
|
|
||||||
unsigned long long size = readLongLong(source);
|
unsigned long long size = readLongLong(source);
|
||||||
|
|
||||||
sink.preallocateContents(size);
|
sink.preallocateContents(size);
|
||||||
|
|
@ -168,18 +165,13 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
|
||||||
readPadding(size, source);
|
readPadding(size, source);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct CaseInsensitiveCompare {
|
||||||
struct CaseInsensitiveCompare
|
bool operator()(const string& a, const string& b) const {
|
||||||
{
|
|
||||||
bool operator() (const string & a, const string & b) const
|
|
||||||
{
|
|
||||||
return strcasecmp(a.c_str(), b.c_str()) < 0;
|
return strcasecmp(a.c_str(), b.c_str()) < 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static void parse(ParseSink& sink, Source& source, const Path& path) {
|
||||||
static void parse(ParseSink & sink, Source & source, const Path & path)
|
|
||||||
{
|
|
||||||
string s;
|
string s;
|
||||||
|
|
||||||
s = readString(source);
|
s = readString(source);
|
||||||
|
|
@ -199,8 +191,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (s == "type") {
|
else if (s == "type") {
|
||||||
if (type != tpUnknown)
|
if (type != tpUnknown) throw badArchive("multiple type fields");
|
||||||
throw badArchive("multiple type fields");
|
|
||||||
string t = readString(source);
|
string t = readString(source);
|
||||||
|
|
||||||
if (t == "regular") {
|
if (t == "regular") {
|
||||||
|
|
@ -217,7 +208,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
|
||||||
type = tpSymlink;
|
type = tpSymlink;
|
||||||
}
|
}
|
||||||
|
|
||||||
else throw badArchive("unknown file type " + t);
|
else
|
||||||
|
throw badArchive("unknown file type " + t);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -246,15 +238,17 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
|
||||||
break;
|
break;
|
||||||
} else if (s == "name") {
|
} else if (s == "name") {
|
||||||
name = readString(source);
|
name = readString(source);
|
||||||
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
|
if (name.empty() || name == "." || name == ".." ||
|
||||||
|
name.find('/') != string::npos ||
|
||||||
|
name.find((char)0) != string::npos)
|
||||||
throw Error(format("NAR contains invalid file name '%1%'") % name);
|
throw Error(format("NAR contains invalid file name '%1%'") % name);
|
||||||
if (name <= prevName)
|
if (name <= prevName) throw Error("NAR directory is not sorted");
|
||||||
throw Error("NAR directory is not sorted");
|
|
||||||
prevName = name;
|
prevName = name;
|
||||||
if (archiveSettings.useCaseHack) {
|
if (archiveSettings.useCaseHack) {
|
||||||
auto i = names.find(name);
|
auto i = names.find(name);
|
||||||
if (i != names.end()) {
|
if (i != names.end()) {
|
||||||
debug(format("case collision between '%1%' and '%2%'") % i->first % name);
|
debug(format("case collision between '%1%' and '%2%'") %
|
||||||
|
i->first % name);
|
||||||
name += caseHackSuffix;
|
name += caseHackSuffix;
|
||||||
name += std::to_string(++i->second);
|
name += std::to_string(++i->second);
|
||||||
} else
|
} else
|
||||||
|
|
@ -278,9 +272,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void parseDump(ParseSink& sink, Source& source) {
|
||||||
void parseDump(ParseSink & sink, Source & source)
|
|
||||||
{
|
|
||||||
string version;
|
string version;
|
||||||
try {
|
try {
|
||||||
version = readString(source, narVersionMagic1.size());
|
version = readString(source, narVersionMagic1.size());
|
||||||
|
|
@ -293,37 +285,30 @@ void parseDump(ParseSink & sink, Source & source)
|
||||||
parse(sink, source, "");
|
parse(sink, source, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct RestoreSink : ParseSink {
|
||||||
struct RestoreSink : ParseSink
|
|
||||||
{
|
|
||||||
Path dstPath;
|
Path dstPath;
|
||||||
AutoCloseFD fd;
|
AutoCloseFD fd;
|
||||||
|
|
||||||
void createDirectory(const Path & path)
|
void createDirectory(const Path& path) {
|
||||||
{
|
|
||||||
Path p = dstPath + path;
|
Path p = dstPath + path;
|
||||||
if (mkdir(p.c_str(), 0777) == -1)
|
if (mkdir(p.c_str(), 0777) == -1)
|
||||||
throw SysError(format("creating directory '%1%'") % p);
|
throw SysError(format("creating directory '%1%'") % p);
|
||||||
};
|
};
|
||||||
|
|
||||||
void createRegularFile(const Path & path)
|
void createRegularFile(const Path& path) {
|
||||||
{
|
|
||||||
Path p = dstPath + path;
|
Path p = dstPath + path;
|
||||||
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
|
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
|
||||||
if (!fd) throw SysError(format("creating file '%1%'") % p);
|
if (!fd) throw SysError(format("creating file '%1%'") % p);
|
||||||
}
|
}
|
||||||
|
|
||||||
void isExecutable()
|
void isExecutable() {
|
||||||
{
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstat(fd.get(), &st) == -1)
|
if (fstat(fd.get(), &st) == -1) throw SysError("fstat");
|
||||||
throw SysError("fstat");
|
|
||||||
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
|
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
|
||||||
throw SysError("fchmod");
|
throw SysError("fchmod");
|
||||||
}
|
}
|
||||||
|
|
||||||
void preallocateContents(unsigned long long len)
|
void preallocateContents(unsigned long long len) {
|
||||||
{
|
|
||||||
#if HAVE_POSIX_FALLOCATE
|
#if HAVE_POSIX_FALLOCATE
|
||||||
if (len) {
|
if (len) {
|
||||||
errno = posix_fallocate(fd.get(), 0, len);
|
errno = posix_fallocate(fd.get(), 0, len);
|
||||||
|
|
@ -337,29 +322,23 @@ struct RestoreSink : ParseSink
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void receiveContents(unsigned char * data, unsigned int len)
|
void receiveContents(unsigned char* data, unsigned int len) {
|
||||||
{
|
|
||||||
writeFull(fd.get(), data, len);
|
writeFull(fd.get(), data, len);
|
||||||
}
|
}
|
||||||
|
|
||||||
void createSymlink(const Path & path, const string & target)
|
void createSymlink(const Path& path, const string& target) {
|
||||||
{
|
|
||||||
Path p = dstPath + path;
|
Path p = dstPath + path;
|
||||||
nix::createSymlink(target, p);
|
nix::createSymlink(target, p);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
void restorePath(const Path& path, Source& source) {
|
||||||
void restorePath(const Path & path, Source & source)
|
|
||||||
{
|
|
||||||
RestoreSink sink;
|
RestoreSink sink;
|
||||||
sink.dstPath = path;
|
sink.dstPath = path;
|
||||||
parseDump(sink, source);
|
parseDump(sink, source);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void copyNAR(Source& source, Sink& sink) {
|
||||||
void copyNAR(Source & source, Sink & sink)
|
|
||||||
{
|
|
||||||
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
|
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
|
||||||
// we should just forward all data directly without parsing.
|
// we should just forward all data directly without parsing.
|
||||||
|
|
||||||
|
|
@ -374,5 +353,4 @@ void copyNAR(Source & source, Sink & sink)
|
||||||
parseDump(parseSink, wrapper);
|
parseDump(parseSink, wrapper);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
15
third_party/nix/src/libutil/archive.hh
vendored
15
third_party/nix/src/libutil/archive.hh
vendored
|
|
@ -1,12 +1,10 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "types.hh"
|
|
||||||
#include "serialise.hh"
|
#include "serialise.hh"
|
||||||
|
#include "types.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* dumpPath creates a Nix archive of the specified path. The format
|
/* dumpPath creates a Nix archive of the specified path. The format
|
||||||
is as follows:
|
is as follows:
|
||||||
|
|
||||||
|
|
@ -44,15 +42,13 @@ namespace nix {
|
||||||
|
|
||||||
`+' denotes string concatenation. */
|
`+' denotes string concatenation. */
|
||||||
|
|
||||||
|
|
||||||
void dumpPath(const Path& path, Sink& sink,
|
void dumpPath(const Path& path, Sink& sink,
|
||||||
PathFilter& filter = defaultPathFilter);
|
PathFilter& filter = defaultPathFilter);
|
||||||
|
|
||||||
void dumpString(const std::string& s, Sink& sink);
|
void dumpString(const std::string& s, Sink& sink);
|
||||||
|
|
||||||
/* FIXME: fix this API, it sucks. */
|
/* FIXME: fix this API, it sucks. */
|
||||||
struct ParseSink
|
struct ParseSink {
|
||||||
{
|
|
||||||
virtual void createDirectory(const Path& path){};
|
virtual void createDirectory(const Path& path){};
|
||||||
|
|
||||||
virtual void createRegularFile(const Path& path){};
|
virtual void createRegularFile(const Path& path){};
|
||||||
|
|
@ -63,8 +59,7 @@ struct ParseSink
|
||||||
virtual void createSymlink(const Path& path, const string& target){};
|
virtual void createSymlink(const Path& path, const string& target){};
|
||||||
};
|
};
|
||||||
|
|
||||||
struct TeeSink : ParseSink
|
struct TeeSink : ParseSink {
|
||||||
{
|
|
||||||
TeeSource source;
|
TeeSource source;
|
||||||
|
|
||||||
TeeSink(Source& source) : source(source) {}
|
TeeSink(Source& source) : source(source) {}
|
||||||
|
|
@ -77,8 +72,6 @@ void restorePath(const Path & path, Source & source);
|
||||||
/* Read a NAR from 'source' and write it to 'sink'. */
|
/* Read a NAR from 'source' and write it to 'sink'. */
|
||||||
void copyNAR(Source& source, Sink& sink);
|
void copyNAR(Source& source, Sink& sink);
|
||||||
|
|
||||||
|
|
||||||
extern const std::string narVersionMagic1;
|
extern const std::string narVersionMagic1;
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
}
|
|
||||||
|
|
|
||||||
81
third_party/nix/src/libutil/args.cc
vendored
81
third_party/nix/src/libutil/args.cc
vendored
|
|
@ -3,34 +3,30 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
Args::FlagMaker Args::mkFlag()
|
Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); }
|
||||||
{
|
|
||||||
return FlagMaker(*this);
|
|
||||||
}
|
|
||||||
|
|
||||||
Args::FlagMaker::~FlagMaker()
|
Args::FlagMaker::~FlagMaker() {
|
||||||
{
|
|
||||||
assert(flag->longName != "");
|
assert(flag->longName != "");
|
||||||
args.longFlags[flag->longName] = flag;
|
args.longFlags[flag->longName] = flag;
|
||||||
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
|
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Args::parseCmdline(const Strings & _cmdline)
|
void Args::parseCmdline(const Strings& _cmdline) {
|
||||||
{
|
|
||||||
Strings pendingArgs;
|
Strings pendingArgs;
|
||||||
bool dashDash = false;
|
bool dashDash = false;
|
||||||
|
|
||||||
Strings cmdline(_cmdline);
|
Strings cmdline(_cmdline);
|
||||||
|
|
||||||
for (auto pos = cmdline.begin(); pos != cmdline.end();) {
|
for (auto pos = cmdline.begin(); pos != cmdline.end();) {
|
||||||
|
|
||||||
auto arg = *pos;
|
auto arg = *pos;
|
||||||
|
|
||||||
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
|
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
|
||||||
`-j3` -> `-j 3`). */
|
`-j3` -> `-j 3`). */
|
||||||
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) {
|
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' &&
|
||||||
|
isalpha(arg[1])) {
|
||||||
*pos = (string) "-" + arg[1];
|
*pos = (string) "-" + arg[1];
|
||||||
auto next = pos; ++next;
|
auto next = pos;
|
||||||
|
++next;
|
||||||
for (unsigned int j = 2; j < arg.length(); j++)
|
for (unsigned int j = 2; j < arg.length(); j++)
|
||||||
if (isalpha(arg[j]))
|
if (isalpha(arg[j]))
|
||||||
cmdline.insert(next, (string) "-" + arg[j]);
|
cmdline.insert(next, (string) "-" + arg[j]);
|
||||||
|
|
@ -44,23 +40,19 @@ void Args::parseCmdline(const Strings & _cmdline)
|
||||||
if (!dashDash && arg == "--") {
|
if (!dashDash && arg == "--") {
|
||||||
dashDash = true;
|
dashDash = true;
|
||||||
++pos;
|
++pos;
|
||||||
}
|
} else if (!dashDash && std::string(arg, 0, 1) == "-") {
|
||||||
else if (!dashDash && std::string(arg, 0, 1) == "-") {
|
|
||||||
if (!processFlag(pos, cmdline.end()))
|
if (!processFlag(pos, cmdline.end()))
|
||||||
throw UsageError(format("unrecognised flag '%1%'") % arg);
|
throw UsageError(format("unrecognised flag '%1%'") % arg);
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
pendingArgs.push_back(*pos++);
|
pendingArgs.push_back(*pos++);
|
||||||
if (processArgs(pendingArgs, false))
|
if (processArgs(pendingArgs, false)) pendingArgs.clear();
|
||||||
pendingArgs.clear();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
processArgs(pendingArgs, true);
|
processArgs(pendingArgs, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Args::printHelp(const string & programName, std::ostream & out)
|
void Args::printHelp(const string& programName, std::ostream& out) {
|
||||||
{
|
|
||||||
std::cout << "Usage: " << programName << " <FLAGS>...";
|
std::cout << "Usage: " << programName << " <FLAGS>...";
|
||||||
for (auto& exp : expectedArgs) {
|
for (auto& exp : expectedArgs) {
|
||||||
std::cout << renderLabels({exp.label});
|
std::cout << renderLabels({exp.label});
|
||||||
|
|
@ -71,8 +63,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
|
||||||
std::cout << "\n";
|
std::cout << "\n";
|
||||||
|
|
||||||
auto s = description();
|
auto s = description();
|
||||||
if (s != "")
|
if (s != "") std::cout << "\nSummary: " << s << ".\n";
|
||||||
std::cout << "\nSummary: " << s << ".\n";
|
|
||||||
|
|
||||||
if (longFlags.size()) {
|
if (longFlags.size()) {
|
||||||
std::cout << "\n";
|
std::cout << "\n";
|
||||||
|
|
@ -81,21 +72,21 @@ void Args::printHelp(const string & programName, std::ostream & out)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Args::printFlags(std::ostream & out)
|
void Args::printFlags(std::ostream& out) {
|
||||||
{
|
|
||||||
Table2 table;
|
Table2 table;
|
||||||
for (auto& flag : longFlags) {
|
for (auto& flag : longFlags) {
|
||||||
if (hiddenCategories.count(flag.second->category)) continue;
|
if (hiddenCategories.count(flag.second->category)) continue;
|
||||||
table.push_back(std::make_pair(
|
table.push_back(std::make_pair(
|
||||||
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
|
(flag.second->shortName
|
||||||
+ "--" + flag.first + renderLabels(flag.second->labels),
|
? std::string("-") + flag.second->shortName + ", "
|
||||||
|
: " ") +
|
||||||
|
"--" + flag.first + renderLabels(flag.second->labels),
|
||||||
flag.second->description));
|
flag.second->description));
|
||||||
}
|
}
|
||||||
printTable(out, table);
|
printTable(out, table);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
|
bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) {
|
||||||
{
|
|
||||||
assert(pos != end);
|
assert(pos != end);
|
||||||
|
|
||||||
auto process = [&](const std::string& name, const Flag& flag) -> bool {
|
auto process = [&](const std::string& name, const Flag& flag) -> bool {
|
||||||
|
|
@ -104,8 +95,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
|
||||||
for (size_t n = 0; n < flag.arity; ++n) {
|
for (size_t n = 0; n < flag.arity; ++n) {
|
||||||
if (pos == end) {
|
if (pos == end) {
|
||||||
if (flag.arity == ArityAny) break;
|
if (flag.arity == ArityAny) break;
|
||||||
throw UsageError(format("flag '%1%' requires %2% argument(s)")
|
throw UsageError(format("flag '%1%' requires %2% argument(s)") % name %
|
||||||
% name % flag.arity);
|
flag.arity);
|
||||||
}
|
}
|
||||||
args.push_back(*pos++);
|
args.push_back(*pos++);
|
||||||
}
|
}
|
||||||
|
|
@ -129,8 +120,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Args::processArgs(const Strings & args, bool finish)
|
bool Args::processArgs(const Strings& args, bool finish) {
|
||||||
{
|
|
||||||
if (expectedArgs.empty()) {
|
if (expectedArgs.empty()) {
|
||||||
if (!args.empty())
|
if (!args.empty())
|
||||||
throw UsageError(format("unexpected argument '%1%'") % args.front());
|
throw UsageError(format("unexpected argument '%1%'") % args.front());
|
||||||
|
|
@ -142,8 +132,7 @@ bool Args::processArgs(const Strings & args, bool finish)
|
||||||
bool res = false;
|
bool res = false;
|
||||||
|
|
||||||
if ((exp.arity == 0 && finish) ||
|
if ((exp.arity == 0 && finish) ||
|
||||||
(exp.arity > 0 && args.size() == exp.arity))
|
(exp.arity > 0 && args.size() == exp.arity)) {
|
||||||
{
|
|
||||||
std::vector<std::string> ss;
|
std::vector<std::string> ss;
|
||||||
for (auto& s : args) ss.push_back(s);
|
for (auto& s : args) ss.push_back(s);
|
||||||
exp.handler(std::move(ss));
|
exp.handler(std::move(ss));
|
||||||
|
|
@ -157,29 +146,26 @@ bool Args::processArgs(const Strings & args, bool finish)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
|
Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) {
|
||||||
{
|
|
||||||
arity(1);
|
arity(1);
|
||||||
label("type");
|
label("type");
|
||||||
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
|
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
|
||||||
handler([ht](std::string s) {
|
handler([ht](std::string s) {
|
||||||
*ht = parseHashType(s);
|
*ht = parseHashType(s);
|
||||||
if (*ht == htUnknown)
|
if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s);
|
||||||
throw UsageError("unknown hash type '%1%'", s);
|
|
||||||
});
|
});
|
||||||
return *this;
|
return *this;
|
||||||
}
|
}
|
||||||
|
|
||||||
Strings argvToStrings(int argc, char * * argv)
|
Strings argvToStrings(int argc, char** argv) {
|
||||||
{
|
|
||||||
Strings args;
|
Strings args;
|
||||||
argc--; argv++;
|
argc--;
|
||||||
|
argv++;
|
||||||
while (argc--) args.push_back(*argv++);
|
while (argc--) args.push_back(*argv++);
|
||||||
return args;
|
return args;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string renderLabels(const Strings & labels)
|
std::string renderLabels(const Strings& labels) {
|
||||||
{
|
|
||||||
std::string res;
|
std::string res;
|
||||||
for (auto label : labels) {
|
for (auto label : labels) {
|
||||||
for (auto& c : label) c = std::toupper(c);
|
for (auto& c : label) c = std::toupper(c);
|
||||||
|
|
@ -188,16 +174,13 @@ std::string renderLabels(const Strings & labels)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void printTable(std::ostream & out, const Table2 & table)
|
void printTable(std::ostream& out, const Table2& table) {
|
||||||
{
|
|
||||||
size_t max = 0;
|
size_t max = 0;
|
||||||
for (auto & row : table)
|
for (auto& row : table) max = std::max(max, row.first.size());
|
||||||
max = std::max(max, row.first.size());
|
|
||||||
for (auto& row : table) {
|
for (auto& row : table) {
|
||||||
out << " " << row.first
|
out << " " << row.first << std::string(max - row.first.size() + 2, ' ')
|
||||||
<< std::string(max - row.first.size() + 2, ' ')
|
|
||||||
<< row.second << "\n";
|
<< row.second << "\n";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
123
third_party/nix/src/libutil/args.hh
vendored
123
third_party/nix/src/libutil/args.hh
vendored
|
|
@ -3,7 +3,6 @@
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
@ -12,10 +11,8 @@ MakeError(UsageError, Error);
|
||||||
|
|
||||||
enum HashType : char;
|
enum HashType : char;
|
||||||
|
|
||||||
class Args
|
class Args {
|
||||||
{
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/* Parse the command line, throwing a UsageError if something goes
|
/* Parse the command line, throwing a UsageError if something goes
|
||||||
wrong. */
|
wrong. */
|
||||||
void parseCmdline(const Strings& cmdline);
|
void parseCmdline(const Strings& cmdline);
|
||||||
|
|
@ -25,12 +22,10 @@ public:
|
||||||
virtual std::string description() { return ""; }
|
virtual std::string description() { return ""; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
static const size_t ArityAny = std::numeric_limits<size_t>::max();
|
static const size_t ArityAny = std::numeric_limits<size_t>::max();
|
||||||
|
|
||||||
/* Flags. */
|
/* Flags. */
|
||||||
struct Flag
|
struct Flag {
|
||||||
{
|
|
||||||
typedef std::shared_ptr<Flag> ptr;
|
typedef std::shared_ptr<Flag> ptr;
|
||||||
std::string longName;
|
std::string longName;
|
||||||
char shortName = 0;
|
char shortName = 0;
|
||||||
|
|
@ -49,8 +44,7 @@ protected:
|
||||||
virtual void printFlags(std::ostream& out);
|
virtual void printFlags(std::ostream& out);
|
||||||
|
|
||||||
/* Positional arguments. */
|
/* Positional arguments. */
|
||||||
struct ExpectedArg
|
struct ExpectedArg {
|
||||||
{
|
|
||||||
std::string label;
|
std::string label;
|
||||||
size_t arity; // 0 = any
|
size_t arity; // 0 = any
|
||||||
bool optional;
|
bool optional;
|
||||||
|
|
@ -64,41 +58,69 @@ protected:
|
||||||
std::set<std::string> hiddenCategories;
|
std::set<std::string> hiddenCategories;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
class FlagMaker {
|
||||||
class FlagMaker
|
|
||||||
{
|
|
||||||
Args& args;
|
Args& args;
|
||||||
Flag::ptr flag;
|
Flag::ptr flag;
|
||||||
friend class Args;
|
friend class Args;
|
||||||
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
|
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
|
||||||
|
|
||||||
public:
|
public:
|
||||||
~FlagMaker();
|
~FlagMaker();
|
||||||
FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
|
FlagMaker& longName(const std::string& s) {
|
||||||
FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
|
flag->longName = s;
|
||||||
FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
|
return *this;
|
||||||
FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
|
};
|
||||||
FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
|
FlagMaker& shortName(char s) {
|
||||||
FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
|
flag->shortName = s;
|
||||||
FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
|
return *this;
|
||||||
FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
|
};
|
||||||
FlagMaker & handler(std::function<void(std::string)> handler) {
|
FlagMaker& description(const std::string& s) {
|
||||||
flag->arity = 1;
|
flag->description = s;
|
||||||
flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& label(const std::string& l) {
|
||||||
|
flag->arity = 1;
|
||||||
|
flag->labels = {l};
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& labels(const Strings& ls) {
|
||||||
|
flag->arity = ls.size();
|
||||||
|
flag->labels = ls;
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& arity(size_t arity) {
|
||||||
|
flag->arity = arity;
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) {
|
||||||
|
flag->handler = handler;
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& handler(std::function<void()> handler) {
|
||||||
|
flag->handler = [handler](std::vector<std::string>) { handler(); };
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& handler(std::function<void(std::string)> handler) {
|
||||||
|
flag->arity = 1;
|
||||||
|
flag->handler = [handler](std::vector<std::string> ss) {
|
||||||
|
handler(std::move(ss[0]));
|
||||||
|
};
|
||||||
|
return *this;
|
||||||
|
};
|
||||||
|
FlagMaker& category(const std::string& s) {
|
||||||
|
flag->category = s;
|
||||||
return *this;
|
return *this;
|
||||||
};
|
};
|
||||||
FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
|
|
||||||
|
|
||||||
template <class T>
|
template <class T>
|
||||||
FlagMaker & dest(T * dest)
|
FlagMaker& dest(T* dest) {
|
||||||
{
|
|
||||||
flag->arity = 1;
|
flag->arity = 1;
|
||||||
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
|
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
|
||||||
return *this;
|
return *this;
|
||||||
};
|
};
|
||||||
|
|
||||||
template <class T>
|
template <class T>
|
||||||
FlagMaker & set(T * dest, const T & val)
|
FlagMaker& set(T* dest, const T& val) {
|
||||||
{
|
|
||||||
flag->arity = 0;
|
flag->arity = 0;
|
||||||
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
|
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
|
||||||
return *this;
|
return *this;
|
||||||
|
|
@ -114,8 +136,7 @@ public:
|
||||||
|
|
||||||
void mkFlag1(char shortName, const std::string& longName,
|
void mkFlag1(char shortName, const std::string& longName,
|
||||||
const std::string& label, const std::string& description,
|
const std::string& label, const std::string& description,
|
||||||
std::function<void(std::string)> fun)
|
std::function<void(std::string)> fun) {
|
||||||
{
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.shortName(shortName)
|
.shortName(shortName)
|
||||||
.longName(longName)
|
.longName(longName)
|
||||||
|
|
@ -126,15 +147,13 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
void mkFlag(char shortName, const std::string& name,
|
void mkFlag(char shortName, const std::string& name,
|
||||||
const std::string & description, bool * dest)
|
const std::string& description, bool* dest) {
|
||||||
{
|
|
||||||
mkFlag(shortName, name, description, dest, true);
|
mkFlag(shortName, name, description, dest, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class T>
|
template <class T>
|
||||||
void mkFlag(char shortName, const std::string & longName, const std::string & description,
|
void mkFlag(char shortName, const std::string& longName,
|
||||||
T * dest, const T & value)
|
const std::string& description, T* dest, const T& value) {
|
||||||
{
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.shortName(shortName)
|
.shortName(shortName)
|
||||||
.longName(longName)
|
.longName(longName)
|
||||||
|
|
@ -144,17 +163,13 @@ public:
|
||||||
|
|
||||||
template <class I>
|
template <class I>
|
||||||
void mkIntFlag(char shortName, const std::string& longName,
|
void mkIntFlag(char shortName, const std::string& longName,
|
||||||
const std::string & description, I * dest)
|
const std::string& description, I* dest) {
|
||||||
{
|
mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; });
|
||||||
mkFlag<I>(shortName, longName, description, [=](I n) {
|
|
||||||
*dest = n;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class I>
|
template <class I>
|
||||||
void mkFlag(char shortName, const std::string& longName,
|
void mkFlag(char shortName, const std::string& longName,
|
||||||
const std::string & description, std::function<void(I)> fun)
|
const std::string& description, std::function<void(I)> fun) {
|
||||||
{
|
|
||||||
mkFlag()
|
mkFlag()
|
||||||
.shortName(shortName)
|
.shortName(shortName)
|
||||||
.longName(longName)
|
.longName(longName)
|
||||||
|
|
@ -164,25 +179,25 @@ public:
|
||||||
.handler([=](std::vector<std::string> ss) {
|
.handler([=](std::vector<std::string> ss) {
|
||||||
I n;
|
I n;
|
||||||
if (!string2Int(ss[0], n))
|
if (!string2Int(ss[0], n))
|
||||||
throw UsageError("flag '--%s' requires a integer argument", longName);
|
throw UsageError("flag '--%s' requires a integer argument",
|
||||||
|
longName);
|
||||||
fun(n);
|
fun(n);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Expect a string argument. */
|
/* Expect a string argument. */
|
||||||
void expectArg(const std::string & label, string * dest, bool optional = false)
|
void expectArg(const std::string& label, string* dest,
|
||||||
{
|
bool optional = false) {
|
||||||
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
|
expectedArgs.push_back(
|
||||||
*dest = ss[0];
|
ExpectedArg{label, 1, optional,
|
||||||
}});
|
[=](std::vector<std::string> ss) { *dest = ss[0]; }});
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Expect 0 or more arguments. */
|
/* Expect 0 or more arguments. */
|
||||||
void expectArgs(const std::string & label, std::vector<std::string> * dest)
|
void expectArgs(const std::string& label, std::vector<std::string>* dest) {
|
||||||
{
|
expectedArgs.push_back(ExpectedArg{
|
||||||
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
|
label, 0, false,
|
||||||
*dest = std::move(ss);
|
[=](std::vector<std::string> ss) { *dest = std::move(ss); }});
|
||||||
}});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
friend class MultiCommand;
|
friend class MultiCommand;
|
||||||
|
|
@ -198,4 +213,4 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
|
||||||
|
|
||||||
void printTable(std::ostream& out, const Table2& table);
|
void printTable(std::ostream& out, const Table2& table);
|
||||||
|
|
||||||
}
|
} // namespace nix
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue