* Sync with the trunk.

This commit is contained in:
Eelco Dolstra 2011-12-16 23:33:01 +00:00
commit 194d21f9f6
45 changed files with 928 additions and 682 deletions

View file

@ -15,7 +15,16 @@ libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_
EXTRA_DIST = schema.sql
AM_CXXFLAGS = -Wall \
${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil
${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil \
-DNIX_STORE_DIR=\"$(storedir)\" \
-DNIX_DATA_DIR=\"$(datadir)\" \
-DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
-DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
-DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
-DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
-DNIX_BIN_DIR=\"$(bindir)\" \
-I$(srcdir)/.. -I$(srcdir)/../libutil \
-I$(srcdir)/../libstore
local-store.lo: schema.sql.hh

View file

@ -1650,6 +1650,9 @@ void DerivationGoal::startBuilder()
(format("nixbld:!:%1%:\n")
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
/* Create /etc/hosts with localhost entry. */
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
/* Bind-mount a user-configurable set of directories from the
host file system. The `/dev/pts' directory must be mounted
separately so that newly-created pseudo-terminals show
@ -2199,9 +2202,7 @@ void SubstitutionGoal::tryNext()
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
printMsg(lvlError,
format("path `%1%' is required, but there is no substituter that can build it")
% storePath);
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
amDone(ecFailed);
return;
}
@ -2232,8 +2233,7 @@ void SubstitutionGoal::referencesValid()
trace("all references realised");
if (nrFailed > 0) {
printMsg(lvlError,
format("some references of path `%1%' could not be realised") % storePath);
debug(format("some references of path `%1%' could not be realised") % storePath);
amDone(ecFailed);
return;
}
@ -2286,9 +2286,7 @@ void SubstitutionGoal::tryToRun()
return;
}
printMsg(lvlInfo,
format("substituting path `%1%' using substituter `%2%'")
% storePath % sub);
printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
logPipe.create();
@ -2364,19 +2362,15 @@ void SubstitutionGoal::finished()
try {
if (!statusOk(status))
throw SubstError(format("builder for `%1%' %2%")
throw SubstError(format("fetching path `%1%' %2%")
% storePath % statusToString(status));
if (!pathExists(storePath))
throw SubstError(
format("substitute did not produce path `%1%'")
% storePath);
throw SubstError(format("substitute did not produce path `%1%'") % storePath);
} catch (SubstError & e) {
printMsg(lvlInfo,
format("substitution of path `%1%' using substituter `%2%' failed: %3%")
% storePath % sub % e.msg());
printMsg(lvlInfo, e.msg());
if (printBuildTrace) {
printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")

View file

@ -1,3 +1,5 @@
#include "config.h"
#include "globals.hh"
#include "util.hh"
@ -138,5 +140,33 @@ void reloadSettings()
settings.clear();
}
void setDefaultsFromEnvironment()
{
/* Setup Nix paths. */
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
} else
substituters = tokenizeString(subs, ":");
/* Get some settings from the configuration file. */
thisSystem = querySetting("system", SYSTEM);
maxBuildJobs = queryIntSetting("build-max-jobs", 1);
buildCores = queryIntSetting("build-cores", 1);
maxSilentTime = queryIntSetting("build-max-silent-time", 0);
buildTimeout = queryIntSetting("build-timeout", 0);
}
}

View file

@ -114,7 +114,9 @@ void overrideSetting(const string & name, const Strings & value);
void reloadSettings();
void setDefaultsFromEnvironment();
}

View file

@ -327,10 +327,9 @@ void LocalStore::openDB(bool create)
if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "setting synchronous mode");
/* Set the SQLite journal mode. WAL mode is fastest, but doesn't
seem entirely stable at the moment (Oct. 2010). Thus, use
truncate mode by default. */
string mode = queryBoolSetting("use-sqlite-wal", false) ? "wal" : "truncate";
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
default. */
string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate";
string prevMode;
{
SQLiteStmt stmt;
@ -367,7 +366,7 @@ void LocalStore::openDB(bool create)
stmtRegisterValidPath.create(db,
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);");
stmtUpdatePathInfo.create(db,
"update ValidPaths set narSize = ? where path = ?;");
"update ValidPaths set narSize = ?, hash = ? where path = ?;");
stmtAddReference.create(db,
"insert or replace into Refs (referrer, reference) values (?, ?);");
stmtQueryPathInfo.create(db,
@ -684,7 +683,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
}
/* Update path info in the database. Currently only updated the
/* Update path info in the database. Currently only updates the
narSize field. */
void LocalStore::updatePathInfo(const ValidPathInfo & info)
{
@ -693,6 +692,7 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info)
stmtUpdatePathInfo.bind64(info.narSize);
else
stmtUpdatePathInfo.bind(); // null
stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash));
stmtUpdatePathInfo.bind(info.path);
if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE)
throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path);
@ -1125,16 +1125,14 @@ struct HashAndWriteSink : Sink
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator ()
(const unsigned char * data, unsigned int len)
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
HashSink hashSinkClone(hashSink);
return hashSinkClone.finish().first;
return hashSink.currentHash().first;
}
};
@ -1180,7 +1178,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
PathSet references;
queryReferences(path, references);
writeStringSet(references, hashAndWriteSink);
writeStrings(references, hashAndWriteSink);
Path deriver = queryDeriver(path);
writeString(deriver, hashAndWriteSink);
@ -1223,11 +1221,11 @@ struct HashAndReadSource : Source
{
hashing = true;
}
virtual void operator ()
(unsigned char * data, unsigned int len)
size_t read(unsigned char * data, size_t len)
{
readSource(data, len);
if (hashing) hashSink(data, len);
size_t n = readSource.read(data, len);
if (hashing) hashSink(data, n);
return n;
}
};
@ -1267,7 +1265,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
Path dstPath = readStorePath(hashAndReadSource);
PathSet references = readStorePaths(hashAndReadSource);
PathSet references = readStorePaths<PathSet>(hashAndReadSource);
Path deriver = readString(hashAndReadSource);
if (deriver != "") assertStorePath(deriver);
@ -1278,7 +1276,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
bool haveSignature = readInt(hashAndReadSource) == 1;
if (requireSignature && !haveSignature)
throw Error("imported archive lacks a signature");
throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
if (haveSignature) {
string signature = readString(hashAndReadSource);
@ -1354,6 +1352,19 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
}
Paths LocalStore::importPaths(bool requireSignature, Source & source)
{
Paths res;
while (true) {
unsigned long long n = readLongLong(source);
if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
res.push_back(importPath(requireSignature, source));
}
return res;
}
void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed,
unsigned long long & blocksFreed)
{
@ -1369,7 +1380,7 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
PathSet referrers; queryReferrers(path, referrers);
referrers.erase(path); /* ignore self-references */
if (!referrers.empty())
throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'")
throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%")
% path % showPaths(referrers));
invalidatePath(path);
}
@ -1409,6 +1420,8 @@ void LocalStore::verifyStore(bool checkContents)
if (checkContents) {
printMsg(lvlInfo, "checking hashes...");
Hash nullHash(htSHA256);
foreach (PathSet::iterator, i, validPaths) {
try {
ValidPathInfo info = queryPathInfo(*i);
@ -1417,17 +1430,30 @@ void LocalStore::verifyStore(bool checkContents)
printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
HashResult current = hashPath(info.hash.type, *i);
if (current.first != info.hash) {
if (info.hash != nullHash && info.hash != current.first) {
printMsg(lvlError, format("path `%1%' was modified! "
"expected hash `%2%', got `%3%'")
% *i % printHash(info.hash) % printHash(current.first));
} else {
bool update = false;
/* Fill in missing hashes. */
if (info.hash == nullHash) {
printMsg(lvlError, format("fixing missing hash on `%1%'") % *i);
info.hash = current.first;
update = true;
}
/* Fill in missing narSize fields (from old stores). */
if (info.narSize == 0) {
printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second);
info.narSize = current.second;
updatePathInfo(info);
update = true;
}
if (update) updatePathInfo(info);
}
} catch (Error & e) {

View file

@ -148,7 +148,7 @@ public:
void exportPath(const Path & path, bool sign,
Sink & sink);
Path importPath(bool requireSignature, Source & source);
Paths importPaths(bool requireSignature, Source & source);
void buildDerivations(const PathSet & drvPaths);
@ -261,6 +261,8 @@ private:
Path createTempDirInStore();
Path importPath(bool requireSignature, Source & source);
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
};

View file

@ -57,11 +57,11 @@ struct RefScanSink : Sink
RefScanSink() : hashSink(htSHA256) { }
void operator () (const unsigned char * data, unsigned int len);
void operator () (const unsigned char * data, size_t len);
};
void RefScanSink::operator () (const unsigned char * data, unsigned int len)
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
hashSink(data, len);

View file

@ -27,13 +27,15 @@ Path readStorePath(Source & from)
}
PathSet readStorePaths(Source & from)
template<class T> T readStorePaths(Source & from)
{
PathSet paths = readStringSet(from);
foreach (PathSet::iterator, i, paths) assertStorePath(*i);
T paths = readStrings<T>(from);
foreach (typename T::iterator, i, paths) assertStorePath(*i);
return paths;
}
template PathSet readStorePaths(Source & from);
RemoteStore::RemoteStore()
{
@ -65,6 +67,7 @@ void RemoteStore::openConnection()
/* Send the magic greeting, check for the reply. */
try {
writeInt(WORKER_MAGIC_1, to);
to.flush();
unsigned int magic = readInt(from);
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
@ -166,6 +169,7 @@ void RemoteStore::connectToDaemon()
RemoteStore::~RemoteStore()
{
try {
to.flush();
fdSocket.close();
if (child != -1)
child.wait(true);
@ -213,7 +217,7 @@ PathSet RemoteStore::queryValidPaths()
openConnection();
writeInt(wopQueryValidPaths, to);
processStderr();
return readStorePaths(from);
return readStorePaths<PathSet>(from);
}
@ -240,7 +244,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path,
if (reply == 0) return false;
info.deriver = readString(from);
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths(from);
info.references = readStorePaths<PathSet>(from);
info.downloadSize = readLongLong(from);
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
return true;
@ -258,7 +262,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
info.deriver = readString(from);
if (info.deriver != "") assertStorePath(info.deriver);
info.hash = parseHash(htSHA256, readString(from));
info.references = readStorePaths(from);
info.references = readStorePaths<PathSet>(from);
info.registrationTime = readInt(from);
info.narSize = readLongLong(from);
return info;
@ -283,7 +287,7 @@ void RemoteStore::queryReferences(const Path & path,
writeInt(wopQueryReferences, to);
writeString(path, to);
processStderr();
PathSet references2 = readStorePaths(from);
PathSet references2 = readStorePaths<PathSet>(from);
references.insert(references2.begin(), references2.end());
}
@ -295,7 +299,7 @@ void RemoteStore::queryReferrers(const Path & path,
writeInt(wopQueryReferrers, to);
writeString(path, to);
processStderr();
PathSet referrers2 = readStorePaths(from);
PathSet referrers2 = readStorePaths<PathSet>(from);
referrers.insert(referrers2.begin(), referrers2.end());
}
@ -318,7 +322,7 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path)
writeInt(wopQueryDerivationOutputs, to);
writeString(path, to);
processStderr();
return readStorePaths(from);
return readStorePaths<PathSet>(from);
}
@ -338,7 +342,7 @@ Path RemoteStore::addToStore(const Path & _srcPath,
openConnection();
Path srcPath(absPath(_srcPath));
writeInt(wopAddToStore, to);
writeString(baseNameOf(srcPath), to);
/* backwards compatibility hack */
@ -358,7 +362,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
writeInt(wopAddTextToStore, to);
writeString(name, to);
writeString(s, to);
writeStringSet(references, to);
writeStrings(references, to);
processStderr();
return readStorePath(from);
@ -377,14 +381,14 @@ void RemoteStore::exportPath(const Path & path, bool sign,
}
Path RemoteStore::importPath(bool requireSignature, Source & source)
Paths RemoteStore::importPaths(bool requireSignature, Source & source)
{
openConnection();
writeInt(wopImportPath, to);
writeInt(wopImportPaths, to);
/* We ignore requireSignature, since the worker forces it to true
anyway. */
anyway. */
processStderr(0, &source);
return readStorePath(from);
return readStorePaths<Paths>(from);
}
@ -392,7 +396,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths)
{
openConnection();
writeInt(wopBuildDerivations, to);
writeStringSet(drvPaths, to);
writeStrings(drvPaths, to);
processStderr();
readInt(from);
}
@ -459,7 +463,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
writeInt(wopCollectGarbage, to);
writeInt(options.action, to);
writeStringSet(options.pathsToDelete, to);
writeStrings(options.pathsToDelete, to);
writeInt(options.ignoreLiveness, to);
writeLongLong(options.maxFreed, to);
writeInt(options.maxLinks, to);
@ -471,7 +475,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
processStderr();
results.paths = readStringSet(from);
results.paths = readStrings<PathSet>(from);
results.bytesFreed = readLongLong(from);
results.blocksFreed = readLongLong(from);
}
@ -482,7 +486,7 @@ PathSet RemoteStore::queryFailedPaths()
openConnection();
writeInt(wopQueryFailedPaths, to);
processStderr();
return readStorePaths(from);
return readStorePaths<PathSet>(from);
}
@ -490,7 +494,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths)
{
openConnection();
writeInt(wopClearFailedPaths, to);
writeStringSet(paths, to);
writeStrings(paths, to);
processStderr();
readInt(from);
}
@ -498,6 +502,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths)
void RemoteStore::processStderr(Sink * sink, Source * source)
{
to.flush();
unsigned int msg;
while ((msg = readInt(from)) == STDERR_NEXT
|| msg == STDERR_READ || msg == STDERR_WRITE) {
@ -508,11 +513,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source)
}
else if (msg == STDERR_READ) {
if (!source) throw Error("no source");
unsigned int len = readInt(from);
size_t len = readInt(from);
unsigned char * buf = new unsigned char[len];
AutoDeleteArray<unsigned char> d(buf);
(*source)(buf, len);
writeString(string((const char *) buf, len), to);
writeString(buf, source->read(buf, len), to);
to.flush();
}
else {
string s = readString(from);

View file

@ -58,7 +58,7 @@ public:
void exportPath(const Path & path, bool sign,
Sink & sink);
Path importPath(bool requireSignature, Source & source);
Paths importPaths(bool requireSignature, Source & source);
void buildDerivations(const PathSet & drvPaths);

View file

@ -298,6 +298,17 @@ string showPaths(const PathSet & paths)
}
void exportPaths(StoreAPI & store, const Paths & paths,
bool sign, Sink & sink)
{
foreach (Paths::const_iterator, i, paths) {
writeInt(1, sink);
store.exportPath(*i, sign, sink);
}
writeInt(0, sink);
}
}

View file

@ -154,9 +154,7 @@ public:
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
If `fixed' is true, then the output of a fixed-output
derivation is pre-loaded into the Nix store. The function
object `filter' can be used to exclude files (see
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
@ -174,9 +172,9 @@ public:
virtual void exportPath(const Path & path, bool sign,
Sink & sink) = 0;
/* Import a NAR dump created by exportPath() into the Nix
store. */
virtual Path importPath(bool requireSignature, Source & source) = 0;
/* Import a sequence of NAR dumps created by exportPaths() into
the Nix store. */
virtual Paths importPaths(bool requireSignature, Source & source) = 0;
/* Ensure that the output paths of the derivation are valid. If
they are already valid, this is a no-op. Otherwise, validity
@ -345,6 +343,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
bool hashGiven = false);
/* Export multiple paths in the format expected by nix-store
--import. */
void exportPaths(StoreAPI & store, const Paths & paths,
bool sign, Sink & sink);
}

View file

@ -8,7 +8,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x108
#define PROTOCOL_VERSION 0x109
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -29,7 +29,6 @@ typedef enum {
wopSyncWithGC = 13,
wopFindRoots = 14,
wopExportPath = 16,
wopImportPath = 17,
wopQueryDeriver = 18,
wopSetOptions = 19,
wopCollectGarbage = 20,
@ -39,7 +38,8 @@ typedef enum {
wopQueryFailedPaths = 24,
wopClearFailedPaths = 25,
wopQueryPathInfo = 26,
wopQueryDerivationOutputNames = 27,
wopImportPaths = 27,
wopQueryDerivationOutputNames = 28,
} WorkerOp;
@ -59,7 +59,7 @@ typedef enum {
Path readStorePath(Source & from);
PathSet readStorePaths(Source & from);
template<class T> T readStorePaths(Source & from);
}