maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into no-manifests

Browse source
This commit is contained in:
Eelco Dolstra 2012-07-26 15:14:33 -04:00
commit 8c79100839
50 changed files with 343 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/libstore/build.cc
View file

@ -1533,7 +1533,7 @@ void DerivationGoal::startBuilder()
/* Create a temporary directory where the build will take
place. */
tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700);
/* For convenience, set an environment pointing to the top build
directory. */
@ -2099,6 +2099,8 @@ void DerivationGoal::computeClosure()
if (allowed.find(*i) == allowed.end())
throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
}
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
}
/* Register each output path as valid, and register the sets of
@ -2182,6 +2184,7 @@ void DerivationGoal::deleteTmpDir(bool force)
% drvPath % tmpDir);
if (buildUser.enabled() && !amPrivileged())
getOwnership(tmpDir);
chmod(tmpDir.c_str(), 0755);
}
else
deletePathWrapped(tmpDir);
@ -2562,6 +2565,8 @@ void SubstitutionGoal::finished()
HashResult hash = hashPath(htSHA256, storePath);
worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
ValidPathInfo info2;
info2.path = storePath;
info2.hash = hash.first;

6
src/libstore/derivations.hh
View file

@ -1,5 +1,4 @@
#ifndef __DERIVATIONS_H
#define __DERIVATIONS_H
#pragma once
#include <map>
@ -81,6 +80,3 @@ typedef std::map<Path, Hash> DrvHashes;
extern DrvHashes drvHashes;
}
#endif /* !__DERIVATIONS_H */

37
src/libstore/gc.cc
View file

@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
{
checkInterrupt();
if (path == linksDir) return true;
struct stat st;
if (lstat(path.c_str(), &st)) {
if (errno == ENOENT) return true;
@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
}
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
void LocalStore::removeUnusedLinks()
{
AutoCloseDir dir = opendir(linksDir.c_str());
if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir)) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
struct stat st;
if (lstat(path.c_str(), &st) == -1)
throw SysError(format("statting `%1%'") % path);
if (st.st_nlink != 1) continue;
printMsg(lvlTalkative, format("deleting unused link `%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError(format("deleting `%1%'") % path);
}
}
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
GCState state(results);
@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
released. */
foreach (PathSet::iterator, i, state.invalidated)
deleteGarbage(state, *i);
/* Clean up the links directory. */
printMsg(lvlError, format("deleting unused links..."));
removeUnusedLinks();
}

6
src/libstore/globals.hh
View file

@ -1,5 +1,4 @@
#ifndef __GLOBALS_H
#define __GLOBALS_H
#pragma once
#include "types.hh"
@ -118,6 +117,3 @@ void setDefaultsFromEnvironment();
}
#endif /* !__GLOBALS_H */

32
src/libstore/local-store.cc
View file

@ -12,6 +12,7 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h>
#include <utime.h>
#include <fcntl.h>
@ -208,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace)
/* Create missing state directories if they don't already exist. */
createDirs(nixStore);
createDirs(linksDir = nixStore + "/.links");
Path profilesDir = nixStateDir + "/profiles";
createDirs(nixStateDir + "/profiles");
createDirs(nixStateDir + "/temproots");
@ -444,7 +446,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
throw SysError(format("changing owner of `%1%' to %2%")
% path % geteuid());
}
if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */
@ -458,14 +460,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
}
if (st.st_mtime != mtimeStore) {
struct utimbuf utimbuf;
utimbuf.actime = st.st_atime;
utimbuf.modtime = mtimeStore;
if (utime(path.c_str(), &utimbuf) == -1)
throw SysError(format("changing modification time of `%1%'") % path);
}
}
if (st.st_mtime != mtimeStore) {
struct timeval times[2];
times[0].tv_sec = st.st_atime;
times[0].tv_usec = 0;
times[1].tv_sec = mtimeStore;
times[1].tv_usec = 0;
#if HAVE_LUTIMES
if (lutimes(path.c_str(), times) == -1)
#else
if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
#endif
throw SysError(format("changing modification time of `%1%'") % path);
}
if (recurse && S_ISDIR(st.st_mode)) {
@ -1134,6 +1142,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
hash.second = dump.size();
} else
hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath); // FIXME: combine with hashPath()
ValidPathInfo info;
info.path = dstPath;
@ -1188,6 +1198,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
canonicalisePathMetaData(dstPath);
HashResult hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath);
ValidPathInfo info;
info.path = dstPath;
@ -1423,6 +1435,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
/* !!! if we were clever, we could prevent the hashPath()
here. */
HashResult hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath); // FIXME: combine with hashPath()
ValidPathInfo info;
info.path = dstPath;

17
src/libstore/local-store.hh
View file

@ -1,5 +1,4 @@
#ifndef __LOCAL_STORE_H
#define __LOCAL_STORE_H
#pragma once
#include <string>
@ -86,6 +85,8 @@ private:
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
RunningSubstituters runningSubstituters;
Path linksDir;
public:
@ -168,8 +169,11 @@ public:
/* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */
void optimiseStore(bool dryRun, OptimiseStats & stats);
void optimiseStore(OptimiseStats & stats);
/* Optimise a single store path. */
void optimisePath(const Path & path);
/* Check the integrity of the Nix store. */
void verifyStore(bool checkContents);
@ -260,6 +264,8 @@ private:
int openGCLock(LockType lockType);
void removeUnusedLinks();
void startSubstituter(const Path & substituter,
RunningSubstituter & runningSubstituter);
@ -268,6 +274,8 @@ private:
Path importPath(bool requireSignature, Source & source);
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
void optimisePath_(OptimiseStats & stats, const Path & path);
};
@ -302,6 +310,3 @@ void deletePathWrapped(const Path & path,
void deletePathWrapped(const Path & path);
}
#endif /* !__LOCAL_STORE_H */

6
src/libstore/misc.hh
View file

@ -1,5 +1,4 @@
#ifndef __MISC_H
#define __MISC_H
#pragma once
#include "derivations.hh"
@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
}
#endif /* !__MISC_H */

222
src/libstore/optimise-store.cc
View file

@ -1,6 +1,7 @@
#include "util.hh"
#include "local-store.hh"
#include "immutable.hh"
#include "globals.hh"
#include <sys/types.h>
#include <sys/stat.h>
@ -12,9 +13,6 @@
namespace nix {
typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
static void makeWritable(const Path & path)
{
struct stat st;
@ -51,140 +49,152 @@ struct MakeImmutable
};
static void hashAndLink(bool dryRun, HashToPath & hashToPath,
OptimiseStats & stats, const Path & path)
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
foreach (Strings::iterator, i, names)
optimisePath_(stats, path + "/" + *i);
return;
}
/* We can hard link regular files and maybe symlinks. */
if (!S_ISREG(st.st_mode)
#if CAN_LINK_SYMLINK
x
&& !S_ISLNK(st.st_mode)
#endif
) return;
/* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under
NixOS (example: $fontconfig/var/cache being modified). Skip
those files. */
those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
return;
}
/* We can hard link regular files and symlinks. */
if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
/* Hash the file. Note that hashPath() returns the hash over the
NAR serialisation, which includes the execute bit on the file.
Thus, executable and non-executable files with the same
contents *won't* be linked (which is good because otherwise the
permissions would be screwed up).
/* Hash the file. Note that hashPath() returns the hash over
the NAR serialisation, which includes the execute bit on
the file. Thus, executable and non-executable files with
the same contents *won't* be linked (which is good because
otherwise the permissions would be screwed up).
Also note that if `path' is a symlink, then we're hashing the
contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first;
stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
Also note that if `path' is a symlink, then we're hashing
the contents of the symlink (i.e. the result of
readlink()), not the contents of the target (which may not
even exist). */
Hash hash = hashPath(htSHA256, path).first;
stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + printHash32(hash);
std::pair<Path, ino_t> prevPath = hashToPath[hash];
if (prevPath.first == "") {
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
return;
}
/* Yes! We've seen a file with the same contents. Replace
the current file with a hard link to that file. */
stats.sameContents++;
if (prevPath.second == st.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
return;
}
if (!dryRun) {
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
makeMutable(path);
MakeImmutable mk1(path);
Path tempLink = (format("%1%.tmp-%2%-%3%")
% path % getpid() % rand()).str();
if (link(path.c_str(), linkPath.c_str()) == -1)
throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
/* Make the containing directory writable, but only if
it's not the store itself (we don't want or need to
mess with its permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and
reset its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
/* If ‘prevPath’ is immutable, we can't create hard links
to it, so make it mutable first (and make it immutable
again when we're done). We also have to make ‘path’
mutable, otherwise rename() will fail to delete it. */
makeMutable(prevPath.first);
MakeImmutable mk1(prevPath.first);
makeMutable(path);
MakeImmutable mk2(path);
if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on
most file systems). This is likely to happen
with empty files. Just start over, creating
links to the current file. */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
return;
}
throw SysError(format("cannot link `%1%' to `%2%'")
% tempLink % prevPath.first);
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (errno == EMLINK) {
/* Some filesystems generate too many links on the
rename, rather than on the original link.
(Probably it temporarily increases the st_nlink
field before decreasing it again.) */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
/* Unlink the temp link. */
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
return;
}
throw SysError(format("cannot rename `%1%' to `%2%'")
% tempLink % path);
}
} else
printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
stats.filesLinked++;
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
return;
}
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
foreach (Strings::iterator, i, names)
hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
/* Yes! We've seen a file with the same contents. Replace the
current file with a hard link to that file. */
struct stat stLink;
if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path `%1%'") % linkPath);
stats.sameContents++;
if (st.st_ino == stLink.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
return;
}
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% nixStore % getpid() % rand()).str();
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
/* If ‘linkPath’ is immutable, we can't create hard links to it,
so make it mutable first (and make it immutable again when
we're done). We also have to make ‘path’ mutable, otherwise
rename() will fail to delete it. */
makeMutable(linkPath);
MakeImmutable mk1(linkPath);
makeMutable(path);
MakeImmutable mk2(path);
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files.
Just shrug and ignore. */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
temporarily increases the st_nlink field before
decreasing it again.) */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
/* Unlink the temp link. */
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
return;
}
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
}
stats.filesLinked++;
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
}
void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
void LocalStore::optimiseStore(OptimiseStats & stats)
{
HashToPath hashToPath;
PathSet paths = queryAllValidPaths();
foreach (PathSet::iterator, i, paths) {
addTempRoot(*i);
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
hashAndLink(dryRun, hashToPath, stats, *i);
optimisePath_(stats, *i);
}
}
void LocalStore::optimisePath(const Path & path)
{
if (queryBoolSetting("auto-optimise-store", true)) {
OptimiseStats stats;
optimisePath_(stats, path);
}
}

6
src/libstore/pathlocks.hh
View file

@ -1,5 +1,4 @@
#ifndef __PATHLOCKS_H
#define __PATHLOCKS_H
#pragma once
#include "types.hh"
@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path);
}
#endif /* !__PATHLOCKS_H */

5
src/libstore/references.hh
View file

@ -1,5 +1,4 @@
#ifndef __REFERENCES_H
#define __REFERENCES_H
#pragma once
#include "types.hh"
#include "hash.hh"
@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs,
HashResult & hash);
}
#endif /* !__REFERENCES_H */

6
src/libstore/remote-store.hh
View file

@ -1,5 +1,4 @@
#ifndef __REMOTE_STORE_H
#define __REMOTE_STORE_H
#pragma once
#include <string>
@ -103,6 +102,3 @@ private:
}
#endif /* !__REMOTE_STORE_H */

6
src/libstore/store-api.hh
View file

@ -1,5 +1,4 @@
#ifndef __STOREAPI_H
#define __STOREAPI_H
#pragma once
#include "hash.hh"
#include "serialise.hh"
@ -362,6 +361,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
}
#endif /* !__STOREAPI_H */

7
src/libstore/worker-protocol.hh
View file

@ -1,6 +1,4 @@
#ifndef __WORKER_PROTOCOL_H
#define __WORKER_PROTOCOL_H
#pragma once
namespace nix {
@ -67,6 +65,3 @@ template<class T> T readStorePaths(Source & from);
}
#endif /* !__WORKER_PROTOCOL_H */