maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

style(3p/nix): Reformat project in Google C++ style

Browse source 
Reformatted with:

    fd . -e hh -e cc | xargs clang-format -i
This commit is contained in:
Vincent Ambo 2020-05-17 16:31:57 +01:00
parent 65a1aae98c
commit 0f2cf531f7
175 changed files with 32126 additions and 34689 deletions
Download patch file Download diff file Expand all files Collapse all files

48
third_party/nix/src/libutil/affinity.cc vendored
View file

@ -1,6 +1,6 @@
#include "affinity.hh"
#include "types.hh"
#include "util.hh"
#include "affinity.hh"
#if __linux__
#include <sched.h>
@ -8,48 +8,40 @@
namespace nix {
#if __linux__
static bool didSaveAffinity = false;
static cpu_set_t savedAffinity;
#endif
void setAffinityTo(int cpu)
{
void setAffinityTo(int cpu) {
#if __linux__
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true;
debug(format("locking this thread to CPU %1%") % cpu);
cpu_set_t newAffinity;
CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printError(format("failed to lock thread to CPU %1%") % cpu);
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true;
debug(format("locking this thread to CPU %1%") % cpu);
cpu_set_t newAffinity;
CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printError(format("failed to lock thread to CPU %1%") % cpu);
#endif
}
int lockToCurrentCPU()
{
int lockToCurrentCPU() {
#if __linux__
int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu);
return cpu;
int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu);
return cpu;
#else
return -1;
return -1;
#endif
}
void restoreAffinity()
{
void restoreAffinity() {
#if __linux__
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
printError("failed to restore affinity %1%");
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
printError("failed to restore affinity %1%");
#endif
}
}
} // namespace nix

2
third_party/nix/src/libutil/affinity.hh vendored
View file

@ -6,4 +6,4 @@ void setAffinityTo(int cpu);
int lockToCurrentCPU();
void restoreAffinity();
}
} // namespace nix

550
third_party/nix/src/libutil/archive.cc vendored
View file

@ -1,32 +1,31 @@
#include <cerrno>
#include <algorithm>
#include <vector>
#include <map>
#include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "archive.hh"
#include <dirent.h>
#include <fcntl.h>
#include "archive.hh"
#include "util.hh"
#include <strings.h> // for strcasecmp
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <cerrno>
#include <map>
#include <vector>
#include "config.hh"
#include "util.hh"
namespace nix {
struct ArchiveSettings : Config
{
Setting<bool> useCaseHack{this,
#if __APPLE__
true,
#else
false,
#endif
struct ArchiveSettings : Config {
Setting<bool> useCaseHack {
this,
#if __APPLE__
true,
#else
false,
#endif
"use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."};
"Whether to enable a Darwin-specific hack for dealing with file name "
"collisions."
};
};
static ArchiveSettings archiveSettings;
@ -37,105 +36,105 @@ const std::string narVersionMagic1 = "nix-archive-1";
static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; };
PathFilter defaultPathFilter = [](const Path&) { return true; };
static void dumpContents(const Path& path, size_t size, Sink& sink) {
sink << "contents" << size;
static void dumpContents(const Path & path, size_t size,
Sink & sink)
{
sink << "contents" << size;
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
std::vector<unsigned char> buf(65536);
size_t left = size;
std::vector<unsigned char> buf(65536);
size_t left = size;
while (left > 0) {
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink(buf.data(), n);
}
while (left > 0) {
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink(buf.data(), n);
}
writePadding(size, sink);
writePadding(size, sink);
}
static void dump(const Path& path, Sink& sink, PathFilter& filter) {
checkInterrupt();
static void dump(const Path & path, Sink & sink, PathFilter & filter)
{
checkInterrupt();
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % path);
sink << "(";
sink << "(";
if (S_ISREG(st.st_mode)) {
sink << "type"
<< "regular";
if (st.st_mode & S_IXUSR)
sink << "executable"
<< "";
dumpContents(path, (size_t)st.st_size, sink);
}
if (S_ISREG(st.st_mode)) {
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
dumpContents(path, (size_t) st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {
sink << "type"
<< "directory";
else if (S_ISDIR(st.st_mode)) {
sink << "type" << "directory";
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
std::map<string, string> unhacked;
for (auto& i : readDirectory(path))
if (archiveSettings.useCaseHack) {
string name(i.name);
size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) {
debug(format("removing case hack suffix from '%1%'") %
(path + "/" + i.name));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'") %
(path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
std::map<string, string> unhacked;
for (auto & i : readDirectory(path))
if (archiveSettings.useCaseHack) {
string name(i.name);
size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) {
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
throw Error(format("file name collision in between '%1%' and '%2%'")
% (path + "/" + unhacked[name]) % (path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
for (auto& i : unhacked)
if (filter(path + "/" + i.first)) {
sink << "entry"
<< "("
<< "name" << i.first << "node";
dump(path + "/" + i.second, sink, filter);
sink << ")";
}
}
for (auto & i : unhacked)
if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node";
dump(path + "/" + i.second, sink, filter);
sink << ")";
}
}
else if (S_ISLNK(st.st_mode))
sink << "type"
<< "symlink"
<< "target" << readLink(path);
else if (S_ISLNK(st.st_mode))
sink << "type" << "symlink" << "target" << readLink(path);
else
throw Error(format("file '%1%' has an unsupported type") % path);
else throw Error(format("file '%1%' has an unsupported type") % path);
sink << ")";
sink << ")";
}
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
sink << narVersionMagic1;
dump(path, sink, filter);
void dumpPath(const Path& path, Sink& sink, PathFilter& filter) {
sink << narVersionMagic1;
dump(path, sink, filter);
}
void dumpString(const std::string & s, Sink & sink)
{
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
void dumpString(const std::string& s, Sink& sink) {
sink << narVersionMagic1 << "("
<< "type"
<< "regular"
<< "contents" << s << ")";
}
static SerialisationError badArchive(string s)
{
return SerialisationError("bad archive: " + s);
static SerialisationError badArchive(string s) {
return SerialisationError("bad archive: " + s);
}
#if 0
static void skipGeneric(Source & source)
{
@ -146,233 +145,212 @@ static void skipGeneric(Source & source)
}
#endif
static void parseContents(ParseSink& sink, Source& source, const Path& path) {
unsigned long long size = readLongLong(source);
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
unsigned long long size = readLongLong(source);
sink.preallocateContents(size);
sink.preallocateContents(size);
unsigned long long left = size;
std::vector<unsigned char> buf(65536);
unsigned long long left = size;
std::vector<unsigned char> buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
if ((unsigned long long)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
}
while (left) {
checkInterrupt();
auto n = buf.size();
if ((unsigned long long)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
}
readPadding(size, source);
readPadding(size, source);
}
struct CaseInsensitiveCompare
{
bool operator() (const string & a, const string & b) const
{
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
struct CaseInsensitiveCompare {
bool operator()(const string& a, const string& b) const {
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};
static void parse(ParseSink& sink, Source& source, const Path& path) {
string s;
static void parse(ParseSink & sink, Source & source, const Path & path)
{
string s;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
std::map<Path, int, CaseInsensitiveCompare> names;
while (1) {
checkInterrupt();
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
if (s == ")") {
break;
}
std::map<Path, int, CaseInsensitiveCompare> names;
else if (s == "type") {
if (type != tpUnknown) throw badArchive("multiple type fields");
string t = readString(source);
while (1) {
if (t == "regular") {
type = tpRegular;
sink.createRegularFile(path);
}
else if (t == "directory") {
sink.createDirectory(path);
type = tpDirectory;
}
else if (t == "symlink") {
type = tpSymlink;
}
else
throw badArchive("unknown file type " + t);
}
else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path);
}
else if (s == "executable" && type == tpRegular) {
auto s = readString(source);
if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
else if (s == "entry" && type == tpDirectory) {
string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
while (1) {
checkInterrupt();
s = readString(source);
if (s == ")") {
break;
}
else if (s == "type") {
if (type != tpUnknown)
throw badArchive("multiple type fields");
string t = readString(source);
if (t == "regular") {
type = tpRegular;
sink.createRegularFile(path);
}
else if (t == "directory") {
sink.createDirectory(path);
type = tpDirectory;
}
else if (t == "symlink") {
type = tpSymlink;
}
else throw badArchive("unknown file type " + t);
}
else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path);
}
else if (s == "executable" && type == tpRegular) {
auto s = readString(source);
if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
else if (s == "entry" && type == tpDirectory) {
string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
while (1) {
checkInterrupt();
s = readString(source);
if (s == ")") {
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") % i->first % name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}
else if (s == "target" && type == tpSymlink) {
string target = readString(source);
sink.createSymlink(path, target);
}
else
throw badArchive("unknown field " + s);
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." ||
name.find('/') != string::npos ||
name.find((char)0) != string::npos)
throw Error(format("NAR contains invalid file name '%1%'") % name);
if (name <= prevName) throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug(format("case collision between '%1%' and '%2%'") %
i->first % name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (s.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}
else if (s == "target" && type == tpSymlink) {
string target = readString(source);
sink.createSymlink(path, target);
}
else
throw badArchive("unknown field " + s);
}
}
void parseDump(ParseSink & sink, Source & source)
{
string version;
try {
version = readString(source, narVersionMagic1.size());
} catch (SerialisationError & e) {
/* This generally means the integer at the start couldn't be
decoded. Ignore and throw the exception below. */
}
if (version != narVersionMagic1)
throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, "");
void parseDump(ParseSink& sink, Source& source) {
string version;
try {
version = readString(source, narVersionMagic1.size());
} catch (SerialisationError& e) {
/* This generally means the integer at the start couldn't be
decoded. Ignore and throw the exception below. */
}
if (version != narVersionMagic1)
throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, "");
}
struct RestoreSink : ParseSink {
Path dstPath;
AutoCloseFD fd;
struct RestoreSink : ParseSink
{
Path dstPath;
AutoCloseFD fd;
void createDirectory(const Path& path) {
Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1)
throw SysError(format("creating directory '%1%'") % p);
};
void createDirectory(const Path & path)
{
Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1)
throw SysError(format("creating directory '%1%'") % p);
};
void createRegularFile(const Path& path) {
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError(format("creating file '%1%'") % p);
}
void createRegularFile(const Path & path)
{
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError(format("creating file '%1%'") % p);
}
void isExecutable() {
struct stat st;
if (fstat(fd.get(), &st) == -1) throw SysError("fstat");
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
throw SysError("fchmod");
}
void isExecutable()
{
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("fstat");
if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
throw SysError("fchmod");
}
void preallocateContents(unsigned long long len)
{
void preallocateContents(unsigned long long len) {
#if HAVE_POSIX_FALLOCATE
if (len) {
errno = posix_fallocate(fd.get(), 0, len);
/* Note that EINVAL may indicate that the underlying
filesystem doesn't support preallocation (e.g. on
OpenSolaris). Since preallocation is just an
optimisation, ignore it. */
if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
throw SysError(format("preallocating file of %1% bytes") % len);
}
if (len) {
errno = posix_fallocate(fd.get(), 0, len);
/* Note that EINVAL may indicate that the underlying
filesystem doesn't support preallocation (e.g. on
OpenSolaris). Since preallocation is just an
optimisation, ignore it. */
if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
throw SysError(format("preallocating file of %1% bytes") % len);
}
#endif
}
}
void receiveContents(unsigned char * data, unsigned int len)
{
writeFull(fd.get(), data, len);
}
void receiveContents(unsigned char* data, unsigned int len) {
writeFull(fd.get(), data, len);
}
void createSymlink(const Path & path, const string & target)
{
Path p = dstPath + path;
nix::createSymlink(target, p);
}
void createSymlink(const Path& path, const string& target) {
Path p = dstPath + path;
nix::createSymlink(target, p);
}
};
void restorePath(const Path & path, Source & source)
{
RestoreSink sink;
sink.dstPath = path;
parseDump(sink, source);
void restorePath(const Path& path, Source& source) {
RestoreSink sink;
sink.dstPath = path;
parseDump(sink, source);
}
void copyNAR(Source& source, Sink& sink) {
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
void copyNAR(Source & source, Sink & sink)
{
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
ParseSink parseSink; /* null sink; just parse the NAR */
ParseSink parseSink; /* null sink; just parse the NAR */
LambdaSource wrapper([&](unsigned char* data, size_t len) {
auto n = source.read(data, len);
sink(data, n);
return n;
});
LambdaSource wrapper([&](unsigned char * data, size_t len) {
auto n = source.read(data, len);
sink(data, n);
return n;
});
parseDump(parseSink, wrapper);
parseDump(parseSink, wrapper);
}
}
} // namespace nix

43
third_party/nix/src/libutil/archive.hh vendored
View file

@ -1,12 +1,10 @@
#pragma once
#include "types.hh"
#include "serialise.hh"
#include "types.hh"
namespace nix {
/* dumpPath creates a Nix archive of the specified path. The format
is as follows:
@ -44,41 +42,36 @@ namespace nix {
`+' denotes string concatenation. */
void dumpPath(const Path& path, Sink& sink,
PathFilter& filter = defaultPathFilter);
void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter);
void dumpString(const std::string & s, Sink & sink);
void dumpString(const std::string& s, Sink& sink);
/* FIXME: fix this API, it sucks. */
struct ParseSink
{
virtual void createDirectory(const Path & path) { };
struct ParseSink {
virtual void createDirectory(const Path& path){};
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { };
virtual void receiveContents(unsigned char * data, unsigned int len) { };
virtual void createRegularFile(const Path& path){};
virtual void isExecutable(){};
virtual void preallocateContents(unsigned long long size){};
virtual void receiveContents(unsigned char* data, unsigned int len){};
virtual void createSymlink(const Path & path, const string & target) { };
virtual void createSymlink(const Path& path, const string& target){};
};
struct TeeSink : ParseSink
{
TeeSource source;
struct TeeSink : ParseSink {
TeeSource source;
TeeSink(Source & source) : source(source) { }
TeeSink(Source& source) : source(source) {}
};
void parseDump(ParseSink & sink, Source & source);
void parseDump(ParseSink& sink, Source& source);
void restorePath(const Path & path, Source & source);
void restorePath(const Path& path, Source& source);
/* Read a NAR from 'source' and write it to 'sink'. */
void copyNAR(Source & source, Sink & sink);
void copyNAR(Source& source, Sink& sink);
extern const std::string narVersionMagic1;
}
} // namespace nix

325
third_party/nix/src/libutil/args.cc vendored
View file

@ -3,201 +3,184 @@
namespace nix {
Args::FlagMaker Args::mkFlag()
{
return FlagMaker(*this);
Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); }
Args::FlagMaker::~FlagMaker() {
assert(flag->longName != "");
args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
}
Args::FlagMaker::~FlagMaker()
{
assert(flag->longName != "");
args.longFlags[flag->longName] = flag;
if (flag->shortName) args.shortFlags[flag->shortName] = flag;
}
void Args::parseCmdline(const Strings& _cmdline) {
Strings pendingArgs;
bool dashDash = false;
void Args::parseCmdline(const Strings & _cmdline)
{
Strings pendingArgs;
bool dashDash = false;
Strings cmdline(_cmdline);
Strings cmdline(_cmdline);
for (auto pos = cmdline.begin(); pos != cmdline.end();) {
auto arg = *pos;
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
auto arg = *pos;
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) {
*pos = (string) "-" + arg[1];
auto next = pos; ++next;
for (unsigned int j = 2; j < arg.length(); j++)
if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]);
else {
cmdline.insert(next, string(arg, j));
break;
}
arg = *pos;
}
if (!dashDash && arg == "--") {
dashDash = true;
++pos;
}
else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg);
}
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' &&
isalpha(arg[1])) {
*pos = (string) "-" + arg[1];
auto next = pos;
++next;
for (unsigned int j = 2; j < arg.length(); j++)
if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]);
else {
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false))
pendingArgs.clear();
cmdline.insert(next, string(arg, j));
break;
}
arg = *pos;
}
processArgs(pendingArgs, true);
if (!dashDash && arg == "--") {
dashDash = true;
++pos;
} else if (!dashDash && std::string(arg, 0, 1) == "-") {
if (!processFlag(pos, cmdline.end()))
throw UsageError(format("unrecognised flag '%1%'") % arg);
} else {
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false)) pendingArgs.clear();
}
}
processArgs(pendingArgs, true);
}
void Args::printHelp(const string & programName, std::ostream & out)
{
std::cout << "Usage: " << programName << " <FLAGS>...";
for (auto & exp : expectedArgs) {
std::cout << renderLabels({exp.label});
// FIXME: handle arity > 1
if (exp.arity == 0) std::cout << "...";
if (exp.optional) std::cout << "?";
}
void Args::printHelp(const string& programName, std::ostream& out) {
std::cout << "Usage: " << programName << " <FLAGS>...";
for (auto& exp : expectedArgs) {
std::cout << renderLabels({exp.label});
// FIXME: handle arity > 1
if (exp.arity == 0) std::cout << "...";
if (exp.optional) std::cout << "?";
}
std::cout << "\n";
auto s = description();
if (s != "") std::cout << "\nSummary: " << s << ".\n";
if (longFlags.size()) {
std::cout << "\n";
std::cout << "Flags:\n";
printFlags(out);
}
}
auto s = description();
if (s != "")
std::cout << "\nSummary: " << s << ".\n";
void Args::printFlags(std::ostream& out) {
Table2 table;
for (auto& flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair(
(flag.second->shortName
? std::string("-") + flag.second->shortName + ", "
: " ") +
"--" + flag.first + renderLabels(flag.second->labels),
flag.second->description));
}
printTable(out, table);
}
if (longFlags.size()) {
std::cout << "\n";
std::cout << "Flags:\n";
printFlags(out);
bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) {
assert(pos != end);
auto process = [&](const std::string& name, const Flag& flag) -> bool {
++pos;
std::vector<std::string> args;
for (size_t n = 0; n < flag.arity; ++n) {
if (pos == end) {
if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)") % name %
flag.arity);
}
args.push_back(*pos++);
}
flag.handler(std::move(args));
return true;
};
if (string(*pos, 0, 2) == "--") {
auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
}
if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
auto c = (*pos)[1];
auto i = shortFlags.find(c);
if (i == shortFlags.end()) return false;
return process(std::string("-") + c, *i->second);
}
return false;
}
void Args::printFlags(std::ostream & out)
{
Table2 table;
for (auto & flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair(
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
+ "--" + flag.first + renderLabels(flag.second->labels),
flag.second->description));
}
printTable(out, table);
bool Args::processArgs(const Strings& args, bool finish) {
if (expectedArgs.empty()) {
if (!args.empty())
throw UsageError(format("unexpected argument '%1%'") % args.front());
return true;
}
auto& exp = expectedArgs.front();
bool res = false;
if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity)) {
std::vector<std::string> ss;
for (auto& s : args) ss.push_back(s);
exp.handler(std::move(ss));
expectedArgs.pop_front();
res = true;
}
if (finish && !expectedArgs.empty() && !expectedArgs.front().optional)
throw UsageError("more arguments are required");
return res;
}
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
{
assert(pos != end);
auto process = [&](const std::string & name, const Flag & flag) -> bool {
++pos;
std::vector<std::string> args;
for (size_t n = 0 ; n < flag.arity; ++n) {
if (pos == end) {
if (flag.arity == ArityAny) break;
throw UsageError(format("flag '%1%' requires %2% argument(s)")
% name % flag.arity);
}
args.push_back(*pos++);
}
flag.handler(std::move(args));
return true;
};
if (string(*pos, 0, 2) == "--") {
auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
}
if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
auto c = (*pos)[1];
auto i = shortFlags.find(c);
if (i == shortFlags.end()) return false;
return process(std::string("-") + c, *i->second);
}
return false;
Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) {
arity(1);
label("type");
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
handler([ht](std::string s) {
*ht = parseHashType(s);
if (*ht == htUnknown) throw UsageError("unknown hash type '%1%'", s);
});
return *this;
}
bool Args::processArgs(const Strings & args, bool finish)
{
if (expectedArgs.empty()) {
if (!args.empty())
throw UsageError(format("unexpected argument '%1%'") % args.front());
return true;
}
auto & exp = expectedArgs.front();
bool res = false;
if ((exp.arity == 0 && finish) ||
(exp.arity > 0 && args.size() == exp.arity))
{
std::vector<std::string> ss;
for (auto & s : args) ss.push_back(s);
exp.handler(std::move(ss));
expectedArgs.pop_front();
res = true;
}
if (finish && !expectedArgs.empty() && !expectedArgs.front().optional)
throw UsageError("more arguments are required");
return res;
Strings argvToStrings(int argc, char** argv) {
Strings args;
argc--;
argv++;
while (argc--) args.push_back(*argv++);
return args;
}
Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
{
arity(1);
label("type");
description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
handler([ht](std::string s) {
*ht = parseHashType(s);
if (*ht == htUnknown)
throw UsageError("unknown hash type '%1%'", s);
});
return *this;
std::string renderLabels(const Strings& labels) {
std::string res;
for (auto label : labels) {
for (auto& c : label) c = std::toupper(c);
res += " <" + label + ">";
}
return res;
}
Strings argvToStrings(int argc, char * * argv)
{
Strings args;
argc--; argv++;
while (argc--) args.push_back(*argv++);
return args;
void printTable(std::ostream& out, const Table2& table) {
size_t max = 0;
for (auto& row : table) max = std::max(max, row.first.size());
for (auto& row : table) {
out << " " << row.first << std::string(max - row.first.size() + 2, ' ')
<< row.second << "\n";
}
}
std::string renderLabels(const Strings & labels)
{
std::string res;
for (auto label : labels) {
for (auto & c : label) c = std::toupper(c);
res += " <" + label + ">";
}
return res;
}
void printTable(std::ostream & out, const Table2 & table)
{
size_t max = 0;
for (auto & row : table)
max = std::max(max, row.first.size());
for (auto & row : table) {
out << " " << row.first
<< std::string(max - row.first.size() + 2, ' ')
<< row.second << "\n";
}
}
}
} // namespace nix

331
third_party/nix/src/libutil/args.hh vendored
View file

@ -3,7 +3,6 @@
#include <iostream>
#include <map>
#include <memory>
#include "util.hh"
namespace nix {
@ -12,190 +11,206 @@ MakeError(UsageError, Error);
enum HashType : char;
class Args
{
public:
class Args {
public:
/* Parse the command line, throwing a UsageError if something goes
wrong. */
void parseCmdline(const Strings& cmdline);
/* Parse the command line, throwing a UsageError if something goes
wrong. */
void parseCmdline(const Strings & cmdline);
virtual void printHelp(const string& programName, std::ostream& out);
virtual void printHelp(const string & programName, std::ostream & out);
virtual std::string description() { return ""; }
virtual std::string description() { return ""; }
protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max();
protected:
/* Flags. */
struct Flag {
typedef std::shared_ptr<Flag> ptr;
std::string longName;
char shortName = 0;
std::string description;
Strings labels;
size_t arity = 0;
std::function<void(std::vector<std::string>)> handler;
std::string category;
};
static const size_t ArityAny = std::numeric_limits<size_t>::max();
std::map<std::string, Flag::ptr> longFlags;
std::map<char, Flag::ptr> shortFlags;
/* Flags. */
struct Flag
{
typedef std::shared_ptr<Flag> ptr;
std::string longName;
char shortName = 0;
std::string description;
Strings labels;
size_t arity = 0;
std::function<void(std::vector<std::string>)> handler;
std::string category;
virtual bool processFlag(Strings::iterator& pos, Strings::iterator end);
virtual void printFlags(std::ostream& out);
/* Positional arguments. */
struct ExpectedArg {
std::string label;
size_t arity; // 0 = any
bool optional;
std::function<void(std::vector<std::string>)> handler;
};
std::list<ExpectedArg> expectedArgs;
virtual bool processArgs(const Strings& args, bool finish);
std::set<std::string> hiddenCategories;
public:
class FlagMaker {
Args& args;
Flag::ptr flag;
friend class Args;
FlagMaker(Args& args) : args(args), flag(std::make_shared<Flag>()){};
public:
~FlagMaker();
FlagMaker& longName(const std::string& s) {
flag->longName = s;
return *this;
};
FlagMaker& shortName(char s) {
flag->shortName = s;
return *this;
};
FlagMaker& description(const std::string& s) {
flag->description = s;
return *this;
};
FlagMaker& label(const std::string& l) {
flag->arity = 1;
flag->labels = {l};
return *this;
};
FlagMaker& labels(const Strings& ls) {
flag->arity = ls.size();
flag->labels = ls;
return *this;
};
FlagMaker& arity(size_t arity) {
flag->arity = arity;
return *this;
};
FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) {
flag->handler = handler;
return *this;
};
FlagMaker& handler(std::function<void()> handler) {
flag->handler = [handler](std::vector<std::string>) { handler(); };
return *this;
};
FlagMaker& handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) {
handler(std::move(ss[0]));
};
return *this;
};
FlagMaker& category(const std::string& s) {
flag->category = s;
return *this;
};
std::map<std::string, Flag::ptr> longFlags;
std::map<char, Flag::ptr> shortFlags;
virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
virtual void printFlags(std::ostream & out);
/* Positional arguments. */
struct ExpectedArg
{
std::string label;
size_t arity; // 0 = any
bool optional;
std::function<void(std::vector<std::string>)> handler;
template <class T>
FlagMaker& dest(T* dest) {
flag->arity = 1;
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this;
};
std::list<ExpectedArg> expectedArgs;
virtual bool processArgs(const Strings & args, bool finish);
std::set<std::string> hiddenCategories;
public:
class FlagMaker
{
Args & args;
Flag::ptr flag;
friend class Args;
FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { };
public:
~FlagMaker();
FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
FlagMaker & handler(std::function<void(std::string)> handler) {
flag->arity = 1;
flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
return *this;
};
FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
template<class T>
FlagMaker & dest(T * dest)
{
flag->arity = 1;
flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
return *this;
};
template<class T>
FlagMaker & set(T * dest, const T & val)
{
flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
return *this;
};
FlagMaker & mkHashTypeFlag(HashType * ht);
template <class T>
FlagMaker& set(T* dest, const T& val) {
flag->arity = 0;
flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
return *this;
};
FlagMaker mkFlag();
FlagMaker& mkHashTypeFlag(HashType* ht);
};
/* Helper functions for constructing flags / positional
arguments. */
FlagMaker mkFlag();
void mkFlag1(char shortName, const std::string & longName,
const std::string & label, const std::string & description,
std::function<void(std::string)> fun)
{
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({label})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) { fun(ss[0]); });
}
/* Helper functions for constructing flags / positional
arguments. */
void mkFlag(char shortName, const std::string & name,
const std::string & description, bool * dest)
{
mkFlag(shortName, name, description, dest, true);
}
void mkFlag1(char shortName, const std::string& longName,
const std::string& label, const std::string& description,
std::function<void(std::string)> fun) {
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({label})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) { fun(ss[0]); });
}
template<class T>
void mkFlag(char shortName, const std::string & longName, const std::string & description,
T * dest, const T & value)
{
mkFlag()
.shortName(shortName)
.longName(longName)
.description(description)
.handler([=](std::vector<std::string> ss) { *dest = value; });
}
void mkFlag(char shortName, const std::string& name,
const std::string& description, bool* dest) {
mkFlag(shortName, name, description, dest, true);
}
template<class I>
void mkIntFlag(char shortName, const std::string & longName,
const std::string & description, I * dest)
{
mkFlag<I>(shortName, longName, description, [=](I n) {
*dest = n;
template <class T>
void mkFlag(char shortName, const std::string& longName,
const std::string& description, T* dest, const T& value) {
mkFlag()
.shortName(shortName)
.longName(longName)
.description(description)
.handler([=](std::vector<std::string> ss) { *dest = value; });
}
template <class I>
void mkIntFlag(char shortName, const std::string& longName,
const std::string& description, I* dest) {
mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; });
}
template <class I>
void mkFlag(char shortName, const std::string& longName,
const std::string& description, std::function<void(I)> fun) {
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({"N"})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) {
I n;
if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument",
longName);
fun(n);
});
}
}
template<class I>
void mkFlag(char shortName, const std::string & longName,
const std::string & description, std::function<void(I)> fun)
{
mkFlag()
.shortName(shortName)
.longName(longName)
.labels({"N"})
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) {
I n;
if (!string2Int(ss[0], n))
throw UsageError("flag '--%s' requires a integer argument", longName);
fun(n);
});
}
/* Expect a string argument. */
void expectArg(const std::string& label, string* dest,
bool optional = false) {
expectedArgs.push_back(
ExpectedArg{label, 1, optional,
[=](std::vector<std::string> ss) { *dest = ss[0]; }});
}
/* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false)
{
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
*dest = ss[0];
}});
}
/* Expect 0 or more arguments. */
void expectArgs(const std::string& label, std::vector<std::string>* dest) {
expectedArgs.push_back(ExpectedArg{
label, 0, false,
[=](std::vector<std::string> ss) { *dest = std::move(ss); }});
}
/* Expect 0 or more arguments. */
void expectArgs(const std::string & label, std::vector<std::string> * dest)
{
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
*dest = std::move(ss);
}});
}
friend class MultiCommand;
friend class MultiCommand;
};
Strings argvToStrings(int argc, char * * argv);
Strings argvToStrings(int argc, char** argv);
/* Helper function for rendering argument labels. */
std::string renderLabels(const Strings & labels);
std::string renderLabels(const Strings& labels);
/* Helper function for printing 2-column tables. */
typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table);
void printTable(std::ostream& out, const Table2& table);
}
} // namespace nix

683
third_party/nix/src/libutil/compression.cc vendored
View file

@ -1,432 +1,379 @@
#include "compression.hh"
#include "util.hh"
#include "finally.hh"
#include "logging.hh"
#include <lzma.h>
#include <bzlib.h>
#include <cstdio>
#include <cstring>
#include <brotli/decode.h>
#include <brotli/encode.h>
#include <bzlib.h>
#include <lzma.h>
#include <cstdio>
#include <cstring>
#include <iostream>
#include "finally.hh"
#include "logging.hh"
#include "util.hh"
namespace nix {
// Don't feed brotli too much at once.
struct ChunkedCompressionSink : CompressionSink
{
uint8_t outbuf[32 * 1024];
struct ChunkedCompressionSink : CompressionSink {
uint8_t outbuf[32 * 1024];
void write(const unsigned char * data, size_t len) override
{
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (len) {
size_t n = std::min(CHUNK_SIZE, len);
writeInternal(data, n);
data += n;
len -= n;
}
void write(const unsigned char* data, size_t len) override {
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (len) {
size_t n = std::min(CHUNK_SIZE, len);
writeInternal(data, n);
data += n;
len -= n;
}
}
virtual void writeInternal(const unsigned char * data, size_t len) = 0;
virtual void writeInternal(const unsigned char* data, size_t len) = 0;
};
struct NoneSink : CompressionSink
{
Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { }
void finish() override { flush(); }
void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
struct NoneSink : CompressionSink {
Sink& nextSink;
NoneSink(Sink& nextSink) : nextSink(nextSink) {}
void finish() override { flush(); }
void write(const unsigned char* data, size_t len) override {
nextSink(data, len);
}
};
struct XzDecompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
struct XzDecompressionSink : CompressionSink {
Sink& nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
lzma_ret ret = lzma_stream_decoder(
&strm, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma decoder");
XzDecompressionSink(Sink& nextSink) : nextSink(nextSink) {
lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma decoder");
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzDecompressionSink() { lzma_end(&strm); }
void finish() override {
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char* data, size_t len) override {
strm.next_in = data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
~XzDecompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char * data, size_t len) override
{
strm.next_in = data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
}
};
struct BzipDecompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
struct BzipDecompressionSink : ChunkedCompressionSink {
Sink& nextSink;
bz_stream strm;
bool finished = false;
BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 decoder");
BzipDecompressionSink(Sink& nextSink) : nextSink(nextSink) {
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 decoder");
strm.next_out = (char *) outbuf;
strm.next_out = (char*)outbuf;
strm.avail_out = sizeof(outbuf);
}
~BzipDecompressionSink() { BZ2_bzDecompressEnd(&strm); }
void finish() override {
flush();
write(nullptr, 0);
}
void writeInternal(const unsigned char* data, size_t len) override {
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char*)data;
strm.avail_in = len;
while (strm.avail_in) {
checkInterrupt();
int ret = BZ2_bzDecompress(&strm);
if (ret != BZ_OK && ret != BZ_STREAM_END)
throw CompressionError("error while decompressing bzip2 file");
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (char*)outbuf;
strm.avail_out = sizeof(outbuf);
}
}
~BzipDecompressionSink()
{
BZ2_bzDecompressEnd(&strm);
}
void finish() override
{
flush();
write(nullptr, 0);
}
void writeInternal(const unsigned char * data, size_t len) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data;
strm.avail_in = len;
while (strm.avail_in) {
checkInterrupt();
int ret = BZ2_bzDecompress(&strm);
if (ret != BZ_OK && ret != BZ_STREAM_END)
throw CompressionError("error while decompressing bzip2 file");
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
}
};
struct BrotliDecompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
BrotliDecoderState * state;
bool finished = false;
struct BrotliDecompressionSink : ChunkedCompressionSink {
Sink& nextSink;
BrotliDecoderState* state;
bool finished = false;
BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
if (!state)
throw CompressionError("unable to initialize brotli decoder");
}
~BrotliDecompressionSink()
{
BrotliDecoderDestroyInstance(state);
}
void finish() override
{
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char * data, size_t len) override
{
const uint8_t * next_in = data;
size_t avail_in = len;
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
checkInterrupt();
if (!BrotliDecoderDecompressStream(state,
&avail_in, &next_in,
&avail_out, &next_out,
nullptr))
throw CompressionError("error while decompressing brotli file");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
next_out = outbuf;
avail_out = sizeof(outbuf);
}
finished = BrotliDecoderIsFinished(state);
}
BrotliDecompressionSink(Sink& nextSink) : nextSink(nextSink) {
state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
if (!state) throw CompressionError("unable to initialize brotli decoder");
}
~BrotliDecompressionSink() { BrotliDecoderDestroyInstance(state); }
void finish() override {
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char* data, size_t len) override {
const uint8_t* next_in = data;
size_t avail_in = len;
uint8_t* next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
checkInterrupt();
if (!BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out,
&next_out, nullptr))
throw CompressionError("error while decompressing brotli file");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
next_out = outbuf;
avail_out = sizeof(outbuf);
}
finished = BrotliDecoderIsFinished(state);
}
}
};
ref<std::string> decompress(const std::string & method, const std::string & in)
{
StringSink ssink;
auto sink = makeDecompressionSink(method, ssink);
(*sink)(in);
sink->finish();
return ssink.s;
ref<std::string> decompress(const std::string& method, const std::string& in) {
StringSink ssink;
auto sink = makeDecompressionSink(method, ssink);
(*sink)(in);
sink->finish();
return ssink.s;
}
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
{
if (method == "none" || method == "")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
ref<CompressionSink> makeDecompressionSink(const std::string& method,
Sink& nextSink) {
if (method == "none" || method == "")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
}
struct XzCompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
struct XzCompressionSink : CompressionSink {
Sink& nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
{
lzma_ret ret;
bool done = false;
XzCompressionSink(Sink& nextSink, bool parallel) : nextSink(nextSink) {
lzma_ret ret;
bool done = false;
if (parallel) {
if (parallel) {
#ifdef HAVE_LZMA_MT
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(&strm, &mt_options);
done = true;
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0) mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(&strm, &mt_options);
done = true;
#else
printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
printMsg(lvlError,
"warning: parallel XZ compression requested but not supported, "
"falling back to single-threaded compression");
#endif
}
}
if (!done)
ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
if (!done) ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter?
// FIXME: apply the x86 BCJ filter?
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzCompressionSink() { lzma_end(&strm); }
void finish() override {
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char* data, size_t len) override {
strm.next_in = data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
~XzCompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char * data, size_t len) override
{
strm.next_in = data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
}
};
struct BzipCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
struct BzipCompressionSink : ChunkedCompressionSink {
Sink& nextSink;
bz_stream strm;
bool finished = false;
BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 encoder");
BzipCompressionSink(Sink& nextSink) : nextSink(nextSink) {
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 encoder");
strm.next_out = (char *) outbuf;
strm.next_out = (char*)outbuf;
strm.avail_out = sizeof(outbuf);
}
~BzipCompressionSink() { BZ2_bzCompressEnd(&strm); }
void finish() override {
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char* data, size_t len) override {
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char*)data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (char*)outbuf;
strm.avail_out = sizeof(outbuf);
}
}
~BzipCompressionSink()
{
BZ2_bzCompressEnd(&strm);
}
void finish() override
{
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char * data, size_t len) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
}
};
struct BrotliCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
BrotliEncoderState *state;
bool finished = false;
struct BrotliCompressionSink : ChunkedCompressionSink {
Sink& nextSink;
uint8_t outbuf[BUFSIZ];
BrotliEncoderState* state;
bool finished = false;
BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
{
state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
if (!state)
throw CompressionError("unable to initialise brotli encoder");
}
~BrotliCompressionSink()
{
BrotliEncoderDestroyInstance(state);
}
void finish() override
{
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char * data, size_t len) override
{
const uint8_t * next_in = data;
size_t avail_in = len;
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
checkInterrupt();
if (!BrotliEncoderCompressStream(state,
data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&avail_in, &next_in,
&avail_out, &next_out,
nullptr))
throw CompressionError("error while compressing brotli compression");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
next_out = outbuf;
avail_out = sizeof(outbuf);
}
finished = BrotliEncoderIsFinished(state);
}
BrotliCompressionSink(Sink& nextSink) : nextSink(nextSink) {
state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
if (!state) throw CompressionError("unable to initialise brotli encoder");
}
~BrotliCompressionSink() { BrotliEncoderDestroyInstance(state); }
void finish() override {
flush();
writeInternal(nullptr, 0);
}
void writeInternal(const unsigned char* data, size_t len) override {
const uint8_t* next_in = data;
size_t avail_in = len;
uint8_t* next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
checkInterrupt();
if (!BrotliEncoderCompressStream(
state, data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&avail_in, &next_in, &avail_out, &next_out, nullptr))
throw CompressionError("error while compressing brotli compression");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
next_out = outbuf;
avail_out = sizeof(outbuf);
}
finished = BrotliEncoderIsFinished(state);
}
}
};
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzCompressionSink>(nextSink, parallel);
else if (method == "bzip2")
return make_ref<BzipCompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else
throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
ref<CompressionSink> makeCompressionSink(const std::string& method,
Sink& nextSink, const bool parallel) {
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzCompressionSink>(nextSink, parallel);
else if (method == "bzip2")
return make_ref<BzipCompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else
throw UnknownCompressionMethod(format("unknown compression method '%s'") %
method);
}
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
{
StringSink ssink;
auto sink = makeCompressionSink(method, ssink, parallel);
(*sink)(in);
sink->finish();
return ssink.s;
ref<std::string> compress(const std::string& method, const std::string& in,
const bool parallel) {
StringSink ssink;
auto sink = makeCompressionSink(method, ssink, parallel);
(*sink)(in);
sink->finish();
return ssink.s;
}
}
} // namespace nix

26
third_party/nix/src/libutil/compression.hh vendored
View file

@ -1,28 +1,30 @@
#pragma once
#include "ref.hh"
#include "types.hh"
#include "serialise.hh"
#include <string>
#include "ref.hh"
#include "serialise.hh"
#include "types.hh"
namespace nix {
struct CompressionSink : BufferedSink
{
virtual void finish() = 0;
struct CompressionSink : BufferedSink {
virtual void finish() = 0;
};
ref<std::string> decompress(const std::string & method, const std::string & in);
ref<std::string> decompress(const std::string& method, const std::string& in);
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<CompressionSink> makeDecompressionSink(const std::string& method,
Sink& nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
ref<std::string> compress(const std::string& method, const std::string& in,
const bool parallel = false);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
ref<CompressionSink> makeCompressionSink(const std::string& method,
Sink& nextSink,
const bool parallel = false);
MakeError(UnknownCompressionMethod, Error);
MakeError(CompressionError, Error);
}
} // namespace nix

484
third_party/nix/src/libutil/config.cc vendored
View file

@ -4,268 +4,251 @@
namespace nix {
bool Config::set(const std::string & name, const std::string & value)
{
auto i = _settings.find(name);
if (i == _settings.end()) return false;
i->second.setting->set(value);
i->second.setting->overriden = true;
return true;
bool Config::set(const std::string& name, const std::string& value) {
auto i = _settings.find(name);
if (i == _settings.end()) return false;
i->second.setting->set(value);
i->second.setting->overriden = true;
return true;
}
void Config::addSetting(AbstractSetting * setting)
{
_settings.emplace(setting->name, Config::SettingData(false, setting));
for (auto & alias : setting->aliases)
_settings.emplace(alias, Config::SettingData(true, setting));
void Config::addSetting(AbstractSetting* setting) {
_settings.emplace(setting->name, Config::SettingData(false, setting));
for (auto& alias : setting->aliases)
_settings.emplace(alias, Config::SettingData(true, setting));
bool set = false;
bool set = false;
auto i = unknownSettings.find(setting->name);
auto i = unknownSettings.find(setting->name);
if (i != unknownSettings.end()) {
setting->set(i->second);
setting->overriden = true;
unknownSettings.erase(i);
set = true;
}
for (auto& alias : setting->aliases) {
auto i = unknownSettings.find(alias);
if (i != unknownSettings.end()) {
if (set)
warn("setting '%s' is set, but it's an alias of '%s' which is also set",
alias, setting->name);
else {
setting->set(i->second);
setting->overriden = true;
unknownSettings.erase(i);
set = true;
}
}
}
}
for (auto & alias : setting->aliases) {
auto i = unknownSettings.find(alias);
if (i != unknownSettings.end()) {
if (set)
warn("setting '%s' is set, but it's an alias of '%s' which is also set",
alias, setting->name);
else {
setting->set(i->second);
setting->overriden = true;
unknownSettings.erase(i);
set = true;
}
void AbstractConfig::warnUnknownSettings() {
for (auto& s : unknownSettings) warn("unknown setting '%s'", s.first);
}
void AbstractConfig::reapplyUnknownSettings() {
auto unknownSettings2 = std::move(unknownSettings);
for (auto& s : unknownSettings2) set(s.first, s.second);
}
void Config::getSettings(std::map<std::string, SettingInfo>& res,
bool overridenOnly) {
for (auto& opt : _settings)
if (!opt.second.isAlias &&
(!overridenOnly || opt.second.setting->overriden))
res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(),
opt.second.setting->description});
}
void AbstractConfig::applyConfigFile(const Path& path) {
try {
string contents = readFile(path);
unsigned int pos = 0;
while (pos < contents.size()) {
string line;
while (pos < contents.size() && contents[pos] != '\n')
line += contents[pos++];
pos++;
string::size_type hash = line.find('#');
if (hash != string::npos) line = string(line, 0, hash);
vector<string> tokens = tokenizeString<vector<string> >(line);
if (tokens.empty()) continue;
if (tokens.size() < 2)
throw UsageError("illegal configuration line '%1%' in '%2%'", line,
path);
auto include = false;
auto ignoreMissing = false;
if (tokens[0] == "include")
include = true;
else if (tokens[0] == "!include") {
include = true;
ignoreMissing = true;
}
if (include) {
if (tokens.size() != 2)
throw UsageError("illegal configuration line '%1%' in '%2%'", line,
path);
auto p = absPath(tokens[1], dirOf(path));
if (pathExists(p)) {
applyConfigFile(p);
} else if (!ignoreMissing) {
throw Error("file '%1%' included from '%2%' not found", p, path);
}
continue;
}
if (tokens[1] != "=")
throw UsageError("illegal configuration line '%1%' in '%2%'", line,
path);
string name = tokens[0];
vector<string>::iterator i = tokens.begin();
advance(i, 2);
set(name,
concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
};
} catch (SysError&) {
}
}
void Config::resetOverriden() {
for (auto& s : _settings) s.second.setting->overriden = false;
}
void Config::toJSON(JSONObject& out) {
for (auto& s : _settings)
if (!s.second.isAlias) {
JSONObject out2(out.object(s.first));
out2.attr("description", s.second.setting->description);
JSONPlaceholder out3(out2.placeholder("value"));
s.second.setting->toJSON(out3);
}
}
void AbstractConfig::warnUnknownSettings()
{
for (auto & s : unknownSettings)
warn("unknown setting '%s'", s.first);
void Config::convertToArgs(Args& args, const std::string& category) {
for (auto& s : _settings)
if (!s.second.isAlias) s.second.setting->convertToArg(args, category);
}
void AbstractConfig::reapplyUnknownSettings()
{
auto unknownSettings2 = std::move(unknownSettings);
for (auto & s : unknownSettings2)
set(s.first, s.second);
AbstractSetting::AbstractSetting(const std::string& name,
const std::string& description,
const std::set<std::string>& aliases)
: name(name), description(description), aliases(aliases) {}
void AbstractSetting::toJSON(JSONPlaceholder& out) { out.write(to_string()); }
void AbstractSetting::convertToArg(Args& args, const std::string& category) {}
template <typename T>
void BaseSetting<T>::toJSON(JSONPlaceholder& out) {
out.write(value);
}
void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
{
for (auto & opt : _settings)
if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
template <typename T>
void BaseSetting<T>::convertToArg(Args& args, const std::string& category) {
args.mkFlag()
.longName(name)
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) {
overriden = true;
set(ss[0]);
})
.category(category);
}
void AbstractConfig::applyConfigFile(const Path & path)
{
try {
string contents = readFile(path);
unsigned int pos = 0;
while (pos < contents.size()) {
string line;
while (pos < contents.size() && contents[pos] != '\n')
line += contents[pos++];
pos++;
string::size_type hash = line.find('#');
if (hash != string::npos)
line = string(line, 0, hash);
vector<string> tokens = tokenizeString<vector<string> >(line);
if (tokens.empty()) continue;
if (tokens.size() < 2)
throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
auto include = false;
auto ignoreMissing = false;
if (tokens[0] == "include")
include = true;
else if (tokens[0] == "!include") {
include = true;
ignoreMissing = true;
}
if (include) {
if (tokens.size() != 2)
throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
auto p = absPath(tokens[1], dirOf(path));
if (pathExists(p)) {
applyConfigFile(p);
} else if (!ignoreMissing) {
throw Error("file '%1%' included from '%2%' not found", p, path);
}
continue;
}
if (tokens[1] != "=")
throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
string name = tokens[0];
vector<string>::iterator i = tokens.begin();
advance(i, 2);
set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
};
} catch (SysError &) { }
template <>
void BaseSetting<std::string>::set(const std::string& str) {
value = str;
}
void Config::resetOverriden()
{
for (auto & s : _settings)
s.second.setting->overriden = false;
template <>
std::string BaseSetting<std::string>::to_string() {
return value;
}
void Config::toJSON(JSONObject & out)
{
for (auto & s : _settings)
if (!s.second.isAlias) {
JSONObject out2(out.object(s.first));
out2.attr("description", s.second.setting->description);
JSONPlaceholder out3(out2.placeholder("value"));
s.second.setting->toJSON(out3);
}
template <typename T>
void BaseSetting<T>::set(const std::string& str) {
static_assert(std::is_integral<T>::value, "Integer required.");
if (!string2Int(str, value))
throw UsageError("setting '%s' has invalid value '%s'", name, str);
}
void Config::convertToArgs(Args & args, const std::string & category)
{
for (auto & s : _settings)
if (!s.second.isAlias)
s.second.setting->convertToArg(args, category);
template <typename T>
std::string BaseSetting<T>::to_string() {
static_assert(std::is_integral<T>::value, "Integer required.");
return std::to_string(value);
}
AbstractSetting::AbstractSetting(
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases)
: name(name), description(description), aliases(aliases)
{
template <>
void BaseSetting<bool>::set(const std::string& str) {
if (str == "true" || str == "yes" || str == "1")
value = true;
else if (str == "false" || str == "no" || str == "0")
value = false;
else
throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
}
void AbstractSetting::toJSON(JSONPlaceholder & out)
{
out.write(to_string());
template <>
std::string BaseSetting<bool>::to_string() {
return value ? "true" : "false";
}
void AbstractSetting::convertToArg(Args & args, const std::string & category)
{
template <>
void BaseSetting<bool>::convertToArg(Args& args, const std::string& category) {
args.mkFlag()
.longName(name)
.description(description)
.handler([=](std::vector<std::string> ss) { override(true); })
.category(category);
args.mkFlag()
.longName("no-" + name)
.description(description)
.handler([=](std::vector<std::string> ss) { override(false); })
.category(category);
}
template<typename T>
void BaseSetting<T>::toJSON(JSONPlaceholder & out)
{
out.write(value);
template <>
void BaseSetting<Strings>::set(const std::string& str) {
value = tokenizeString<Strings>(str);
}
template<typename T>
void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
{
args.mkFlag()
.longName(name)
.description(description)
.arity(1)
.handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); })
.category(category);
template <>
std::string BaseSetting<Strings>::to_string() {
return concatStringsSep(" ", value);
}
template<> void BaseSetting<std::string>::set(const std::string & str)
{
value = str;
template <>
void BaseSetting<Strings>::toJSON(JSONPlaceholder& out) {
JSONList list(out.list());
for (auto& s : value) list.elem(s);
}
template<> std::string BaseSetting<std::string>::to_string()
{
return value;
template <>
void BaseSetting<StringSet>::set(const std::string& str) {
value = tokenizeString<StringSet>(str);
}
template<typename T>
void BaseSetting<T>::set(const std::string & str)
{
static_assert(std::is_integral<T>::value, "Integer required.");
if (!string2Int(str, value))
throw UsageError("setting '%s' has invalid value '%s'", name, str);
template <>
std::string BaseSetting<StringSet>::to_string() {
return concatStringsSep(" ", value);
}
template<typename T>
std::string BaseSetting<T>::to_string()
{
static_assert(std::is_integral<T>::value, "Integer required.");
return std::to_string(value);
}
template<> void BaseSetting<bool>::set(const std::string & str)
{
if (str == "true" || str == "yes" || str == "1")
value = true;
else if (str == "false" || str == "no" || str == "0")
value = false;
else
throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
}
template<> std::string BaseSetting<bool>::to_string()
{
return value ? "true" : "false";
}
template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category)
{
args.mkFlag()
.longName(name)
.description(description)
.handler([=](std::vector<std::string> ss) { override(true); })
.category(category);
args.mkFlag()
.longName("no-" + name)
.description(description)
.handler([=](std::vector<std::string> ss) { override(false); })
.category(category);
}
template<> void BaseSetting<Strings>::set(const std::string & str)
{
value = tokenizeString<Strings>(str);
}
template<> std::string BaseSetting<Strings>::to_string()
{
return concatStringsSep(" ", value);
}
template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out)
{
JSONList list(out.list());
for (auto & s : value)
list.elem(s);
}
template<> void BaseSetting<StringSet>::set(const std::string & str)
{
value = tokenizeString<StringSet>(str);
}
template<> std::string BaseSetting<StringSet>::to_string()
{
return concatStringsSep(" ", value);
}
template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out)
{
JSONList list(out.list());
for (auto & s : value)
list.elem(s);
template <>
void BaseSetting<StringSet>::toJSON(JSONPlaceholder& out) {
JSONList list(out.list());
for (auto& s : value) list.elem(s);
}
template class BaseSetting<int>;
@ -279,60 +262,51 @@ template class BaseSetting<std::string>;
template class BaseSetting<Strings>;
template class BaseSetting<StringSet>;
void PathSetting::set(const std::string & str)
{
if (str == "") {
if (allowEmpty)
value = "";
else
throw UsageError("setting '%s' cannot be empty", name);
} else
value = canonPath(str);
void PathSetting::set(const std::string& str) {
if (str == "") {
if (allowEmpty)
value = "";
else
throw UsageError("setting '%s' cannot be empty", name);
} else
value = canonPath(str);
}
bool GlobalConfig::set(const std::string & name, const std::string & value)
{
for (auto & config : *configRegistrations)
if (config->set(name, value)) return true;
bool GlobalConfig::set(const std::string& name, const std::string& value) {
for (auto& config : *configRegistrations)
if (config->set(name, value)) return true;
unknownSettings.emplace(name, value);
unknownSettings.emplace(name, value);
return false;
return false;
}
void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
{
for (auto & config : *configRegistrations)
config->getSettings(res, overridenOnly);
void GlobalConfig::getSettings(std::map<std::string, SettingInfo>& res,
bool overridenOnly) {
for (auto& config : *configRegistrations)
config->getSettings(res, overridenOnly);
}
void GlobalConfig::resetOverriden()
{
for (auto & config : *configRegistrations)
config->resetOverriden();
void GlobalConfig::resetOverriden() {
for (auto& config : *configRegistrations) config->resetOverriden();
}
void GlobalConfig::toJSON(JSONObject & out)
{
for (auto & config : *configRegistrations)
config->toJSON(out);
void GlobalConfig::toJSON(JSONObject& out) {
for (auto& config : *configRegistrations) config->toJSON(out);
}
void GlobalConfig::convertToArgs(Args & args, const std::string & category)
{
for (auto & config : *configRegistrations)
config->convertToArgs(args, category);
void GlobalConfig::convertToArgs(Args& args, const std::string& category) {
for (auto& config : *configRegistrations)
config->convertToArgs(args, category);
}
GlobalConfig globalConfig;
GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations;
GlobalConfig::ConfigRegistrations* GlobalConfig::configRegistrations;
GlobalConfig::Register::Register(Config * config)
{
if (!configRegistrations)
configRegistrations = new ConfigRegistrations;
configRegistrations->emplace_back(config);
GlobalConfig::Register::Register(Config* config) {
if (!configRegistrations) configRegistrations = new ConfigRegistrations;
configRegistrations->emplace_back(config);
}
}
} // namespace nix

291
third_party/nix/src/libutil/config.hh vendored
View file

@ -1,6 +1,5 @@
#include <map>
#include <set>
#include "types.hh"
#pragma once
@ -12,38 +11,34 @@ class AbstractSetting;
class JSONPlaceholder;
class JSONObject;
class AbstractConfig
{
protected:
StringMap unknownSettings;
class AbstractConfig {
protected:
StringMap unknownSettings;
AbstractConfig(const StringMap & initials = {})
: unknownSettings(initials)
{ }
AbstractConfig(const StringMap& initials = {}) : unknownSettings(initials) {}
public:
public:
virtual bool set(const std::string& name, const std::string& value) = 0;
virtual bool set(const std::string & name, const std::string & value) = 0;
struct SettingInfo {
std::string value;
std::string description;
};
struct SettingInfo
{
std::string value;
std::string description;
};
virtual void getSettings(std::map<std::string, SettingInfo>& res,
bool overridenOnly = false) = 0;
virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
void applyConfigFile(const Path& path);
void applyConfigFile(const Path & path);
virtual void resetOverriden() = 0;
virtual void resetOverriden() = 0;
virtual void toJSON(JSONObject& out) = 0;
virtual void toJSON(JSONObject & out) = 0;
virtual void convertToArgs(Args& args, const std::string& category) = 0;
virtual void convertToArgs(Args & args, const std::string & category) = 0;
void warnUnknownSettings();
void warnUnknownSettings();
void reapplyUnknownSettings();
void reapplyUnknownSettings();
};
/* A class to simplify providing configuration settings. The typical
@ -61,201 +56,171 @@ public:
};
*/
class Config : public AbstractConfig
{
friend class AbstractSetting;
class Config : public AbstractConfig {
friend class AbstractSetting;
public:
public:
struct SettingData {
bool isAlias;
AbstractSetting* setting;
SettingData(bool isAlias, AbstractSetting* setting)
: isAlias(isAlias), setting(setting) {}
};
struct SettingData
{
bool isAlias;
AbstractSetting * setting;
SettingData(bool isAlias, AbstractSetting * setting)
: isAlias(isAlias), setting(setting)
{ }
};
typedef std::map<std::string, SettingData> Settings;
typedef std::map<std::string, SettingData> Settings;
private:
Settings _settings;
private:
public:
Config(const StringMap& initials = {}) : AbstractConfig(initials) {}
Settings _settings;
bool set(const std::string& name, const std::string& value) override;
public:
void addSetting(AbstractSetting* setting);
Config(const StringMap & initials = {})
: AbstractConfig(initials)
{ }
void getSettings(std::map<std::string, SettingInfo>& res,
bool overridenOnly = false) override;
bool set(const std::string & name, const std::string & value) override;
void resetOverriden() override;
void addSetting(AbstractSetting * setting);
void toJSON(JSONObject& out) override;
void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
void resetOverriden() override;
void toJSON(JSONObject & out) override;
void convertToArgs(Args & args, const std::string & category) override;
void convertToArgs(Args& args, const std::string& category) override;
};
class AbstractSetting
{
friend class Config;
class AbstractSetting {
friend class Config;
public:
public:
const std::string name;
const std::string description;
const std::set<std::string> aliases;
const std::string name;
const std::string description;
const std::set<std::string> aliases;
int created = 123;
int created = 123;
bool overriden = false;
bool overriden = false;
protected:
AbstractSetting(const std::string& name, const std::string& description,
const std::set<std::string>& aliases);
protected:
virtual ~AbstractSetting() {
// Check against a gcc miscompilation causing our constructor
// not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431).
assert(created == 123);
}
AbstractSetting(
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases);
virtual void set(const std::string& value) = 0;
virtual ~AbstractSetting()
{
// Check against a gcc miscompilation causing our constructor
// not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431).
assert(created == 123);
}
virtual std::string to_string() = 0;
virtual void set(const std::string & value) = 0;
virtual void toJSON(JSONPlaceholder& out);
virtual std::string to_string() = 0;
virtual void convertToArg(Args& args, const std::string& category);
virtual void toJSON(JSONPlaceholder & out);
virtual void convertToArg(Args & args, const std::string & category);
bool isOverriden() { return overriden; }
bool isOverriden() { return overriden; }
};
/* A setting of type T. */
template<typename T>
class BaseSetting : public AbstractSetting
{
protected:
template <typename T>
class BaseSetting : public AbstractSetting {
protected:
T value;
T value;
public:
BaseSetting(const T& def, const std::string& name,
const std::string& description,
const std::set<std::string>& aliases = {})
: AbstractSetting(name, description, aliases), value(def) {}
public:
operator const T&() const { return value; }
operator T&() { return value; }
const T& get() const { return value; }
bool operator==(const T& v2) const { return value == v2; }
bool operator!=(const T& v2) const { return value != v2; }
void operator=(const T& v) { assign(v); }
virtual void assign(const T& v) { value = v; }
BaseSetting(const T & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: AbstractSetting(name, description, aliases)
, value(def)
{ }
void set(const std::string& str) override;
operator const T &() const { return value; }
operator T &() { return value; }
const T & get() const { return value; }
bool operator ==(const T & v2) const { return value == v2; }
bool operator !=(const T & v2) const { return value != v2; }
void operator =(const T & v) { assign(v); }
virtual void assign(const T & v) { value = v; }
virtual void override(const T& v) {
overriden = true;
value = v;
}
void set(const std::string & str) override;
std::string to_string() override;
virtual void override(const T & v)
{
overriden = true;
value = v;
}
void convertToArg(Args& args, const std::string& category) override;
std::string to_string() override;
void convertToArg(Args & args, const std::string & category) override;
void toJSON(JSONPlaceholder & out) override;
void toJSON(JSONPlaceholder& out) override;
};
template<typename T>
std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt)
{
str << (const T &) opt;
return str;
template <typename T>
std::ostream& operator<<(std::ostream& str, const BaseSetting<T>& opt) {
str << (const T&)opt;
return str;
}
template<typename T>
bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; }
template <typename T>
bool operator==(const T& v1, const BaseSetting<T>& v2) {
return v1 == (const T&)v2;
}
template<typename T>
class Setting : public BaseSetting<T>
{
public:
Setting(Config * options,
const T & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<T>(def, name, description, aliases)
{
options->addSetting(this);
}
template <typename T>
class Setting : public BaseSetting<T> {
public:
Setting(Config* options, const T& def, const std::string& name,
const std::string& description,
const std::set<std::string>& aliases = {})
: BaseSetting<T>(def, name, description, aliases) {
options->addSetting(this);
}
void operator =(const T & v) { this->assign(v); }
void operator=(const T& v) { this->assign(v); }
};
/* A special setting for Paths. These are automatically canonicalised
(e.g. "/foo//bar/" becomes "/foo/bar"). */
class PathSetting : public BaseSetting<Path>
{
bool allowEmpty;
class PathSetting : public BaseSetting<Path> {
bool allowEmpty;
public:
public:
PathSetting(Config* options, bool allowEmpty, const Path& def,
const std::string& name, const std::string& description,
const std::set<std::string>& aliases = {})
: BaseSetting<Path>(def, name, description, aliases),
allowEmpty(allowEmpty) {
options->addSetting(this);
}
PathSetting(Config * options,
bool allowEmpty,
const Path & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<Path>(def, name, description, aliases)
, allowEmpty(allowEmpty)
{
options->addSetting(this);
}
void set(const std::string& str) override;
void set(const std::string & str) override;
Path operator+(const char* p) const { return value + p; }
Path operator +(const char * p) const { return value + p; }
void operator =(const Path & v) { this->assign(v); }
void operator=(const Path& v) { this->assign(v); }
};
struct GlobalConfig : public AbstractConfig
{
typedef std::vector<Config*> ConfigRegistrations;
static ConfigRegistrations * configRegistrations;
struct GlobalConfig : public AbstractConfig {
typedef std::vector<Config*> ConfigRegistrations;
static ConfigRegistrations* configRegistrations;
bool set(const std::string & name, const std::string & value) override;
bool set(const std::string& name, const std::string& value) override;
void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
void getSettings(std::map<std::string, SettingInfo>& res,
bool overridenOnly = false) override;
void resetOverriden() override;
void resetOverriden() override;
void toJSON(JSONObject & out) override;
void toJSON(JSONObject& out) override;
void convertToArgs(Args & args, const std::string & category) override;
void convertToArgs(Args& args, const std::string& category) override;
struct Register
{
Register(Config * config);
};
struct Register {
Register(Config* config);
};
};
extern GlobalConfig globalConfig;
}
} // namespace nix

13
third_party/nix/src/libutil/finally.hh vendored
View file

@ -3,12 +3,11 @@
#include <functional>
/* A trivial class to run a function at the end of a scope. */
class Finally
{
private:
std::function<void()> fun;
class Finally {
private:
std::function<void()> fun;
public:
Finally(std::function<void()> fun) : fun(fun) { }
~Finally() { fun(); }
public:
Finally(std::function<void()> fun) : fun(fun) {}
~Finally() { fun(); }
};

526
third_party/nix/src/libutil/hash.cc vendored
View file

@ -1,355 +1,323 @@
#include <iostream>
#include <cstring>
#include "hash.hh"
#include <fcntl.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
#include "hash.hh"
#include "archive.hh"
#include "util.hh"
#include "istringstream_nocopy.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <cstring>
#include <iostream>
#include "archive.hh"
#include "istringstream_nocopy.hh"
#include "util.hh"
namespace nix {
void Hash::init()
{
if (type == htMD5) hashSize = md5HashSize;
else if (type == htSHA1) hashSize = sha1HashSize;
else if (type == htSHA256) hashSize = sha256HashSize;
else if (type == htSHA512) hashSize = sha512HashSize;
else abort();
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
void Hash::init() {
if (type == htMD5)
hashSize = md5HashSize;
else if (type == htSHA1)
hashSize = sha1HashSize;
else if (type == htSHA256)
hashSize = sha256HashSize;
else if (type == htSHA512)
hashSize = sha512HashSize;
else
abort();
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
}
bool Hash::operator == (const Hash & h2) const
{
if (hashSize != h2.hashSize) return false;
for (unsigned int i = 0; i < hashSize; i++)
if (hash[i] != h2.hash[i]) return false;
return true;
bool Hash::operator==(const Hash& h2) const {
if (hashSize != h2.hashSize) return false;
for (unsigned int i = 0; i < hashSize; i++)
if (hash[i] != h2.hash[i]) return false;
return true;
}
bool Hash::operator!=(const Hash& h2) const { return !(*this == h2); }
bool Hash::operator != (const Hash & h2) const
{
return !(*this == h2);
bool Hash::operator<(const Hash& h) const {
if (hashSize < h.hashSize) return true;
if (hashSize > h.hashSize) return false;
for (unsigned int i = 0; i < hashSize; i++) {
if (hash[i] < h.hash[i]) return true;
if (hash[i] > h.hash[i]) return false;
}
return false;
}
bool Hash::operator < (const Hash & h) const
{
if (hashSize < h.hashSize) return true;
if (hashSize > h.hashSize) return false;
for (unsigned int i = 0; i < hashSize; i++) {
if (hash[i] < h.hash[i]) return true;
if (hash[i] > h.hash[i]) return false;
}
return false;
}
const string base16Chars = "0123456789abcdef";
static string printHash16(const Hash & hash)
{
char buf[hash.hashSize * 2];
for (unsigned int i = 0; i < hash.hashSize; i++) {
buf[i * 2] = base16Chars[hash.hash[i] >> 4];
buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f];
}
return string(buf, hash.hashSize * 2);
static string printHash16(const Hash& hash) {
char buf[hash.hashSize * 2];
for (unsigned int i = 0; i < hash.hashSize; i++) {
buf[i * 2] = base16Chars[hash.hash[i] >> 4];
buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f];
}
return string(buf, hash.hashSize * 2);
}
// omitted: E O U T
const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
static string printHash32(const Hash& hash) {
assert(hash.hashSize);
size_t len = hash.base32Len();
assert(len);
static string printHash32(const Hash & hash)
{
assert(hash.hashSize);
size_t len = hash.base32Len();
assert(len);
string s;
s.reserve(len);
string s;
s.reserve(len);
for (int n = (int)len - 1; n >= 0; n--) {
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
unsigned char c =
(hash.hash[i] >> j) |
(i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
s.push_back(base32Chars[c & 0x1f]);
}
for (int n = (int) len - 1; n >= 0; n--) {
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
unsigned char c =
(hash.hash[i] >> j)
| (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
s.push_back(base32Chars[c & 0x1f]);
}
return s;
return s;
}
string printHash16or32(const Hash & hash)
{
return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
string printHash16or32(const Hash& hash) {
return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
}
std::string Hash::to_string(Base base, bool includeType) const
{
std::string s;
if (base == SRI || includeType) {
s += printHashType(type);
s += base == SRI ? '-' : ':';
}
switch (base) {
std::string Hash::to_string(Base base, bool includeType) const {
std::string s;
if (base == SRI || includeType) {
s += printHashType(type);
s += base == SRI ? '-' : ':';
}
switch (base) {
case Base16:
s += printHash16(*this);
break;
s += printHash16(*this);
break;
case Base32:
s += printHash32(*this);
break;
s += printHash32(*this);
break;
case Base64:
case SRI:
s += base64Encode(std::string((const char *) hash, hashSize));
break;
}
return s;
s += base64Encode(std::string((const char*)hash, hashSize));
break;
}
return s;
}
Hash::Hash(const std::string& s, HashType type) : type(type) {
size_t pos = 0;
bool isSRI = false;
Hash::Hash(const std::string & s, HashType type)
: type(type)
{
size_t pos = 0;
bool isSRI = false;
auto sep = s.find(':');
if (sep == string::npos) {
sep = s.find('-');
if (sep != string::npos) {
isSRI = true;
} else if (type == htUnknown)
throw BadHash("hash '%s' does not include a type", s);
}
auto sep = s.find(':');
if (sep == string::npos) {
sep = s.find('-');
if (sep != string::npos) {
string hts = string(s, 0, sep);
this->type = parseHashType(hts);
if (this->type == htUnknown)
throw BadHash("unknown hash type '%s'", hts);
if (type != htUnknown && type != this->type)
throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
pos = sep + 1;
isSRI = true;
} else if (type == htUnknown)
throw BadHash("hash '%s' does not include a type", s);
}
if (sep != string::npos) {
string hts = string(s, 0, sep);
this->type = parseHashType(hts);
if (this->type == htUnknown) throw BadHash("unknown hash type '%s'", hts);
if (type != htUnknown && type != this->type)
throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
pos = sep + 1;
}
init();
size_t size = s.size() - pos;
if (!isSRI && size == base16Len()) {
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
throw BadHash("invalid base-16 hash '%s'", s);
};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] = parseHexDigit(s[pos + i * 2]) << 4 |
parseHexDigit(s[pos + i * 2 + 1]);
}
}
init();
else if (!isSRI && size == base32Len()) {
for (unsigned int n = 0; n < size; ++n) {
char c = s[pos + size - n - 1];
unsigned char digit;
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", s);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
hash[i] |= digit << j;
size_t size = s.size() - pos;
if (!isSRI && size == base16Len()) {
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
throw BadHash("invalid base-16 hash '%s'", s);
};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] =
parseHexDigit(s[pos + i * 2]) << 4
| parseHexDigit(s[pos + i * 2 + 1]);
}
if (i < hashSize - 1) {
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j)) throw BadHash("invalid base-32 hash '%s'", s);
}
}
}
else if (!isSRI && size == base32Len()) {
else if (isSRI || size == base64Len()) {
auto d = base64Decode(std::string(s, pos));
if (d.size() != hashSize)
throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
for (unsigned int n = 0; n < size; ++n) {
char c = s[pos + size - n - 1];
unsigned char digit;
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32)
throw BadHash("invalid base-32 hash '%s'", s);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
hash[i] |= digit << j;
if (i < hashSize - 1) {
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j))
throw BadHash("invalid base-32 hash '%s'", s);
}
}
}
else if (isSRI || size == base64Len()) {
auto d = base64Decode(std::string(s, pos));
if (d.size() != hashSize)
throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
else
throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
else
throw BadHash("hash '%s' has wrong length for hash type '%s'", s,
printHashType(type));
}
union Ctx
{
MD5_CTX md5;
SHA_CTX sha1;
SHA256_CTX sha256;
SHA512_CTX sha512;
union Ctx {
MD5_CTX md5;
SHA_CTX sha1;
SHA256_CTX sha256;
SHA512_CTX sha512;
};
static void start(HashType ht, Ctx & ctx)
{
if (ht == htMD5) MD5_Init(&ctx.md5);
else if (ht == htSHA1) SHA1_Init(&ctx.sha1);
else if (ht == htSHA256) SHA256_Init(&ctx.sha256);
else if (ht == htSHA512) SHA512_Init(&ctx.sha512);
static void start(HashType ht, Ctx& ctx) {
if (ht == htMD5)
MD5_Init(&ctx.md5);
else if (ht == htSHA1)
SHA1_Init(&ctx.sha1);
else if (ht == htSHA256)
SHA256_Init(&ctx.sha256);
else if (ht == htSHA512)
SHA512_Init(&ctx.sha512);
}
static void update(HashType ht, Ctx & ctx,
const unsigned char * bytes, size_t len)
{
if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len);
else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len);
else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len);
else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len);
static void update(HashType ht, Ctx& ctx, const unsigned char* bytes,
size_t len) {
if (ht == htMD5)
MD5_Update(&ctx.md5, bytes, len);
else if (ht == htSHA1)
SHA1_Update(&ctx.sha1, bytes, len);
else if (ht == htSHA256)
SHA256_Update(&ctx.sha256, bytes, len);
else if (ht == htSHA512)
SHA512_Update(&ctx.sha512, bytes, len);
}
static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
{
if (ht == htMD5) MD5_Final(hash, &ctx.md5);
else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1);
else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256);
else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512);
static void finish(HashType ht, Ctx& ctx, unsigned char* hash) {
if (ht == htMD5)
MD5_Final(hash, &ctx.md5);
else if (ht == htSHA1)
SHA1_Final(hash, &ctx.sha1);
else if (ht == htSHA256)
SHA256_Final(hash, &ctx.sha256);
else if (ht == htSHA512)
SHA512_Final(hash, &ctx.sha512);
}
Hash hashString(HashType ht, const string & s)
{
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
update(ht, ctx, (const unsigned char *) s.data(), s.length());
finish(ht, ctx, hash.hash);
return hash;
Hash hashString(HashType ht, const string& s) {
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
update(ht, ctx, (const unsigned char*)s.data(), s.length());
finish(ht, ctx, hash.hash);
return hash;
}
Hash hashFile(HashType ht, const Path& path) {
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
Hash hashFile(HashType ht, const Path & path)
{
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
std::vector<unsigned char> buf(8192);
ssize_t n;
while ((n = read(fd.get(), buf.data(), buf.size()))) {
checkInterrupt();
if (n == -1) throw SysError(format("reading file '%1%'") % path);
update(ht, ctx, buf.data(), n);
}
std::vector<unsigned char> buf(8192);
ssize_t n;
while ((n = read(fd.get(), buf.data(), buf.size()))) {
checkInterrupt();
if (n == -1) throw SysError(format("reading file '%1%'") % path);
update(ht, ctx, buf.data(), n);
}
finish(ht, ctx, hash.hash);
return hash;
finish(ht, ctx, hash.hash);
return hash;
}
HashSink::HashSink(HashType ht) : ht(ht)
{
ctx = new Ctx;
bytes = 0;
start(ht, *ctx);
HashSink::HashSink(HashType ht) : ht(ht) {
ctx = new Ctx;
bytes = 0;
start(ht, *ctx);
}
HashSink::~HashSink()
{
bufPos = 0;
delete ctx;
HashSink::~HashSink() {
bufPos = 0;
delete ctx;
}
void HashSink::write(const unsigned char * data, size_t len)
{
bytes += len;
update(ht, *ctx, data, len);
void HashSink::write(const unsigned char* data, size_t len) {
bytes += len;
update(ht, *ctx, data, len);
}
HashResult HashSink::finish()
{
flush();
Hash hash(ht);
nix::finish(ht, *ctx, hash.hash);
return HashResult(hash, bytes);
HashResult HashSink::finish() {
flush();
Hash hash(ht);
nix::finish(ht, *ctx, hash.hash);
return HashResult(hash, bytes);
}
HashResult HashSink::currentHash()
{
flush();
Ctx ctx2 = *ctx;
Hash hash(ht);
nix::finish(ht, ctx2, hash.hash);
return HashResult(hash, bytes);
HashResult HashSink::currentHash() {
flush();
Ctx ctx2 = *ctx;
Hash hash(ht);
nix::finish(ht, ctx2, hash.hash);
return HashResult(hash, bytes);
}
HashResult hashPath(
HashType ht, const Path & path, PathFilter & filter)
{
HashSink sink(ht);
dumpPath(path, sink, filter);
return sink.finish();
HashResult hashPath(HashType ht, const Path& path, PathFilter& filter) {
HashSink sink(ht);
dumpPath(path, sink, filter);
return sink.finish();
}
Hash compressHash(const Hash & hash, unsigned int newSize)
{
Hash h;
h.hashSize = newSize;
for (unsigned int i = 0; i < hash.hashSize; ++i)
h.hash[i % newSize] ^= hash.hash[i];
return h;
Hash compressHash(const Hash& hash, unsigned int newSize) {
Hash h;
h.hashSize = newSize;
for (unsigned int i = 0; i < hash.hashSize; ++i)
h.hash[i % newSize] ^= hash.hash[i];
return h;
}
HashType parseHashType(const string & s)
{
if (s == "md5") return htMD5;
else if (s == "sha1") return htSHA1;
else if (s == "sha256") return htSHA256;
else if (s == "sha512") return htSHA512;
else return htUnknown;
HashType parseHashType(const string& s) {
if (s == "md5")
return htMD5;
else if (s == "sha1")
return htSHA1;
else if (s == "sha256")
return htSHA256;
else if (s == "sha512")
return htSHA512;
else
return htUnknown;
}
string printHashType(HashType ht)
{
if (ht == htMD5) return "md5";
else if (ht == htSHA1) return "sha1";
else if (ht == htSHA256) return "sha256";
else if (ht == htSHA512) return "sha512";
else abort();
string printHashType(HashType ht) {
if (ht == htMD5)
return "md5";
else if (ht == htSHA1)
return "sha1";
else if (ht == htSHA256)
return "sha256";
else if (ht == htSHA512)
return "sha512";
else
abort();
}
}
} // namespace nix

129
third_party/nix/src/libutil/hash.hh vendored
View file

@ -1,18 +1,14 @@
#pragma once
#include "types.hh"
#include "serialise.hh"
#include "types.hh"
namespace nix {
MakeError(BadHash, Error);
enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
const int md5HashSize = 16;
const int sha1HashSize = 20;
const int sha256HashSize = 32;
@ -22,110 +18,95 @@ extern const string base32Chars;
enum Base : int { Base64, Base32, Base16, SRI };
struct Hash {
static const unsigned int maxHashSize = 64;
unsigned int hashSize = 0;
unsigned char hash[maxHashSize] = {};
struct Hash
{
static const unsigned int maxHashSize = 64;
unsigned int hashSize = 0;
unsigned char hash[maxHashSize] = {};
HashType type = htUnknown;
HashType type = htUnknown;
/* Create an unset hash object. */
Hash(){};
/* Create an unset hash object. */
Hash() { };
/* Create a zero-filled hash object. */
Hash(HashType type) : type(type) { init(); };
/* Create a zero-filled hash object. */
Hash(HashType type) : type(type) { init(); };
/* Initialize the hash from a string representation, in the format
"[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
Subresource Integrity hash expression). If the 'type' argument
is htUnknown, then the hash type must be specified in the
string. */
Hash(const std::string& s, HashType type = htUnknown);
/* Initialize the hash from a string representation, in the format
"[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
Subresource Integrity hash expression). If the 'type' argument
is htUnknown, then the hash type must be specified in the
string. */
Hash(const std::string & s, HashType type = htUnknown);
void init();
void init();
/* Check whether a hash is set. */
operator bool() const { return type != htUnknown; }
/* Check whether a hash is set. */
operator bool () const { return type != htUnknown; }
/* Check whether two hash are equal. */
bool operator==(const Hash& h2) const;
/* Check whether two hash are equal. */
bool operator == (const Hash & h2) const;
/* Check whether two hash are not equal. */
bool operator!=(const Hash& h2) const;
/* Check whether two hash are not equal. */
bool operator != (const Hash & h2) const;
/* For sorting. */
bool operator<(const Hash& h) const;
/* For sorting. */
bool operator < (const Hash & h) const;
/* Returns the length of a base-16 representation of this hash. */
size_t base16Len() const { return hashSize * 2; }
/* Returns the length of a base-16 representation of this hash. */
size_t base16Len() const
{
return hashSize * 2;
}
/* Returns the length of a base-32 representation of this hash. */
size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; }
/* Returns the length of a base-32 representation of this hash. */
size_t base32Len() const
{
return (hashSize * 8 - 1) / 5 + 1;
}
/* Returns the length of a base-64 representation of this hash. */
size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; }
/* Returns the length of a base-64 representation of this hash. */
size_t base64Len() const
{
return ((4 * hashSize / 3) + 3) & ~3;
}
/* Return a string representation of the hash, in base-16, base-32
or base-64. By default, this is prefixed by the hash type
(e.g. "sha256:"). */
std::string to_string(Base base = Base32, bool includeType = true) const;
/* Return a string representation of the hash, in base-16, base-32
or base-64. By default, this is prefixed by the hash type
(e.g. "sha256:"). */
std::string to_string(Base base = Base32, bool includeType = true) const;
};
/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
string printHash16or32(const Hash & hash);
string printHash16or32(const Hash& hash);
/* Compute the hash of the given string. */
Hash hashString(HashType ht, const string & s);
Hash hashString(HashType ht, const string& s);
/* Compute the hash of the given file. */
Hash hashFile(HashType ht, const Path & path);
Hash hashFile(HashType ht, const Path& path);
/* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */
typedef std::pair<Hash, unsigned long long> HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
HashResult hashPath(HashType ht, const Path& path,
PathFilter& filter = defaultPathFilter);
/* Compress a hash to the specified number of bytes by cyclically
XORing bytes together. */
Hash compressHash(const Hash & hash, unsigned int newSize);
Hash compressHash(const Hash& hash, unsigned int newSize);
/* Parse a string representing a hash type. */
HashType parseHashType(const string & s);
HashType parseHashType(const string& s);
/* And the reverse. */
string printHashType(HashType ht);
union Ctx;
class HashSink : public BufferedSink
{
private:
HashType ht;
Ctx * ctx;
unsigned long long bytes;
class HashSink : public BufferedSink {
private:
HashType ht;
Ctx* ctx;
unsigned long long bytes;
public:
HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink();
void write(const unsigned char * data, size_t len);
HashResult finish();
HashResult currentHash();
public:
HashSink(HashType ht);
HashSink(const HashSink& h);
~HashSink();
void write(const unsigned char* data, size_t len);
HashResult finish();
HashResult currentHash();
};
}
} // namespace nix

118
third_party/nix/src/libutil/istringstream_nocopy.hh vendored
View file

@ -5,88 +5,78 @@
#pragma once
#include <string>
#include <iostream>
#include <string>
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
{
public:
typedef std::basic_string<CharT, Traits, Allocator> string_type;
template <class CharT, class Traits = std::char_traits<CharT>,
class Allocator = std::allocator<CharT>>
class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> {
public:
typedef std::basic_string<CharT, Traits, Allocator> string_type;
typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
private:
const string_type & s;
private:
const string_type& s;
off_type off;
off_type off;
public:
basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
{
public:
basic_istringbuf_nocopy(const string_type& s) : s{s}, off{0} {}
private:
pos_type seekoff(off_type off, std::ios_base::seekdir dir,
std::ios_base::openmode which) {
if (which & std::ios_base::in) {
this->off =
dir == std::ios_base::beg
? off
: (dir == std::ios_base::end ? s.size() + off : this->off + off);
}
return pos_type(this->off);
}
private:
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
{
if (which & std::ios_base::in) {
this->off = dir == std::ios_base::beg
? off
: (dir == std::ios_base::end
? s.size() + off
: this->off + off);
}
return pos_type(this->off);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which) {
return seekoff(pos, std::ios_base::beg, which);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which)
{
return seekoff(pos, std::ios_base::beg, which);
}
std::streamsize showmanyc() { return s.size() - off; }
std::streamsize showmanyc()
{
return s.size() - off;
}
int_type underflow() {
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off]);
}
int_type underflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off]);
}
int_type uflow() {
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off++]);
}
int_type uflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off++]);
}
int_type pbackfail(int_type ch)
{
if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
return traits_type::eof();
return traits_type::to_int_type(s[--off]);
}
int_type pbackfail(int_type ch) {
if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
return traits_type::eof();
return traits_type::to_int_type(s[--off]);
}
};
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
{
typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
buf_type buf;
public:
basic_istringstream_nocopy(const typename buf_type::string_type & s) :
std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
template <class CharT, class Traits = std::char_traits<CharT>,
class Allocator = std::allocator<CharT>>
class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> {
typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
buf_type buf;
public:
basic_istringstream_nocopy(const typename buf_type::string_type& s)
: std::basic_iostream<CharT, Traits>(&buf), buf(s){};
};
typedef basic_istringstream_nocopy<char> istringstream_nocopy;

280
third_party/nix/src/libutil/json.cc vendored
View file

@ -1,174 +1,184 @@
#include "json.hh"
#include <iomanip>
#include <cstring>
#include <iomanip>
namespace nix {
void toJSON(std::ostream & str, const char * start, const char * end)
{
str << '"';
for (auto i = start; i != end; i++)
if (*i == '\"' || *i == '\\') str << '\\' << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i >= 0 && *i < 32)
str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec;
else str << *i;
str << '"';
void toJSON(std::ostream& str, const char* start, const char* end) {
str << '"';
for (auto i = start; i != end; i++)
if (*i == '\"' || *i == '\\')
str << '\\' << *i;
else if (*i == '\n')
str << "\\n";
else if (*i == '\r')
str << "\\r";
else if (*i == '\t')
str << "\\t";
else if (*i >= 0 && *i < 32)
str << "\\u" << std::setfill('0') << std::setw(4) << std::hex
<< (uint16_t)*i << std::dec;
else
str << *i;
str << '"';
}
void toJSON(std::ostream & str, const char * s)
{
if (!s) str << "null"; else toJSON(str, s, s + strlen(s));
}
template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; }
template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; }
template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; }
template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
{
toJSON(str, s.c_str(), s.c_str() + s.size());
}
template<> void toJSON<bool>(std::ostream & str, const bool & b)
{
str << (b ? "true" : "false");
}
template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b)
{
void toJSON(std::ostream& str, const char* s) {
if (!s)
str << "null";
else
toJSON(str, s, s + strlen(s));
}
JSONWriter::JSONWriter(std::ostream & str, bool indent)
: state(new JSONState(str, indent))
{
state->stack++;
template <>
void toJSON<int>(std::ostream& str, const int& n) {
str << n;
}
template <>
void toJSON<unsigned int>(std::ostream& str, const unsigned int& n) {
str << n;
}
template <>
void toJSON<long>(std::ostream& str, const long& n) {
str << n;
}
template <>
void toJSON<unsigned long>(std::ostream& str, const unsigned long& n) {
str << n;
}
template <>
void toJSON<long long>(std::ostream& str, const long long& n) {
str << n;
}
template <>
void toJSON<unsigned long long>(std::ostream& str,
const unsigned long long& n) {
str << n;
}
template <>
void toJSON<float>(std::ostream& str, const float& n) {
str << n;
}
template <>
void toJSON<double>(std::ostream& str, const double& n) {
str << n;
}
JSONWriter::JSONWriter(JSONState * state)
: state(state)
{
state->stack++;
template <>
void toJSON<std::string>(std::ostream& str, const std::string& s) {
toJSON(str, s.c_str(), s.c_str() + s.size());
}
JSONWriter::~JSONWriter()
{
if (state) {
assertActive();
state->stack--;
if (state->stack == 0) delete state;
}
template <>
void toJSON<bool>(std::ostream& str, const bool& b) {
str << (b ? "true" : "false");
}
void JSONWriter::comma()
{
template <>
void toJSON<std::nullptr_t>(std::ostream& str, const std::nullptr_t& b) {
str << "null";
}
JSONWriter::JSONWriter(std::ostream& str, bool indent)
: state(new JSONState(str, indent)) {
state->stack++;
}
JSONWriter::JSONWriter(JSONState* state) : state(state) { state->stack++; }
JSONWriter::~JSONWriter() {
if (state) {
assertActive();
if (first) {
first = false;
} else {
state->str << ',';
}
if (state->indent) indent();
state->stack--;
if (state->stack == 0) delete state;
}
}
void JSONWriter::indent()
{
state->str << '\n' << std::string(state->depth * 2, ' ');
void JSONWriter::comma() {
assertActive();
if (first) {
first = false;
} else {
state->str << ',';
}
if (state->indent) indent();
}
void JSONList::open()
{
state->depth++;
state->str << '[';
void JSONWriter::indent() {
state->str << '\n' << std::string(state->depth * 2, ' ');
}
JSONList::~JSONList()
{
void JSONList::open() {
state->depth++;
state->str << '[';
}
JSONList::~JSONList() {
state->depth--;
if (state->indent && !first) indent();
state->str << "]";
}
JSONList JSONList::list() {
comma();
return JSONList(state);
}
JSONObject JSONList::object() {
comma();
return JSONObject(state);
}
JSONPlaceholder JSONList::placeholder() {
comma();
return JSONPlaceholder(state);
}
void JSONObject::open() {
state->depth++;
state->str << '{';
}
JSONObject::~JSONObject() {
if (state) {
state->depth--;
if (state->indent && !first) indent();
state->str << "]";
state->str << "}";
}
}
JSONList JSONList::list()
{
comma();
return JSONList(state);
void JSONObject::attr(const std::string& s) {
comma();
toJSON(state->str, s);
state->str << ':';
if (state->indent) state->str << ' ';
}
JSONObject JSONList::object()
{
comma();
return JSONObject(state);
JSONList JSONObject::list(const std::string& name) {
attr(name);
return JSONList(state);
}
JSONPlaceholder JSONList::placeholder()
{
comma();
return JSONPlaceholder(state);
JSONObject JSONObject::object(const std::string& name) {
attr(name);
return JSONObject(state);
}
void JSONObject::open()
{
state->depth++;
state->str << '{';
JSONPlaceholder JSONObject::placeholder(const std::string& name) {
attr(name);
return JSONPlaceholder(state);
}
JSONObject::~JSONObject()
{
if (state) {
state->depth--;
if (state->indent && !first) indent();
state->str << "}";
}
JSONList JSONPlaceholder::list() {
assertValid();
first = false;
return JSONList(state);
}
void JSONObject::attr(const std::string & s)
{
comma();
toJSON(state->str, s);
state->str << ':';
if (state->indent) state->str << ' ';
JSONObject JSONPlaceholder::object() {
assertValid();
first = false;
return JSONObject(state);
}
JSONList JSONObject::list(const std::string & name)
{
attr(name);
return JSONList(state);
}
JSONObject JSONObject::object(const std::string & name)
{
attr(name);
return JSONObject(state);
}
JSONPlaceholder JSONObject::placeholder(const std::string & name)
{
attr(name);
return JSONPlaceholder(state);
}
JSONList JSONPlaceholder::list()
{
assertValid();
first = false;
return JSONList(state);
}
JSONObject JSONPlaceholder::object()
{
assertValid();
first = false;
return JSONObject(state);
}
}
} // namespace nix

223
third_party/nix/src/libutil/json.hh vendored
View file

@ -1,189 +1,142 @@
#pragma once
#include <cassert>
#include <iostream>
#include <vector>
#include <cassert>
namespace nix {
void toJSON(std::ostream & str, const char * start, const char * end);
void toJSON(std::ostream & str, const char * s);
void toJSON(std::ostream& str, const char* start, const char* end);
void toJSON(std::ostream& str, const char* s);
template<typename T>
void toJSON(std::ostream & str, const T & n);
template <typename T>
void toJSON(std::ostream& str, const T& n);
class JSONWriter
{
protected:
class JSONWriter {
protected:
struct JSONState {
std::ostream& str;
bool indent;
size_t depth = 0;
size_t stack = 0;
JSONState(std::ostream& str, bool indent) : str(str), indent(indent) {}
~JSONState() { assert(stack == 0); }
};
struct JSONState
{
std::ostream & str;
bool indent;
size_t depth = 0;
size_t stack = 0;
JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
~JSONState()
{
assert(stack == 0);
}
};
JSONState* state;
JSONState * state;
bool first = true;
bool first = true;
JSONWriter(std::ostream& str, bool indent);
JSONWriter(std::ostream & str, bool indent);
JSONWriter(JSONState* state);
JSONWriter(JSONState * state);
~JSONWriter();
~JSONWriter();
void assertActive() { assert(state->stack != 0); }
void assertActive()
{
assert(state->stack != 0);
}
void comma();
void comma();
void indent();
void indent();
};
class JSONObject;
class JSONPlaceholder;
class JSONList : JSONWriter
{
private:
class JSONList : JSONWriter {
private:
friend class JSONObject;
friend class JSONPlaceholder;
friend class JSONObject;
friend class JSONPlaceholder;
void open();
void open();
JSONList(JSONState* state) : JSONWriter(state) { open(); }
JSONList(JSONState * state)
: JSONWriter(state)
{
open();
}
public:
JSONList(std::ostream& str, bool indent = false) : JSONWriter(str, indent) {
open();
}
public:
~JSONList();
JSONList(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
open();
}
template <typename T>
JSONList& elem(const T& v) {
comma();
toJSON(state->str, v);
return *this;
}
~JSONList();
JSONList list();
template<typename T>
JSONList & elem(const T & v)
{
comma();
toJSON(state->str, v);
return *this;
}
JSONObject object();
JSONList list();
JSONObject object();
JSONPlaceholder placeholder();
JSONPlaceholder placeholder();
};
class JSONObject : JSONWriter
{
private:
class JSONObject : JSONWriter {
private:
friend class JSONList;
friend class JSONPlaceholder;
friend class JSONList;
friend class JSONPlaceholder;
void open();
void open();
JSONObject(JSONState* state) : JSONWriter(state) { open(); }
JSONObject(JSONState * state)
: JSONWriter(state)
{
open();
}
void attr(const std::string& s);
void attr(const std::string & s);
public:
JSONObject(std::ostream& str, bool indent = false) : JSONWriter(str, indent) {
open();
}
public:
JSONObject(const JSONObject& obj) = delete;
JSONObject(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
open();
}
JSONObject(JSONObject&& obj) : JSONWriter(obj.state) { obj.state = 0; }
JSONObject(const JSONObject & obj) = delete;
~JSONObject();
JSONObject(JSONObject && obj)
: JSONWriter(obj.state)
{
obj.state = 0;
}
template <typename T>
JSONObject& attr(const std::string& name, const T& v) {
attr(name);
toJSON(state->str, v);
return *this;
}
~JSONObject();
JSONList list(const std::string& name);
template<typename T>
JSONObject & attr(const std::string & name, const T & v)
{
attr(name);
toJSON(state->str, v);
return *this;
}
JSONObject object(const std::string& name);
JSONList list(const std::string & name);
JSONObject object(const std::string & name);
JSONPlaceholder placeholder(const std::string & name);
JSONPlaceholder placeholder(const std::string& name);
};
class JSONPlaceholder : JSONWriter
{
class JSONPlaceholder : JSONWriter {
private:
friend class JSONList;
friend class JSONObject;
private:
JSONPlaceholder(JSONState* state) : JSONWriter(state) {}
friend class JSONList;
friend class JSONObject;
void assertValid() {
assertActive();
assert(first);
}
JSONPlaceholder(JSONState * state)
: JSONWriter(state)
{
}
public:
JSONPlaceholder(std::ostream& str, bool indent = false)
: JSONWriter(str, indent) {}
void assertValid()
{
assertActive();
assert(first);
}
~JSONPlaceholder() { assert(!first || std::uncaught_exception()); }
public:
template <typename T>
void write(const T& v) {
assertValid();
first = false;
toJSON(state->str, v);
}
JSONPlaceholder(std::ostream & str, bool indent = false)
: JSONWriter(str, indent)
{
}
JSONList list();
~JSONPlaceholder()
{
assert(!first || std::uncaught_exception());
}
template<typename T>
void write(const T & v)
{
assertValid();
first = false;
toJSON(state->str, v);
}
JSONList list();
JSONObject object();
JSONObject object();
};
}
} // namespace nix

47
third_party/nix/src/libutil/lazy.hh vendored
View file

@ -12,37 +12,32 @@ namespace nix {
thread-safe way) on first use, that is, when var() is first
called. If the initialiser code throws an exception, then all
subsequent calls to var() will rethrow that exception. */
template<typename T>
class Lazy
{
template <typename T>
class Lazy {
typedef std::function<T()> Init;
typedef std::function<T()> Init;
Init init;
Init init;
std::once_flag done;
std::once_flag done;
T value;
T value;
std::exception_ptr ex;
std::exception_ptr ex;
public:
Lazy(Init init) : init(init) {}
public:
Lazy(Init init) : init(init)
{ }
const T & operator () ()
{
std::call_once(done, [&]() {
try {
value = init();
} catch (...) {
ex = std::current_exception();
}
});
if (ex) std::rethrow_exception(ex);
return value;
}
const T& operator()() {
std::call_once(done, [&]() {
try {
value = init();
} catch (...) {
ex = std::current_exception();
}
});
if (ex) std::rethrow_exception(ex);
return value;
}
};
}
} // namespace nix

369
third_party/nix/src/libutil/logging.cc vendored
View file

@ -1,242 +1,227 @@
#include "logging.hh"
#include "util.hh"
#include <atomic>
#include <nlohmann/json.hpp>
#include "util.hh"
namespace nix {
static thread_local ActivityId curActivity = 0;
ActivityId getCurActivity()
{
return curActivity;
}
void setCurActivity(const ActivityId activityId)
{
curActivity = activityId;
ActivityId getCurActivity() { return curActivity; }
void setCurActivity(const ActivityId activityId) { curActivity = activityId; }
Logger* logger = makeDefaultLogger();
void Logger::warn(const std::string& msg) {
log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
}
Logger * logger = makeDefaultLogger();
class SimpleLogger : public Logger {
public:
bool systemd, tty;
void Logger::warn(const std::string & msg)
{
log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
}
SimpleLogger() {
systemd = getEnv("IN_SYSTEMD") == "1";
tty = isatty(STDERR_FILENO);
}
class SimpleLogger : public Logger
{
public:
void log(Verbosity lvl, const FormatOrString& fs) override {
if (lvl > verbosity) return;
bool systemd, tty;
std::string prefix;
SimpleLogger()
{
systemd = getEnv("IN_SYSTEMD") == "1";
tty = isatty(STDERR_FILENO);
if (systemd) {
char c;
switch (lvl) {
case lvlError:
c = '3';
break;
case lvlWarn:
c = '4';
break;
case lvlInfo:
c = '5';
break;
case lvlTalkative:
case lvlChatty:
c = '6';
break;
default:
c = '7';
}
prefix = std::string("<") + c + ">";
}
void log(Verbosity lvl, const FormatOrString & fs) override
{
if (lvl > verbosity) return;
writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
}
std::string prefix;
if (systemd) {
char c;
switch (lvl) {
case lvlError: c = '3'; break;
case lvlWarn: c = '4'; break;
case lvlInfo: c = '5'; break;
case lvlTalkative: case lvlChatty: c = '6'; break;
default: c = '7';
}
prefix = std::string("<") + c + ">";
}
writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
}
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string & s, const Fields & fields, ActivityId parent)
override
{
if (lvl <= verbosity && !s.empty())
log(lvl, s + "...");
}
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string& s, const Fields& fields,
ActivityId parent) override {
if (lvl <= verbosity && !s.empty()) log(lvl, s + "...");
}
};
Verbosity verbosity = lvlInfo;
void warnOnce(bool & haveWarned, const FormatOrString & fs)
{
if (!haveWarned) {
warn(fs.s);
haveWarned = true;
}
void warnOnce(bool& haveWarned, const FormatOrString& fs) {
if (!haveWarned) {
warn(fs.s);
haveWarned = true;
}
}
void writeToStderr(const string & s)
{
try {
writeFull(STDERR_FILENO, s, false);
} catch (SysError & e) {
/* Ignore failing writes to stderr. We need to ignore write
errors to ensure that cleanup code that logs to stderr runs
to completion if the other side of stderr has been closed
unexpectedly. */
}
void writeToStderr(const string& s) {
try {
writeFull(STDERR_FILENO, s, false);
} catch (SysError& e) {
/* Ignore failing writes to stderr. We need to ignore write
errors to ensure that cleanup code that logs to stderr runs
to completion if the other side of stderr has been closed
unexpectedly. */
}
}
Logger * makeDefaultLogger()
{
return new SimpleLogger();
Logger* makeDefaultLogger() { return new SimpleLogger(); }
std::atomic<uint64_t> nextId{(uint64_t)getpid() << 32};
Activity::Activity(Logger& logger, Verbosity lvl, ActivityType type,
const std::string& s, const Logger::Fields& fields,
ActivityId parent)
: logger(logger), id(nextId++) {
logger.startActivity(id, lvl, type, s, fields, parent);
}
std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
struct JSONLogger : Logger {
Logger& prevLogger;
Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
const std::string & s, const Logger::Fields & fields, ActivityId parent)
: logger(logger), id(nextId++)
{
logger.startActivity(id, lvl, type, s, fields, parent);
}
JSONLogger(Logger& prevLogger) : prevLogger(prevLogger) {}
struct JSONLogger : Logger
{
Logger & prevLogger;
void addFields(nlohmann::json& json, const Fields& fields) {
if (fields.empty()) return;
auto& arr = json["fields"] = nlohmann::json::array();
for (auto& f : fields)
if (f.type == Logger::Field::tInt)
arr.push_back(f.i);
else if (f.type == Logger::Field::tString)
arr.push_back(f.s);
else
abort();
}
JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
void write(const nlohmann::json& json) {
prevLogger.log(lvlError, "@nix " + json.dump());
}
void addFields(nlohmann::json & json, const Fields & fields)
{
if (fields.empty()) return;
auto & arr = json["fields"] = nlohmann::json::array();
for (auto & f : fields)
if (f.type == Logger::Field::tInt)
arr.push_back(f.i);
else if (f.type == Logger::Field::tString)
arr.push_back(f.s);
else
abort();
}
void log(Verbosity lvl, const FormatOrString& fs) override {
nlohmann::json json;
json["action"] = "msg";
json["level"] = lvl;
json["msg"] = fs.s;
write(json);
}
void write(const nlohmann::json & json)
{
prevLogger.log(lvlError, "@nix " + json.dump());
}
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string& s, const Fields& fields,
ActivityId parent) override {
nlohmann::json json;
json["action"] = "start";
json["id"] = act;
json["level"] = lvl;
json["type"] = type;
json["text"] = s;
addFields(json, fields);
// FIXME: handle parent
write(json);
}
void log(Verbosity lvl, const FormatOrString & fs) override
{
nlohmann::json json;
json["action"] = "msg";
json["level"] = lvl;
json["msg"] = fs.s;
write(json);
}
void stopActivity(ActivityId act) override {
nlohmann::json json;
json["action"] = "stop";
json["id"] = act;
write(json);
}
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string & s, const Fields & fields, ActivityId parent) override
{
nlohmann::json json;
json["action"] = "start";
json["id"] = act;
json["level"] = lvl;
json["type"] = type;
json["text"] = s;
addFields(json, fields);
// FIXME: handle parent
write(json);
}
void stopActivity(ActivityId act) override
{
nlohmann::json json;
json["action"] = "stop";
json["id"] = act;
write(json);
}
void result(ActivityId act, ResultType type, const Fields & fields) override
{
nlohmann::json json;
json["action"] = "result";
json["id"] = act;
json["type"] = type;
addFields(json, fields);
write(json);
}
void result(ActivityId act, ResultType type, const Fields& fields) override {
nlohmann::json json;
json["action"] = "result";
json["id"] = act;
json["type"] = type;
addFields(json, fields);
write(json);
}
};
Logger * makeJSONLogger(Logger & prevLogger)
{
return new JSONLogger(prevLogger);
Logger* makeJSONLogger(Logger& prevLogger) {
return new JSONLogger(prevLogger);
}
static Logger::Fields getFields(nlohmann::json & json)
{
Logger::Fields fields;
for (auto & f : json) {
if (f.type() == nlohmann::json::value_t::number_unsigned)
fields.emplace_back(Logger::Field(f.get<uint64_t>()));
else if (f.type() == nlohmann::json::value_t::string)
fields.emplace_back(Logger::Field(f.get<std::string>()));
else throw Error("unsupported JSON type %d", (int) f.type());
}
return fields;
static Logger::Fields getFields(nlohmann::json& json) {
Logger::Fields fields;
for (auto& f : json) {
if (f.type() == nlohmann::json::value_t::number_unsigned)
fields.emplace_back(Logger::Field(f.get<uint64_t>()));
else if (f.type() == nlohmann::json::value_t::string)
fields.emplace_back(Logger::Field(f.get<std::string>()));
else
throw Error("unsupported JSON type %d", (int)f.type());
}
return fields;
}
bool handleJSONLogMessage(const std::string & msg,
const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted)
{
if (!hasPrefix(msg, "@nix ")) return false;
bool handleJSONLogMessage(const std::string& msg, const Activity& act,
std::map<ActivityId, Activity>& activities,
bool trusted) {
if (!hasPrefix(msg, "@nix ")) return false;
try {
auto json = nlohmann::json::parse(std::string(msg, 5));
try {
auto json = nlohmann::json::parse(std::string(msg, 5));
std::string action = json["action"];
std::string action = json["action"];
if (action == "start") {
auto type = (ActivityType) json["type"];
if (trusted || type == actDownload)
activities.emplace(std::piecewise_construct,
std::forward_as_tuple(json["id"]),
std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
json["text"], getFields(json["fields"]), act.id));
}
else if (action == "stop")
activities.erase((ActivityId) json["id"]);
else if (action == "result") {
auto i = activities.find((ActivityId) json["id"]);
if (i != activities.end())
i->second.result((ResultType) json["type"], getFields(json["fields"]));
}
else if (action == "setPhase") {
std::string phase = json["phase"];
act.result(resSetPhase, phase);
}
else if (action == "msg") {
std::string msg = json["msg"];
logger->log((Verbosity) json["level"], msg);
}
} catch (std::exception & e) {
printError("bad log message from builder: %s", e.what());
if (action == "start") {
auto type = (ActivityType)json["type"];
if (trusted || type == actDownload)
activities.emplace(
std::piecewise_construct, std::forward_as_tuple(json["id"]),
std::forward_as_tuple(*logger, (Verbosity)json["level"], type,
json["text"], getFields(json["fields"]),
act.id));
}
return true;
else if (action == "stop")
activities.erase((ActivityId)json["id"]);
else if (action == "result") {
auto i = activities.find((ActivityId)json["id"]);
if (i != activities.end())
i->second.result((ResultType)json["type"], getFields(json["fields"]));
}
else if (action == "setPhase") {
std::string phase = json["phase"];
act.result(resSetPhase, phase);
}
else if (action == "msg") {
std::string msg = json["msg"];
logger->log((Verbosity)json["level"], msg);
}
} catch (std::exception& e) {
printError("bad log message from builder: %s", e.what());
}
return true;
}
Activity::~Activity() {
try {
logger.stopActivity(id);
} catch (...) {
ignoreException();
}
try {
logger.stopActivity(id);
} catch (...) {
ignoreException();
}
}
}
} // namespace nix

210
third_party/nix/src/libutil/logging.hh vendored
View file

@ -5,151 +5,148 @@
namespace nix {
typedef enum {
lvlError = 0,
lvlWarn,
lvlInfo,
lvlTalkative,
lvlChatty,
lvlDebug,
lvlVomit
lvlError = 0,
lvlWarn,
lvlInfo,
lvlTalkative,
lvlChatty,
lvlDebug,
lvlVomit
} Verbosity;
typedef enum {
actUnknown = 0,
actCopyPath = 100,
actDownload = 101,
actRealise = 102,
actCopyPaths = 103,
actBuilds = 104,
actBuild = 105,
actOptimiseStore = 106,
actVerifyPaths = 107,
actSubstitute = 108,
actQueryPathInfo = 109,
actPostBuildHook = 110,
actUnknown = 0,
actCopyPath = 100,
actDownload = 101,
actRealise = 102,
actCopyPaths = 103,
actBuilds = 104,
actBuild = 105,
actOptimiseStore = 106,
actVerifyPaths = 107,
actSubstitute = 108,
actQueryPathInfo = 109,
actPostBuildHook = 110,
} ActivityType;
typedef enum {
resFileLinked = 100,
resBuildLogLine = 101,
resUntrustedPath = 102,
resCorruptedPath = 103,
resSetPhase = 104,
resProgress = 105,
resSetExpected = 106,
resPostBuildLogLine = 107,
resFileLinked = 100,
resBuildLogLine = 101,
resUntrustedPath = 102,
resCorruptedPath = 103,
resSetPhase = 104,
resProgress = 105,
resSetExpected = 106,
resPostBuildLogLine = 107,
} ResultType;
typedef uint64_t ActivityId;
class Logger
{
friend struct Activity;
class Logger {
friend struct Activity;
public:
public:
struct Field {
// FIXME: use std::variant.
enum { tInt = 0, tString = 1 } type;
uint64_t i = 0;
std::string s;
Field(const std::string& s) : type(tString), s(s) {}
Field(const char* s) : type(tString), s(s) {}
Field(const uint64_t& i) : type(tInt), i(i) {}
};
struct Field
{
// FIXME: use std::variant.
enum { tInt = 0, tString = 1 } type;
uint64_t i = 0;
std::string s;
Field(const std::string & s) : type(tString), s(s) { }
Field(const char * s) : type(tString), s(s) { }
Field(const uint64_t & i) : type(tInt), i(i) { }
};
typedef std::vector<Field> Fields;
typedef std::vector<Field> Fields;
virtual ~Logger() {}
virtual ~Logger() { }
virtual void log(Verbosity lvl, const FormatOrString& fs) = 0;
virtual void log(Verbosity lvl, const FormatOrString & fs) = 0;
void log(const FormatOrString& fs) { log(lvlInfo, fs); }
void log(const FormatOrString & fs)
{
log(lvlInfo, fs);
}
virtual void warn(const std::string& msg);
virtual void warn(const std::string & msg);
virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string& s, const Fields& fields,
ActivityId parent){};
virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
const std::string & s, const Fields & fields, ActivityId parent) { };
virtual void stopActivity(ActivityId act){};
virtual void stopActivity(ActivityId act) { };
virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
virtual void result(ActivityId act, ResultType type, const Fields& fields){};
};
ActivityId getCurActivity();
void setCurActivity(const ActivityId activityId);
struct Activity
{
Logger & logger;
struct Activity {
Logger& logger;
const ActivityId id;
const ActivityId id;
Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "",
const Logger::Fields & fields = {}, ActivityId parent = getCurActivity());
Activity(Logger& logger, Verbosity lvl, ActivityType type,
const std::string& s = "", const Logger::Fields& fields = {},
ActivityId parent = getCurActivity());
Activity(Logger & logger, ActivityType type,
const Logger::Fields & fields = {}, ActivityId parent = getCurActivity())
: Activity(logger, lvlError, type, "", fields, parent) { };
Activity(Logger& logger, ActivityType type, const Logger::Fields& fields = {},
ActivityId parent = getCurActivity())
: Activity(logger, lvlError, type, "", fields, parent){};
Activity(const Activity & act) = delete;
Activity(const Activity& act) = delete;
~Activity();
~Activity();
void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
{ result(resProgress, done, expected, running, failed); }
void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0,
uint64_t failed = 0) const {
result(resProgress, done, expected, running, failed);
}
void setExpected(ActivityType type2, uint64_t expected) const
{ result(resSetExpected, type2, expected); }
void setExpected(ActivityType type2, uint64_t expected) const {
result(resSetExpected, type2, expected);
}
template<typename... Args>
void result(ResultType type, const Args & ... args) const
{
Logger::Fields fields;
nop{(fields.emplace_back(Logger::Field(args)), 1)...};
result(type, fields);
}
template <typename... Args>
void result(ResultType type, const Args&... args) const {
Logger::Fields fields;
nop{(fields.emplace_back(Logger::Field(args)), 1)...};
result(type, fields);
}
void result(ResultType type, const Logger::Fields & fields) const
{
logger.result(id, type, fields);
}
void result(ResultType type, const Logger::Fields& fields) const {
logger.result(id, type, fields);
}
friend class Logger;
friend class Logger;
};
struct PushActivity
{
const ActivityId prevAct;
PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); }
~PushActivity() { setCurActivity(prevAct); }
struct PushActivity {
const ActivityId prevAct;
PushActivity(ActivityId act) : prevAct(getCurActivity()) {
setCurActivity(act);
}
~PushActivity() { setCurActivity(prevAct); }
};
extern Logger * logger;
extern Logger* logger;
Logger * makeDefaultLogger();
Logger* makeDefaultLogger();
Logger * makeJSONLogger(Logger & prevLogger);
Logger* makeJSONLogger(Logger& prevLogger);
bool handleJSONLogMessage(const std::string & msg,
const Activity & act, std::map<ActivityId, Activity> & activities,
bool trusted);
bool handleJSONLogMessage(const std::string& msg, const Activity& act,
std::map<ActivityId, Activity>& activities,
bool trusted);
extern Verbosity verbosity; /* suppress msgs > this */
/* Print a message if the current log level is at least the specified
level. Note that this has to be implemented as a macro to ensure
that the arguments are evaluated lazily. */
#define printMsg(level, args...) \
do { \
if (level <= nix::verbosity) { \
logger->log(level, fmt(args)); \
} \
} while (0)
#define printMsg(level, args...) \
do { \
if (level <= nix::verbosity) { \
logger->log(level, fmt(args)); \
} \
} while (0)
#define printError(args...) printMsg(lvlError, args)
#define printInfo(args...) printMsg(lvlInfo, args)
@ -157,16 +154,15 @@ extern Verbosity verbosity; /* suppress msgs > this */
#define debug(args...) printMsg(lvlDebug, args)
#define vomit(args...) printMsg(lvlVomit, args)
template<typename... Args>
inline void warn(const std::string & fs, Args... args)
{
boost::format f(fs);
nop{boost::io::detail::feed(f, args)...};
logger->warn(f.str());
template <typename... Args>
inline void warn(const std::string& fs, Args... args) {
boost::format f(fs);
nop{boost::io::detail::feed(f, args)...};
logger->warn(f.str());
}
void warnOnce(bool & haveWarned, const FormatOrString & fs);
void warnOnce(bool& haveWarned, const FormatOrString& fs);
void writeToStderr(const string & s);
void writeToStderr(const string& s);
}
} // namespace nix

126
third_party/nix/src/libutil/lru-cache.hh vendored
View file

@ -1,92 +1,84 @@
#pragma once
#include <map>
#include <list>
#include <map>
#include <optional>
namespace nix {
/* A simple least-recently used cache. Not thread-safe. */
template<typename Key, typename Value>
class LRUCache
{
private:
template <typename Key, typename Value>
class LRUCache {
private:
size_t capacity;
size_t capacity;
// Stupid wrapper to get around circular dependency between Data
// and LRU.
struct LRUIterator;
// Stupid wrapper to get around circular dependency between Data
// and LRU.
struct LRUIterator;
using Data = std::map<Key, std::pair<LRUIterator, Value>>;
using LRU = std::list<typename Data::iterator>;
using Data = std::map<Key, std::pair<LRUIterator, Value>>;
using LRU = std::list<typename Data::iterator>;
struct LRUIterator {
typename LRU::iterator it;
};
struct LRUIterator { typename LRU::iterator it; };
Data data;
LRU lru;
Data data;
LRU lru;
public:
LRUCache(size_t capacity) : capacity(capacity) {}
public:
/* Insert or upsert an item in the cache. */
void upsert(const Key& key, const Value& value) {
if (capacity == 0) return;
LRUCache(size_t capacity) : capacity(capacity) { }
erase(key);
/* Insert or upsert an item in the cache. */
void upsert(const Key & key, const Value & value)
{
if (capacity == 0) return;
erase(key);
if (data.size() >= capacity) {
/* Retire the oldest item. */
auto oldest = lru.begin();
data.erase(*oldest);
lru.erase(oldest);
}
auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
assert(res.second);
auto & i(res.first);
auto j = lru.insert(lru.end(), i);
i->second.first.it = j;
if (data.size() >= capacity) {
/* Retire the oldest item. */
auto oldest = lru.begin();
data.erase(*oldest);
lru.erase(oldest);
}
bool erase(const Key & key)
{
auto i = data.find(key);
if (i == data.end()) return false;
lru.erase(i->second.first.it);
data.erase(i);
return true;
}
auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
assert(res.second);
auto& i(res.first);
/* Look up an item in the cache. If it exists, it becomes the most
recently used item. */
std::optional<Value> get(const Key & key)
{
auto i = data.find(key);
if (i == data.end()) return {};
auto j = lru.insert(lru.end(), i);
/* Move this item to the back of the LRU list. */
lru.erase(i->second.first.it);
auto j = lru.insert(lru.end(), i);
i->second.first.it = j;
i->second.first.it = j;
}
return i->second.second;
}
bool erase(const Key& key) {
auto i = data.find(key);
if (i == data.end()) return false;
lru.erase(i->second.first.it);
data.erase(i);
return true;
}
size_t size()
{
return data.size();
}
/* Look up an item in the cache. If it exists, it becomes the most
recently used item. */
std::optional<Value> get(const Key& key) {
auto i = data.find(key);
if (i == data.end()) return {};
void clear()
{
data.clear();
lru.clear();
}
/* Move this item to the back of the LRU list. */
lru.erase(i->second.first.it);
auto j = lru.insert(lru.end(), i);
i->second.first.it = j;
return i->second.second;
}
size_t size() { return data.size(); }
void clear() {
data.clear();
lru.clear();
}
};
}
} // namespace nix

84
third_party/nix/src/libutil/monitor-fd.hh vendored
View file

@ -1,58 +1,52 @@
#pragma once
#include <thread>
#include <atomic>
#include <cstdlib>
#include <poll.h>
#include <signal.h>
#include <sys/types.h>
#include <unistd.h>
#include <signal.h>
#include <atomic>
#include <cstdlib>
#include <thread>
namespace nix {
class MonitorFdHup {
private:
std::thread thread;
class MonitorFdHup
{
private:
std::thread thread;
public:
MonitorFdHup(int fd) {
thread = std::thread([fd]() {
while (true) {
/* Wait indefinitely until a POLLHUP occurs. */
struct pollfd fds[1];
fds[0].fd = fd;
/* This shouldn't be necessary, but macOS doesn't seem to
like a zeroed out events field.
See rdar://37537852.
*/
fds[0].events = POLLHUP;
auto count = poll(fds, 1, -1);
if (count == -1) abort(); // can't happen
/* This shouldn't happen, but can on macOS due to a bug.
See rdar://37550628.
public:
MonitorFdHup(int fd)
{
thread = std::thread([fd]() {
while (true) {
/* Wait indefinitely until a POLLHUP occurs. */
struct pollfd fds[1];
fds[0].fd = fd;
/* This shouldn't be necessary, but macOS doesn't seem to
like a zeroed out events field.
See rdar://37537852.
*/
fds[0].events = POLLHUP;
auto count = poll(fds, 1, -1);
if (count == -1) abort(); // can't happen
/* This shouldn't happen, but can on macOS due to a bug.
See rdar://37550628.
This may eventually need a delay or further
coordination with the main thread if spinning proves
too harmful.
*/
if (count == 0) continue;
assert(fds[0].revents & POLLHUP);
triggerInterrupt();
break;
}
});
};
This may eventually need a delay or further
coordination with the main thread if spinning proves
too harmful.
*/
if (count == 0) continue;
assert(fds[0].revents & POLLHUP);
triggerInterrupt();
break;
}
});
};
~MonitorFdHup()
{
pthread_cancel(thread.native_handle());
thread.join();
}
~MonitorFdHup() {
pthread_cancel(thread.native_handle());
thread.join();
}
};
}
} // namespace nix

252
third_party/nix/src/libutil/pool.hh vendored
View file

@ -1,13 +1,12 @@
#pragma once
#include <cassert>
#include <functional>
#include <limits>
#include <list>
#include <memory>
#include <cassert>
#include "sync.hh"
#include "ref.hh"
#include "sync.hh"
namespace nix {
@ -29,159 +28,140 @@ namespace nix {
*/
template <class R>
class Pool
{
public:
class Pool {
public:
/* A function that produces new instances of R on demand. */
typedef std::function<ref<R>()> Factory;
/* A function that produces new instances of R on demand. */
typedef std::function<ref<R>()> Factory;
/* A function that checks whether an instance of R is still
usable. Unusable instances are removed from the pool. */
typedef std::function<bool(const ref<R>&)> Validator;
/* A function that checks whether an instance of R is still
usable. Unusable instances are removed from the pool. */
typedef std::function<bool(const ref<R> &)> Validator;
private:
Factory factory;
Validator validator;
private:
struct State {
size_t inUse = 0;
size_t max;
std::vector<ref<R>> idle;
};
Factory factory;
Validator validator;
Sync<State> state;
struct State
{
size_t inUse = 0;
size_t max;
std::vector<ref<R>> idle;
};
std::condition_variable wakeup;
Sync<State> state;
public:
Pool(
size_t max = std::numeric_limits<size_t>::max(),
const Factory& factory = []() { return make_ref<R>(); },
const Validator& validator = [](ref<R> r) { return true; })
: factory(factory), validator(validator) {
auto state_(state.lock());
state_->max = max;
}
std::condition_variable wakeup;
void incCapacity() {
auto state_(state.lock());
state_->max++;
/* we could wakeup here, but this is only used when we're
* about to nest Pool usages, and we want to save the slot for
* the nested use if we can
*/
}
public:
void decCapacity() {
auto state_(state.lock());
state_->max--;
}
Pool(size_t max = std::numeric_limits<size_t>::max(),
const Factory & factory = []() { return make_ref<R>(); },
const Validator & validator = [](ref<R> r) { return true; })
: factory(factory)
, validator(validator)
{
auto state_(state.lock());
state_->max = max;
~Pool() {
auto state_(state.lock());
assert(!state_->inUse);
state_->max = 0;
state_->idle.clear();
}
class Handle {
private:
Pool& pool;
std::shared_ptr<R> r;
bool bad = false;
friend Pool;
Handle(Pool& pool, std::shared_ptr<R> r) : pool(pool), r(r) {}
public:
Handle(Handle&& h) : pool(h.pool), r(h.r) { h.r.reset(); }
Handle(const Handle& l) = delete;
~Handle() {
if (!r) return;
{
auto state_(pool.state.lock());
if (!bad) state_->idle.push_back(ref<R>(r));
assert(state_->inUse);
state_->inUse--;
}
pool.wakeup.notify_one();
}
void incCapacity()
R* operator->() { return &*r; }
R& operator*() { return *r; }
void markBad() { bad = true; }
};
Handle get() {
{
auto state_(state.lock());
state_->max++;
/* we could wakeup here, but this is only used when we're
* about to nest Pool usages, and we want to save the slot for
* the nested use if we can
*/
}
auto state_(state.lock());
void decCapacity()
{
auto state_(state.lock());
state_->max--;
}
/* If we're over the maximum number of instance, we need
to wait until a slot becomes available. */
while (state_->idle.empty() && state_->inUse >= state_->max)
state_.wait(wakeup);
~Pool()
{
auto state_(state.lock());
assert(!state_->inUse);
state_->max = 0;
state_->idle.clear();
}
class Handle
{
private:
Pool & pool;
std::shared_ptr<R> r;
bool bad = false;
friend Pool;
Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { }
public:
Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); }
Handle(const Handle & l) = delete;
~Handle()
{
if (!r) return;
{
auto state_(pool.state.lock());
if (!bad)
state_->idle.push_back(ref<R>(r));
assert(state_->inUse);
state_->inUse--;
}
pool.wakeup.notify_one();
while (!state_->idle.empty()) {
auto p = state_->idle.back();
state_->idle.pop_back();
if (validator(p)) {
state_->inUse++;
return Handle(*this, p);
}
}
R * operator -> () { return &*r; }
R & operator * () { return *r; }
void markBad() { bad = true; }
};
Handle get()
{
{
auto state_(state.lock());
/* If we're over the maximum number of instance, we need
to wait until a slot becomes available. */
while (state_->idle.empty() && state_->inUse >= state_->max)
state_.wait(wakeup);
while (!state_->idle.empty()) {
auto p = state_->idle.back();
state_->idle.pop_back();
if (validator(p)) {
state_->inUse++;
return Handle(*this, p);
}
}
state_->inUse++;
}
/* We need to create a new instance. Because that might take a
while, we don't hold the lock in the meantime. */
try {
Handle h(*this, factory());
return h;
} catch (...) {
auto state_(state.lock());
state_->inUse--;
wakeup.notify_one();
throw;
}
state_->inUse++;
}
size_t count()
{
auto state_(state.lock());
return state_->idle.size() + state_->inUse;
/* We need to create a new instance. Because that might take a
while, we don't hold the lock in the meantime. */
try {
Handle h(*this, factory());
return h;
} catch (...) {
auto state_(state.lock());
state_->inUse--;
wakeup.notify_one();
throw;
}
}
size_t capacity()
{
return state.lock()->max;
}
size_t count() {
auto state_(state.lock());
return state_->idle.size() + state_->inUse;
}
void flushBad()
{
auto state_(state.lock());
std::vector<ref<R>> left;
for (auto & p : state_->idle)
if (validator(p))
left.push_back(p);
std::swap(state_->idle, left);
}
size_t capacity() { return state.lock()->max; }
void flushBad() {
auto state_(state.lock());
std::vector<ref<R>> left;
for (auto& p : state_->idle)
if (validator(p)) left.push_back(p);
std::swap(state_->idle, left);
}
};
}
} // namespace nix

105
third_party/nix/src/libutil/ref.hh vendored
View file

@ -1,92 +1,61 @@
#pragma once
#include <memory>
#include <exception>
#include <memory>
#include <stdexcept>
namespace nix {
/* A simple non-nullable reference-counted pointer. Actually a wrapper
around std::shared_ptr that prevents non-null constructions. */
template<typename T>
class ref
{
private:
template <typename T>
class ref {
private:
std::shared_ptr<T> p;
std::shared_ptr<T> p;
public:
ref<T>(const ref<T>& r) : p(r.p) {}
public:
explicit ref<T>(const std::shared_ptr<T>& p) : p(p) {
if (!p) throw std::invalid_argument("null pointer cast to ref");
}
ref<T>(const ref<T> & r)
: p(r.p)
{ }
explicit ref<T>(T* p) : p(p) {
if (!p) throw std::invalid_argument("null pointer cast to ref");
}
explicit ref<T>(const std::shared_ptr<T> & p)
: p(p)
{
if (!p)
throw std::invalid_argument("null pointer cast to ref");
}
T* operator->() const { return &*p; }
explicit ref<T>(T * p)
: p(p)
{
if (!p)
throw std::invalid_argument("null pointer cast to ref");
}
T& operator*() const { return *p; }
T* operator ->() const
{
return &*p;
}
operator std::shared_ptr<T>() const { return p; }
T& operator *() const
{
return *p;
}
std::shared_ptr<T> get_ptr() const { return p; }
operator std::shared_ptr<T> () const
{
return p;
}
template <typename T2>
ref<T2> cast() const {
return ref<T2>(std::dynamic_pointer_cast<T2>(p));
}
std::shared_ptr<T> get_ptr() const
{
return p;
}
template <typename T2>
std::shared_ptr<T2> dynamic_pointer_cast() const {
return std::dynamic_pointer_cast<T2>(p);
}
template<typename T2>
ref<T2> cast() const
{
return ref<T2>(std::dynamic_pointer_cast<T2>(p));
}
template<typename T2>
std::shared_ptr<T2> dynamic_pointer_cast() const
{
return std::dynamic_pointer_cast<T2>(p);
}
template<typename T2>
operator ref<T2> () const
{
return ref<T2>((std::shared_ptr<T2>) p);
}
private:
template<typename T2, typename... Args>
friend ref<T2>
make_ref(Args&&... args);
template <typename T2>
operator ref<T2>() const {
return ref<T2>((std::shared_ptr<T2>)p);
}
private:
template <typename T2, typename... Args>
friend ref<T2> make_ref(Args&&... args);
};
template<typename T, typename... Args>
inline ref<T>
make_ref(Args&&... args)
{
auto p = std::make_shared<T>(std::forward<Args>(args)...);
return ref<T>(p);
template <typename T, typename... Args>
inline ref<T> make_ref(Args&&... args) {
auto p = std::make_shared<T>(std::forward<Args>(args)...);
return ref<T>(p);
}
}
} // namespace nix

460
third_party/nix/src/libutil/serialise.cc vendored
View file

@ -1,323 +1,277 @@
#include "serialise.hh"
#include "util.hh"
#include <cstring>
#include <cerrno>
#include <memory>
#include <boost/coroutine2/coroutine.hpp>
#include <cerrno>
#include <cstring>
#include <memory>
#include "util.hh"
namespace nix {
void BufferedSink::operator()(const unsigned char* data, size_t len) {
if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
void BufferedSink::operator () (const unsigned char * data, size_t len)
{
if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
while (len) {
/* Optimisation: bypass the buffer if the data exceeds the
buffer size. */
if (bufPos + len >= bufSize) {
flush();
write(data, len);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
when it's full. */
size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
memcpy(buffer.get() + bufPos, data, n);
data += n; bufPos += n; len -= n;
if (bufPos == bufSize) flush();
while (len) {
/* Optimisation: bypass the buffer if the data exceeds the
buffer size. */
if (bufPos + len >= bufSize) {
flush();
write(data, len);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
when it's full. */
size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
memcpy(buffer.get() + bufPos, data, n);
data += n;
bufPos += n;
len -= n;
if (bufPos == bufSize) flush();
}
}
void BufferedSink::flush()
{
if (bufPos == 0) return;
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
write(buffer.get(), n);
void BufferedSink::flush() {
if (bufPos == 0) return;
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
write(buffer.get(), n);
}
FdSink::~FdSink()
{
try { flush(); } catch (...) { ignoreException(); }
FdSink::~FdSink() {
try {
flush();
} catch (...) {
ignoreException();
}
}
size_t threshold = 256 * 1024 * 1024;
static void warnLargeDump()
{
printError("warning: dumping very large path (> 256 MiB); this may run out of memory");
static void warnLargeDump() {
printError(
"warning: dumping very large path (> 256 MiB); this may run out of "
"memory");
}
void FdSink::write(const unsigned char * data, size_t len)
{
written += len;
static bool warned = false;
if (warn && !warned) {
if (written > threshold) {
warnLargeDump();
warned = true;
}
void FdSink::write(const unsigned char* data, size_t len) {
written += len;
static bool warned = false;
if (warn && !warned) {
if (written > threshold) {
warnLargeDump();
warned = true;
}
}
try {
writeFull(fd, data, len);
} catch (SysError& e) {
_good = false;
throw;
}
}
bool FdSink::good() { return _good; }
void Source::operator()(unsigned char* data, size_t len) {
while (len) {
size_t n = read(data, len);
data += n;
len -= n;
}
}
std::string Source::drain() {
std::string s;
std::vector<unsigned char> buf(8192);
while (true) {
size_t n;
try {
writeFull(fd, data, len);
} catch (SysError & e) {
_good = false;
throw;
n = read(buf.data(), buf.size());
s.append((char*)buf.data(), n);
} catch (EndOfFile&) {
break;
}
}
return s;
}
size_t BufferedSource::read(unsigned char* data, size_t len) {
if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
bool FdSink::good()
{
return _good;
if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize);
/* Copy out the data in the buffer. */
size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
memcpy(data, buffer.get() + bufPosOut, n);
bufPosOut += n;
if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
return n;
}
bool BufferedSource::hasData() { return bufPosOut < bufPosIn; }
void Source::operator () (unsigned char * data, size_t len)
{
while (len) {
size_t n = read(data, len);
data += n; len -= n;
}
size_t FdSource::readUnbuffered(unsigned char* data, size_t len) {
ssize_t n;
do {
checkInterrupt();
n = ::read(fd, (char*)data, len);
} while (n == -1 && errno == EINTR);
if (n == -1) {
_good = false;
throw SysError("reading from file");
}
if (n == 0) {
_good = false;
throw EndOfFile("unexpected end-of-file");
}
read += n;
return n;
}
bool FdSource::good() { return _good; }
std::string Source::drain()
{
std::string s;
std::vector<unsigned char> buf(8192);
while (true) {
size_t n;
try {
n = read(buf.data(), buf.size());
s.append((char *) buf.data(), n);
} catch (EndOfFile &) {
break;
}
}
return s;
size_t StringSource::read(unsigned char* data, size_t len) {
if (pos == s.size()) throw EndOfFile("end of string reached");
size_t n = s.copy((char*)data, len, pos);
pos += n;
return n;
}
size_t BufferedSource::read(unsigned char * data, size_t len)
{
if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize);
/* Copy out the data in the buffer. */
size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
memcpy(data, buffer.get() + bufPosOut, n);
bufPosOut += n;
if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
return n;
}
bool BufferedSource::hasData()
{
return bufPosOut < bufPosIn;
}
size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
{
ssize_t n;
do {
checkInterrupt();
n = ::read(fd, (char *) data, len);
} while (n == -1 && errno == EINTR);
if (n == -1) { _good = false; throw SysError("reading from file"); }
if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
read += n;
return n;
}
bool FdSource::good()
{
return _good;
}
size_t StringSource::read(unsigned char * data, size_t len)
{
if (pos == s.size()) throw EndOfFile("end of string reached");
size_t n = s.copy((char *) data, len, pos);
pos += n;
return n;
}
#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
#error Coroutines are broken in this version of Boost!
#endif
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
{
struct SinkToSource : Source
{
typedef boost::coroutines2::coroutine<std::string> coro_t;
std::unique_ptr<Source> sinkToSource(std::function<void(Sink&)> fun,
std::function<void()> eof) {
struct SinkToSource : Source {
typedef boost::coroutines2::coroutine<std::string> coro_t;
std::function<void(Sink &)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
bool started = false;
std::function<void(Sink&)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
bool started = false;
SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)
{
}
SinkToSource(std::function<void(Sink&)> fun, std::function<void()> eof)
: fun(fun), eof(eof) {}
std::string cur;
size_t pos = 0;
std::string cur;
size_t pos = 0;
size_t read(unsigned char * data, size_t len) override
{
if (!coro)
coro = coro_t::pull_type([&](coro_t::push_type & yield) {
LambdaSink sink([&](const unsigned char * data, size_t len) {
if (len) yield(std::string((const char *) data, len));
});
fun(sink);
});
size_t read(unsigned char* data, size_t len) override {
if (!coro)
coro = coro_t::pull_type([&](coro_t::push_type& yield) {
LambdaSink sink([&](const unsigned char* data, size_t len) {
if (len) yield(std::string((const char*)data, len));
});
fun(sink);
});
if (!*coro) { eof(); abort(); }
if (!*coro) {
eof();
abort();
}
if (pos == cur.size()) {
if (!cur.empty()) (*coro)();
cur = coro->get();
pos = 0;
}
if (pos == cur.size()) {
if (!cur.empty()) (*coro)();
cur = coro->get();
pos = 0;
}
auto n = std::min(cur.size() - pos, len);
memcpy(data, (unsigned char *) cur.data() + pos, n);
pos += n;
auto n = std::min(cur.size() - pos, len);
memcpy(data, (unsigned char*)cur.data() + pos, n);
pos += n;
return n;
}
};
return std::make_unique<SinkToSource>(fun, eof);
}
void writePadding(size_t len, Sink & sink)
{
if (len % 8) {
unsigned char zero[8];
memset(zero, 0, sizeof(zero));
sink(zero, 8 - (len % 8));
return n;
}
};
return std::make_unique<SinkToSource>(fun, eof);
}
void writeString(const unsigned char * buf, size_t len, Sink & sink)
{
sink << len;
sink(buf, len);
writePadding(len, sink);
void writePadding(size_t len, Sink& sink) {
if (len % 8) {
unsigned char zero[8];
memset(zero, 0, sizeof(zero));
sink(zero, 8 - (len % 8));
}
}
Sink & operator << (Sink & sink, const string & s)
{
writeString((const unsigned char *) s.data(), s.size(), sink);
return sink;
void writeString(const unsigned char* buf, size_t len, Sink& sink) {
sink << len;
sink(buf, len);
writePadding(len, sink);
}
template<class T> void writeStrings(const T & ss, Sink & sink)
{
sink << ss.size();
for (auto & i : ss)
sink << i;
Sink& operator<<(Sink& sink, const string& s) {
writeString((const unsigned char*)s.data(), s.size(), sink);
return sink;
}
Sink & operator << (Sink & sink, const Strings & s)
{
writeStrings(s, sink);
return sink;
template <class T>
void writeStrings(const T& ss, Sink& sink) {
sink << ss.size();
for (auto& i : ss) sink << i;
}
Sink & operator << (Sink & sink, const StringSet & s)
{
writeStrings(s, sink);
return sink;
Sink& operator<<(Sink& sink, const Strings& s) {
writeStrings(s, sink);
return sink;
}
void readPadding(size_t len, Source & source)
{
if (len % 8) {
unsigned char zero[8];
size_t n = 8 - (len % 8);
source(zero, n);
for (unsigned int i = 0; i < n; i++)
if (zero[i]) throw SerialisationError("non-zero padding");
}
Sink& operator<<(Sink& sink, const StringSet& s) {
writeStrings(s, sink);
return sink;
}
size_t readString(unsigned char * buf, size_t max, Source & source)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
source(buf, len);
readPadding(len, source);
return len;
void readPadding(size_t len, Source& source) {
if (len % 8) {
unsigned char zero[8];
size_t n = 8 - (len % 8);
source(zero, n);
for (unsigned int i = 0; i < n; i++)
if (zero[i]) throw SerialisationError("non-zero padding");
}
}
string readString(Source & source, size_t max)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
std::string res(len, 0);
source((unsigned char*) res.data(), len);
readPadding(len, source);
return res;
size_t readString(unsigned char* buf, size_t max, Source& source) {
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
source(buf, len);
readPadding(len, source);
return len;
}
Source & operator >> (Source & in, string & s)
{
s = readString(in);
return in;
string readString(Source& source, size_t max) {
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
std::string res(len, 0);
source((unsigned char*)res.data(), len);
readPadding(len, source);
return res;
}
template<class T> T readStrings(Source & source)
{
auto count = readNum<size_t>(source);
T ss;
while (count--)
ss.insert(ss.end(), readString(source));
return ss;
Source& operator>>(Source& in, string& s) {
s = readString(in);
return in;
}
template Paths readStrings(Source & source);
template PathSet readStrings(Source & source);
void StringSink::operator () (const unsigned char * data, size_t len)
{
static bool warned = false;
if (!warned && s->size() > threshold) {
warnLargeDump();
warned = true;
}
s->append((const char *) data, len);
template <class T>
T readStrings(Source& source) {
auto count = readNum<size_t>(source);
T ss;
while (count--) ss.insert(ss.end(), readString(source));
return ss;
}
template Paths readStrings(Source& source);
template PathSet readStrings(Source& source);
void StringSink::operator()(const unsigned char* data, size_t len) {
static bool warned = false;
if (!warned && s->size() > threshold) {
warnLargeDump();
warned = true;
}
s->append((const char*)data, len);
}
} // namespace nix

418
third_party/nix/src/libutil/serialise.hh vendored
View file

@ -1,337 +1,285 @@
#pragma once
#include <memory>
#include "types.hh"
#include "util.hh"
namespace nix {
/* Abstract destination of binary data. */
struct Sink
{
virtual ~Sink() { }
virtual void operator () (const unsigned char * data, size_t len) = 0;
virtual bool good() { return true; }
struct Sink {
virtual ~Sink() {}
virtual void operator()(const unsigned char* data, size_t len) = 0;
virtual bool good() { return true; }
void operator () (const std::string & s)
{
(*this)((const unsigned char *) s.data(), s.size());
}
void operator()(const std::string& s) {
(*this)((const unsigned char*)s.data(), s.size());
}
};
/* A buffered abstract sink. */
struct BufferedSink : Sink
{
size_t bufSize, bufPos;
std::unique_ptr<unsigned char[]> buffer;
struct BufferedSink : Sink {
size_t bufSize, bufPos;
std::unique_ptr<unsigned char[]> buffer;
BufferedSink(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPos(0), buffer(nullptr) { }
BufferedSink(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPos(0), buffer(nullptr) {}
void operator () (const unsigned char * data, size_t len) override;
void operator()(const unsigned char* data, size_t len) override;
void operator () (const std::string & s)
{
Sink::operator()(s);
}
void operator()(const std::string& s) { Sink::operator()(s); }
void flush();
void flush();
virtual void write(const unsigned char * data, size_t len) = 0;
virtual void write(const unsigned char* data, size_t len) = 0;
};
/* Abstract source of binary data. */
struct Source
{
virtual ~Source() { }
struct Source {
virtual ~Source() {}
/* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’.
It blocks until all the requested data is available, or throws
an error if it is not going to be available. */
void operator () (unsigned char * data, size_t len);
/* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’.
It blocks until all the requested data is available, or throws
an error if it is not going to be available. */
void operator()(unsigned char* data, size_t len);
/* Store up to ‘len’ in the buffer pointed to by ‘data’, and
return the number of bytes stored. It blocks until at least
one byte is available. */
virtual size_t read(unsigned char * data, size_t len) = 0;
/* Store up to ‘len’ in the buffer pointed to by ‘data’, and
return the number of bytes stored. It blocks until at least
one byte is available. */
virtual size_t read(unsigned char* data, size_t len) = 0;
virtual bool good() { return true; }
virtual bool good() { return true; }
std::string drain();
std::string drain();
};
/* A buffered abstract source. */
struct BufferedSource : Source
{
size_t bufSize, bufPosIn, bufPosOut;
std::unique_ptr<unsigned char[]> buffer;
struct BufferedSource : Source {
size_t bufSize, bufPosIn, bufPosOut;
std::unique_ptr<unsigned char[]> buffer;
BufferedSource(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { }
BufferedSource(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) {}
size_t read(unsigned char * data, size_t len) override;
size_t read(unsigned char* data, size_t len) override;
bool hasData();
bool hasData();
protected:
/* Underlying read call, to be overridden. */
virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
protected:
/* Underlying read call, to be overridden. */
virtual size_t readUnbuffered(unsigned char* data, size_t len) = 0;
};
/* A sink that writes data to a file descriptor. */
struct FdSink : BufferedSink
{
int fd;
bool warn = false;
size_t written = 0;
struct FdSink : BufferedSink {
int fd;
bool warn = false;
size_t written = 0;
FdSink() : fd(-1) { }
FdSink(int fd) : fd(fd) { }
FdSink(FdSink&&) = default;
FdSink() : fd(-1) {}
FdSink(int fd) : fd(fd) {}
FdSink(FdSink&&) = default;
FdSink& operator=(FdSink && s)
{
flush();
fd = s.fd;
s.fd = -1;
warn = s.warn;
written = s.written;
return *this;
}
FdSink& operator=(FdSink&& s) {
flush();
fd = s.fd;
s.fd = -1;
warn = s.warn;
written = s.written;
return *this;
}
~FdSink();
~FdSink();
void write(const unsigned char * data, size_t len) override;
void write(const unsigned char* data, size_t len) override;
bool good() override;
bool good() override;
private:
bool _good = true;
private:
bool _good = true;
};
/* A source that reads data from a file descriptor. */
struct FdSource : BufferedSource
{
int fd;
size_t read = 0;
struct FdSource : BufferedSource {
int fd;
size_t read = 0;
FdSource() : fd(-1) { }
FdSource(int fd) : fd(fd) { }
FdSource(FdSource&&) = default;
FdSource() : fd(-1) {}
FdSource(int fd) : fd(fd) {}
FdSource(FdSource&&) = default;
FdSource& operator=(FdSource && s)
{
fd = s.fd;
s.fd = -1;
read = s.read;
return *this;
}
FdSource& operator=(FdSource&& s) {
fd = s.fd;
s.fd = -1;
read = s.read;
return *this;
}
bool good() override;
protected:
size_t readUnbuffered(unsigned char * data, size_t len) override;
private:
bool _good = true;
bool good() override;
protected:
size_t readUnbuffered(unsigned char* data, size_t len) override;
private:
bool _good = true;
};
/* A sink that writes data to a string. */
struct StringSink : Sink
{
ref<std::string> s;
StringSink() : s(make_ref<std::string>()) { };
StringSink(ref<std::string> s) : s(s) { };
void operator () (const unsigned char * data, size_t len) override;
struct StringSink : Sink {
ref<std::string> s;
StringSink() : s(make_ref<std::string>()){};
StringSink(ref<std::string> s) : s(s){};
void operator()(const unsigned char* data, size_t len) override;
};
/* A source that reads data from a string. */
struct StringSource : Source
{
const string & s;
size_t pos;
StringSource(const string & _s) : s(_s), pos(0) { }
size_t read(unsigned char * data, size_t len) override;
struct StringSource : Source {
const string& s;
size_t pos;
StringSource(const string& _s) : s(_s), pos(0) {}
size_t read(unsigned char* data, size_t len) override;
};
/* Adapter class of a Source that saves all data read to `s'. */
struct TeeSource : Source
{
Source & orig;
ref<std::string> data;
TeeSource(Source & orig)
: orig(orig), data(make_ref<std::string>()) { }
size_t read(unsigned char * data, size_t len)
{
size_t n = orig.read(data, len);
this->data->append((const char *) data, n);
return n;
}
struct TeeSource : Source {
Source& orig;
ref<std::string> data;
TeeSource(Source& orig) : orig(orig), data(make_ref<std::string>()) {}
size_t read(unsigned char* data, size_t len) {
size_t n = orig.read(data, len);
this->data->append((const char*)data, n);
return n;
}
};
/* A reader that consumes the original Source until 'size'. */
struct SizedSource : Source
{
Source & orig;
size_t remain;
SizedSource(Source & orig, size_t size)
: orig(orig), remain(size) { }
size_t read(unsigned char * data, size_t len)
{
if (this->remain <= 0) {
throw EndOfFile("sized: unexpected end-of-file");
}
len = std::min(len, this->remain);
size_t n = this->orig.read(data, len);
this->remain -= n;
return n;
struct SizedSource : Source {
Source& orig;
size_t remain;
SizedSource(Source& orig, size_t size) : orig(orig), remain(size) {}
size_t read(unsigned char* data, size_t len) {
if (this->remain <= 0) {
throw EndOfFile("sized: unexpected end-of-file");
}
len = std::min(len, this->remain);
size_t n = this->orig.read(data, len);
this->remain -= n;
return n;
}
/* Consume the original source until no remain data is left to consume. */
size_t drainAll()
{
std::vector<unsigned char> buf(8192);
size_t sum = 0;
while (this->remain > 0) {
size_t n = read(buf.data(), buf.size());
sum += n;
}
return sum;
/* Consume the original source until no remain data is left to consume. */
size_t drainAll() {
std::vector<unsigned char> buf(8192);
size_t sum = 0;
while (this->remain > 0) {
size_t n = read(buf.data(), buf.size());
sum += n;
}
return sum;
}
};
/* Convert a function into a sink. */
struct LambdaSink : Sink
{
typedef std::function<void(const unsigned char *, size_t)> lambda_t;
struct LambdaSink : Sink {
typedef std::function<void(const unsigned char*, size_t)> lambda_t;
lambda_t lambda;
lambda_t lambda;
LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
LambdaSink(const lambda_t& lambda) : lambda(lambda) {}
virtual void operator () (const unsigned char * data, size_t len)
{
lambda(data, len);
}
virtual void operator()(const unsigned char* data, size_t len) {
lambda(data, len);
}
};
/* Convert a function into a source. */
struct LambdaSource : Source
{
typedef std::function<size_t(unsigned char *, size_t)> lambda_t;
struct LambdaSource : Source {
typedef std::function<size_t(unsigned char*, size_t)> lambda_t;
lambda_t lambda;
lambda_t lambda;
LambdaSource(const lambda_t & lambda) : lambda(lambda) { }
LambdaSource(const lambda_t& lambda) : lambda(lambda) {}
size_t read(unsigned char * data, size_t len) override
{
return lambda(data, len);
}
size_t read(unsigned char* data, size_t len) override {
return lambda(data, len);
}
};
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof = []() {
throw EndOfFile("coroutine has finished");
std::function<void(Sink&)> fun, std::function<void()> eof = []() {
throw EndOfFile("coroutine has finished");
});
void writePadding(size_t len, Sink& sink);
void writeString(const unsigned char* buf, size_t len, Sink& sink);
void writePadding(size_t len, Sink & sink);
void writeString(const unsigned char * buf, size_t len, Sink & sink);
inline Sink & operator << (Sink & sink, uint64_t n)
{
unsigned char buf[8];
buf[0] = n & 0xff;
buf[1] = (n >> 8) & 0xff;
buf[2] = (n >> 16) & 0xff;
buf[3] = (n >> 24) & 0xff;
buf[4] = (n >> 32) & 0xff;
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char) (n >> 56) & 0xff;
sink(buf, sizeof(buf));
return sink;
inline Sink& operator<<(Sink& sink, uint64_t n) {
unsigned char buf[8];
buf[0] = n & 0xff;
buf[1] = (n >> 8) & 0xff;
buf[2] = (n >> 16) & 0xff;
buf[3] = (n >> 24) & 0xff;
buf[4] = (n >> 32) & 0xff;
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char)(n >> 56) & 0xff;
sink(buf, sizeof(buf));
return sink;
}
Sink & operator << (Sink & sink, const string & s);
Sink & operator << (Sink & sink, const Strings & s);
Sink & operator << (Sink & sink, const StringSet & s);
Sink& operator<<(Sink& sink, const string& s);
Sink& operator<<(Sink& sink, const Strings& s);
Sink& operator<<(Sink& sink, const StringSet& s);
MakeError(SerialisationError, Error)
template <typename T>
T readNum(Source& source) {
unsigned char buf[8];
source(buf, sizeof(buf));
template<typename T>
T readNum(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
uint64_t n =
((unsigned long long)buf[0]) | ((unsigned long long)buf[1] << 8) |
((unsigned long long)buf[2] << 16) | ((unsigned long long)buf[3] << 24) |
((unsigned long long)buf[4] << 32) | ((unsigned long long)buf[5] << 40) |
((unsigned long long)buf[6] << 48) | ((unsigned long long)buf[7] << 56);
uint64_t n =
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type '%s'",
n, typeid(T).name());
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
return (T) n;
return (T)n;
}
inline unsigned int readInt(Source & source)
{
return readNum<unsigned int>(source);
inline unsigned int readInt(Source& source) {
return readNum<unsigned int>(source);
}
inline uint64_t readLongLong(Source & source)
{
return readNum<uint64_t>(source);
inline uint64_t readLongLong(Source& source) {
return readNum<uint64_t>(source);
}
void readPadding(size_t len, Source& source);
size_t readString(unsigned char* buf, size_t max, Source& source);
string readString(Source& source,
size_t max = std::numeric_limits<size_t>::max());
template <class T>
T readStrings(Source& source);
void readPadding(size_t len, Source & source);
size_t readString(unsigned char * buf, size_t max, Source & source);
string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
template<class T> T readStrings(Source & source);
Source& operator>>(Source& in, string& s);
Source & operator >> (Source & in, string & s);
template<typename T>
Source & operator >> (Source & in, T & n)
{
n = readNum<T>(in);
return in;
template <typename T>
Source& operator>>(Source& in, T& n) {
n = readNum<T>(in);
return in;
}
template<typename T>
Source & operator >> (Source & in, bool & b)
{
b = readNum<uint64_t>(in);
return in;
template <typename T>
Source& operator>>(Source& in, bool& b) {
b = readNum<uint64_t>(in);
return in;
}
}
} // namespace nix

106
third_party/nix/src/libutil/sync.hh vendored
View file

@ -1,9 +1,9 @@
#pragma once
#include <cassert>
#include <condition_variable>
#include <cstdlib>
#include <mutex>
#include <condition_variable>
#include <cassert>
namespace nix {
@ -23,66 +23,62 @@ namespace nix {
scope.
*/
template<class T, class M = std::mutex>
class Sync
{
private:
M mutex;
T data;
template <class T, class M = std::mutex>
class Sync {
private:
M mutex;
T data;
public:
public:
Sync() {}
Sync(const T& data) : data(data) {}
Sync(T&& data) noexcept : data(std::move(data)) {}
Sync() { }
Sync(const T & data) : data(data) { }
Sync(T && data) noexcept : data(std::move(data)) { }
class Lock {
private:
Sync* s;
std::unique_lock<M> lk;
friend Sync;
Lock(Sync* s) : s(s), lk(s->mutex) {}
class Lock
{
private:
Sync * s;
std::unique_lock<M> lk;
friend Sync;
Lock(Sync * s) : s(s), lk(s->mutex) { }
public:
Lock(Lock && l) : s(l.s) { abort(); }
Lock(const Lock & l) = delete;
~Lock() { }
T * operator -> () { return &s->data; }
T & operator * () { return s->data; }
public:
Lock(Lock&& l) : s(l.s) { abort(); }
Lock(const Lock& l) = delete;
~Lock() {}
T* operator->() { return &s->data; }
T& operator*() { return s->data; }
void wait(std::condition_variable & cv)
{
assert(s);
cv.wait(lk);
}
void wait(std::condition_variable& cv) {
assert(s);
cv.wait(lk);
}
template<class Rep, class Period>
std::cv_status wait_for(std::condition_variable & cv,
const std::chrono::duration<Rep, Period> & duration)
{
assert(s);
return cv.wait_for(lk, duration);
}
template <class Rep, class Period>
std::cv_status wait_for(
std::condition_variable& cv,
const std::chrono::duration<Rep, Period>& duration) {
assert(s);
return cv.wait_for(lk, duration);
}
template<class Rep, class Period, class Predicate>
bool wait_for(std::condition_variable & cv,
const std::chrono::duration<Rep, Period> & duration,
Predicate pred)
{
assert(s);
return cv.wait_for(lk, duration, pred);
}
template <class Rep, class Period, class Predicate>
bool wait_for(std::condition_variable& cv,
const std::chrono::duration<Rep, Period>& duration,
Predicate pred) {
assert(s);
return cv.wait_for(lk, duration, pred);
}
template<class Clock, class Duration>
std::cv_status wait_until(std::condition_variable & cv,
const std::chrono::time_point<Clock, Duration> & duration)
{
assert(s);
return cv.wait_until(lk, duration);
}
};
template <class Clock, class Duration>
std::cv_status wait_until(
std::condition_variable& cv,
const std::chrono::time_point<Clock, Duration>& duration) {
assert(s);
return cv.wait_until(lk, duration);
}
};
Lock lock() { return Lock(this); }
Lock lock() { return Lock(this); }
};
}
} // namespace nix

261
third_party/nix/src/libutil/thread-pool.cc vendored
View file

@ -3,154 +3,141 @@
namespace nix {
ThreadPool::ThreadPool(size_t _maxThreads)
: maxThreads(_maxThreads)
{
restoreAffinity(); // FIXME
ThreadPool::ThreadPool(size_t _maxThreads) : maxThreads(_maxThreads) {
restoreAffinity(); // FIXME
if (!maxThreads) {
maxThreads = std::thread::hardware_concurrency();
if (!maxThreads) maxThreads = 1;
}
if (!maxThreads) {
maxThreads = std::thread::hardware_concurrency();
if (!maxThreads) maxThreads = 1;
}
debug("starting pool of %d threads", maxThreads - 1);
debug("starting pool of %d threads", maxThreads - 1);
}
ThreadPool::~ThreadPool()
{
shutdown();
}
ThreadPool::~ThreadPool() { shutdown(); }
void ThreadPool::shutdown()
{
std::vector<std::thread> workers;
{
auto state(state_.lock());
quit = true;
std::swap(workers, state->workers);
}
if (workers.empty()) return;
debug("reaping %d worker threads", workers.size());
work.notify_all();
for (auto & thr : workers)
thr.join();
}
void ThreadPool::enqueue(const work_t & t)
{
void ThreadPool::shutdown() {
std::vector<std::thread> workers;
{
auto state(state_.lock());
if (quit)
throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down");
state->pending.push(t);
/* Note: process() also executes items, so count it as a worker. */
if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads)
state->workers.emplace_back(&ThreadPool::doWork, this, false);
work.notify_one();
quit = true;
std::swap(workers, state->workers);
}
if (workers.empty()) return;
debug("reaping %d worker threads", workers.size());
work.notify_all();
for (auto& thr : workers) thr.join();
}
void ThreadPool::process()
{
state_.lock()->draining = true;
void ThreadPool::enqueue(const work_t& t) {
auto state(state_.lock());
if (quit)
throw ThreadPoolShutDown(
"cannot enqueue a work item while the thread pool is shutting down");
state->pending.push(t);
/* Note: process() also executes items, so count it as a worker. */
if (state->pending.size() > state->workers.size() + 1 &&
state->workers.size() + 1 < maxThreads)
state->workers.emplace_back(&ThreadPool::doWork, this, false);
work.notify_one();
}
void ThreadPool::process() {
state_.lock()->draining = true;
/* Do work until no more work is pending or active. */
try {
doWork(true);
auto state(state_.lock());
assert(quit);
if (state->exception) std::rethrow_exception(state->exception);
} catch (...) {
/* In the exceptional case, some workers may still be
active. They may be referencing the stack frame of the
caller. So wait for them to finish. (~ThreadPool also does
this, but it might be destroyed after objects referenced by
the work item lambdas.) */
shutdown();
throw;
}
}
void ThreadPool::doWork(bool mainThread) {
if (!mainThread) interruptCheck = [&]() { return (bool)quit; };
bool didWork = false;
std::exception_ptr exc;
while (true) {
work_t w;
{
auto state(state_.lock());
if (didWork) {
assert(state->active);
state->active--;
if (exc) {
if (!state->exception) {
state->exception = exc;
// Tell the other workers to quit.
quit = true;
work.notify_all();
} else {
/* Print the exception, since we can't
propagate it. */
try {
std::rethrow_exception(exc);
} catch (std::exception& e) {
if (!dynamic_cast<Interrupted*>(&e) &&
!dynamic_cast<ThreadPoolShutDown*>(&e))
ignoreException();
} catch (...) {
}
}
}
}
/* Wait until a work item is available or we're asked to
quit. */
while (true) {
if (quit) return;
if (!state->pending.empty()) break;
/* If there are no active or pending items, and the
main thread is running process(), then no new items
can be added. So exit. */
if (!state->active && state->draining) {
quit = true;
work.notify_all();
return;
}
state.wait(work);
}
w = std::move(state->pending.front());
state->pending.pop();
state->active++;
}
/* Do work until no more work is pending or active. */
try {
doWork(true);
auto state(state_.lock());
assert(quit);
if (state->exception)
std::rethrow_exception(state->exception);
w();
} catch (...) {
/* In the exceptional case, some workers may still be
active. They may be referencing the stack frame of the
caller. So wait for them to finish. (~ThreadPool also does
this, but it might be destroyed after objects referenced by
the work item lambdas.) */
shutdown();
throw;
exc = std::current_exception();
}
didWork = true;
}
}
void ThreadPool::doWork(bool mainThread)
{
if (!mainThread)
interruptCheck = [&]() { return (bool) quit; };
bool didWork = false;
std::exception_ptr exc;
while (true) {
work_t w;
{
auto state(state_.lock());
if (didWork) {
assert(state->active);
state->active--;
if (exc) {
if (!state->exception) {
state->exception = exc;
// Tell the other workers to quit.
quit = true;
work.notify_all();
} else {
/* Print the exception, since we can't
propagate it. */
try {
std::rethrow_exception(exc);
} catch (std::exception & e) {
if (!dynamic_cast<Interrupted*>(&e) &&
!dynamic_cast<ThreadPoolShutDown*>(&e))
ignoreException();
} catch (...) {
}
}
}
}
/* Wait until a work item is available or we're asked to
quit. */
while (true) {
if (quit) return;
if (!state->pending.empty()) break;
/* If there are no active or pending items, and the
main thread is running process(), then no new items
can be added. So exit. */
if (!state->active && state->draining) {
quit = true;
work.notify_all();
return;
}
state.wait(work);
}
w = std::move(state->pending.front());
state->pending.pop();
state->active++;
}
try {
w();
} catch (...) {
exc = std::current_exception();
}
didWork = true;
}
}
}
} // namespace nix

196
third_party/nix/src/libutil/thread-pool.hh vendored
View file

@ -1,143 +1,129 @@
#pragma once
#include <atomic>
#include <functional>
#include <map>
#include <queue>
#include <thread>
#include "sync.hh"
#include "util.hh"
#include <queue>
#include <functional>
#include <thread>
#include <map>
#include <atomic>
namespace nix {
MakeError(ThreadPoolShutDown, Error)
/* A simple thread pool that executes a queue of work items
(lambdas). */
class ThreadPool
{
public:
/* A simple thread pool that executes a queue of work items
(lambdas). */
class ThreadPool {
public:
ThreadPool(size_t maxThreads = 0);
ThreadPool(size_t maxThreads = 0);
~ThreadPool();
~ThreadPool();
// FIXME: use std::packaged_task?
typedef std::function<void()> work_t;
// FIXME: use std::packaged_task?
typedef std::function<void()> work_t;
/* Enqueue a function to be executed by the thread pool. */
void enqueue(const work_t& t);
/* Enqueue a function to be executed by the thread pool. */
void enqueue(const work_t & t);
/* Execute work items until the queue is empty. Note that work
items are allowed to add new items to the queue; this is
handled correctly. Queue processing stops prematurely if any
work item throws an exception. This exception is propagated to
the calling thread. If multiple work items throw an exception
concurrently, only one item is propagated; the others are
printed on stderr and otherwise ignored. */
void process();
/* Execute work items until the queue is empty. Note that work
items are allowed to add new items to the queue; this is
handled correctly. Queue processing stops prematurely if any
work item throws an exception. This exception is propagated to
the calling thread. If multiple work items throw an exception
concurrently, only one item is propagated; the others are
printed on stderr and otherwise ignored. */
void process();
private:
size_t maxThreads;
private:
struct State {
std::queue<work_t> pending;
size_t active = 0;
std::exception_ptr exception;
std::vector<std::thread> workers;
bool draining = false;
};
size_t maxThreads;
std::atomic_bool quit{false};
struct State
{
std::queue<work_t> pending;
size_t active = 0;
std::exception_ptr exception;
std::vector<std::thread> workers;
bool draining = false;
};
Sync<State> state_;
std::atomic_bool quit{false};
std::condition_variable work;
Sync<State> state_;
void doWork(bool mainThread);
std::condition_variable work;
void doWork(bool mainThread);
void shutdown();
void shutdown();
};
/* Process in parallel a set of items of type T that have a partial
ordering between them. Thus, any item is only processed after all
its dependencies have been processed. */
template<typename T>
void processGraph(
ThreadPool & pool,
const std::set<T> & nodes,
std::function<std::set<T>(const T &)> getEdges,
std::function<void(const T &)> processNode)
{
struct Graph {
std::set<T> left;
std::map<T, std::set<T>> refs, rrefs;
};
template <typename T>
void processGraph(ThreadPool& pool, const std::set<T>& nodes,
std::function<std::set<T>(const T&)> getEdges,
std::function<void(const T&)> processNode) {
struct Graph {
std::set<T> left;
std::map<T, std::set<T>> refs, rrefs;
};
Sync<Graph> graph_(Graph{nodes, {}, {}});
Sync<Graph> graph_(Graph{nodes, {}, {}});
std::function<void(const T &)> worker;
std::function<void(const T&)> worker;
worker = [&](const T & node) {
worker = [&](const T& node) {
{
auto graph(graph_.lock());
auto i = graph->refs.find(node);
if (i == graph->refs.end()) goto getRefs;
goto doWork;
}
{
auto graph(graph_.lock());
auto i = graph->refs.find(node);
if (i == graph->refs.end())
goto getRefs;
goto doWork;
getRefs : {
auto refs = getEdges(node);
refs.erase(node);
{
auto graph(graph_.lock());
for (auto& ref : refs)
if (graph->left.count(ref)) {
graph->refs[node].insert(ref);
graph->rrefs[ref].insert(node);
}
if (graph->refs[node].empty()) goto doWork;
}
}
getRefs:
{
auto refs = getEdges(node);
refs.erase(node);
return;
{
auto graph(graph_.lock());
for (auto & ref : refs)
if (graph->left.count(ref)) {
graph->refs[node].insert(ref);
graph->rrefs[ref].insert(node);
}
if (graph->refs[node].empty())
goto doWork;
}
}
doWork:
processNode(node);
return;
/* Enqueue work for all nodes that were waiting on this one
and have no unprocessed dependencies. */
{
auto graph(graph_.lock());
for (auto& rref : graph->rrefs[node]) {
auto& refs(graph->refs[rref]);
auto i = refs.find(node);
assert(i != refs.end());
refs.erase(i);
if (refs.empty()) pool.enqueue(std::bind(worker, rref));
}
graph->left.erase(node);
graph->refs.erase(node);
graph->rrefs.erase(node);
}
};
doWork:
processNode(node);
for (auto& node : nodes) pool.enqueue(std::bind(worker, std::ref(node)));
/* Enqueue work for all nodes that were waiting on this one
and have no unprocessed dependencies. */
{
auto graph(graph_.lock());
for (auto & rref : graph->rrefs[node]) {
auto & refs(graph->refs[rref]);
auto i = refs.find(node);
assert(i != refs.end());
refs.erase(i);
if (refs.empty())
pool.enqueue(std::bind(worker, rref));
}
graph->left.erase(node);
graph->refs.erase(node);
graph->rrefs.erase(node);
}
};
pool.process();
for (auto & node : nodes)
pool.enqueue(std::bind(worker, std::ref(node)));
pool.process();
if (!graph_.lock()->left.empty())
throw Error("graph processing incomplete (cyclic reference?)");
if (!graph_.lock()->left.empty())
throw Error("graph processing incomplete (cyclic reference?)");
}
}
} // namespace nix

143
third_party/nix/src/libutil/types.hh vendored
View file

@ -1,15 +1,12 @@
#pragma once
#include "ref.hh"
#include <string>
#include <list>
#include <set>
#include <memory>
#include <map>
#include <boost/format.hpp>
#include <list>
#include <map>
#include <memory>
#include <set>
#include <string>
#include "ref.hh"
/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
* its (virtual) destructor and what() in c++11 mode, in violation of spec
@ -20,131 +17,103 @@
#endif
#endif
namespace nix {
/* Inherit some names from other namespaces for convenience. */
using std::string;
using boost::format;
using std::list;
using std::set;
using std::string;
using std::vector;
using boost::format;
/* A variadic template that does nothing. Useful to call a function
for all variadic arguments but ignoring the result. */
struct nop { template<typename... T> nop(T...) {} };
struct FormatOrString
{
string s;
FormatOrString(const string & s) : s(s) { };
FormatOrString(const format & f) : s(f.str()) { };
FormatOrString(const char * s) : s(s) { };
struct nop {
template <typename... T>
nop(T...) {}
};
struct FormatOrString {
string s;
FormatOrString(const string& s) : s(s){};
FormatOrString(const format& f) : s(f.str()){};
FormatOrString(const char* s) : s(s){};
};
/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is
equivalent to ‘boost::format(format) % a_0 % ... %
... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
takes place). */
inline std::string fmt(const std::string & s)
{
return s;
}
inline std::string fmt(const std::string& s) { return s; }
inline std::string fmt(const char * s)
{
return s;
}
inline std::string fmt(const char* s) { return s; }
inline std::string fmt(const FormatOrString & fs)
{
return fs.s;
}
inline std::string fmt(const FormatOrString& fs) { return fs.s; }
template<typename... Args>
inline std::string fmt(const std::string & fs, Args... args)
{
boost::format f(fs);
f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
nop{boost::io::detail::feed(f, args)...};
return f.str();
template <typename... Args>
inline std::string fmt(const std::string& fs, Args... args) {
boost::format f(fs);
f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
nop{boost::io::detail::feed(f, args)...};
return f.str();
}
/* BaseError should generally not be caught, as it has Interrupted as
a subclass. Catch Error instead. */
class BaseError : public std::exception
{
protected:
string prefix_; // used for location traces etc.
string err;
public:
unsigned int status = 1; // exit status
class BaseError : public std::exception {
protected:
string prefix_; // used for location traces etc.
string err;
template<typename... Args>
BaseError(unsigned int status, Args... args)
: err(fmt(args...))
, status(status)
{
}
public:
unsigned int status = 1; // exit status
template<typename... Args>
BaseError(Args... args)
: err(fmt(args...))
{
}
template <typename... Args>
BaseError(unsigned int status, Args... args)
: err(fmt(args...)), status(status) {}
template <typename... Args>
BaseError(Args... args) : err(fmt(args...)) {}
#ifdef EXCEPTION_NEEDS_THROW_SPEC
~BaseError() throw () { };
const char * what() const throw () { return err.c_str(); }
~BaseError() throw(){};
const char* what() const throw() { return err.c_str(); }
#else
const char * what() const noexcept { return err.c_str(); }
const char* what() const noexcept { return err.c_str(); }
#endif
const string & msg() const { return err; }
const string & prefix() const { return prefix_; }
BaseError & addPrefix(const FormatOrString & fs);
const string& msg() const { return err; }
const string& prefix() const { return prefix_; }
BaseError& addPrefix(const FormatOrString& fs);
};
#define MakeError(newClass, superClass) \
class newClass : public superClass \
{ \
public: \
using superClass::superClass; \
};
class newClass : public superClass { \
public: \
using superClass::superClass; \
};
MakeError(Error, BaseError)
class SysError : public Error
{
public:
int errNo;
class SysError : public Error {
public:
int errNo;
template<typename... Args>
SysError(Args... args)
: Error(addErrno(fmt(args...)))
{ }
template <typename... Args>
SysError(Args... args) : Error(addErrno(fmt(args...))) {}
private:
std::string addErrno(const std::string & s);
private:
std::string addErrno(const std::string& s);
};
typedef list<string> Strings;
typedef set<string> StringSet;
typedef std::map<std::string, std::string> StringMap;
/* Paths are just strings. */
typedef string Path;
typedef list<Path> Paths;
typedef set<Path> PathSet;
}
} // namespace nix

2161
third_party/nix/src/libutil/util.cc vendored
View file

File diff suppressed because it is too large Load diff

470
third_party/nix/src/libutil/util.hh vendored
View file

@ -1,21 +1,19 @@
#pragma once
#include "types.hh"
#include "logging.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <signal.h>
#include <functional>
#include <limits>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdio>
#include <map>
#include <sstream>
#include <optional>
#include <functional>
#include <future>
#include <limits>
#include <map>
#include <optional>
#include <sstream>
#include "logging.hh"
#include "types.hh"
#ifndef HAVE_STRUCT_DIRENT_D_TYPE
#define DT_UNKNOWN 0
@ -29,13 +27,11 @@ namespace nix {
struct Sink;
struct Source;
/* The system for which Nix is compiled. */
extern const std::string nativeSystem;
/* Return an environment variable. */
string getEnv(const string & key, const string & def = "");
string getEnv(const string& key, const string& def = "");
/* Get the entire environment. */
std::map<std::string, std::string> getEnv();
@ -52,62 +48,61 @@ Path absPath(Path path, Path dir = "");
double or trailing slashes. Optionally resolves all symlink
components such that each component of the resulting path is *not*
a symbolic link. */
Path canonPath(const Path & path, bool resolveSymlinks = false);
Path canonPath(const Path& path, bool resolveSymlinks = false);
/* Return the directory part of the given canonical path, i.e.,
everything before the final `/'. If the path is the root or an
immediate child thereof (e.g., `/foo'), this means an empty string
is returned. */
Path dirOf(const Path & path);
Path dirOf(const Path& path);
/* Return the base name of the given canonical path, i.e., everything
following the final `/'. */
string baseNameOf(const Path & path);
string baseNameOf(const Path& path);
/* Check whether 'path' is a descendant of 'dir'. */
bool isInDir(const Path & path, const Path & dir);
bool isInDir(const Path& path, const Path& dir);
/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */
bool isDirOrInDir(const Path & path, const Path & dir);
bool isDirOrInDir(const Path& path, const Path& dir);
/* Get status of `path'. */
struct stat lstat(const Path & path);
struct stat lstat(const Path& path);
/* Return true iff the given path exists. */
bool pathExists(const Path & path);
bool pathExists(const Path& path);
/* Read the contents (target) of a symbolic link. The result is not
in any way canonicalised. */
Path readLink(const Path & path);
Path readLink(const Path& path);
bool isLink(const Path & path);
bool isLink(const Path& path);
/* Read the contents of a directory. The entries `.' and `..' are
removed. */
struct DirEntry
{
string name;
ino_t ino;
unsigned char type; // one of DT_*
DirEntry(const string & name, ino_t ino, unsigned char type)
: name(name), ino(ino), type(type) { }
struct DirEntry {
string name;
ino_t ino;
unsigned char type; // one of DT_*
DirEntry(const string& name, ino_t ino, unsigned char type)
: name(name), ino(ino), type(type) {}
};
typedef vector<DirEntry> DirEntries;
DirEntries readDirectory(const Path & path);
DirEntries readDirectory(const Path& path);
unsigned char getFileType(const Path & path);
unsigned char getFileType(const Path& path);
/* Read the contents of a file into a string. */
string readFile(int fd);
string readFile(const Path & path, bool drain = false);
void readFile(const Path & path, Sink & sink);
string readFile(const Path& path, bool drain = false);
void readFile(const Path& path, Sink& sink);
/* Write a string to a file. */
void writeFile(const Path & path, const string & s, mode_t mode = 0666);
void writeFile(const Path& path, const string& s, mode_t mode = 0666);
void writeFile(const Path & path, Source & source, mode_t mode = 0666);
void writeFile(const Path& path, Source& source, mode_t mode = 0666);
/* Read a line from a file descriptor. */
string readLine(int fd);
@ -118,13 +113,14 @@ void writeLine(int fd, string s);
/* Delete a path; i.e., in the case of a directory, it is deleted
recursively. It's not an error if the path does not exist. The
second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
void deletePath(const Path& path);
void deletePath(const Path & path, unsigned long long & bytesFreed);
void deletePath(const Path& path, unsigned long long& bytesFreed);
/* Create a temporary directory. */
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
Path createTempDir(const Path& tmpRoot = "", const Path& prefix = "nix",
bool includePid = true, bool useGlobalCounter = true,
mode_t mode = 0755);
std::string getUserName();
@ -145,178 +141,161 @@ Path getDataDir();
/* Create a directory and all its parents, if necessary. Returns the
list of created directories, in order of creation. */
Paths createDirs(const Path & path);
Paths createDirs(const Path& path);
/* Create a symlink. */
void createSymlink(const Path & target, const Path & link);
void createSymlink(const Path& target, const Path& link);
/* Atomically create or replace a symlink. */
void replaceSymlink(const Path & target, const Path & link);
void replaceSymlink(const Path& target, const Path& link);
/* Wrappers arount read()/write() that read/write exactly the
requested number of bytes. */
void readFull(int fd, unsigned char * buf, size_t count);
void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
void writeFull(int fd, const string & s, bool allowInterrupts = true);
void readFull(int fd, unsigned char* buf, size_t count);
void writeFull(int fd, const unsigned char* buf, size_t count,
bool allowInterrupts = true);
void writeFull(int fd, const string& s, bool allowInterrupts = true);
MakeError(EndOfFile, Error)
/* Read a file descriptor until EOF occurs. */
string drainFD(int fd, bool block = true);
/* Read a file descriptor until EOF occurs. */
string drainFD(int fd, bool block = true);
void drainFD(int fd, Sink & sink, bool block = true);
void drainFD(int fd, Sink& sink, bool block = true);
/* Automatic cleanup of resources. */
class AutoDelete {
Path path;
bool del;
bool recursive;
class AutoDelete
{
Path path;
bool del;
bool recursive;
public:
AutoDelete();
AutoDelete(const Path & p, bool recursive = true);
~AutoDelete();
void cancel();
void reset(const Path & p, bool recursive = true);
operator Path() const { return path; }
public:
AutoDelete();
AutoDelete(const Path& p, bool recursive = true);
~AutoDelete();
void cancel();
void reset(const Path& p, bool recursive = true);
operator Path() const { return path; }
};
class AutoCloseFD {
int fd;
void close();
class AutoCloseFD
{
int fd;
void close();
public:
AutoCloseFD();
AutoCloseFD(int fd);
AutoCloseFD(const AutoCloseFD & fd) = delete;
AutoCloseFD(AutoCloseFD&& fd);
~AutoCloseFD();
AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
AutoCloseFD& operator =(AutoCloseFD&& fd);
int get() const;
explicit operator bool() const;
int release();
public:
AutoCloseFD();
AutoCloseFD(int fd);
AutoCloseFD(const AutoCloseFD& fd) = delete;
AutoCloseFD(AutoCloseFD&& fd);
~AutoCloseFD();
AutoCloseFD& operator=(const AutoCloseFD& fd) = delete;
AutoCloseFD& operator=(AutoCloseFD&& fd);
int get() const;
explicit operator bool() const;
int release();
};
class Pipe
{
public:
AutoCloseFD readSide, writeSide;
void create();
class Pipe {
public:
AutoCloseFD readSide, writeSide;
void create();
};
struct DIRDeleter
{
void operator()(DIR * dir) const {
closedir(dir);
}
struct DIRDeleter {
void operator()(DIR* dir) const { closedir(dir); }
};
typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
class Pid {
pid_t pid = -1;
bool separatePG = false;
int killSignal = SIGKILL;
class Pid
{
pid_t pid = -1;
bool separatePG = false;
int killSignal = SIGKILL;
public:
Pid();
Pid(pid_t pid);
~Pid();
void operator =(pid_t pid);
operator pid_t();
int kill();
int wait();
public:
Pid();
Pid(pid_t pid);
~Pid();
void operator=(pid_t pid);
operator pid_t();
int kill();
int wait();
void setSeparatePG(bool separatePG);
void setKillSignal(int signal);
pid_t release();
void setSeparatePG(bool separatePG);
void setKillSignal(int signal);
pid_t release();
};
/* Kill all processes running under the specified uid by sending them
a SIGKILL. */
void killUser(uid_t uid);
/* Fork a process that runs the given function, and return the child
pid to the caller. */
struct ProcessOptions
{
string errorPrefix = "error: ";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = true;
struct ProcessOptions {
string errorPrefix = "error: ";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = true;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
pid_t startProcess(std::function<void()> fun,
const ProcessOptions& options = ProcessOptions());
/* Run a program and return its stdout in a string (i.e., like the
shell backtick operator). */
string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(),
const std::optional<std::string> & input = {});
const Strings& args = Strings(),
const std::optional<std::string>& input = {});
struct RunOptions
{
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
Path program;
bool searchPath = true;
Strings args;
std::optional<std::string> input;
Source * standardIn = nullptr;
Sink * standardOut = nullptr;
bool mergeStderrToStdout = false;
bool _killStderr = false;
struct RunOptions {
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
Path program;
bool searchPath = true;
Strings args;
std::optional<std::string> input;
Source* standardIn = nullptr;
Sink* standardOut = nullptr;
bool mergeStderrToStdout = false;
bool _killStderr = false;
RunOptions(const Path & program, const Strings & args)
: program(program), args(args) { };
RunOptions(const Path& program, const Strings& args)
: program(program), args(args){};
RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
RunOptions& killStderr(bool v) {
_killStderr = true;
return *this;
}
};
std::pair<int, std::string> runProgram(const RunOptions & options);
std::pair<int, std::string> runProgram(const RunOptions& options);
void runProgram2(const RunOptions & options);
void runProgram2(const RunOptions& options);
class ExecError : public Error {
public:
int status;
class ExecError : public Error
{
public:
int status;
template<typename... Args>
ExecError(int status, Args... args)
: Error(args...), status(status)
{ }
template <typename... Args>
ExecError(int status, Args... args) : Error(args...), status(status) {}
};
/* Convert a list of strings to a null-terminated vector of char
*'s. The result must not be accessed beyond the lifetime of the
list of strings. */
std::vector<char *> stringsToCharPtrs(const Strings & ss);
std::vector<char*> stringsToCharPtrs(const Strings& ss);
/* Close all file descriptors except those listed in the given set.
Good practice in child processes. */
void closeMostFDs(const set<int> & exceptions);
void closeMostFDs(const set<int>& exceptions);
/* Set the close-on-exec flag for the given file descriptor. */
void closeOnExec(int fd);
/* User interruption. */
extern bool _isInterrupted;
@ -327,40 +306,32 @@ void setInterruptThrown();
void _interrupted();
void inline checkInterrupt()
{
if (_isInterrupted || (interruptCheck && interruptCheck()))
_interrupted();
void inline checkInterrupt() {
if (_isInterrupted || (interruptCheck && interruptCheck())) _interrupted();
}
MakeError(Interrupted, BaseError)
MakeError(FormatError, Error)
MakeError(FormatError, Error)
/* String tokenizer. */
template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");
/* String tokenizer. */
template <class C>
C tokenizeString(const string& s, const string& separators = " \t\n\r");
/* Concatenate the given strings with a separator between the
elements. */
string concatStringsSep(const string & sep, const Strings & ss);
string concatStringsSep(const string & sep, const StringSet & ss);
string concatStringsSep(const string& sep, const Strings& ss);
string concatStringsSep(const string& sep, const StringSet& ss);
/* Remove trailing whitespace from a string. */
string chomp(const string & s);
string chomp(const string& s);
/* Remove whitespace from the start and end of a string. */
string trim(const string & s, const string & whitespace = " \n\r\t");
string trim(const string& s, const string& whitespace = " \n\r\t");
/* Replace all occurrences of a string inside another string. */
string replaceStrings(const std::string & s,
const std::string & from, const std::string & to);
string replaceStrings(const std::string& s, const std::string& from,
const std::string& to);
/* Convert the exit status of a child as returned by wait() into an
error string. */
@ -368,47 +339,40 @@ string statusToString(int status);
bool statusOk(int status);
/* Parse a string into an integer. */
template<class N> bool string2Int(const string & s, N & n)
{
if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
return false;
std::istringstream str(s);
str >> n;
return str && str.get() == EOF;
template <class N>
bool string2Int(const string& s, N& n) {
if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
return false;
std::istringstream str(s);
str >> n;
return str && str.get() == EOF;
}
/* Parse a string into a float. */
template<class N> bool string2Float(const string & s, N & n)
{
std::istringstream str(s);
str >> n;
return str && str.get() == EOF;
template <class N>
bool string2Float(const string& s, N& n) {
std::istringstream str(s);
str >> n;
return str && str.get() == EOF;
}
/* Return true iff `s' starts with `prefix'. */
bool hasPrefix(const string & s, const string & prefix);
bool hasPrefix(const string& s, const string& prefix);
/* Return true iff `s' ends in `suffix'. */
bool hasSuffix(const string & s, const string & suffix);
bool hasSuffix(const string& s, const string& suffix);
/* Convert a string to lower case. */
std::string toLower(const std::string & s);
std::string toLower(const std::string& s);
/* Escape a string as a shell word. */
std::string shellEscape(const std::string & s);
std::string shellEscape(const std::string& s);
/* Exception handling in destructors: print an error message, then
ignore the exception. */
void ignoreException();
/* Some ANSI escape sequences. */
#define ANSI_NORMAL "\e[0m"
#define ANSI_BOLD "\e[1m"
@ -417,71 +381,61 @@ void ignoreException();
#define ANSI_GREEN "\e[32;1m"
#define ANSI_BLUE "\e[34;1m"
/* Truncate a string to 'width' printable characters. If 'filterAll'
is true, all ANSI escape sequences are filtered out. Otherwise,
some escape sequences (such as colour setting) are copied but not
included in the character count. Also, tabs are expanded to
spaces. */
std::string filterANSIEscapes(const std::string & s,
bool filterAll = false,
std::string filterANSIEscapes(
const std::string& s, bool filterAll = false,
unsigned int width = std::numeric_limits<unsigned int>::max());
/* Base64 encoding/decoding. */
string base64Encode(const string & s);
string base64Decode(const string & s);
string base64Encode(const string& s);
string base64Decode(const string& s);
/* Get a value for the specified key from an associate container, or a
default value if the key doesn't exist. */
template <class T>
string get(const T & map, const string & key, const string & def = "")
{
auto i = map.find(key);
return i == map.end() ? def : i->second;
string get(const T& map, const string& key, const string& def = "") {
auto i = map.find(key);
return i == map.end() ? def : i->second;
}
/* A callback is a wrapper around a lambda that accepts a valid of
type T or an exception. (We abuse std::future<T> to pass the value or
exception.) */
template<typename T>
class Callback
{
std::function<void(std::future<T>)> fun;
std::atomic_flag done = ATOMIC_FLAG_INIT;
template <typename T>
class Callback {
std::function<void(std::future<T>)> fun;
std::atomic_flag done = ATOMIC_FLAG_INIT;
public:
public:
Callback(std::function<void(std::future<T>)> fun) : fun(fun) {}
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
Callback(Callback&& callback) : fun(std::move(callback.fun)) {
auto prev = callback.done.test_and_set();
if (prev) done.test_and_set();
}
Callback(Callback && callback) : fun(std::move(callback.fun))
{
auto prev = callback.done.test_and_set();
if (prev) done.test_and_set();
}
void operator()(T&& t) noexcept {
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_value(std::move(t));
fun(promise.get_future());
}
void operator()(T && t) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_value(std::move(t));
fun(promise.get_future());
}
void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_exception(exc);
fun(promise.get_future());
}
void rethrow(
const std::exception_ptr& exc = std::current_exception()) noexcept {
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_exception(exc);
fun(promise.get_future());
}
};
/* Start a thread that handles various signals. Also block those signals
on the current thread (and thus any threads created by it). */
void startSignalHandlerThread();
@ -489,9 +443,8 @@ void startSignalHandlerThread();
/* Restore default signal handling. */
void restoreSignals();
struct InterruptCallback
{
virtual ~InterruptCallback() { };
struct InterruptCallback {
virtual ~InterruptCallback(){};
};
/* Register a function that gets called on SIGINT (in a non-signal
@ -504,39 +457,34 @@ void triggerInterrupt();
/* A RAII class that causes the current thread to receive SIGUSR1 when
the signal handler thread receives SIGINT. That is, this allows
SIGINT to be multiplexed to multiple threads. */
struct ReceiveInterrupts
{
pthread_t target;
std::unique_ptr<InterruptCallback> callback;
struct ReceiveInterrupts {
pthread_t target;
std::unique_ptr<InterruptCallback> callback;
ReceiveInterrupts()
: target(pthread_self())
, callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
{ }
ReceiveInterrupts()
: target(pthread_self()), callback(createInterruptCallback([&]() {
pthread_kill(target, SIGUSR1);
})) {}
};
/* A RAII helper that increments a counter on construction and
decrements it on destruction. */
template<typename T>
struct MaintainCount
{
T & counter;
long delta;
MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; }
~MaintainCount() { counter -= delta; }
template <typename T>
struct MaintainCount {
T& counter;
long delta;
MaintainCount(T& counter, long delta = 1) : counter(counter), delta(delta) {
counter += delta;
}
~MaintainCount() { counter -= delta; }
};
/* Return the number of rows and columns of the terminal. */
std::pair<unsigned short, unsigned short> getWindowSize();
/* Used in various places. */
typedef std::function<bool(const Path & path)> PathFilter;
typedef std::function<bool(const Path& path)> PathFilter;
extern PathFilter defaultPathFilter;
}
} // namespace nix

131
third_party/nix/src/libutil/xml-writer.cc vendored
View file

@ -1,94 +1,77 @@
#include "xml-writer.hh"
#include <assert.h>
#include "xml-writer.hh"
namespace nix {
XMLWriter::XMLWriter(bool indent, std::ostream & output)
: output(output), indent(indent)
{
output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl;
closed = false;
XMLWriter::XMLWriter(bool indent, std::ostream& output)
: output(output), indent(indent) {
output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl;
closed = false;
}
XMLWriter::~XMLWriter() { close(); }
XMLWriter::~XMLWriter()
{
close();
void XMLWriter::close() {
if (closed) return;
while (!pendingElems.empty()) closeElement();
closed = true;
}
void XMLWriter::close()
{
if (closed) return;
while (!pendingElems.empty()) closeElement();
closed = true;
void XMLWriter::indent_(size_t depth) {
if (!indent) return;
output << string(depth * 2, ' ');
}
void XMLWriter::indent_(size_t depth)
{
if (!indent) return;
output << string(depth * 2, ' ');
void XMLWriter::openElement(const string& name, const XMLAttrs& attrs) {
assert(!closed);
indent_(pendingElems.size());
output << "<" << name;
writeAttrs(attrs);
output << ">";
if (indent) output << std::endl;
pendingElems.push_back(name);
}
void XMLWriter::openElement(const string & name,
const XMLAttrs & attrs)
{
assert(!closed);
indent_(pendingElems.size());
output << "<" << name;
writeAttrs(attrs);
output << ">";
if (indent) output << std::endl;
pendingElems.push_back(name);
void XMLWriter::closeElement() {
assert(!pendingElems.empty());
indent_(pendingElems.size() - 1);
output << "</" << pendingElems.back() << ">";
if (indent) output << std::endl;
pendingElems.pop_back();
if (pendingElems.empty()) closed = true;
}
void XMLWriter::closeElement()
{
assert(!pendingElems.empty());
indent_(pendingElems.size() - 1);
output << "</" << pendingElems.back() << ">";
if (indent) output << std::endl;
pendingElems.pop_back();
if (pendingElems.empty()) closed = true;
void XMLWriter::writeEmptyElement(const string& name, const XMLAttrs& attrs) {
assert(!closed);
indent_(pendingElems.size());
output << "<" << name;
writeAttrs(attrs);
output << " />";
if (indent) output << std::endl;
}
void XMLWriter::writeEmptyElement(const string & name,
const XMLAttrs & attrs)
{
assert(!closed);
indent_(pendingElems.size());
output << "<" << name;
writeAttrs(attrs);
output << " />";
if (indent) output << std::endl;
}
void XMLWriter::writeAttrs(const XMLAttrs & attrs)
{
for (auto & i : attrs) {
output << " " << i.first << "=\"";
for (size_t j = 0; j < i.second.size(); ++j) {
char c = i.second[j];
if (c == '"') output << "&quot;";
else if (c == '<') output << "&lt;";
else if (c == '>') output << "&gt;";
else if (c == '&') output << "&amp;";
/* Escape newlines to prevent attribute normalisation (see
XML spec, section 3.3.3. */
else if (c == '\n') output << "&#xA;";
else output << c;
}
output << "\"";
void XMLWriter::writeAttrs(const XMLAttrs& attrs) {
for (auto& i : attrs) {
output << " " << i.first << "=\"";
for (size_t j = 0; j < i.second.size(); ++j) {
char c = i.second[j];
if (c == '"')
output << "&quot;";
else if (c == '<')
output << "&lt;";
else if (c == '>')
output << "&gt;";
else if (c == '&')
output << "&amp;";
/* Escape newlines to prevent attribute normalisation (see
XML spec, section 3.3.3. */
else if (c == '\n')
output << "&#xA;";
else
output << c;
}
output << "\"";
}
}
}
} // namespace nix

75
third_party/nix/src/libutil/xml-writer.hh vendored
View file

@ -1,69 +1,56 @@
#pragma once
#include <iostream>
#include <string>
#include <list>
#include <map>
#include <string>
namespace nix {
using std::string;
using std::map;
using std::list;
using std::map;
using std::string;
typedef map<string, string> XMLAttrs;
class XMLWriter {
private:
std::ostream& output;
class XMLWriter
{
private:
bool indent;
bool closed;
std::ostream & output;
list<string> pendingElems;
bool indent;
bool closed;
public:
XMLWriter(bool indent, std::ostream& output);
~XMLWriter();
list<string> pendingElems;
void close();
public:
void openElement(const string& name, const XMLAttrs& attrs = XMLAttrs());
void closeElement();
XMLWriter(bool indent, std::ostream & output);
~XMLWriter();
void writeEmptyElement(const string& name,
const XMLAttrs& attrs = XMLAttrs());
void close();
private:
void writeAttrs(const XMLAttrs& attrs);
void openElement(const string & name,
const XMLAttrs & attrs = XMLAttrs());
void closeElement();
void writeEmptyElement(const string & name,
const XMLAttrs & attrs = XMLAttrs());
private:
void writeAttrs(const XMLAttrs & attrs);
void indent_(size_t depth);
void indent_(size_t depth);
};
class XMLOpenElement {
private:
XMLWriter& writer;
class XMLOpenElement
{
private:
XMLWriter & writer;
public:
XMLOpenElement(XMLWriter & writer, const string & name,
const XMLAttrs & attrs = XMLAttrs())
: writer(writer)
{
writer.openElement(name, attrs);
}
~XMLOpenElement()
{
writer.closeElement();
}
public:
XMLOpenElement(XMLWriter& writer, const string& name,
const XMLAttrs& attrs = XMLAttrs())
: writer(writer) {
writer.openElement(name, attrs);
}
~XMLOpenElement() { writer.closeElement(); }
};
}
} // namespace nix