maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplify the callback mechanism

Browse source
This commit is contained in:
Eelco Dolstra 2018-03-27 22:16:01 +02:00
parent 1672bcd230
commit 81ea8bd5ce
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
16 changed files with 152 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/libstore/binary-cache-store.cc
View file

@ -58,12 +58,13 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
{
std::promise<std::shared_ptr<std::string>> promise;
getFile(path,
[&](std::shared_ptr<std::string> result) {
promise.set_value(result);
},
[&](std::exception_ptr exc) {
promise.set_exception(exc);
});
{[&](std::future<std::shared_ptr<std::string>> result) {
try {
promise.set_value(result.get());
} catch (...) {
promise.set_exception(std::current_exception());
}
}});
return promise.get_future().get();
}
@ -218,8 +219,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
}
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure)
Callback<std::shared_ptr<ValidPathInfo>> callback)
{
auto uri = getUri();
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
@ -229,17 +229,22 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
auto narInfoFile = narInfoFileFor(storePath);
getFile(narInfoFile,
[=](std::shared_ptr<std::string> data) {
if (!data) return success(0);
{[=](std::future<std::shared_ptr<std::string>> fut) {
try {
auto data = fut.get();
stats.narInfoRead++;
if (!data) return callback(nullptr);
callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>)
std::make_shared<NarInfo>(*this, *data, narInfoFile));
stats.narInfoRead++;
(void) act; // force Activity into this lambda to ensure it stays alive
},
failure);
callback((std::shared_ptr<ValidPathInfo>)
std::make_shared<NarInfo>(*this, *data, narInfoFile));
(void) act; // force Activity into this lambda to ensure it stays alive
} catch (...) {
callback.rethrow();
}
}});
}
Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,

6
src/libstore/binary-cache-store.hh
View file

@ -41,8 +41,7 @@ public:
/* Return the contents of the specified file, or null if it
doesn't exist. */
virtual void getFile(const std::string & path,
std::function<void(std::shared_ptr<std::string>)> success,
std::function<void(std::exception_ptr exc)> failure) = 0;
Callback<std::shared_ptr<std::string>> callback) = 0;
std::shared_ptr<std::string> getFile(const std::string & path);
@ -71,8 +70,7 @@ public:
{ unsupported(); }
void queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure) override;
Callback<std::shared_ptr<ValidPathInfo>> callback) override;
void queryReferrers(const Path & path,
PathSet & referrers) override

37
src/libstore/download.cc
View file

@ -81,8 +81,7 @@ struct CurlDownloader : public Downloader
DownloadResult result;
Activity act;
bool done = false; // whether either the success or failure function has been called
std::function<void(const DownloadResult &)> success;
std::function<void(std::exception_ptr exc)> failure;
Callback<DownloadResult> callback;
CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object
std::string status;
@ -97,10 +96,13 @@ struct CurlDownloader : public Downloader
std::string encoding;
DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
DownloadItem(CurlDownloader & downloader,
const DownloadRequest & request,
Callback<DownloadResult> callback)
: downloader(downloader)
, request(request)
, act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct)
, callback(callback)
{
if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@ -129,7 +131,7 @@ struct CurlDownloader : public Downloader
{
assert(!done);
done = true;
callFailure(failure, std::make_exception_ptr(e));
callback.rethrow(std::make_exception_ptr(e));
}
size_t writeCallback(void * contents, size_t size, size_t nmemb)
@ -316,11 +318,11 @@ struct CurlDownloader : public Downloader
try {
if (request.decompress)
result.data = decodeContent(encoding, ref<std::string>(result.data));
callSuccess(success, failure, const_cast<const DownloadResult &>(result));
act.progress(result.data->size(), result.data->size());
callback(std::move(result));
} catch (...) {
done = true;
callFailure(failure, std::current_exception());
callback.rethrow();
}
} else {
// We treat most errors as transient, but won't retry when hopeless
@ -570,13 +572,12 @@ struct CurlDownloader : public Downloader
}
void enqueueDownload(const DownloadRequest & request,
std::function<void(const DownloadResult &)> success,
std::function<void(std::exception_ptr exc)> failure) override
Callback<DownloadResult> callback) override
{
/* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) {
// FIXME: do this on a worker thread
sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
try {
#ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
auto slash = request.uri.find('/', 5);
@ -590,18 +591,15 @@ struct CurlDownloader : public Downloader
if (!s3Res.data)
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data;
return res;
callback(std::move(res));
#else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
#endif
});
} catch (...) { callback.rethrow(); }
return;
}
auto item = std::make_shared<DownloadItem>(*this, request);
item->success = success;
item->failure = failure;
enqueueItem(item);
enqueueItem(std::make_shared<DownloadItem>(*this, request, callback));
}
};
@ -622,8 +620,13 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
{
auto promise = std::make_shared<std::promise<DownloadResult>>();
enqueueDownload(request,
[promise](const DownloadResult & result) { promise->set_value(result); },
[promise](std::exception_ptr exc) { promise->set_exception(exc); });
{[promise](std::future<DownloadResult> fut) {
try {
promise->set_value(fut.get());
} catch (...) {
promise->set_exception(std::current_exception());
}
}});
return promise->get_future();
}

3
src/libstore/download.hh
View file

@ -42,8 +42,7 @@ struct Downloader
the download. The future may throw a DownloadError
exception. */
virtual void enqueueDownload(const DownloadRequest & request,
std::function<void(const DownloadResult &)> success,
std::function<void(std::exception_ptr exc)> failure) = 0;
Callback<DownloadResult> callback) = 0;
std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);

18
src/libstore/http-binary-cache-store.cc
View file

@ -78,27 +78,23 @@ protected:
}
void getFile(const std::string & path,
std::function<void(std::shared_ptr<std::string>)> success,
std::function<void(std::exception_ptr exc)> failure) override
Callback<std::shared_ptr<std::string>> callback) override
{
DownloadRequest request(cacheUri + "/" + path);
request.tries = 8;
getDownloader()->enqueueDownload(request,
[success](const DownloadResult & result) {
success(result.data);
},
[success, failure](std::exception_ptr exc) {
{[callback](std::future<DownloadResult> result) {
try {
std::rethrow_exception(exc);
callback(result.get().data);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return success(0);
failure(exc);
return callback(std::shared_ptr<std::string>());
callback.rethrow();
} catch (...) {
failure(exc);
callback.rethrow();
}
});
}});
}
};

11
src/libstore/legacy-ssh-store.cc
View file

@ -84,10 +84,9 @@ struct LegacySSHStore : public Store
}
void queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure) override
Callback<std::shared_ptr<ValidPathInfo>> callback) override
{
sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
try {
auto conn(connections->get());
debug("querying remote host '%s' for info on '%s'", host, path);
@ -97,7 +96,7 @@ struct LegacySSHStore : public Store
auto info = std::make_shared<ValidPathInfo>();
conn->from >> info->path;
if (info->path.empty()) return nullptr;
if (info->path.empty()) return callback(nullptr);
assert(path == info->path);
PathSet references;
@ -116,8 +115,8 @@ struct LegacySSHStore : public Store
auto s = readString(conn->from);
assert(s == "");
return info;
});
callback(std::move(info));
} catch (...) { callback.rethrow(); }
}
void addToStore(const ValidPathInfo & info, Source & source,

17
src/libstore/local-binary-cache-store.cc
View file

@ -35,17 +35,14 @@ protected:
const std::string & mimeType) override;
void getFile(const std::string & path,
std::function<void(std::shared_ptr<std::string>)> success,
std::function<void(std::exception_ptr exc)> failure) override
Callback<std::shared_ptr<std::string>> callback) override
{
sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
try {
return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path));
} catch (SysError & e) {
if (e.errNo == ENOENT) return std::shared_ptr<std::string>();
throw;
}
});
try {
// FIXME: O(n) space
callback(std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)));
} catch (SysError & e) {
if (e.errNo == ENOENT) callback(nullptr); else callback.rethrow();
} catch (...) { callback.rethrow(); }
}
PathSet queryAllValidPaths() override

13
src/libstore/local-store.cc
View file

@ -629,17 +629,15 @@ uint64_t LocalStore::addValidPath(State & state,
void LocalStore::queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure)
Callback<std::shared_ptr<ValidPathInfo>> callback)
{
sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
try {
auto info = std::make_shared<ValidPathInfo>();
info->path = path;
assertStorePath(path);
return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
auto state(_state.lock());
/* Get the path info. */
@ -679,8 +677,9 @@ void LocalStore::queryPathInfoUncached(const Path & path,
info->references.insert(useQueryReferences.getStr(0));
return info;
});
});
}));
} catch (...) { callback.rethrow(); }
}

3
src/libstore/local-store.hh
View file

@ -127,8 +127,7 @@ public:
PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure) override;
Callback<std::shared_ptr<ValidPathInfo>> callback) override;
void queryReferrers(const Path & path, PathSet & referrers) override;

17
src/libstore/misc.cc
View file

@ -33,9 +33,11 @@ void Store::computeFSClosure(const PathSet & startPaths,
state->pending++;
}
queryPathInfo(path,
[&, path](ref<ValidPathInfo> info) {
// FIXME: calls to isValidPath() should be async
queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
if (flipDirection) {
@ -75,14 +77,13 @@ void Store::computeFSClosure(const PathSet & startPaths,
if (!--state->pending) done.notify_one();
}
},
[&, path](std::exception_ptr exc) {
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = exc;
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
});
};
}});
};
for (auto & startPath : startPaths)

9
src/libstore/remote-store.cc
View file

@ -294,10 +294,9 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
void RemoteStore::queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure)
Callback<std::shared_ptr<ValidPathInfo>> callback)
{
sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
try {
auto conn(connections->get());
conn->to << wopQueryPathInfo << path;
try {
@ -324,8 +323,8 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
info->sigs = readStrings<StringSet>(conn->from);
conn->from >> info->ca;
}
return info;
});
callback(std::move(info));
} catch (...) { callback.rethrow(); }
}

3
src/libstore/remote-store.hh
View file

@ -40,8 +40,7 @@ public:
PathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure) override;
Callback<std::shared_ptr<ValidPathInfo>> callback) override;
void queryReferrers(const Path & path, PathSet & referrers) override;

9
src/libstore/s3-binary-cache-store.cc
View file

@ -365,10 +365,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
}
void getFile(const std::string & path,
std::function<void(std::shared_ptr<std::string>)> success,
std::function<void(std::exception_ptr exc)> failure) override
Callback<std::shared_ptr<std::string>> callback) override
{
sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
try {
stats.get++;
auto res = s3Helper.getObject(bucketName, path);
@ -380,8 +379,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
return res.data;
});
callback(std::move(res.data));
} catch (...) { callback.rethrow(); }
}
PathSet queryAllValidPaths() override

88
src/libstore/store-api.cc
View file

@ -305,20 +305,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
std::promise<ref<ValidPathInfo>> promise;
queryPathInfo(storePath,
[&](ref<ValidPathInfo> info) {
promise.set_value(info);
},
[&](std::exception_ptr exc) {
promise.set_exception(exc);
});
{[&](std::future<ref<ValidPathInfo>> result) {
try {
promise.set_value(result.get());
} catch (...) {
promise.set_exception(std::current_exception());
}
}});
return promise.get_future().get();
}
void Store::queryPathInfo(const Path & storePath,
std::function<void(ref<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure)
Callback<ref<ValidPathInfo>> callback)
{
auto hashPart = storePathToHash(storePath);
@ -330,7 +330,7 @@ void Store::queryPathInfo(const Path & storePath,
stats.narInfoReadAverted++;
if (!*res)
throw InvalidPath(format("path '%s' is not valid") % storePath);
return success(ref<ValidPathInfo>(*res));
return callback(ref<ValidPathInfo>(*res));
}
}
@ -346,35 +346,36 @@ void Store::queryPathInfo(const Path & storePath,
(res.second->path != storePath && storePathToName(storePath) != ""))
throw InvalidPath(format("path '%s' is not valid") % storePath);
}
return success(ref<ValidPathInfo>(res.second));
return callback(ref<ValidPathInfo>(res.second));
}
}
} catch (std::exception & e) {
return callFailure(failure);
}
} catch (...) { return callback.rethrow(); }
queryPathInfoUncached(storePath,
[this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
{[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) {
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, info);
try {
auto info = fut.get();
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, info);
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, info);
if (!info
|| (info->path != storePath && storePathToName(storePath) != ""))
{
stats.narInfoMissing++;
return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath)));
}
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, info);
}
callSuccess(success, failure, ref<ValidPathInfo>(info));
if (!info
|| (info->path != storePath && storePathToName(storePath) != ""))
{
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
}
}, failure);
callback(ref<ValidPathInfo>(info));
} catch (...) { callback.rethrow(); }
}});
}
@ -394,26 +395,19 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
auto doQuery = [&](const Path & path ) {
checkInterrupt();
queryPathInfo(path,
[path, &state_, &wakeup](ref<ValidPathInfo> info) {
auto state(state_.lock());
queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
auto state(state_.lock());
try {
auto info = fut.get();
state->valid.insert(path);
assert(state->left);
if (!--state->left)
wakeup.notify_one();
},
[path, &state_, &wakeup](std::exception_ptr exc) {
auto state(state_.lock());
try {
std::rethrow_exception(exc);
} catch (InvalidPath &) {
} catch (...) {
state->exc = exc;
}
assert(state->left);
if (!--state->left)
wakeup.notify_one();
});
} catch (InvalidPath &) {
} catch (...) {
state->exc = std::current_exception();
}
assert(state->left);
if (!--state->left)
wakeup.notify_one();
}});
};
for (auto & path : paths)

6
src/libstore/store-api.hh
View file

@ -355,14 +355,12 @@ public:
/* Asynchronous version of queryPathInfo(). */
void queryPathInfo(const Path & path,
std::function<void(ref<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure);
Callback<ref<ValidPathInfo>> callback);
protected:
virtual void queryPathInfoUncached(const Path & path,
std::function<void(std::shared_ptr<ValidPathInfo>)> success,
std::function<void(std::exception_ptr exc)> failure) = 0;
Callback<std::shared_ptr<ValidPathInfo>> callback) = 0;
public:

55
src/libutil/util.hh
View file

@ -15,6 +15,7 @@
#include <map>
#include <sstream>
#include <experimental/optional>
#include <future>
#ifndef HAVE_STRUCT_DIRENT_D_TYPE
#define DT_UNKNOWN 0
@ -424,44 +425,30 @@ string get(const T & map, const string & key, const string & def = "")
}
/* Call ‘failure’ with the current exception as argument. If ‘failure’
throws an exception, abort the program. */
void callFailure(const std::function<void(std::exception_ptr exc)> & failure,
std::exception_ptr exc = std::current_exception());
/* Evaluate the function ‘f’. If it returns a value, call ‘success’
with that value as its argument. If it or ‘success’ throws an
exception, call ‘failure’. If ‘failure’ throws an exception, abort
the program. */
template<class T>
void sync2async(
const std::function<void(T)> & success,
const std::function<void(std::exception_ptr exc)> & failure,
const std::function<T()> & f)
/* A callback is a wrapper around a lambda that accepts a valid of
type T or an exception. (We abuse std::future<T> to pass the value or
exception.) */
template<typename T>
struct Callback
{
try {
success(f());
} catch (...) {
callFailure(failure);
}
}
std::function<void(std::future<T>)> fun;
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
/* Call the function ‘success’. If it throws an exception, call
‘failure’. If that throws an exception, abort the program. */
template<class T>
void callSuccess(
const std::function<void(T)> & success,
const std::function<void(std::exception_ptr exc)> & failure,
T && arg)
{
try {
success(arg);
} catch (...) {
callFailure(failure);
void operator()(T && t) const
{
std::promise<T> promise;
promise.set_value(std::move(t));
fun(promise.get_future());
}
}
void rethrow(const std::exception_ptr & exc = std::current_exception()) const
{
std::promise<T> promise;
promise.set_exception(exc);
fun(promise.get_future());
}
};
/* Start a thread that handles various signals. Also block those signals