maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

importPaths(): Optionally add NARs to binary cache accessor

Browse source 
This enables an optimisation in hydra-queue-runner, preventing a
download of a NAR it just uploaded to the cache when reading files
like hydra-build-products.
This commit is contained in:
Eelco Dolstra 2016-02-26 15:20:10 +01:00
parent 00b2c05749
commit cca4a8dc1a
10 changed files with 58 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

2
perl/lib/Nix/Store.xs
View file

@ -186,7 +186,7 @@ void importPaths(int fd)
PPCODE:
try {
FdSource source(fd);
store()->importPaths(false, source);
store()->importPaths(false, source, 0);
} catch (Error & e) {
croak("%s", e.what());
}

66
src/libstore/binary-cache-store.cc
View file

@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
sink << exportMagic << storePath << res.references << res.deriver << 0;
}
Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{
assert(!requireSignature);
Paths res;
@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
unsigned long long n = readLongLong(source);
if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
res.push_back(importPath(source));
res.push_back(importPath(source, accessor));
}
return res;
}
@ -215,34 +216,6 @@ struct NopSink : ParseSink
{
};
Path BinaryCacheStore::importPath(Source & source)
{
/* FIXME: some cut&paste of LocalStore::importPath(). */
/* Extract the NAR from the source. */
TeeSource tee(source);
NopSink sink;
parseDump(sink, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info;
info.path = readStorePath(source);
info.references = readStorePaths<PathSet>(source);
readString(source); // deriver, don't care
bool haveSignature = readInt(source) == 1;
assert(!haveSignature);
addToCache(info, tee.data);
return info.path;
}
ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
{
return ValidPathInfo(readNarInfo(storePath));
@ -416,4 +389,37 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
}
Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor)
{
/* FIXME: some cut&paste of LocalStore::importPath(). */
/* Extract the NAR from the source. */
TeeSource tee(source);
NopSink sink;
parseDump(sink, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info;
info.path = readStorePath(source);
info.references = readStorePaths<PathSet>(source);
readString(source); // deriver, don't care
bool haveSignature = readInt(source) == 1;
assert(!haveSignature);
addToCache(info, tee.data);
auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
if (accessor_)
// FIXME: more gratuitous string copying
accessor_->nars.emplace(info.path, makeNarAccessor(make_ref<std::string>(tee.data)));
return info.path;
}
}

5
src/libstore/binary-cache-store.hh
View file

@ -127,9 +127,10 @@ public:
void exportPath(const Path & path, bool sign, Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override;
Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
Path importPath(Source & source);
Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor);
void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;

3
src/libstore/local-store.cc
View file

@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
}
Paths LocalStore::importPaths(bool requireSignature, Source & source)
Paths LocalStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{
Paths res;
while (true) {

3
src/libstore/local-store.hh
View file

@ -147,7 +147,8 @@ public:
void exportPath(const Path & path, bool sign,
Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override;
Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override;

3
src/libstore/remote-store.cc
View file

@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign,
}
Paths RemoteStore::importPaths(bool requireSignature, Source & source)
Paths RemoteStore::importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor)
{
auto conn(connections->get());
conn->to << wopImportPaths;

3
src/libstore/remote-store.hh
View file

@ -65,7 +65,8 @@ public:
void exportPath(const Path & path, bool sign,
Sink & sink) override;
Paths importPaths(bool requireSignature, Source & source) override;
Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override;

7
src/libstore/store-api.hh
View file

@ -227,8 +227,11 @@ public:
void exportPaths(const Paths & paths, bool sign, Sink & sink);
/* Import a sequence of NAR dumps created by exportPaths() into
the Nix store. */
virtual Paths importPaths(bool requireSignature, Source & source) = 0;
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
virtual Paths importPaths(bool requireSignature, Source & source,
std::shared_ptr<FSAccessor> accessor) = 0;
/* For each path, if it's a derivation, build it. Building a
derivation means ensuring that the output paths are valid. If

2
src/nix-daemon/nix-daemon.cc
View file

@ -310,7 +310,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
case wopImportPaths: {
startWork();
TunnelSource source(from);
Paths paths = store->importPaths(!trusted, source);
Paths paths = store->importPaths(!trusted, source, 0);
stopWork();
to << paths;
break;

4
src/nix-store/nix-store.cc
View file

@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO);
Paths paths = store->importPaths(requireSignature, source);
Paths paths = store->importPaths(requireSignature, source, 0);
for (auto & i : paths)
cout << format("%1%\n") % i << std::flush;
@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed");
store->importPaths(false, in);
store->importPaths(false, in, 0);
out << 1; // indicate success
break;
}