Add NAR / Store accessor abstraction
This is primary to allow hydra-queue-runner to extract files like "nix-support/hydra-build-products" from NARs in binary caches.
This commit is contained in:
		
							parent
							
								
									152b1d6bf9
								
							
						
					
					
						commit
						1042c10fd0
					
				
					 11 changed files with 352 additions and 13 deletions
				
			
		|  | @ -1,12 +1,13 @@ | |||
| #include "binary-cache-store.hh" | ||||
| #include "sync.hh" | ||||
| 
 | ||||
| #include "archive.hh" | ||||
| #include "binary-cache-store.hh" | ||||
| #include "compression.hh" | ||||
| #include "derivations.hh" | ||||
| #include "fs-accessor.hh" | ||||
| #include "globals.hh" | ||||
| #include "nar-info.hh" | ||||
| #include "sync.hh" | ||||
| #include "worker-protocol.hh" | ||||
| #include "nar-accessor.hh" | ||||
| 
 | ||||
| #include <chrono> | ||||
| 
 | ||||
|  | @ -122,7 +123,8 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) | |||
| 
 | ||||
|     auto narInfoFile = narInfoFileFor(storePath); | ||||
|     auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile); | ||||
|     assert(narInfo->path == storePath); | ||||
|     if (narInfo->path != storePath) | ||||
|         throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath); | ||||
| 
 | ||||
|     stats.narInfoRead++; | ||||
| 
 | ||||
|  | @ -142,6 +144,9 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) | |||
| 
 | ||||
| bool BinaryCacheStore::isValidPath(const Path & storePath) | ||||
| { | ||||
|     // FIXME: this only checks whether a .narinfo with a matching hash
 | ||||
|     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
 | ||||
|     // though they shouldn't. Not easily fixed.
 | ||||
|     return fileExists(narInfoFileFor(storePath)); | ||||
| } | ||||
| 
 | ||||
|  | @ -344,4 +349,71 @@ void BinaryCacheStore::ensurePath(const Path & path) | |||
|     buildPaths({path}); | ||||
| } | ||||
| 
 | ||||
| /* Given requests for a path /nix/store/<x>/<y>, this accessor will
 | ||||
|    first download the NAR for /nix/store/<x> from the binary cache, | ||||
|    build a NAR accessor for that NAR, and use that to access <y>. */ | ||||
| struct BinaryCacheStoreAccessor : public FSAccessor | ||||
| { | ||||
|     ref<BinaryCacheStore> store; | ||||
| 
 | ||||
|     std::map<Path, ref<FSAccessor>> nars; | ||||
| 
 | ||||
|     BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) | ||||
|         : store(store) | ||||
|     { | ||||
|     } | ||||
| 
 | ||||
|     std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) | ||||
|     { | ||||
|         auto path = canonPath(path_); | ||||
| 
 | ||||
|         auto storePath = toStorePath(path); | ||||
|         std::string restPath = std::string(path, storePath.size()); | ||||
| 
 | ||||
|         if (!store->isValidPath(storePath)) | ||||
|             throw Error(format("path ‘%1%’ is not a valid store path") % storePath); | ||||
| 
 | ||||
|         auto i = nars.find(storePath); | ||||
|         if (i != nars.end()) return {i->second, restPath}; | ||||
| 
 | ||||
|         StringSink sink; | ||||
|         store->exportPath(storePath, false, sink); | ||||
| 
 | ||||
|         // FIXME: gratuitous string copying.
 | ||||
|         auto accessor = makeNarAccessor(make_ref<std::string>(sink.s)); | ||||
|         nars.emplace(storePath, accessor); | ||||
|         return {accessor, restPath}; | ||||
|     } | ||||
| 
 | ||||
|     Stat stat(const Path & path) override | ||||
|     { | ||||
|         auto res = fetch(path); | ||||
|         return res.first->stat(res.second); | ||||
|     } | ||||
| 
 | ||||
|     StringSet readDirectory(const Path & path) override | ||||
|     { | ||||
|         auto res = fetch(path); | ||||
|         return res.first->readDirectory(res.second); | ||||
|     } | ||||
| 
 | ||||
|     std::string readFile(const Path & path) override | ||||
|     { | ||||
|         auto res = fetch(path); | ||||
|         return res.first->readFile(res.second); | ||||
|     } | ||||
| 
 | ||||
|     std::string readLink(const Path & path) override | ||||
|     { | ||||
|         auto res = fetch(path); | ||||
|         return res.first->readLink(res.second); | ||||
|     } | ||||
| }; | ||||
| 
 | ||||
| ref<FSAccessor> BinaryCacheStore::getFSAccessor() | ||||
| { | ||||
|     return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>( | ||||
|             std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); | ||||
| } | ||||
| 
 | ||||
| } | ||||
|  |  | |||
|  | @ -125,8 +125,7 @@ public: | |||
|     Path addTextToStore(const string & name, const string & s, | ||||
|         const PathSet & references, bool repair = false) override; | ||||
| 
 | ||||
|     void exportPath(const Path & path, bool sign, | ||||
|         Sink & sink) override; | ||||
|     void exportPath(const Path & path, bool sign, Sink & sink) override; | ||||
| 
 | ||||
|     Paths importPaths(bool requireSignature, Source & source) override; | ||||
| 
 | ||||
|  | @ -167,6 +166,8 @@ public: | |||
|     bool verifyStore(bool checkContents, bool repair) override | ||||
|     { return true; } | ||||
| 
 | ||||
|     ref<FSAccessor> getFSAccessor() override; | ||||
| 
 | ||||
| }; | ||||
| 
 | ||||
| } | ||||
|  |  | |||
							
								
								
									
										30
									
								
								src/libstore/fs-accessor.hh
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										30
									
								
								src/libstore/fs-accessor.hh
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,30 @@ | |||
| #pragma once | ||||
| 
 | ||||
| #include "types.hh" | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| /* An abstract class for accessing a filesystem-like structure, such
 | ||||
|    as a (possibly remote) Nix store or the contents of a NAR file. */ | ||||
| class FSAccessor | ||||
| { | ||||
| public: | ||||
|     enum Type { tMissing, tRegular, tSymlink, tDirectory }; | ||||
| 
 | ||||
|     struct Stat | ||||
|     { | ||||
|         Type type; | ||||
|         uint64_t fileSize; // regular files only
 | ||||
|         bool isExecutable; // regular files only
 | ||||
|     }; | ||||
| 
 | ||||
|     virtual Stat stat(const Path & path) = 0; | ||||
| 
 | ||||
|     virtual StringSet readDirectory(const Path & path) = 0; | ||||
| 
 | ||||
|     virtual std::string readFile(const Path & path) = 0; | ||||
| 
 | ||||
|     virtual std::string readLink(const Path & path) = 0; | ||||
| }; | ||||
| 
 | ||||
| } | ||||
							
								
								
									
										71
									
								
								src/libstore/local-fs-store.cc
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										71
									
								
								src/libstore/local-fs-store.cc
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,71 @@ | |||
| #include "fs-accessor.hh" | ||||
| #include "store-api.hh" | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| struct LocalStoreAccessor : public FSAccessor | ||||
| { | ||||
|     ref<Store> store; | ||||
| 
 | ||||
|     LocalStoreAccessor(ref<Store> store) : store(store) { } | ||||
| 
 | ||||
|     void assertStore(const Path & path) | ||||
|     { | ||||
|         Path storePath = toStorePath(path); | ||||
|         if (!store->isValidPath(storePath)) | ||||
|             throw Error(format("path ‘%1%’ is not a valid store path") % storePath); | ||||
|     } | ||||
| 
 | ||||
|     FSAccessor::Stat stat(const Path & path) override | ||||
|     { | ||||
|         assertStore(path); | ||||
| 
 | ||||
|         struct stat st; | ||||
|         if (lstat(path.c_str(), &st)) { | ||||
|             if (errno == ENOENT) return {Type::tMissing, 0, false}; | ||||
|             throw SysError(format("getting status of ‘%1%’") % path); | ||||
|         } | ||||
| 
 | ||||
|         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) | ||||
|             throw Error(format("file ‘%1%’ has unsupported type") % path); | ||||
| 
 | ||||
|         return { | ||||
|             S_ISREG(st.st_mode) ? Type::tRegular : | ||||
|             S_ISLNK(st.st_mode) ? Type::tSymlink : | ||||
|             Type::tDirectory, | ||||
|             S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, | ||||
|             S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; | ||||
|     } | ||||
| 
 | ||||
|     StringSet readDirectory(const Path & path) override | ||||
|     { | ||||
|         assertStore(path); | ||||
| 
 | ||||
|         auto entries = nix::readDirectory(path); | ||||
| 
 | ||||
|         StringSet res; | ||||
|         for (auto & entry : entries) | ||||
|             res.insert(entry.name); | ||||
| 
 | ||||
|         return res; | ||||
|     } | ||||
| 
 | ||||
|     std::string readFile(const Path & path) override | ||||
|     { | ||||
|         assertStore(path); | ||||
|         return nix::readFile(path); | ||||
|     } | ||||
| 
 | ||||
|     std::string readLink(const Path & path) override | ||||
|     { | ||||
|         assertStore(path); | ||||
|         return nix::readLink(path); | ||||
|     } | ||||
| }; | ||||
| 
 | ||||
| ref<FSAccessor> LocalFSStore::getFSAccessor() | ||||
| { | ||||
|     return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this())); | ||||
| } | ||||
| 
 | ||||
| } | ||||
|  | @ -80,7 +80,7 @@ struct SQLiteStmt | |||
| }; | ||||
| 
 | ||||
| 
 | ||||
| class LocalStore : public Store | ||||
| class LocalStore : public LocalFSStore | ||||
| { | ||||
| private: | ||||
|     typedef std::map<Path, RunningSubstituter> RunningSubstituters; | ||||
|  | @ -170,14 +170,11 @@ public: | |||
|        files with the same contents. */ | ||||
|     void optimiseStore(OptimiseStats & stats); | ||||
| 
 | ||||
|     /* Generic variant of the above method.  */ | ||||
|     void optimiseStore() override; | ||||
| 
 | ||||
|     /* Optimise a single store path. */ | ||||
|     void optimisePath(const Path & path); | ||||
| 
 | ||||
|     /* Check the integrity of the Nix store.  Returns true if errors
 | ||||
|        remain. */ | ||||
|     bool verifyStore(bool checkContents, bool repair) override; | ||||
| 
 | ||||
|     /* Register the validity of a path, i.e., that `path' exists, that
 | ||||
|  |  | |||
							
								
								
									
										142
									
								
								src/libstore/nar-accessor.cc
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										142
									
								
								src/libstore/nar-accessor.cc
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,142 @@ | |||
| #include "nar-accessor.hh" | ||||
| #include "archive.hh" | ||||
| 
 | ||||
| #include <map> | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| struct NarMember | ||||
| { | ||||
|     FSAccessor::Type type; | ||||
| 
 | ||||
|     bool isExecutable; | ||||
| 
 | ||||
|     /* If this is a regular file, position of the contents of this
 | ||||
|        file in the NAR. */ | ||||
|     size_t start, size; | ||||
| 
 | ||||
|     std::string target; | ||||
| }; | ||||
| 
 | ||||
| struct NarIndexer : ParseSink, StringSource | ||||
| { | ||||
|     // FIXME: should store this as a tree. Now we're vulnerable to
 | ||||
|     // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
 | ||||
|     typedef std::map<Path, NarMember> Members; | ||||
|     Members members; | ||||
| 
 | ||||
|     Path currentPath; | ||||
|     std::string currentStart; | ||||
|     bool isExec; | ||||
| 
 | ||||
|     NarIndexer(const std::string & nar) : StringSource(nar) | ||||
|     { | ||||
|     } | ||||
| 
 | ||||
|     void createDirectory(const Path & path) | ||||
|     { | ||||
|         members.emplace(path, | ||||
|             NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); | ||||
|     } | ||||
| 
 | ||||
|     void createRegularFile(const Path & path) override | ||||
|     { | ||||
|         currentPath = path; | ||||
|     } | ||||
| 
 | ||||
|     void isExecutable() | ||||
|     { | ||||
|         isExec = true; | ||||
|     } | ||||
| 
 | ||||
|     void preallocateContents(unsigned long long size) override | ||||
|     { | ||||
|         assert(currentPath != ""); | ||||
|         currentStart = string(s, pos, 16); | ||||
|         members.emplace(currentPath, | ||||
|             NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); | ||||
|     } | ||||
| 
 | ||||
|     void receiveContents(unsigned char * data, unsigned int len) override | ||||
|     { | ||||
|         // Sanity check
 | ||||
|         if (!currentStart.empty()) { | ||||
|             assert(len < 16 || currentStart == string((char *) data, 16)); | ||||
|             currentStart.clear(); | ||||
|         } | ||||
|     } | ||||
| 
 | ||||
|     void createSymlink(const Path & path, const string & target) override | ||||
|     { | ||||
|         members.emplace(path, | ||||
|             NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); | ||||
|     } | ||||
| 
 | ||||
|     Members::iterator find(const Path & path) | ||||
|     { | ||||
|         auto i = members.find(path); | ||||
|         if (i == members.end()) | ||||
|             throw Error(format("NAR file does not contain path ‘%1%’") % path); | ||||
|         return i; | ||||
|     } | ||||
| }; | ||||
| 
 | ||||
| struct NarAccessor : public FSAccessor | ||||
| { | ||||
|     ref<const std::string> nar; | ||||
|     NarIndexer indexer; | ||||
| 
 | ||||
|     NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar) | ||||
|     { | ||||
|         parseDump(indexer, indexer); | ||||
|     } | ||||
| 
 | ||||
|     Stat stat(const Path & path) override | ||||
|     { | ||||
|         auto i = indexer.members.find(path); | ||||
|         if (i == indexer.members.end()) | ||||
|             return {FSAccessor::Type::tMissing, 0, false}; | ||||
|         return {i->second.type, i->second.size, i->second.isExecutable}; | ||||
|     } | ||||
| 
 | ||||
|     StringSet readDirectory(const Path & path) override | ||||
|     { | ||||
|         auto i = indexer.find(path); | ||||
| 
 | ||||
|         if (i->second.type != FSAccessor::Type::tDirectory) | ||||
|             throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); | ||||
| 
 | ||||
|         ++i; | ||||
|         StringSet res; | ||||
|         while (i != indexer.members.end() && isInDir(i->first, path)) { | ||||
|             // FIXME: really bad performance.
 | ||||
|             if (i->first.find('/', path.size() + 1) == std::string::npos) | ||||
|                 res.insert(std::string(i->first, path.size() + 1)); | ||||
|             ++i; | ||||
|         } | ||||
|         return res; | ||||
|     } | ||||
| 
 | ||||
|     std::string readFile(const Path & path) override | ||||
|     { | ||||
|         auto i = indexer.find(path); | ||||
|         if (i->second.type != FSAccessor::Type::tRegular) | ||||
|             throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); | ||||
|         return std::string(*nar, i->second.start, i->second.size); | ||||
|     } | ||||
| 
 | ||||
|     std::string readLink(const Path & path) override | ||||
|     { | ||||
|         auto i = indexer.find(path); | ||||
|         if (i->second.type != FSAccessor::Type::tSymlink) | ||||
|             throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); | ||||
|         return i->second.target; | ||||
|     } | ||||
| }; | ||||
| 
 | ||||
| ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) | ||||
| { | ||||
|     return make_ref<NarAccessor>(nar); | ||||
| } | ||||
| 
 | ||||
| } | ||||
							
								
								
									
										11
									
								
								src/libstore/nar-accessor.hh
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										11
									
								
								src/libstore/nar-accessor.hh
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,11 @@ | |||
| #pragma once | ||||
| 
 | ||||
| #include "fs-accessor.hh" | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| /* Return an object that provides access to the contents of a NAR
 | ||||
|    file. */ | ||||
| ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); | ||||
| 
 | ||||
| } | ||||
|  | @ -16,7 +16,9 @@ struct FdSource; | |||
| template<typename T> class Pool; | ||||
| 
 | ||||
| 
 | ||||
| class RemoteStore : public Store | ||||
| /* FIXME: RemoteStore is a misnomer - should be something like
 | ||||
|    DaemonStore. */ | ||||
| class RemoteStore : public LocalFSStore | ||||
| { | ||||
| public: | ||||
| 
 | ||||
|  |  | |||
|  | @ -332,7 +332,9 @@ ref<Store> openStoreAt(const std::string & uri) | |||
| 
 | ||||
|     enum { mDaemon, mLocal, mAuto } mode; | ||||
| 
 | ||||
|     mode = uri == "daemon" ? mDaemon : mAuto; | ||||
|     mode = | ||||
|         uri == "daemon" ? mDaemon : | ||||
|         uri == "local" ? mLocal : mAuto; | ||||
| 
 | ||||
|     if (mode == mAuto) { | ||||
|         if (LocalStore::haveWriteAccess()) | ||||
|  |  | |||
|  | @ -140,9 +140,10 @@ struct BuildResult | |||
| 
 | ||||
| struct BasicDerivation; | ||||
| struct Derivation; | ||||
| struct FSAccessor; | ||||
| 
 | ||||
| 
 | ||||
| class Store | ||||
| class Store : public std::enable_shared_from_this<Store> | ||||
| { | ||||
| public: | ||||
| 
 | ||||
|  | @ -314,6 +315,9 @@ public: | |||
|        remain. */ | ||||
|     virtual bool verifyStore(bool checkContents, bool repair) = 0; | ||||
| 
 | ||||
|     /* Return an object to access files in the Nix store. */ | ||||
|     virtual ref<FSAccessor> getFSAccessor() = 0; | ||||
| 
 | ||||
|     /* Utility functions. */ | ||||
| 
 | ||||
|     /* Read a derivation, after ensuring its existence through
 | ||||
|  | @ -345,6 +349,12 @@ public: | |||
| }; | ||||
| 
 | ||||
| 
 | ||||
| class LocalFSStore : public Store | ||||
| { | ||||
|     ref<FSAccessor> getFSAccessor() override; | ||||
| }; | ||||
| 
 | ||||
| 
 | ||||
| /* !!! These should be part of the store API, I guess. */ | ||||
| 
 | ||||
| /* Throw an exception if `path' is not directly in the Nix store. */ | ||||
|  |  | |||
|  | @ -57,6 +57,7 @@ void dumpPath(const Path & path, Sink & sink, | |||
| 
 | ||||
| void dumpString(const std::string & s, Sink & sink); | ||||
| 
 | ||||
| /* FIXME: fix this API, it sucks. */ | ||||
| struct ParseSink | ||||
| { | ||||
|     virtual void createDirectory(const Path & path) { }; | ||||
|  |  | |||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue