Handle importing NARs containing files greater than 4 GiB
Also templatize readInt() to work for various integer types.
This commit is contained in:
		
							parent
							
								
									0780805246
								
							
						
					
					
						commit
						c4a40949d9
					
				
					 7 changed files with 84 additions and 81 deletions
				
			
		|  | @ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const | ||||||
| Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) | Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) | ||||||
| { | { | ||||||
|     drv.outputs.clear(); |     drv.outputs.clear(); | ||||||
|     auto nr = readInt(in); |     auto nr = readNum<size_t>(in); | ||||||
|     for (unsigned int n = 0; n < nr; n++) { |     for (size_t n = 0; n < nr; n++) { | ||||||
|         auto name = readString(in); |         auto name = readString(in); | ||||||
|         DerivationOutput o; |         DerivationOutput o; | ||||||
|         in >> o.path >> o.hashAlgo >> o.hash; |         in >> o.path >> o.hashAlgo >> o.hash; | ||||||
|  | @ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) | ||||||
|     in >> drv.platform >> drv.builder; |     in >> drv.platform >> drv.builder; | ||||||
|     drv.args = readStrings<Strings>(in); |     drv.args = readStrings<Strings>(in); | ||||||
| 
 | 
 | ||||||
|     nr = readInt(in); |     nr = readNum<size_t>(in); | ||||||
|     for (unsigned int n = 0; n < nr; n++) { |     for (size_t n = 0; n < nr; n++) { | ||||||
|         auto key = readString(in); |         auto key = readString(in); | ||||||
|         auto value = readString(in); |         auto value = readString(in); | ||||||
|         drv.env[key] = value; |         drv.env[key] = value; | ||||||
|  |  | ||||||
|  | @ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, | ||||||
| { | { | ||||||
|     Paths res; |     Paths res; | ||||||
|     while (true) { |     while (true) { | ||||||
|         unsigned long long n = readLongLong(source); |         auto n = readNum<uint64_t>(source); | ||||||
|         if (n == 0) break; |         if (n == 0) break; | ||||||
|         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); |         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn) | ||||||
|         unsigned int magic = readInt(conn.from); |         unsigned int magic = readInt(conn.from); | ||||||
|         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); |         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); | ||||||
| 
 | 
 | ||||||
|         conn.daemonVersion = readInt(conn.from); |         conn.from >> conn.daemonVersion; | ||||||
|         if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) |         if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) | ||||||
|             throw Error("Nix daemon protocol version not supported"); |             throw Error("Nix daemon protocol version not supported"); | ||||||
|         if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) |         if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) | ||||||
|  | @ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path) | ||||||
|     auto conn(connections->get()); |     auto conn(connections->get()); | ||||||
|     conn->to << wopIsValidPath << path; |     conn->to << wopIsValidPath << path; | ||||||
|     conn->processStderr(); |     conn->processStderr(); | ||||||
|     unsigned int reply = readInt(conn->from); |     return readInt(conn->from); | ||||||
|     return reply != 0; |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, | ||||||
| 
 | 
 | ||||||
|         conn->to << wopQuerySubstitutablePathInfos << paths; |         conn->to << wopQuerySubstitutablePathInfos << paths; | ||||||
|         conn->processStderr(); |         conn->processStderr(); | ||||||
|         unsigned int count = readInt(conn->from); |         size_t count = readNum<size_t>(conn->from); | ||||||
|         for (unsigned int n = 0; n < count; n++) { |         for (size_t n = 0; n < count; n++) { | ||||||
|             Path path = readStorePath(*this, conn->from); |             Path path = readStorePath(*this, conn->from); | ||||||
|             SubstitutablePathInfo & info(infos[path]); |             SubstitutablePathInfo & info(infos[path]); | ||||||
|             info.deriver = readString(conn->from); |             info.deriver = readString(conn->from); | ||||||
|  | @ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path, | ||||||
|             throw; |             throw; | ||||||
|         } |         } | ||||||
|         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { |         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { | ||||||
|             bool valid = readInt(conn->from) != 0; |             bool valid; conn->from >> valid; | ||||||
|             if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); |             if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); | ||||||
|         } |         } | ||||||
|         auto info = std::make_shared<ValidPathInfo>(); |         auto info = std::make_shared<ValidPathInfo>(); | ||||||
|  | @ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path, | ||||||
|         if (info->deriver != "") assertStorePath(info->deriver); |         if (info->deriver != "") assertStorePath(info->deriver); | ||||||
|         info->narHash = parseHash(htSHA256, readString(conn->from)); |         info->narHash = parseHash(htSHA256, readString(conn->from)); | ||||||
|         info->references = readStorePaths<PathSet>(*this, conn->from); |         info->references = readStorePaths<PathSet>(*this, conn->from); | ||||||
|         info->registrationTime = readInt(conn->from); |         conn->from >> info->registrationTime >> info->narSize; | ||||||
|         info->narSize = readLongLong(conn->from); |  | ||||||
|         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { |         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { | ||||||
|             info->ultimate = readInt(conn->from) != 0; |             conn->from >> info->ultimate; | ||||||
|             info->sigs = readStrings<StringSet>(conn->from); |             info->sigs = readStrings<StringSet>(conn->from); | ||||||
|             info->ca = readString(conn->from); |             conn->from >> info->ca; | ||||||
|         } |         } | ||||||
|         return info; |         return info; | ||||||
|     }); |     }); | ||||||
|  | @ -515,7 +513,7 @@ Roots RemoteStore::findRoots() | ||||||
|     auto conn(connections->get()); |     auto conn(connections->get()); | ||||||
|     conn->to << wopFindRoots; |     conn->to << wopFindRoots; | ||||||
|     conn->processStderr(); |     conn->processStderr(); | ||||||
|     unsigned int count = readInt(conn->from); |     size_t count = readNum<size_t>(conn->from); | ||||||
|     Roots result; |     Roots result; | ||||||
|     while (count--) { |     while (count--) { | ||||||
|         Path link = readString(conn->from); |         Path link = readString(conn->from); | ||||||
|  | @ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair) | ||||||
|     auto conn(connections->get()); |     auto conn(connections->get()); | ||||||
|     conn->to << wopVerifyStore << checkContents << repair; |     conn->to << wopVerifyStore << checkContents << repair; | ||||||
|     conn->processStderr(); |     conn->processStderr(); | ||||||
|     return readInt(conn->from) != 0; |     return readInt(conn->from); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) | ||||||
|         } |         } | ||||||
|         else if (msg == STDERR_READ) { |         else if (msg == STDERR_READ) { | ||||||
|             if (!source) throw Error("no source"); |             if (!source) throw Error("no source"); | ||||||
|             size_t len = readInt(from); |             size_t len = readNum<size_t>(from); | ||||||
|             auto buf = std::make_unique<unsigned char[]>(len); |             auto buf = std::make_unique<unsigned char[]>(len); | ||||||
|             writeString(buf.get(), source->read(buf.get(), len), to); |             writeString(buf.get(), source->read(buf.get(), len), to); | ||||||
|             to.flush(); |             to.flush(); | ||||||
|  |  | ||||||
|  | @ -194,39 +194,9 @@ void readPadding(size_t len, Source & source) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| unsigned int readInt(Source & source) |  | ||||||
| { |  | ||||||
|     unsigned char buf[8]; |  | ||||||
|     source(buf, sizeof(buf)); |  | ||||||
|     if (buf[4] || buf[5] || buf[6] || buf[7]) |  | ||||||
|         throw SerialisationError("implementation cannot deal with > 32-bit integers"); |  | ||||||
|     return |  | ||||||
|         buf[0] | |  | ||||||
|         (buf[1] << 8) | |  | ||||||
|         (buf[2] << 16) | |  | ||||||
|         (buf[3] << 24); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| unsigned long long readLongLong(Source & source) |  | ||||||
| { |  | ||||||
|     unsigned char buf[8]; |  | ||||||
|     source(buf, sizeof(buf)); |  | ||||||
|     return |  | ||||||
|         ((unsigned long long) buf[0]) | |  | ||||||
|         ((unsigned long long) buf[1] << 8) | |  | ||||||
|         ((unsigned long long) buf[2] << 16) | |  | ||||||
|         ((unsigned long long) buf[3] << 24) | |  | ||||||
|         ((unsigned long long) buf[4] << 32) | |  | ||||||
|         ((unsigned long long) buf[5] << 40) | |  | ||||||
|         ((unsigned long long) buf[6] << 48) | |  | ||||||
|         ((unsigned long long) buf[7] << 56); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| size_t readString(unsigned char * buf, size_t max, Source & source) | size_t readString(unsigned char * buf, size_t max, Source & source) | ||||||
| { | { | ||||||
|     size_t len = readInt(source); |     auto len = readNum<size_t>(source); | ||||||
|     if (len > max) throw Error("string is too long"); |     if (len > max) throw Error("string is too long"); | ||||||
|     source(buf, len); |     source(buf, len); | ||||||
|     readPadding(len, source); |     readPadding(len, source); | ||||||
|  | @ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source) | ||||||
| 
 | 
 | ||||||
| string readString(Source & source) | string readString(Source & source) | ||||||
| { | { | ||||||
|     size_t len = readInt(source); |     auto len = readNum<size_t>(source); | ||||||
|     auto buf = std::make_unique<unsigned char[]>(len); |     auto buf = std::make_unique<unsigned char[]>(len); | ||||||
|     source(buf.get(), len); |     source(buf.get(), len); | ||||||
|     readPadding(len, source); |     readPadding(len, source); | ||||||
|  | @ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| Source & operator >> (Source & in, unsigned int & n) |  | ||||||
| { |  | ||||||
|     n = readInt(in); |  | ||||||
|     return in; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| template<class T> T readStrings(Source & source) | template<class T> T readStrings(Source & source) | ||||||
| { | { | ||||||
|     unsigned int count = readInt(source); |     auto count = readNum<size_t>(source); | ||||||
|     T ss; |     T ss; | ||||||
|     while (count--) |     while (count--) | ||||||
|         ss.insert(ss.end(), readString(source)); |         ss.insert(ss.end(), readString(source)); | ||||||
|  |  | ||||||
|  | @ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s); | ||||||
| Sink & operator << (Sink & sink, const StringSet & s); | Sink & operator << (Sink & sink, const StringSet & s); | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | MakeError(SerialisationError, Error) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | template<typename T> | ||||||
|  | T readNum(Source & source) | ||||||
|  | { | ||||||
|  |     unsigned char buf[8]; | ||||||
|  |     source(buf, sizeof(buf)); | ||||||
|  | 
 | ||||||
|  |     uint64_t n = | ||||||
|  |         ((unsigned long long) buf[0]) | | ||||||
|  |         ((unsigned long long) buf[1] << 8) | | ||||||
|  |         ((unsigned long long) buf[2] << 16) | | ||||||
|  |         ((unsigned long long) buf[3] << 24) | | ||||||
|  |         ((unsigned long long) buf[4] << 32) | | ||||||
|  |         ((unsigned long long) buf[5] << 40) | | ||||||
|  |         ((unsigned long long) buf[6] << 48) | | ||||||
|  |         ((unsigned long long) buf[7] << 56); | ||||||
|  | 
 | ||||||
|  |     if (n > std::numeric_limits<T>::max()) | ||||||
|  |         throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name()); | ||||||
|  | 
 | ||||||
|  |     return n; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | inline unsigned int readInt(Source & source) | ||||||
|  | { | ||||||
|  |     return readNum<unsigned int>(source); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | inline uint64_t readLongLong(Source & source) | ||||||
|  | { | ||||||
|  |     return readNum<uint64_t>(source); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| void readPadding(size_t len, Source & source); | void readPadding(size_t len, Source & source); | ||||||
| unsigned int readInt(Source & source); |  | ||||||
| unsigned long long readLongLong(Source & source); |  | ||||||
| size_t readString(unsigned char * buf, size_t max, Source & source); | size_t readString(unsigned char * buf, size_t max, Source & source); | ||||||
| string readString(Source & source); | string readString(Source & source); | ||||||
| template<class T> T readStrings(Source & source); | template<class T> T readStrings(Source & source); | ||||||
| 
 | 
 | ||||||
| Source & operator >> (Source & in, string & s); | Source & operator >> (Source & in, string & s); | ||||||
| Source & operator >> (Source & in, unsigned int & n); |  | ||||||
| 
 | 
 | ||||||
|  | template<typename T> | ||||||
|  | Source & operator >> (Source & in, T & n) | ||||||
|  | { | ||||||
|  |     n = readNum<T>(in); | ||||||
|  |     return in; | ||||||
|  | } | ||||||
| 
 | 
 | ||||||
| MakeError(SerialisationError, Error) | template<typename T> | ||||||
|  | Source & operator >> (Source & in, bool & b) | ||||||
|  | { | ||||||
|  |     b = readNum<uint64_t>(in); | ||||||
|  |     return in; | ||||||
|  | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -273,10 +273,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     case wopAddToStore: { |     case wopAddToStore: { | ||||||
|         string baseName = readString(from); |         bool fixed, recursive; | ||||||
|         bool fixed = readInt(from) == 1; /* obsolete */ |         std::string s, baseName; | ||||||
|         bool recursive = readInt(from) == 1; |         from >> baseName >> fixed /* obsolete */ >> recursive >> s; | ||||||
|         string s = readString(from); |  | ||||||
|         /* Compatibility hack. */ |         /* Compatibility hack. */ | ||||||
|         if (!fixed) { |         if (!fixed) { | ||||||
|             s = "sha256"; |             s = "sha256"; | ||||||
|  | @ -340,7 +339,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|         PathSet drvs = readStorePaths<PathSet>(*store, from); |         PathSet drvs = readStorePaths<PathSet>(*store, from); | ||||||
|         BuildMode mode = bmNormal; |         BuildMode mode = bmNormal; | ||||||
|         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { |         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { | ||||||
|             mode = (BuildMode)readInt(from); |             mode = (BuildMode) readInt(from); | ||||||
| 
 | 
 | ||||||
|             /* Repairing is not atomic, so disallowed for "untrusted"
 |             /* Repairing is not atomic, so disallowed for "untrusted"
 | ||||||
|                clients.  */ |                clients.  */ | ||||||
|  | @ -417,8 +416,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|         GCOptions options; |         GCOptions options; | ||||||
|         options.action = (GCOptions::GCAction) readInt(from); |         options.action = (GCOptions::GCAction) readInt(from); | ||||||
|         options.pathsToDelete = readStorePaths<PathSet>(*store, from); |         options.pathsToDelete = readStorePaths<PathSet>(*store, from); | ||||||
|         options.ignoreLiveness = readInt(from); |         from >> options.ignoreLiveness >> options.maxFreed; | ||||||
|         options.maxFreed = readLongLong(from); |  | ||||||
|         // obsolete fields
 |         // obsolete fields
 | ||||||
|         readInt(from); |         readInt(from); | ||||||
|         readInt(from); |         readInt(from); | ||||||
|  | @ -438,8 +436,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     case wopSetOptions: { |     case wopSetOptions: { | ||||||
|         settings.keepFailed = readInt(from) != 0; |         from >> settings.keepFailed; | ||||||
|         settings.keepGoing = readInt(from) != 0; |         from >> settings.keepGoing; | ||||||
|         settings.set("build-fallback", readInt(from) ? "true" : "false"); |         settings.set("build-fallback", readInt(from) ? "true" : "false"); | ||||||
|         verbosity = (Verbosity) readInt(from); |         verbosity = (Verbosity) readInt(from); | ||||||
|         settings.set("build-max-jobs", std::to_string(readInt(from))); |         settings.set("build-max-jobs", std::to_string(readInt(from))); | ||||||
|  | @ -539,8 +537,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|         break; |         break; | ||||||
| 
 | 
 | ||||||
|     case wopVerifyStore: { |     case wopVerifyStore: { | ||||||
|         bool checkContents = readInt(from) != 0; |         bool checkContents, repair; | ||||||
|         bool repair = readInt(from) != 0; |         from >> checkContents >> repair; | ||||||
|         startWork(); |         startWork(); | ||||||
|         if (repair && !trusted) |         if (repair && !trusted) | ||||||
|             throw Error("you are not privileged to repair paths"); |             throw Error("you are not privileged to repair paths"); | ||||||
|  | @ -573,19 +571,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe | ||||||
|     case wopAddToStoreNar: { |     case wopAddToStoreNar: { | ||||||
|         ValidPathInfo info; |         ValidPathInfo info; | ||||||
|         info.path = readStorePath(*store, from); |         info.path = readStorePath(*store, from); | ||||||
|         info.deriver = readString(from); |         from >> info.deriver; | ||||||
|         if (!info.deriver.empty()) |         if (!info.deriver.empty()) | ||||||
|             store->assertStorePath(info.deriver); |             store->assertStorePath(info.deriver); | ||||||
|         info.narHash = parseHash(htSHA256, readString(from)); |         info.narHash = parseHash(htSHA256, readString(from)); | ||||||
|         info.references = readStorePaths<PathSet>(*store, from); |         info.references = readStorePaths<PathSet>(*store, from); | ||||||
|         info.registrationTime = readInt(from); |         from >> info.registrationTime >> info.narSize >> info.ultimate; | ||||||
|         info.narSize = readLongLong(from); |  | ||||||
|         info.ultimate = readLongLong(from); |  | ||||||
|         info.sigs = readStrings<StringSet>(from); |         info.sigs = readStrings<StringSet>(from); | ||||||
|         info.ca = readString(from); |         from >> info.ca; | ||||||
|         auto nar = make_ref<std::string>(readString(from)); |         auto nar = make_ref<std::string>(readString(from)); | ||||||
|         auto repair = readInt(from) ? true : false; |         bool repair, dontCheckSigs; | ||||||
|         auto dontCheckSigs = readInt(from) ? true : false; |         from >> repair >> dontCheckSigs; | ||||||
|         if (!trusted && dontCheckSigs) |         if (!trusted && dontCheckSigs) | ||||||
|             dontCheckSigs = false; |             dontCheckSigs = false; | ||||||
|         startWork(); |         startWork(); | ||||||
|  |  | ||||||
|  | @ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs) | ||||||
|         settings.maxSilentTime = readInt(in); |         settings.maxSilentTime = readInt(in); | ||||||
|         settings.buildTimeout = readInt(in); |         settings.buildTimeout = readInt(in); | ||||||
|         if (GET_PROTOCOL_MINOR(clientVersion) >= 2) |         if (GET_PROTOCOL_MINOR(clientVersion) >= 2) | ||||||
|             settings.maxLogSize = readInt(in); |             in >> settings.maxLogSize; | ||||||
|         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { |         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { | ||||||
|             settings.set("build-repeat", std::to_string(readInt(in))); |             settings.set("build-repeat", std::to_string(readInt(in))); | ||||||
|             settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); |             settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue