Don't require --fallback to recover from disappeared binary cache NARs
This commit is contained in:
		
							parent
							
								
									691b7582c7
								
							
						
					
					
						commit
						4ac4f675df
					
				
					 4 changed files with 41 additions and 17 deletions
				
			
		|  | @ -218,7 +218,11 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) | ||||||
|     auto info = queryPathInfo(storePath).cast<const NarInfo>(); |     auto info = queryPathInfo(storePath).cast<const NarInfo>(); | ||||||
| 
 | 
 | ||||||
|     auto source = sinkToSource([this, url{info->url}](Sink & sink) { |     auto source = sinkToSource([this, url{info->url}](Sink & sink) { | ||||||
|  |         try { | ||||||
|             getFile(url, sink); |             getFile(url, sink); | ||||||
|  |         } catch (NoSuchBinaryCacheFile & e) { | ||||||
|  |             throw SubstituteGone(e.what()); | ||||||
|  |         } | ||||||
|     }); |     }); | ||||||
| 
 | 
 | ||||||
|     stats.narRead++; |     stats.narRead++; | ||||||
|  |  | ||||||
|  | @ -733,7 +733,7 @@ private: | ||||||
| 
 | 
 | ||||||
|     /* Whether to retry substituting the outputs after building the
 |     /* Whether to retry substituting the outputs after building the
 | ||||||
|        inputs. */ |        inputs. */ | ||||||
|     bool retrySubstitution = false; |     bool retrySubstitution; | ||||||
| 
 | 
 | ||||||
|     /* The derivation stored at drvPath. */ |     /* The derivation stored at drvPath. */ | ||||||
|     std::unique_ptr<BasicDerivation> drv; |     std::unique_ptr<BasicDerivation> drv; | ||||||
|  | @ -1123,6 +1123,8 @@ void DerivationGoal::haveDerivation() | ||||||
| { | { | ||||||
|     trace("have derivation"); |     trace("have derivation"); | ||||||
| 
 | 
 | ||||||
|  |     retrySubstitution = false; | ||||||
|  | 
 | ||||||
|     for (auto & i : drv->outputs) |     for (auto & i : drv->outputs) | ||||||
|         worker.store.addTempRoot(i.second.path); |         worker.store.addTempRoot(i.second.path); | ||||||
| 
 | 
 | ||||||
|  | @ -1161,7 +1163,7 @@ void DerivationGoal::outputsSubstituted() | ||||||
|     /*  If the substitutes form an incomplete closure, then we should
 |     /*  If the substitutes form an incomplete closure, then we should
 | ||||||
|         build the dependencies of this derivation, but after that, we |         build the dependencies of this derivation, but after that, we | ||||||
|         can still use the substitutes for this derivation itself. */ |         can still use the substitutes for this derivation itself. */ | ||||||
|     if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; |     if (nrIncompleteClosure > 0) retrySubstitution = true; | ||||||
| 
 | 
 | ||||||
|     nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; |     nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; | ||||||
| 
 | 
 | ||||||
|  | @ -3524,8 +3526,8 @@ private: | ||||||
|     /* The current substituter. */ |     /* The current substituter. */ | ||||||
|     std::shared_ptr<Store> sub; |     std::shared_ptr<Store> sub; | ||||||
| 
 | 
 | ||||||
|     /* Whether any substituter can realise this path. */ |     /* Whether a substituter failed. */ | ||||||
|     bool hasSubstitute; |     bool substituterFailed = false; | ||||||
| 
 | 
 | ||||||
|     /* Path info returned by the substituter's query info operation. */ |     /* Path info returned by the substituter's query info operation. */ | ||||||
|     std::shared_ptr<const ValidPathInfo> info; |     std::shared_ptr<const ValidPathInfo> info; | ||||||
|  | @ -3589,7 +3591,6 @@ public: | ||||||
| 
 | 
 | ||||||
| SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) | SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) | ||||||
|     : Goal(worker) |     : Goal(worker) | ||||||
|     , hasSubstitute(false) |  | ||||||
|     , repair(repair) |     , repair(repair) | ||||||
| { | { | ||||||
|     this->storePath = storePath; |     this->storePath = storePath; | ||||||
|  | @ -3653,9 +3654,9 @@ void SubstitutionGoal::tryNext() | ||||||
|         /* Hack: don't indicate failure if there were no substituters.
 |         /* Hack: don't indicate failure if there were no substituters.
 | ||||||
|            In that case the calling derivation should just do a |            In that case the calling derivation should just do a | ||||||
|            build. */ |            build. */ | ||||||
|         amDone(hasSubstitute ? ecFailed : ecNoSubstituters); |         amDone(substituterFailed ? ecFailed : ecNoSubstituters); | ||||||
| 
 | 
 | ||||||
|         if (hasSubstitute) { |         if (substituterFailed) { | ||||||
|             worker.failedSubstitutions++; |             worker.failedSubstitutions++; | ||||||
|             worker.updateProgress(); |             worker.updateProgress(); | ||||||
|         } |         } | ||||||
|  | @ -3691,8 +3692,6 @@ void SubstitutionGoal::tryNext() | ||||||
| 
 | 
 | ||||||
|     worker.updateProgress(); |     worker.updateProgress(); | ||||||
| 
 | 
 | ||||||
|     hasSubstitute = true; |  | ||||||
| 
 |  | ||||||
|     /* Bail out early if this substituter lacks a valid
 |     /* Bail out early if this substituter lacks a valid
 | ||||||
|        signature. LocalStore::addToStore() also checks for this, but |        signature. LocalStore::addToStore() also checks for this, but | ||||||
|        only after we've downloaded the path. */ |        only after we've downloaded the path. */ | ||||||
|  | @ -3807,8 +3806,19 @@ void SubstitutionGoal::finished() | ||||||
|         state = &SubstitutionGoal::init; |         state = &SubstitutionGoal::init; | ||||||
|         worker.waitForAWhile(shared_from_this()); |         worker.waitForAWhile(shared_from_this()); | ||||||
|         return; |         return; | ||||||
|     } catch (Error & e) { |     } catch (std::exception & e) { | ||||||
|         printError(e.msg()); |         printError(e.what()); | ||||||
|  | 
 | ||||||
|  |         /* Cause the parent build to fail unless --fallback is given,
 | ||||||
|  |            or the substitute has disappeared. The latter case behaves | ||||||
|  |            the same as the substitute never having existed in the | ||||||
|  |            first place. */ | ||||||
|  |         try { | ||||||
|  |             throw; | ||||||
|  |         } catch (SubstituteGone &) { | ||||||
|  |         } catch (...) { | ||||||
|  |             substituterFailed = true; | ||||||
|  |         } | ||||||
| 
 | 
 | ||||||
|         /* Try the next substitute. */ |         /* Try the next substitute. */ | ||||||
|         state = &SubstitutionGoal::tryNext; |         state = &SubstitutionGoal::tryNext; | ||||||
|  |  | ||||||
|  | @ -22,6 +22,7 @@ MakeError(SubstError, Error) | ||||||
| MakeError(BuildError, Error) /* denotes a permanent build failure */ | MakeError(BuildError, Error) /* denotes a permanent build failure */ | ||||||
| MakeError(InvalidPath, Error) | MakeError(InvalidPath, Error) | ||||||
| MakeError(Unsupported, Error) | MakeError(Unsupported, Error) | ||||||
|  | MakeError(SubstituteGone, Error) | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| struct BasicDerivation; | struct BasicDerivation; | ||||||
|  |  | ||||||
|  | @ -76,19 +76,28 @@ if nix-store --substituters "file://$cacheDir" -r $outPath; then | ||||||
| fi | fi | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| # Test whether fallback works if we have cached info but the | # Test whether fallback works if a NAR has disappeared. This does not require --fallback. | ||||||
| # corresponding NAR has disappeared. |  | ||||||
| clearStore | clearStore | ||||||
| 
 | 
 | ||||||
| nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info |  | ||||||
| 
 |  | ||||||
| mkdir $cacheDir/tmp |  | ||||||
| mv $cacheDir/nar $cacheDir/nar2 | mv $cacheDir/nar $cacheDir/nar2 | ||||||
| 
 | 
 | ||||||
|  | nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result | ||||||
|  | 
 | ||||||
|  | mv $cacheDir/nar2 $cacheDir/nar | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | # Test whether fallback works if a NAR is corrupted. This does require --fallback. | ||||||
|  | clearStore | ||||||
|  | 
 | ||||||
|  | mv $cacheDir/nar $cacheDir/nar2 | ||||||
|  | mkdir $cacheDir/nar | ||||||
|  | for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done | ||||||
|  | 
 | ||||||
| (! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) | (! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) | ||||||
| 
 | 
 | ||||||
| nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback | nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback | ||||||
| 
 | 
 | ||||||
|  | rm -rf $cacheDir/nar | ||||||
| mv $cacheDir/nar2 $cacheDir/nar | mv $cacheDir/nar2 $cacheDir/nar | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue