* A flag `--keep-going / -k' to keep building goals if one fails, as
much as possible. (This is similar to GNU Make's `-k' flag.) * Refactoring to implement this: previously we just bombed out when a build failed, but now we have to clean up. In particular this means that goals must be freed quickly --- they shouldn't hang around until the worker exits. So the worker now maintains weak pointers in order not to prevent garbage collection. * Documented the `-k' and `-j' flags.
This commit is contained in:
		
							parent
							
								
									e4883211f9
								
							
						
					
					
						commit
						b113edeab7
					
				
					 10 changed files with 209 additions and 144 deletions
				
			
		|  | @ -4,5 +4,14 @@ | ||||||
| <arg rep='repeat'><option>-v</option></arg> | <arg rep='repeat'><option>-v</option></arg> | ||||||
| <arg><option>--build-output</option></arg> | <arg><option>--build-output</option></arg> | ||||||
| <arg><option>-B</option></arg> | <arg><option>-B</option></arg> | ||||||
|  | <arg> | ||||||
|  |   <group choice='req'> | ||||||
|  |     <arg choice='plain'><option>--max-jobs</option></arg> | ||||||
|  |     <arg choice='plain'><option>-j</option></arg> | ||||||
|  |   </group> | ||||||
|  |   <replaceable>number</replaceable> | ||||||
|  | </arg> | ||||||
|  | <arg><option>--keep-going</option></arg> | ||||||
|  | <arg><option>-k</option></arg> | ||||||
| <arg><option>--keep-failed</option></arg> | <arg><option>--keep-failed</option></arg> | ||||||
| <arg><option>-K</option></arg> | <arg><option>-K</option></arg> | ||||||
|  |  | ||||||
|  | @ -108,6 +108,33 @@ | ||||||
| </varlistentry> | </varlistentry> | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | <varlistentry> | ||||||
|  |   <term><option>--max-jobs</option> / <option>-j</option></term> | ||||||
|  |   <listitem> | ||||||
|  |     <para> | ||||||
|  |       Sets the maximum number of build jobs that Nix will perform in | ||||||
|  |       parallel to the specified number.  The default is 1.  A higher | ||||||
|  |       value is useful on SMP systems or to exploit I/O latency. | ||||||
|  |     </para> | ||||||
|  |   </listitem> | ||||||
|  | </varlistentry> | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | <varlistentry> | ||||||
|  |   <term><option>--keep-going</option> / <option>-k</option></term> | ||||||
|  |   <listitem> | ||||||
|  |     <para> | ||||||
|  |       Keep going in case of failed builds, to the greatest extent | ||||||
|  |       possible.  That is, if building an input of some derivation | ||||||
|  |       fails, Nix will still build the other inputs, but not the | ||||||
|  |       derivation itself.  Without this option, Nix stops if any build | ||||||
|  |       fails (except for builds of substitutes), possibly killing | ||||||
|  |       builds in progress (in case of parallel or distributed builds). | ||||||
|  |     </para> | ||||||
|  |   </listitem> | ||||||
|  | </varlistentry> | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| <varlistentry> | <varlistentry> | ||||||
|   <term><option>--keep-failed</option> / <option>-K</option></term> |   <term><option>--keep-failed</option> / <option>-K</option></term> | ||||||
|   <listitem> |   <listitem> | ||||||
|  |  | ||||||
|  | @ -106,3 +106,4 @@ The hook `nix-mode-hook' is run when Nix mode is started. | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| (setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist)) | (setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist)) | ||||||
|  | (setq auto-mode-alist (cons '("\\.nix.in\\'" . nix-mode) auto-mode-alist)) | ||||||
|  |  | ||||||
|  | @ -137,6 +137,8 @@ static void initAndRun(int argc, char * * argv) | ||||||
|         } |         } | ||||||
|         else if (arg == "--keep-failed" || arg == "-K") |         else if (arg == "--keep-failed" || arg == "-K") | ||||||
|             keepFailed = true; |             keepFailed = true; | ||||||
|  |         else if (arg == "--keep-going" || arg == "-k") | ||||||
|  |             keepGoing = true; | ||||||
|         else if (arg == "--max-jobs" || arg == "-j") { |         else if (arg == "--max-jobs" || arg == "-j") { | ||||||
|             ++i; |             ++i; | ||||||
|             if (i == args.end()) throw UsageError("`--max-jobs' requires an argument"); |             if (i == args.end()) throw UsageError("`--max-jobs' requires an argument"); | ||||||
|  |  | ||||||
|  | @ -8,6 +8,8 @@ string nixDBPath = "/UNINIT"; | ||||||
| 
 | 
 | ||||||
| bool keepFailed = false; | bool keepFailed = false; | ||||||
| 
 | 
 | ||||||
|  | bool keepGoing = false; | ||||||
|  | 
 | ||||||
| Verbosity buildVerbosity = lvlDebug; | Verbosity buildVerbosity = lvlDebug; | ||||||
| 
 | 
 | ||||||
| unsigned int maxBuildJobs = 1; | unsigned int maxBuildJobs = 1; | ||||||
|  |  | ||||||
|  | @ -29,6 +29,10 @@ extern string nixDBPath; | ||||||
| /* Whether to keep temporary directories of failed builds. */ | /* Whether to keep temporary directories of failed builds. */ | ||||||
| extern bool keepFailed; | extern bool keepFailed; | ||||||
| 
 | 
 | ||||||
|  | /* Whether to keep building subgoals when a sibling (another subgoal
 | ||||||
|  |    of the same goal) fails. */ | ||||||
|  | extern bool keepGoing; | ||||||
|  | 
 | ||||||
| /* Verbosity level for build output. */ | /* Verbosity level for build output. */ | ||||||
| extern Verbosity buildVerbosity; | extern Verbosity buildVerbosity; | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -1,5 +1,6 @@ | ||||||
| #include <map> | #include <map> | ||||||
| #include <boost/shared_ptr.hpp> | #include <boost/shared_ptr.hpp> | ||||||
|  | #include <boost/weak_ptr.hpp> | ||||||
| #include <boost/enable_shared_from_this.hpp> | #include <boost/enable_shared_from_this.hpp> | ||||||
| 
 | 
 | ||||||
| #include <sys/types.h> | #include <sys/types.h> | ||||||
|  | @ -26,13 +27,14 @@ class Worker; | ||||||
| /* A pointer to a goal. */ | /* A pointer to a goal. */ | ||||||
| class Goal; | class Goal; | ||||||
| typedef shared_ptr<Goal> GoalPtr; | typedef shared_ptr<Goal> GoalPtr; | ||||||
|  | typedef weak_ptr<Goal> WeakGoalPtr; | ||||||
| 
 | 
 | ||||||
| /* A set of goals. */ | /* Set of goals. */ | ||||||
| typedef set<GoalPtr> Goals; | typedef set<GoalPtr> Goals; | ||||||
|  | typedef set<WeakGoalPtr> WeakGoals; | ||||||
| 
 | 
 | ||||||
| /* A map of paths to goals (and the other way around). */ | /* A map of paths to goals (and the other way around). */ | ||||||
| typedef map<Path, GoalPtr> GoalMap; | typedef map<Path, WeakGoalPtr> WeakGoalMap; | ||||||
| typedef map<GoalPtr, Path> GoalMapRev; |  | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -43,8 +45,12 @@ protected: | ||||||
|     /* Backlink to the worker. */ |     /* Backlink to the worker. */ | ||||||
|     Worker & worker; |     Worker & worker; | ||||||
| 
 | 
 | ||||||
|     /* Goals waiting for this one to finish. */ |     /* Goals that this goal is waiting for. */ | ||||||
|     Goals waiters; |     Goals waitees; | ||||||
|  | 
 | ||||||
|  |     /* Goals waiting for this one to finish.  Must use weak pointers
 | ||||||
|  |        here to prevent cycles. */ | ||||||
|  |     WeakGoals waiters; | ||||||
| 
 | 
 | ||||||
|     /* Number of goals we are waiting for. */ |     /* Number of goals we are waiting for. */ | ||||||
|     unsigned int nrWaitees; |     unsigned int nrWaitees; | ||||||
|  | @ -75,20 +81,15 @@ protected: | ||||||
| public: | public: | ||||||
|     virtual void work() = 0; |     virtual void work() = 0; | ||||||
| 
 | 
 | ||||||
|     virtual string name() |     virtual string name() = 0; | ||||||
|     { |  | ||||||
|         return "(noname)"; |  | ||||||
|     } |  | ||||||
| 
 | 
 | ||||||
|     void addWaiter(GoalPtr waiter); |     void addWaitee(GoalPtr waitee); | ||||||
| 
 | 
 | ||||||
|     virtual void waiteeDone(bool success); |     virtual void waiteeDone(GoalPtr waitee, bool success); | ||||||
| 
 | 
 | ||||||
|  |     void trace(const format & f); | ||||||
|  |      | ||||||
| protected: | protected: | ||||||
|     virtual void waiteeFailed() |  | ||||||
|     { |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     void amDone(bool success = true); |     void amDone(bool success = true); | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
|  | @ -97,7 +98,7 @@ protected: | ||||||
|    belongs, and a file descriptor for receiving log data. */ |    belongs, and a file descriptor for receiving log data. */ | ||||||
| struct Child | struct Child | ||||||
| { | { | ||||||
|     GoalPtr goal; |     WeakGoalPtr goal; | ||||||
|     int fdOutput; |     int fdOutput; | ||||||
|     bool inBuildSlot; |     bool inBuildSlot; | ||||||
| }; | }; | ||||||
|  | @ -110,14 +111,17 @@ class Worker | ||||||
| { | { | ||||||
| private: | private: | ||||||
| 
 | 
 | ||||||
|     /* The goals of the worker. */ |     /* Note: the worker should only have strong pointers to the
 | ||||||
|     Goals goals; |        top-level goals. */ | ||||||
|  | 
 | ||||||
|  |     /* The top-level goals of the worker. */ | ||||||
|  |     Goals topGoals; | ||||||
| 
 | 
 | ||||||
|     /* Goals that are ready to do some work. */ |     /* Goals that are ready to do some work. */ | ||||||
|     Goals awake; |     WeakGoals awake; | ||||||
| 
 | 
 | ||||||
|     /* Goals waiting for a build slot. */ |     /* Goals waiting for a build slot. */ | ||||||
|     Goals wantingToBuild; |     WeakGoals wantingToBuild; | ||||||
| 
 | 
 | ||||||
|     /* Child processes currently running. */ |     /* Child processes currently running. */ | ||||||
|     Children children; |     Children children; | ||||||
|  | @ -128,24 +132,21 @@ private: | ||||||
| 
 | 
 | ||||||
|     /* Maps used to prevent multiple instantiation of a goal for the
 |     /* Maps used to prevent multiple instantiation of a goal for the
 | ||||||
|        same expression / path. */ |        same expression / path. */ | ||||||
|     GoalMap normalisationGoals; |     WeakGoalMap normalisationGoals; | ||||||
|     GoalMapRev normalisationGoalsRev; |     WeakGoalMap realisationGoals; | ||||||
|     GoalMap realisationGoals; |     WeakGoalMap substitutionGoals; | ||||||
|     GoalMapRev realisationGoalsRev; |  | ||||||
|     GoalMap substitutionGoals; |  | ||||||
|     GoalMapRev substitutionGoalsRev; |  | ||||||
| 
 | 
 | ||||||
| public: | public: | ||||||
| 
 | 
 | ||||||
|     Worker(); |     Worker(); | ||||||
|     ~Worker(); |     ~Worker(); | ||||||
| 
 | 
 | ||||||
|     /* Add a goal. */ |     /* Make a goal (with caching). */ | ||||||
|     void addNormalisationGoal(const Path & nePath, GoalPtr waiter); |     GoalPtr makeNormalisationGoal(const Path & nePath); | ||||||
|     void addRealisationGoal(const Path & nePath, GoalPtr waiter); |     GoalPtr makeRealisationGoal(const Path & nePath); | ||||||
|     void addSubstitutionGoal(const Path & storePath, GoalPtr waiter); |     GoalPtr makeSubstitutionGoal(const Path & storePath); | ||||||
| 
 | 
 | ||||||
|     /* Remove a finished goal. */ |     /* Remove a dead goal. */ | ||||||
|     void removeGoal(GoalPtr goal); |     void removeGoal(GoalPtr goal); | ||||||
| 
 | 
 | ||||||
|     /* Wake up a goal (i.e., there is something for it to do). */ |     /* Wake up a goal (i.e., there is something for it to do). */ | ||||||
|  | @ -162,8 +163,9 @@ public: | ||||||
|     /* Add a goal to the set of goals waiting for a build slot. */ |     /* Add a goal to the set of goals waiting for a build slot. */ | ||||||
|     void waitForBuildSlot(GoalPtr goal); |     void waitForBuildSlot(GoalPtr goal); | ||||||
|      |      | ||||||
|     /* Loop until all goals have been realised. */ |     /* Loop until the specified top-level goal has finished.  Returns
 | ||||||
|     void run(); |        true if it has finished succesfully. */ | ||||||
|  |     bool run(GoalPtr topGoal); | ||||||
| 
 | 
 | ||||||
|     /* Wait for input to become available. */ |     /* Wait for input to become available. */ | ||||||
|     void waitForInput(); |     void waitForInput(); | ||||||
|  | @ -188,35 +190,44 @@ public: | ||||||
| //////////////////////////////////////////////////////////////////////
 | //////////////////////////////////////////////////////////////////////
 | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Goal::addWaiter(GoalPtr waiter) | void Goal::addWaitee(GoalPtr waitee) | ||||||
| { | { | ||||||
|     waiters.insert(waiter); |     waitees.insert(waitee); | ||||||
|  |     waitee->waiters.insert(shared_from_this()); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Goal::waiteeDone(bool success) | void Goal::waiteeDone(GoalPtr waitee, bool success) | ||||||
| { | { | ||||||
|  |     assert(waitees.find(waitee) != waitees.end()); | ||||||
|  |     waitees.erase(waitee); | ||||||
|     assert(nrWaitees > 0); |     assert(nrWaitees > 0); | ||||||
|     /* Note: waiteeFailed should never call amDone()! */ |     if (!success) ++nrFailed; | ||||||
|     if (!success) { |     if (!--nrWaitees || (!success && !keepGoing)) | ||||||
|         ++nrFailed; |         worker.wakeUp(shared_from_this()); | ||||||
|         waiteeFailed(); |  | ||||||
|     } |  | ||||||
|     if (!--nrWaitees) worker.wakeUp(shared_from_this()); |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Goal::amDone(bool success) | void Goal::amDone(bool success) | ||||||
| { | { | ||||||
|     printMsg(lvlVomit, "done"); |     trace("done"); | ||||||
|     assert(!done); |     assert(!done); | ||||||
|     done = true; |     done = true; | ||||||
|     for (Goals::iterator i = waiters.begin(); i != waiters.end(); ++i) |     for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) { | ||||||
|         (*i)->waiteeDone(success); |         GoalPtr goal = i->lock(); | ||||||
|  |         if (goal) goal->waiteeDone(shared_from_this(), success); | ||||||
|  |     } | ||||||
|  |     waiters.clear(); | ||||||
|     worker.removeGoal(shared_from_this()); |     worker.removeGoal(shared_from_this()); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | void Goal::trace(const format & f) | ||||||
|  | { | ||||||
|  |     debug(format("%1%: %2%") % name() % f); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| 
 | 
 | ||||||
| //////////////////////////////////////////////////////////////////////
 | //////////////////////////////////////////////////////////////////////
 | ||||||
| 
 | 
 | ||||||
|  | @ -356,12 +367,7 @@ private: | ||||||
|     /* Delete the temporary directory, if we have one. */ |     /* Delete the temporary directory, if we have one. */ | ||||||
|     void deleteTmpDir(bool force); |     void deleteTmpDir(bool force); | ||||||
| 
 | 
 | ||||||
|     string name() |     string name(); | ||||||
|     { |  | ||||||
|         return nePath; |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     void trace(const format & f); |  | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -375,6 +381,8 @@ NormalisationGoal::NormalisationGoal(const Path & _nePath, Worker & _worker) | ||||||
| 
 | 
 | ||||||
| NormalisationGoal::~NormalisationGoal() | NormalisationGoal::~NormalisationGoal() | ||||||
| { | { | ||||||
|  |     if (pid != -1) worker.childTerminated(pid); | ||||||
|  |      | ||||||
|     /* Careful: we should never ever throw an exception from a
 |     /* Careful: we should never ever throw an exception from a
 | ||||||
|        destructor. */ |        destructor. */ | ||||||
|     try { |     try { | ||||||
|  | @ -407,7 +415,7 @@ void NormalisationGoal::init() | ||||||
|        exists.  If it doesn't, it may be created through a |        exists.  If it doesn't, it may be created through a | ||||||
|        substitute. */ |        substitute. */ | ||||||
|     resetWaitees(1); |     resetWaitees(1); | ||||||
|     worker.addSubstitutionGoal(nePath, shared_from_this()); |     addWaitee(worker.makeSubstitutionGoal(nePath)); | ||||||
| 
 | 
 | ||||||
|     state = &NormalisationGoal::haveStoreExpr; |     state = &NormalisationGoal::haveStoreExpr; | ||||||
| } | } | ||||||
|  | @ -440,7 +448,7 @@ void NormalisationGoal::haveStoreExpr() | ||||||
|     /* Inputs must be normalised before we can build this goal. */ |     /* Inputs must be normalised before we can build this goal. */ | ||||||
|     for (PathSet::iterator i = expr.derivation.inputs.begin(); |     for (PathSet::iterator i = expr.derivation.inputs.begin(); | ||||||
|          i != expr.derivation.inputs.end(); ++i) |          i != expr.derivation.inputs.end(); ++i) | ||||||
|         worker.addNormalisationGoal(*i, shared_from_this()); |         addWaitee(worker.makeNormalisationGoal(*i)); | ||||||
| 
 | 
 | ||||||
|     resetWaitees(expr.derivation.inputs.size()); |     resetWaitees(expr.derivation.inputs.size()); | ||||||
| 
 | 
 | ||||||
|  | @ -469,7 +477,7 @@ void NormalisationGoal::inputNormalised() | ||||||
|         if (querySuccessor(neInput, nfInput)) |         if (querySuccessor(neInput, nfInput)) | ||||||
|             neInput = nfInput; |             neInput = nfInput; | ||||||
|         /* Otherwise the input must be a closure. */ |         /* Otherwise the input must be a closure. */ | ||||||
|         worker.addRealisationGoal(neInput, shared_from_this()); |         addWaitee(worker.makeRealisationGoal(neInput)); | ||||||
|     } |     } | ||||||
|      |      | ||||||
|     resetWaitees(expr.derivation.inputs.size()); |     resetWaitees(expr.derivation.inputs.size()); | ||||||
|  | @ -1153,9 +1161,9 @@ void NormalisationGoal::deleteTmpDir(bool force) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void NormalisationGoal::trace(const format & f) | string NormalisationGoal::name() | ||||||
| { | { | ||||||
|     debug(format("normalisation of `%1%': %2%") % nePath % f); |     return (format("normalisation of `%1%'") % nePath).str(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1186,7 +1194,7 @@ public: | ||||||
|     void haveStoreExpr(); |     void haveStoreExpr(); | ||||||
|     void elemFinished(); |     void elemFinished(); | ||||||
| 
 | 
 | ||||||
|     void trace(const format & f); |     string name(); | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1217,7 +1225,7 @@ void RealisationGoal::init() | ||||||
|        exists.  If it doesn't, it may be created through a |        exists.  If it doesn't, it may be created through a | ||||||
|        substitute. */ |        substitute. */ | ||||||
|     resetWaitees(1); |     resetWaitees(1); | ||||||
|     worker.addSubstitutionGoal(nePath, shared_from_this()); |     addWaitee(worker.makeSubstitutionGoal(nePath)); | ||||||
| 
 | 
 | ||||||
|     state = &RealisationGoal::haveStoreExpr; |     state = &RealisationGoal::haveStoreExpr; | ||||||
| } | } | ||||||
|  | @ -1248,7 +1256,7 @@ void RealisationGoal::haveStoreExpr() | ||||||
|        through a substitute. */ |        through a substitute. */ | ||||||
|     for (ClosureElems::const_iterator i = expr.closure.elems.begin(); |     for (ClosureElems::const_iterator i = expr.closure.elems.begin(); | ||||||
|          i != expr.closure.elems.end(); ++i) |          i != expr.closure.elems.end(); ++i) | ||||||
|         worker.addSubstitutionGoal(i->first, shared_from_this()); |         addWaitee(worker.makeSubstitutionGoal(i->first)); | ||||||
|      |      | ||||||
|     resetWaitees(expr.closure.elems.size()); |     resetWaitees(expr.closure.elems.size()); | ||||||
| 
 | 
 | ||||||
|  | @ -1274,9 +1282,9 @@ void RealisationGoal::elemFinished() | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void RealisationGoal::trace(const format & f) | string RealisationGoal::name() | ||||||
| { | { | ||||||
|     debug(format("realisation of `%1%': %2%") % nePath % f); |     return (format("realisation of `%1%'") % nePath).str(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1313,6 +1321,7 @@ private: | ||||||
| 
 | 
 | ||||||
| public: | public: | ||||||
|     SubstitutionGoal(const Path & _nePath, Worker & _worker); |     SubstitutionGoal(const Path & _nePath, Worker & _worker); | ||||||
|  |     ~SubstitutionGoal(); | ||||||
| 
 | 
 | ||||||
|     void work(); |     void work(); | ||||||
| 
 | 
 | ||||||
|  | @ -1324,7 +1333,7 @@ public: | ||||||
|     void tryToRun(); |     void tryToRun(); | ||||||
|     void finished(); |     void finished(); | ||||||
| 
 | 
 | ||||||
|     void trace(const format & f); |     string name(); | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1336,6 +1345,12 @@ SubstitutionGoal::SubstitutionGoal(const Path & _storePath, Worker & _worker) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | SubstitutionGoal::~SubstitutionGoal() | ||||||
|  | { | ||||||
|  |     if (pid != -1) worker.childTerminated(pid); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| void SubstitutionGoal::work() | void SubstitutionGoal::work() | ||||||
| { | { | ||||||
|     (this->*state)(); |     (this->*state)(); | ||||||
|  | @ -1377,7 +1392,7 @@ void SubstitutionGoal::tryNext() | ||||||
|     subs.pop_front(); |     subs.pop_front(); | ||||||
| 
 | 
 | ||||||
|     /* Normalise the substitute store expression. */ |     /* Normalise the substitute store expression. */ | ||||||
|     worker.addNormalisationGoal(sub.storeExpr, shared_from_this()); |     addWaitee(worker.makeNormalisationGoal(sub.storeExpr)); | ||||||
|     resetWaitees(1); |     resetWaitees(1); | ||||||
| 
 | 
 | ||||||
|     state = &SubstitutionGoal::exprNormalised; |     state = &SubstitutionGoal::exprNormalised; | ||||||
|  | @ -1396,7 +1411,7 @@ void SubstitutionGoal::exprNormalised() | ||||||
|     /* Realise the substitute store expression. */ |     /* Realise the substitute store expression. */ | ||||||
|     if (!querySuccessor(sub.storeExpr, nfSub)) |     if (!querySuccessor(sub.storeExpr, nfSub)) | ||||||
|         nfSub = sub.storeExpr; |         nfSub = sub.storeExpr; | ||||||
|     worker.addRealisationGoal(nfSub, shared_from_this()); |     addWaitee(worker.makeRealisationGoal(nfSub)); | ||||||
| 
 | 
 | ||||||
|     resetWaitees(1); |     resetWaitees(1); | ||||||
| 
 | 
 | ||||||
|  | @ -1557,9 +1572,9 @@ void SubstitutionGoal::finished() | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void SubstitutionGoal::trace(const format & f) | string SubstitutionGoal::name() | ||||||
| { | { | ||||||
|     debug(format("substitution of `%1%': %2%") % storePath % f); |     return (format("substitution of `%1%'") % storePath).str(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1585,7 +1600,7 @@ public: | ||||||
|         abort(); |         abort(); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     void waiteeDone(bool success) |     void waiteeDone(GoalPtr waitee, bool success) | ||||||
|     { |     { | ||||||
|         if (!success) this->success = false; |         if (!success) this->success = false; | ||||||
|     } |     } | ||||||
|  | @ -1594,6 +1609,11 @@ public: | ||||||
|     { |     { | ||||||
|         return success; |         return success; | ||||||
|     } |     } | ||||||
|  | 
 | ||||||
|  |     string name() | ||||||
|  |     { | ||||||
|  |         return "pseudo-goal"; | ||||||
|  |     } | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1616,71 +1636,67 @@ Worker::Worker() | ||||||
| Worker::~Worker() | Worker::~Worker() | ||||||
| { | { | ||||||
|     working = false; |     working = false; | ||||||
|  | 
 | ||||||
|  |     /* Explicitly get rid of all strong pointers now.  After this all
 | ||||||
|  |        goals that refer to this worker should be gone.  (Otherwise we | ||||||
|  |        are in trouble, since goals may call childTerminated() etc. in | ||||||
|  |        their destructors). */ | ||||||
|  |     topGoals.clear(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| template<class T> | template<class T> | ||||||
| static void addGoal(const Path & path, GoalPtr waiter, | static GoalPtr addGoal(const Path & path, | ||||||
|     Worker & worker, Goals & goals, |     Worker & worker, WeakGoalMap & goalMap) | ||||||
|     GoalMap & goalMap, GoalMapRev & goalMapRev) |  | ||||||
| { | { | ||||||
|     GoalPtr goal; |     GoalPtr goal = goalMap[path].lock(); | ||||||
|     goal = goalMap[path]; |  | ||||||
|     if (!goal) { |     if (!goal) { | ||||||
|         goal = GoalPtr(new T(path, worker)); |         goal = GoalPtr(new T(path, worker)); | ||||||
|         goals.insert(goal); |  | ||||||
|         goalMap[path] = goal; |         goalMap[path] = goal; | ||||||
|         goalMapRev[goal] = path; |  | ||||||
|         worker.wakeUp(goal); |         worker.wakeUp(goal); | ||||||
|     } |     } | ||||||
|     if (waiter) goal->addWaiter(waiter); |     return goal; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::addNormalisationGoal(const Path & nePath, GoalPtr waiter) | GoalPtr Worker::makeNormalisationGoal(const Path & nePath) | ||||||
| { | { | ||||||
|     addGoal<NormalisationGoal>(nePath, waiter, *this, goals, |     return addGoal<NormalisationGoal>(nePath, *this, normalisationGoals); | ||||||
|         normalisationGoals, normalisationGoalsRev); |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::addRealisationGoal(const Path & nePath, GoalPtr waiter) | GoalPtr Worker::makeRealisationGoal(const Path & nePath) | ||||||
| { | { | ||||||
|     addGoal<RealisationGoal>(nePath, waiter, *this, goals, |     return addGoal<RealisationGoal>(nePath, *this, realisationGoals); | ||||||
|         realisationGoals, realisationGoalsRev); |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::addSubstitutionGoal(const Path & storePath, GoalPtr waiter) | GoalPtr Worker::makeSubstitutionGoal(const Path & storePath) | ||||||
| { | { | ||||||
|     addGoal<SubstitutionGoal>(storePath, waiter, *this, goals, |     return addGoal<SubstitutionGoal>(storePath, *this, substitutionGoals); | ||||||
|         substitutionGoals, substitutionGoalsRev); |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| static void removeGoal(GoalPtr goal, | static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) | ||||||
|     GoalMap & goalMap, GoalMapRev & goalMapRev) |  | ||||||
| { | { | ||||||
|     GoalMapRev::iterator i = goalMapRev.find(goal); |     /* !!! For now we just let dead goals accumulate.  We should
 | ||||||
|     if (i != goalMapRev.end()) { |        probably periodically sweep the goalMap to remove dead | ||||||
|         goalMapRev.erase(i); |        goals. */ | ||||||
|         goalMap.erase(i->second); |  | ||||||
|     } |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::removeGoal(GoalPtr goal) | void Worker::removeGoal(GoalPtr goal) | ||||||
| { | { | ||||||
|     goals.erase(goal); |     topGoals.erase(goal); | ||||||
|     ::removeGoal(goal, normalisationGoals, normalisationGoalsRev); |     ::removeGoal(goal, normalisationGoals); | ||||||
|     ::removeGoal(goal, realisationGoals, realisationGoalsRev); |     ::removeGoal(goal, realisationGoals); | ||||||
|     ::removeGoal(goal, substitutionGoals, substitutionGoalsRev); |     ::removeGoal(goal, substitutionGoals); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::wakeUp(GoalPtr goal) | void Worker::wakeUp(GoalPtr goal) | ||||||
| { | { | ||||||
|     printMsg(lvlVomit, "wake up"); |     goal->trace("woken up"); | ||||||
|     awake.insert(goal); |     awake.insert(goal); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | @ -1716,9 +1732,12 @@ void Worker::childTerminated(pid_t pid) | ||||||
|     children.erase(pid); |     children.erase(pid); | ||||||
| 
 | 
 | ||||||
|     /* Wake up goals waiting for a build slot. */ |     /* Wake up goals waiting for a build slot. */ | ||||||
|     for (Goals::iterator i = wantingToBuild.begin(); |     for (WeakGoals::iterator i = wantingToBuild.begin(); | ||||||
|          i != wantingToBuild.end(); ++i) |          i != wantingToBuild.end(); ++i) | ||||||
|         wakeUp(*i); |     { | ||||||
|  |         GoalPtr goal = i->lock(); | ||||||
|  |         if (goal) wakeUp(goal); | ||||||
|  |     } | ||||||
|     wantingToBuild.clear(); |     wantingToBuild.clear(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | @ -1733,8 +1752,18 @@ void Worker::waitForBuildSlot(GoalPtr goal) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| void Worker::run() | bool Worker::run(GoalPtr topGoal) | ||||||
| { | { | ||||||
|  |     assert(topGoal); | ||||||
|  |      | ||||||
|  |     /* Wrap the specified top-level goal in a pseudo-goal so that we
 | ||||||
|  |        can check whether it succeeded. */ | ||||||
|  |     shared_ptr<PseudoGoal> pseudo(new PseudoGoal(*this)); | ||||||
|  |     pseudo->addWaitee(topGoal); | ||||||
|  |      | ||||||
|  |     /* For now, we have only one top-level goal. */ | ||||||
|  |     topGoals.insert(topGoal); | ||||||
|  |      | ||||||
|     startNest(nest, lvlDebug, format("entered goal loop")); |     startNest(nest, lvlDebug, format("entered goal loop")); | ||||||
| 
 | 
 | ||||||
|     while (1) { |     while (1) { | ||||||
|  | @ -1743,18 +1772,16 @@ void Worker::run() | ||||||
| 
 | 
 | ||||||
|         /* Call every wake goal. */ |         /* Call every wake goal. */ | ||||||
|         while (!awake.empty()) { |         while (!awake.empty()) { | ||||||
|             Goals awake2(awake); /* !!! why is this necessary? */ |             WeakGoals awake2(awake); | ||||||
|             awake.clear(); |             awake.clear(); | ||||||
|             for (Goals::iterator i = awake2.begin(); i != awake2.end(); ++i) { |             for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) { | ||||||
|                 printMsg(lvlVomit, |  | ||||||
|                     format("running goal (%1% left)") % goals.size()); |  | ||||||
|                 checkInterrupt(); |                 checkInterrupt(); | ||||||
|                 GoalPtr goal = *i; |                 GoalPtr goal = i->lock(); | ||||||
|                 goal->work(); |                 if (goal) goal->work(); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|         if (goals.empty()) break; |         if (topGoals.empty()) break; | ||||||
| 
 | 
 | ||||||
|         /* !!! not when we're polling */ |         /* !!! not when we're polling */ | ||||||
|         assert(!children.empty()); |         assert(!children.empty()); | ||||||
|  | @ -1763,9 +1790,14 @@ void Worker::run() | ||||||
|         waitForInput(); |         waitForInput(); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     assert(awake.empty()); |     /* If --keep-going is not set, it's possible that the main goal
 | ||||||
|     assert(wantingToBuild.empty()); |        exited while some of its subgoals were still active.  But if | ||||||
|     assert(children.empty()); |        --keep-going *is* set, then they must all be finished now. */ | ||||||
|  |     assert(!keepGoing || awake.empty()); | ||||||
|  |     assert(!keepGoing || wantingToBuild.empty()); | ||||||
|  |     assert(!keepGoing || children.empty()); | ||||||
|  | 
 | ||||||
|  |     return pseudo->isOkay(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -1801,22 +1833,23 @@ void Worker::waitForInput() | ||||||
|          i != children.end(); ++i) |          i != children.end(); ++i) | ||||||
|     { |     { | ||||||
|         checkInterrupt(); |         checkInterrupt(); | ||||||
|         GoalPtr goal = i->second.goal; |         GoalPtr goal = i->second.goal.lock(); | ||||||
|  |         assert(goal); | ||||||
|         int fd = i->second.fdOutput; |         int fd = i->second.fdOutput; | ||||||
|         if (FD_ISSET(fd, &fds)) { |         if (FD_ISSET(fd, &fds)) { | ||||||
|             unsigned char buffer[4096]; |             unsigned char buffer[4096]; | ||||||
|             ssize_t rd = read(fd, buffer, sizeof(buffer)); |             ssize_t rd = read(fd, buffer, sizeof(buffer)); | ||||||
|             if (rd == -1) { |             if (rd == -1) { | ||||||
|                 if (errno != EINTR) |                 if (errno != EINTR) | ||||||
|                     throw SysError(format("reading from `%1%'") |                     throw SysError(format("reading from %1%") | ||||||
|                         % goal->name()); |                         % goal->name()); | ||||||
|             } else if (rd == 0) { |             } else if (rd == 0) { | ||||||
|                 debug(format("EOF on `%1%'") % goal->name()); |                 debug(format("%1%: got EOF") % goal->name()); | ||||||
|                 wakeUp(goal); |                 wakeUp(goal); | ||||||
|             } else { |             } else { | ||||||
|                 printMsg(lvlVomit, format("read %1% bytes from `%2%'") |                 printMsg(lvlVomit, format("%1%: read %2% bytes") | ||||||
|                     % rd % goal->name()); |                     % goal->name() % rd); | ||||||
| //                 writeFull(goal.fdLogFile, buffer, rd);
 | //                 writeFull(goal.fdLogFile, buffer, rd); !!!
 | ||||||
|                 if (verbosity >= buildVerbosity) |                 if (verbosity >= buildVerbosity) | ||||||
|                     writeFull(STDERR_FILENO, buffer, rd); |                     writeFull(STDERR_FILENO, buffer, rd); | ||||||
|             } |             } | ||||||
|  | @ -1833,11 +1866,7 @@ Path normaliseStoreExpr(const Path & nePath) | ||||||
|     startNest(nest, lvlDebug, format("normalising `%1%'") % nePath); |     startNest(nest, lvlDebug, format("normalising `%1%'") % nePath); | ||||||
| 
 | 
 | ||||||
|     Worker worker; |     Worker worker; | ||||||
|     shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker)); |     if (!worker.run(worker.makeNormalisationGoal(nePath))) | ||||||
|     worker.addNormalisationGoal(nePath, pseudo); |  | ||||||
|     worker.run(); |  | ||||||
| 
 |  | ||||||
|     if (!pseudo->isOkay()) |  | ||||||
|         throw Error(format("normalisation of store expression `%1%' failed") % nePath); |         throw Error(format("normalisation of store expression `%1%' failed") % nePath); | ||||||
| 
 | 
 | ||||||
|     Path nfPath; |     Path nfPath; | ||||||
|  | @ -1851,11 +1880,7 @@ void realiseClosure(const Path & nePath) | ||||||
|     startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); |     startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); | ||||||
| 
 | 
 | ||||||
|     Worker worker; |     Worker worker; | ||||||
|     shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker)); |     if (!worker.run(worker.makeRealisationGoal(nePath))) | ||||||
|     worker.addRealisationGoal(nePath, pseudo); |  | ||||||
|     worker.run(); |  | ||||||
| 
 |  | ||||||
|     if (!pseudo->isOkay()) |  | ||||||
|         throw Error(format("realisation of closure `%1%' failed") % nePath); |         throw Error(format("realisation of closure `%1%' failed") % nePath); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | @ -1866,10 +1891,6 @@ void ensurePath(const Path & path) | ||||||
|     if (isValidPath(path)) return; |     if (isValidPath(path)) return; | ||||||
| 
 | 
 | ||||||
|     Worker worker; |     Worker worker; | ||||||
|     shared_ptr<PseudoGoal> pseudo(new PseudoGoal(worker)); |     if (!worker.run(worker.makeSubstitutionGoal(path))) | ||||||
|     worker.addSubstitutionGoal(path, pseudo); |  | ||||||
|     worker.run(); |  | ||||||
| 
 |  | ||||||
|     if (!pseudo->isOkay()) |  | ||||||
|         throw Error(format("path `%1%' does not exist and cannot be created") % path); |         throw Error(format("path `%1%' does not exist and cannot be created") % path); | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -492,20 +492,19 @@ void Pid::kill() | ||||||
| { | { | ||||||
|     if (pid == -1) return; |     if (pid == -1) return; | ||||||
|      |      | ||||||
|     printMsg(lvlError, format("killing child process %1%") % pid); |     printMsg(lvlError, format("killing process %1%") % pid); | ||||||
| 
 | 
 | ||||||
|     /* Send a KILL signal to the child.  If it has its own process
 |     /* Send a KILL signal to the child.  If it has its own process
 | ||||||
|        group, send the signal to every process in the child process |        group, send the signal to every process in the child process | ||||||
|        group (which hopefully includes *all* its children). */ |        group (which hopefully includes *all* its children). */ | ||||||
|     if (::kill(separatePG ? -pid : pid, SIGKILL) != 0) |     if (::kill(separatePG ? -pid : pid, SIGKILL) != 0) | ||||||
|         printMsg(lvlError, format("killing process %1%") % pid); |         printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg())); | ||||||
|     else { | 
 | ||||||
|         /* Wait until the child dies, disregarding the exit status. */ |     /* Wait until the child dies, disregarding the exit status. */ | ||||||
|         int status; |     int status; | ||||||
|         while (waitpid(pid, &status, 0) == -1) |     while (waitpid(pid, &status, 0) == -1) | ||||||
|             if (errno != EINTR) printMsg(lvlError, |         if (errno != EINTR) printMsg(lvlError, | ||||||
|                 format("waiting for process %1%") % pid); |             (SysError(format("waiting for process %1%") % pid).msg())); | ||||||
|     } |  | ||||||
| 
 | 
 | ||||||
|     pid = -1; |     pid = -1; | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -21,9 +21,8 @@ substitutes.sh: substitutes.nix substituter.nix | ||||||
| substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix | substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix | ||||||
| fall-back.sh: fall-back.nix | fall-back.sh: fall-back.nix | ||||||
| 
 | 
 | ||||||
| #TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \ | TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \ | ||||||
| #  build-hook.sh substitutes.sh substitutes2.sh |   build-hook.sh substitutes.sh substitutes2.sh | ||||||
| TESTS = init.sh fall-back.sh |  | ||||||
| 
 | 
 | ||||||
| XFAIL_TESTS = | XFAIL_TESTS = | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -2,11 +2,12 @@ storeExpr=$($TOP/src/nix-instantiate/nix-instantiate fall-back.nix) | ||||||
| 
 | 
 | ||||||
| echo "store expr is $storeExpr" | echo "store expr is $storeExpr" | ||||||
| 
 | 
 | ||||||
| # Register a non-existant successor. | # Register a non-existant successor (and a nox-existant substitute). | ||||||
| suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store | suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store | ||||||
|  | (echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute | ||||||
| $TOP/src/nix-store/nix-store --successor $storeExpr $suc | $TOP/src/nix-store/nix-store --successor $storeExpr $suc | ||||||
| 
 | 
 | ||||||
| outPath=$($TOP/src/nix-store/nix-store -qnfvvvvv "$storeExpr") | outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr") | ||||||
| 
 | 
 | ||||||
| echo "output path is $outPath" | echo "output path is $outPath" | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue