* `nix-store --build' now builds its arguments in parallel instead of
sequentially (within the limits set by `--jobs'). This should greatly improve the utilisation of the build farm when doing Nixpkgs builds.
This commit is contained in:
		
							parent
							
								
									06c77bf7a8
								
							
						
					
					
						commit
						ef5f254a55
					
				
					 4 changed files with 37 additions and 24 deletions
				
			
		| 
						 | 
					@ -163,7 +163,7 @@ public:
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
    /* Loop until the specified top-level goal has finished.  Returns
 | 
					    /* Loop until the specified top-level goal has finished.  Returns
 | 
				
			||||||
       true if it has finished succesfully. */
 | 
					       true if it has finished succesfully. */
 | 
				
			||||||
    bool run(GoalPtr topGoal);
 | 
					    bool run(const Goals & topGoals);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    /* Wait for input to become available. */
 | 
					    /* Wait for input to become available. */
 | 
				
			||||||
    void waitForInput();
 | 
					    void waitForInput();
 | 
				
			||||||
| 
						 | 
					@ -697,7 +697,7 @@ string showPaths(const PathSet & paths)
 | 
				
			||||||
         i != paths.end(); ++i)
 | 
					         i != paths.end(); ++i)
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
        if (s.size() != 0) s += ", ";
 | 
					        if (s.size() != 0) s += ", ";
 | 
				
			||||||
        s += *i;
 | 
					        s += "`" + *i + "'";
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
    return s;
 | 
					    return s;
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
| 
						 | 
					@ -915,7 +915,7 @@ bool DerivationGoal::prepareBuild()
 | 
				
			||||||
void DerivationGoal::startBuilder()
 | 
					void DerivationGoal::startBuilder()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
    startNest(nest, lvlInfo,
 | 
					    startNest(nest, lvlInfo,
 | 
				
			||||||
        format("building path(s) `%1%'") % showPaths(outputPaths(drv.outputs)))
 | 
					        format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
    /* Right platform? */
 | 
					    /* Right platform? */
 | 
				
			||||||
    if (drv.platform != thisSystem)
 | 
					    if (drv.platform != thisSystem)
 | 
				
			||||||
| 
						 | 
					@ -1664,17 +1664,18 @@ void Worker::waitForBuildSlot(GoalPtr goal, bool reallyWait)
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
bool Worker::run(GoalPtr topGoal)
 | 
					bool Worker::run(const Goals & _topGoals)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
    assert(topGoal);
 | 
					 | 
				
			||||||
    
 | 
					 | 
				
			||||||
    /* Wrap the specified top-level goal in a pseudo-goal so that we
 | 
					    /* Wrap the specified top-level goal in a pseudo-goal so that we
 | 
				
			||||||
       can check whether it succeeded. */
 | 
					       can check whether it succeeded. */
 | 
				
			||||||
    shared_ptr<PseudoGoal> pseudo(new PseudoGoal(*this));
 | 
					    shared_ptr<PseudoGoal> pseudo(new PseudoGoal(*this));
 | 
				
			||||||
    pseudo->addWaitee(topGoal);
 | 
					    for (Goals::iterator i = _topGoals.begin();
 | 
				
			||||||
    
 | 
					         i != _topGoals.end(); ++i)
 | 
				
			||||||
    /* For now, we have only one top-level goal. */
 | 
					    {
 | 
				
			||||||
    topGoals.insert(topGoal);
 | 
					        assert(*i);
 | 
				
			||||||
 | 
					        pseudo->addWaitee(*i);
 | 
				
			||||||
 | 
					        topGoals.insert(*i);
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
    startNest(nest, lvlDebug, format("entered goal loop"));
 | 
					    startNest(nest, lvlDebug, format("entered goal loop"));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1773,13 +1774,20 @@ void Worker::waitForInput()
 | 
				
			||||||
//////////////////////////////////////////////////////////////////////
 | 
					//////////////////////////////////////////////////////////////////////
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void buildDerivation(const Path & drvPath)
 | 
					void buildDerivations(const PathSet & drvPaths)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
    startNest(nest, lvlDebug, format("building `%1%'") % drvPath);
 | 
					    startNest(nest, lvlDebug,
 | 
				
			||||||
 | 
					        format("building %1%") % showPaths(drvPaths));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    Worker worker;
 | 
					    Worker worker;
 | 
				
			||||||
    if (!worker.run(worker.makeDerivationGoal(drvPath)))
 | 
					
 | 
				
			||||||
        throw Error(format("build of derivation `%1%' failed") % drvPath);
 | 
					    Goals goals;
 | 
				
			||||||
 | 
					    for (PathSet::const_iterator i = drvPaths.begin();
 | 
				
			||||||
 | 
					         i != drvPaths.end(); ++i)
 | 
				
			||||||
 | 
					        goals.insert(worker.makeDerivationGoal(*i));
 | 
				
			||||||
 | 
					    
 | 
				
			||||||
 | 
					    if (!worker.run(goals))
 | 
				
			||||||
 | 
					        throw Error(format("build failed"));
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1789,6 +1797,8 @@ void ensurePath(const Path & path)
 | 
				
			||||||
    if (isValidPath(path)) return;
 | 
					    if (isValidPath(path)) return;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    Worker worker;
 | 
					    Worker worker;
 | 
				
			||||||
    if (!worker.run(worker.makeSubstitutionGoal(path)))
 | 
					    Goals goals;
 | 
				
			||||||
 | 
					    goals.insert(worker.makeSubstitutionGoal(path));
 | 
				
			||||||
 | 
					    if (!worker.run(goals))
 | 
				
			||||||
        throw Error(format("path `%1%' does not exist and cannot be created") % path);
 | 
					        throw Error(format("path `%1%' does not exist and cannot be created") % path);
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -3,13 +3,13 @@
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#include "storeexpr.hh"
 | 
					#include "storeexpr.hh"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
/* Perform the specified derivation, if necessary.  That is, do
 | 
					/* Perform the specified derivations, if necessary.  That is, do
 | 
				
			||||||
   whatever is necessary to create the output paths of the
 | 
					   whatever is necessary to create the output paths of the derivation.
 | 
				
			||||||
   derivation.  If the output paths already exists, we're done.  If
 | 
					   If the output paths already exists, we're done.  If they have
 | 
				
			||||||
   they have substitutes, we can use those instead.  Otherwise, the
 | 
					   substitutes, we can use those instead.  Otherwise, the build action
 | 
				
			||||||
   build action described by the derivation is performed, after
 | 
					   described by the derivation is performed, after recursively
 | 
				
			||||||
   recursively building any sub-derivations. */
 | 
					   building any sub-derivations. */
 | 
				
			||||||
void buildDerivation(const Path & drvPath);
 | 
					void buildDerivations(const PathSet & drvPaths);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
/* Ensure that a path exists, possibly by instantiating it by
 | 
					/* Ensure that a path exists, possibly by instantiating it by
 | 
				
			||||||
   realising a substitute. */
 | 
					   realising a substitute. */
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -223,7 +223,9 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs,
 | 
				
			||||||
    
 | 
					    
 | 
				
			||||||
    /* Realise the resulting store expression. */
 | 
					    /* Realise the resulting store expression. */
 | 
				
			||||||
    debug(format("building user environment"));
 | 
					    debug(format("building user environment"));
 | 
				
			||||||
    buildDerivation(topLevelDrv.drvPath);
 | 
					    PathSet drvPaths;
 | 
				
			||||||
 | 
					    drvPaths.insert(topLevelDrv.drvPath);
 | 
				
			||||||
 | 
					    buildDerivations(drvPaths);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    /* Switch the current user environment to the output path. */
 | 
					    /* Switch the current user environment to the output path. */
 | 
				
			||||||
    debug(format("switching to new user environment"));
 | 
					    debug(format("switching to new user environment"));
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -32,10 +32,11 @@ static void opBuild(Strings opFlags, Strings opArgs)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
    if (!opFlags.empty()) throw UsageError("unknown flag");
 | 
					    if (!opFlags.empty()) throw UsageError("unknown flag");
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    buildDerivations(PathSet(opArgs.begin(), opArgs.end()));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    for (Strings::iterator i = opArgs.begin();
 | 
					    for (Strings::iterator i = opArgs.begin();
 | 
				
			||||||
         i != opArgs.end(); i++)
 | 
					         i != opArgs.end(); i++)
 | 
				
			||||||
    {
 | 
					    {
 | 
				
			||||||
        buildDerivation(*i);
 | 
					 | 
				
			||||||
        Derivation drv = derivationFromPath(*i);
 | 
					        Derivation drv = derivationFromPath(*i);
 | 
				
			||||||
        cout << format("%1%\n") % findOutput(drv, "out");
 | 
					        cout << format("%1%\n") % findOutput(drv, "out");
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue