Run the daemon worker on the same CPU as the client
On a system with multiple CPUs, running Nix operations through the daemon is significantly slower than "direct" mode: $ NIX_REMOTE= nix-instantiate '<nixos>' -A system real 0m0.974s user 0m0.875s sys 0m0.088s $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m2.118s user 0m1.463s sys 0m0.218s The main reason seems to be that the client and the worker get moved to a different CPU after every call to the worker. This patch adds a hack to lock them to the same CPU. With this, the overhead of going through the daemon is very small: $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m1.074s user 0m0.809s sys 0m0.098s
This commit is contained in:
		
							parent
							
								
									263d668222
								
							
						
					
					
						commit
						a583a2bc59
					
				
					 9 changed files with 92 additions and 4 deletions
				
			
		|  | @ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], | |||
| AC_CHECK_FUNCS([lutimes]) | ||||
| 
 | ||||
| 
 | ||||
| # Check for sched_setaffinity. | ||||
| AC_CHECK_FUNCS([sched_setaffinity]) | ||||
| 
 | ||||
| 
 | ||||
| # Check whether the store optimiser can optimise symlinks. | ||||
| AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) | ||||
| ln -s bla tmp_link | ||||
|  |  | |||
|  | @ -7,6 +7,7 @@ | |||
| #include "local-store.hh" | ||||
| #include "util.hh" | ||||
| #include "archive.hh" | ||||
| #include "affinity.hh" | ||||
| 
 | ||||
| #include <map> | ||||
| #include <sstream> | ||||
|  | @ -366,6 +367,8 @@ void Goal::trace(const format & f) | |||
| /* Common initialisation performed in child processes. */ | ||||
| static void commonChildInit(Pipe & logPipe) | ||||
| { | ||||
|     restoreAffinity(); | ||||
| 
 | ||||
|     /* Put the child in a separate session (and thus a separate
 | ||||
|        process group) so that it has no controlling terminal (meaning | ||||
|        that e.g. ssh cannot open /dev/tty) and it doesn't receive | ||||
|  | @ -568,6 +571,7 @@ static void runSetuidHelper(const string & command, | |||
|             args.push_back(0); | ||||
| 
 | ||||
|             restoreSIGPIPE(); | ||||
|             restoreAffinity(); | ||||
| 
 | ||||
|             execve(program.c_str(), (char * *) &args[0], 0); | ||||
|             throw SysError(format("executing `%1%'") % program); | ||||
|  |  | |||
|  | @ -5,6 +5,7 @@ | |||
| #include "pathlocks.hh" | ||||
| #include "worker-protocol.hh" | ||||
| #include "derivations.hh" | ||||
| #include "affinity.hh" | ||||
| 
 | ||||
| #include <iostream> | ||||
| #include <algorithm> | ||||
|  | @ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & | |||
| 
 | ||||
|     case 0: /* child */ | ||||
|         try { | ||||
|             restoreAffinity(); | ||||
|             if (dup2(toPipe.readSide, STDIN_FILENO) == -1) | ||||
|                 throw SysError("dupping stdin"); | ||||
|             if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) | ||||
|  |  | |||
|  | @ -3,6 +3,7 @@ | |||
| #include "remote-store.hh" | ||||
| #include "worker-protocol.hh" | ||||
| #include "archive.hh" | ||||
| #include "affinity.hh" | ||||
| #include "globals.hh" | ||||
| 
 | ||||
| #include <sys/types.h> | ||||
|  | @ -15,7 +16,6 @@ | |||
| #include <unistd.h> | ||||
| #include <cstring> | ||||
| 
 | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| 
 | ||||
|  | @ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace) | |||
|         if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) | ||||
|             throw Error("Nix daemon protocol version not supported"); | ||||
|         writeInt(PROTOCOL_VERSION, to); | ||||
| 
 | ||||
|         if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { | ||||
|             int cpu = lockToCurrentCPU(); | ||||
|             if (cpu != -1) { | ||||
|                 writeInt(1, to); | ||||
|                 writeInt(cpu, to); | ||||
|             } else | ||||
|                 writeInt(0, to); | ||||
|         } | ||||
| 
 | ||||
|         if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) | ||||
|             writeInt(reserveSpace, to); | ||||
| 
 | ||||
|         processStderr(); | ||||
|     } | ||||
|     catch (Error & e) { | ||||
|  |  | |||
|  | @ -6,7 +6,7 @@ namespace nix { | |||
| #define WORKER_MAGIC_1 0x6e697863 | ||||
| #define WORKER_MAGIC_2 0x6478696f | ||||
| 
 | ||||
| #define PROTOCOL_VERSION 0x10d | ||||
| #define PROTOCOL_VERSION 0x10e | ||||
| #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) | ||||
| #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) | ||||
| 
 | ||||
|  |  | |||
|  | @ -1,12 +1,12 @@ | |||
| pkglib_LTLIBRARIES = libutil.la | ||||
| 
 | ||||
| libutil_la_SOURCES = util.cc hash.cc serialise.cc \ | ||||
|   archive.cc xml-writer.cc | ||||
|   archive.cc xml-writer.cc affinity.cc | ||||
| 
 | ||||
| libutil_la_LIBADD = ../boost/format/libformat.la | ||||
| 
 | ||||
| pkginclude_HEADERS = util.hh hash.hh serialise.hh \ | ||||
|   archive.hh xml-writer.hh types.hh | ||||
|   archive.hh xml-writer.hh types.hh affinity.hh | ||||
| 
 | ||||
| if !HAVE_OPENSSL | ||||
| libutil_la_SOURCES += \ | ||||
|  |  | |||
							
								
								
									
										54
									
								
								src/libutil/affinity.cc
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										54
									
								
								src/libutil/affinity.cc
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,54 @@ | |||
| #include "types.hh" | ||||
| #include "util.hh" | ||||
| #include "affinity.hh" | ||||
| 
 | ||||
| #if HAVE_SCHED_H | ||||
| #include <sched.h> | ||||
| #endif | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| 
 | ||||
| static bool didSaveAffinity = false; | ||||
| static cpu_set_t savedAffinity; | ||||
| 
 | ||||
| 
 | ||||
| void setAffinityTo(int cpu) | ||||
| { | ||||
| #if HAVE_SCHED_SETAFFINITY | ||||
|     if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; | ||||
|     didSaveAffinity = true; | ||||
|     printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); | ||||
|     cpu_set_t newAffinity; | ||||
|     CPU_ZERO(&newAffinity); | ||||
|     CPU_SET(cpu, &newAffinity); | ||||
|     if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) | ||||
|         printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); | ||||
| #endif | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
| int lockToCurrentCPU() | ||||
| { | ||||
| #if HAVE_SCHED_SETAFFINITY | ||||
|     if (getEnv("NIX_AFFINITY_HACK", "1") == "1") { | ||||
|         int cpu = sched_getcpu(); | ||||
|         if (cpu != -1) setAffinityTo(cpu); | ||||
|         return cpu; | ||||
|     } | ||||
| #endif | ||||
|     return -1; | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
| void restoreAffinity() | ||||
| { | ||||
| #if HAVE_SCHED_SETAFFINITY | ||||
|     if (!didSaveAffinity) return; | ||||
|     if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) | ||||
|         printMsg(lvlError, "failed to restore affinity %1%"); | ||||
| #endif | ||||
| } | ||||
| 
 | ||||
| 
 | ||||
| } | ||||
							
								
								
									
										9
									
								
								src/libutil/affinity.hh
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								src/libutil/affinity.hh
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,9 @@ | |||
| #pragma once | ||||
| 
 | ||||
| namespace nix { | ||||
| 
 | ||||
| void setAffinityTo(int cpu); | ||||
| int lockToCurrentCPU(); | ||||
| void restoreAffinity(); | ||||
| 
 | ||||
| } | ||||
|  | @ -4,6 +4,7 @@ | |||
| #include "serialise.hh" | ||||
| #include "worker-protocol.hh" | ||||
| #include "archive.hh" | ||||
| #include "affinity.hh" | ||||
| #include "globals.hh" | ||||
| 
 | ||||
| #include <cstring> | ||||
|  | @ -671,6 +672,9 @@ static void processConnection(bool trusted) | |||
|     to.flush(); | ||||
|     unsigned int clientVersion = readInt(from); | ||||
| 
 | ||||
|     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) | ||||
|         setAffinityTo(readInt(from)); | ||||
| 
 | ||||
|     bool reserveSpace = true; | ||||
|     if (GET_PROTOCOL_MINOR(clientVersion) >= 11) | ||||
|         reserveSpace = readInt(from) != 0; | ||||
|  |  | |||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue