Run the daemon worker on the same CPU as the client
On a system with multiple CPUs, running Nix operations through the daemon is significantly slower than "direct" mode: $ NIX_REMOTE= nix-instantiate '<nixos>' -A system real 0m0.974s user 0m0.875s sys 0m0.088s $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m2.118s user 0m1.463s sys 0m0.218s The main reason seems to be that the client and the worker get moved to a different CPU after every call to the worker. This patch adds a hack to lock them to the same CPU. With this, the overhead of going through the daemon is very small: $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m1.074s user 0m0.809s sys 0m0.098s
This commit is contained in:
		
							parent
							
								
									263d668222
								
							
						
					
					
						commit
						a583a2bc59
					
				
					 9 changed files with 92 additions and 4 deletions
				
			
		|  | @ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], | ||||||
| AC_CHECK_FUNCS([lutimes]) | AC_CHECK_FUNCS([lutimes]) | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | # Check for sched_setaffinity. | ||||||
|  | AC_CHECK_FUNCS([sched_setaffinity]) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| # Check whether the store optimiser can optimise symlinks. | # Check whether the store optimiser can optimise symlinks. | ||||||
| AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) | AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) | ||||||
| ln -s bla tmp_link | ln -s bla tmp_link | ||||||
|  |  | ||||||
|  | @ -7,6 +7,7 @@ | ||||||
| #include "local-store.hh" | #include "local-store.hh" | ||||||
| #include "util.hh" | #include "util.hh" | ||||||
| #include "archive.hh" | #include "archive.hh" | ||||||
|  | #include "affinity.hh" | ||||||
| 
 | 
 | ||||||
| #include <map> | #include <map> | ||||||
| #include <sstream> | #include <sstream> | ||||||
|  | @ -366,6 +367,8 @@ void Goal::trace(const format & f) | ||||||
| /* Common initialisation performed in child processes. */ | /* Common initialisation performed in child processes. */ | ||||||
| static void commonChildInit(Pipe & logPipe) | static void commonChildInit(Pipe & logPipe) | ||||||
| { | { | ||||||
|  |     restoreAffinity(); | ||||||
|  | 
 | ||||||
|     /* Put the child in a separate session (and thus a separate
 |     /* Put the child in a separate session (and thus a separate
 | ||||||
|        process group) so that it has no controlling terminal (meaning |        process group) so that it has no controlling terminal (meaning | ||||||
|        that e.g. ssh cannot open /dev/tty) and it doesn't receive |        that e.g. ssh cannot open /dev/tty) and it doesn't receive | ||||||
|  | @ -568,6 +571,7 @@ static void runSetuidHelper(const string & command, | ||||||
|             args.push_back(0); |             args.push_back(0); | ||||||
| 
 | 
 | ||||||
|             restoreSIGPIPE(); |             restoreSIGPIPE(); | ||||||
|  |             restoreAffinity(); | ||||||
| 
 | 
 | ||||||
|             execve(program.c_str(), (char * *) &args[0], 0); |             execve(program.c_str(), (char * *) &args[0], 0); | ||||||
|             throw SysError(format("executing `%1%'") % program); |             throw SysError(format("executing `%1%'") % program); | ||||||
|  |  | ||||||
|  | @ -5,6 +5,7 @@ | ||||||
| #include "pathlocks.hh" | #include "pathlocks.hh" | ||||||
| #include "worker-protocol.hh" | #include "worker-protocol.hh" | ||||||
| #include "derivations.hh" | #include "derivations.hh" | ||||||
|  | #include "affinity.hh" | ||||||
| 
 | 
 | ||||||
| #include <iostream> | #include <iostream> | ||||||
| #include <algorithm> | #include <algorithm> | ||||||
|  | @ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & | ||||||
| 
 | 
 | ||||||
|     case 0: /* child */ |     case 0: /* child */ | ||||||
|         try { |         try { | ||||||
|  |             restoreAffinity(); | ||||||
|             if (dup2(toPipe.readSide, STDIN_FILENO) == -1) |             if (dup2(toPipe.readSide, STDIN_FILENO) == -1) | ||||||
|                 throw SysError("dupping stdin"); |                 throw SysError("dupping stdin"); | ||||||
|             if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) |             if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) | ||||||
|  |  | ||||||
|  | @ -3,6 +3,7 @@ | ||||||
| #include "remote-store.hh" | #include "remote-store.hh" | ||||||
| #include "worker-protocol.hh" | #include "worker-protocol.hh" | ||||||
| #include "archive.hh" | #include "archive.hh" | ||||||
|  | #include "affinity.hh" | ||||||
| #include "globals.hh" | #include "globals.hh" | ||||||
| 
 | 
 | ||||||
| #include <sys/types.h> | #include <sys/types.h> | ||||||
|  | @ -15,7 +16,6 @@ | ||||||
| #include <unistd.h> | #include <unistd.h> | ||||||
| #include <cstring> | #include <cstring> | ||||||
| 
 | 
 | ||||||
| 
 |  | ||||||
| namespace nix { | namespace nix { | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | @ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace) | ||||||
|         if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) |         if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) | ||||||
|             throw Error("Nix daemon protocol version not supported"); |             throw Error("Nix daemon protocol version not supported"); | ||||||
|         writeInt(PROTOCOL_VERSION, to); |         writeInt(PROTOCOL_VERSION, to); | ||||||
|  | 
 | ||||||
|  |         if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { | ||||||
|  |             int cpu = lockToCurrentCPU(); | ||||||
|  |             if (cpu != -1) { | ||||||
|  |                 writeInt(1, to); | ||||||
|  |                 writeInt(cpu, to); | ||||||
|  |             } else | ||||||
|  |                 writeInt(0, to); | ||||||
|  |         } | ||||||
|  | 
 | ||||||
|         if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) |         if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) | ||||||
|             writeInt(reserveSpace, to); |             writeInt(reserveSpace, to); | ||||||
|  | 
 | ||||||
|         processStderr(); |         processStderr(); | ||||||
|     } |     } | ||||||
|     catch (Error & e) { |     catch (Error & e) { | ||||||
|  |  | ||||||
|  | @ -6,7 +6,7 @@ namespace nix { | ||||||
| #define WORKER_MAGIC_1 0x6e697863 | #define WORKER_MAGIC_1 0x6e697863 | ||||||
| #define WORKER_MAGIC_2 0x6478696f | #define WORKER_MAGIC_2 0x6478696f | ||||||
| 
 | 
 | ||||||
| #define PROTOCOL_VERSION 0x10d | #define PROTOCOL_VERSION 0x10e | ||||||
| #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) | #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) | ||||||
| #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) | #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -1,12 +1,12 @@ | ||||||
| pkglib_LTLIBRARIES = libutil.la | pkglib_LTLIBRARIES = libutil.la | ||||||
| 
 | 
 | ||||||
| libutil_la_SOURCES = util.cc hash.cc serialise.cc \ | libutil_la_SOURCES = util.cc hash.cc serialise.cc \ | ||||||
|   archive.cc xml-writer.cc |   archive.cc xml-writer.cc affinity.cc | ||||||
| 
 | 
 | ||||||
| libutil_la_LIBADD = ../boost/format/libformat.la | libutil_la_LIBADD = ../boost/format/libformat.la | ||||||
| 
 | 
 | ||||||
| pkginclude_HEADERS = util.hh hash.hh serialise.hh \ | pkginclude_HEADERS = util.hh hash.hh serialise.hh \ | ||||||
|   archive.hh xml-writer.hh types.hh |   archive.hh xml-writer.hh types.hh affinity.hh | ||||||
| 
 | 
 | ||||||
| if !HAVE_OPENSSL | if !HAVE_OPENSSL | ||||||
| libutil_la_SOURCES += \ | libutil_la_SOURCES += \ | ||||||
|  |  | ||||||
							
								
								
									
										54
									
								
								src/libutil/affinity.cc
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										54
									
								
								src/libutil/affinity.cc
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,54 @@ | ||||||
|  | #include "types.hh" | ||||||
|  | #include "util.hh" | ||||||
|  | #include "affinity.hh" | ||||||
|  | 
 | ||||||
|  | #if HAVE_SCHED_H | ||||||
|  | #include <sched.h> | ||||||
|  | #endif | ||||||
|  | 
 | ||||||
|  | namespace nix { | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | static bool didSaveAffinity = false; | ||||||
|  | static cpu_set_t savedAffinity; | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | void setAffinityTo(int cpu) | ||||||
|  | { | ||||||
|  | #if HAVE_SCHED_SETAFFINITY | ||||||
|  |     if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; | ||||||
|  |     didSaveAffinity = true; | ||||||
|  |     printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); | ||||||
|  |     cpu_set_t newAffinity; | ||||||
|  |     CPU_ZERO(&newAffinity); | ||||||
|  |     CPU_SET(cpu, &newAffinity); | ||||||
|  |     if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) | ||||||
|  |         printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); | ||||||
|  | #endif | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | int lockToCurrentCPU() | ||||||
|  | { | ||||||
|  | #if HAVE_SCHED_SETAFFINITY | ||||||
|  |     if (getEnv("NIX_AFFINITY_HACK", "1") == "1") { | ||||||
|  |         int cpu = sched_getcpu(); | ||||||
|  |         if (cpu != -1) setAffinityTo(cpu); | ||||||
|  |         return cpu; | ||||||
|  |     } | ||||||
|  | #endif | ||||||
|  |     return -1; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | void restoreAffinity() | ||||||
|  | { | ||||||
|  | #if HAVE_SCHED_SETAFFINITY | ||||||
|  |     if (!didSaveAffinity) return; | ||||||
|  |     if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) | ||||||
|  |         printMsg(lvlError, "failed to restore affinity %1%"); | ||||||
|  | #endif | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | } | ||||||
							
								
								
									
										9
									
								
								src/libutil/affinity.hh
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								src/libutil/affinity.hh
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,9 @@ | ||||||
|  | #pragma once | ||||||
|  | 
 | ||||||
|  | namespace nix { | ||||||
|  | 
 | ||||||
|  | void setAffinityTo(int cpu); | ||||||
|  | int lockToCurrentCPU(); | ||||||
|  | void restoreAffinity(); | ||||||
|  | 
 | ||||||
|  | } | ||||||
|  | @ -4,6 +4,7 @@ | ||||||
| #include "serialise.hh" | #include "serialise.hh" | ||||||
| #include "worker-protocol.hh" | #include "worker-protocol.hh" | ||||||
| #include "archive.hh" | #include "archive.hh" | ||||||
|  | #include "affinity.hh" | ||||||
| #include "globals.hh" | #include "globals.hh" | ||||||
| 
 | 
 | ||||||
| #include <cstring> | #include <cstring> | ||||||
|  | @ -671,6 +672,9 @@ static void processConnection(bool trusted) | ||||||
|     to.flush(); |     to.flush(); | ||||||
|     unsigned int clientVersion = readInt(from); |     unsigned int clientVersion = readInt(from); | ||||||
| 
 | 
 | ||||||
|  |     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) | ||||||
|  |         setAffinityTo(readInt(from)); | ||||||
|  | 
 | ||||||
|     bool reserveSpace = true; |     bool reserveSpace = true; | ||||||
|     if (GET_PROTOCOL_MINOR(clientVersion) >= 11) |     if (GET_PROTOCOL_MINOR(clientVersion) >= 11) | ||||||
|         reserveSpace = readInt(from) != 0; |         reserveSpace = readInt(from) != 0; | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue