Changes imported from Abseil "staging" branch:
- 28631b3dbc582cb88a637cc4c70886e38a4be0cf Refactoring to support production kernel Futex implementa... by Derek Mauro <dmauro@google.com> - 411e7bb779c32bbc02b5fa6f516087a02fcb0812 Update comments about leap smearing. by Abseil Team <absl-team@google.com> GitOrigin-RevId: 28631b3dbc582cb88a637cc4c70886e38a4be0cf Change-Id: I0506aa2705212cd466460cae60182b0c2c667972
This commit is contained in:
		
							parent
							
								
									778abb7c27
								
							
						
					
					
						commit
						9e94e488f5
					
				
					 4 changed files with 55 additions and 33 deletions
				
			
		|  | @ -39,6 +39,7 @@ | |||
| namespace absl { | ||||
| namespace synchronization_internal { | ||||
| 
 | ||||
| class Futex; | ||||
| class Waiter; | ||||
| 
 | ||||
| class KernelTimeout { | ||||
|  | @ -139,6 +140,7 @@ class KernelTimeout { | |||
|   } | ||||
| #endif | ||||
| 
 | ||||
|   friend class Futex; | ||||
|   friend class Waiter; | ||||
| }; | ||||
| 
 | ||||
|  |  | |||
|  | @ -39,10 +39,12 @@ | |||
| 
 | ||||
| #include <atomic> | ||||
| #include <cassert> | ||||
| #include <cstdint> | ||||
| 
 | ||||
| #include "absl/base/internal/malloc_extension.h" | ||||
| #include "absl/base/internal/raw_logging.h" | ||||
| #include "absl/base/internal/thread_identity.h" | ||||
| #include "absl/base/optimization.h" | ||||
| #include "absl/synchronization/internal/kernel_timeout.h" | ||||
| 
 | ||||
| namespace absl { | ||||
|  | @ -82,6 +84,42 @@ static void MaybeBecomeIdle() { | |||
| #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF | ||||
| #endif | ||||
| #endif | ||||
| class Futex { | ||||
|  public: | ||||
|   static int WaitUntil(std::atomic<int32_t> *v, int32_t val, | ||||
|                        KernelTimeout t) { | ||||
|     int err = 0; | ||||
|     if (t.has_timeout()) { | ||||
|       // https://locklessinc.com/articles/futex_cheat_sheet/
 | ||||
|       // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time.
 | ||||
|       struct timespec abs_timeout = t.MakeAbsTimespec(); | ||||
|       // Atomically check that the futex value is still 0, and if it
 | ||||
|       // is, sleep until abs_timeout or until woken by FUTEX_WAKE.
 | ||||
|       err = syscall( | ||||
|           SYS_futex, reinterpret_cast<int32_t *>(v), | ||||
|           FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, | ||||
|           &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); | ||||
|     } else { | ||||
|       // Atomically check that the futex value is still 0, and if it
 | ||||
|       // is, sleep until woken by FUTEX_WAKE.
 | ||||
|       err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), | ||||
|                     FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr); | ||||
|     } | ||||
|     if (err != 0) { | ||||
|       err = -errno; | ||||
|     } | ||||
|     return err; | ||||
|   } | ||||
| 
 | ||||
|   static int Wake(std::atomic<int32_t> *v, int32_t count) { | ||||
|     int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), | ||||
|                       FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); | ||||
|     if (ABSL_PREDICT_FALSE(err < 0)) { | ||||
|       err = -errno; | ||||
|     } | ||||
|     return err; | ||||
|   } | ||||
| }; | ||||
| 
 | ||||
| void Waiter::Init() { | ||||
|   futex_.store(0, std::memory_order_relaxed); | ||||
|  | @ -91,7 +129,7 @@ bool Waiter::Wait(KernelTimeout t) { | |||
|   // Loop until we can atomically decrement futex from a positive
 | ||||
|   // value, waiting on a futex while we believe it is zero.
 | ||||
|   while (true) { | ||||
|     int x = futex_.load(std::memory_order_relaxed); | ||||
|     int32_t x = futex_.load(std::memory_order_relaxed); | ||||
|     if (x != 0) { | ||||
|       if (!futex_.compare_exchange_weak(x, x - 1, | ||||
|                                         std::memory_order_acquire, | ||||
|  | @ -101,30 +139,14 @@ bool Waiter::Wait(KernelTimeout t) { | |||
|       return true;  // Consumed a wakeup, we are done.
 | ||||
|     } | ||||
| 
 | ||||
|     int err = 0; | ||||
|     if (t.has_timeout()) { | ||||
|       // https://locklessinc.com/articles/futex_cheat_sheet/
 | ||||
|       // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time.
 | ||||
|       struct timespec abs_timeout = t.MakeAbsTimespec(); | ||||
|       // Atomically check that the futex value is still 0, and if it
 | ||||
|       // is, sleep until abs_timeout or until woken by FUTEX_WAKE.
 | ||||
|       err = syscall( | ||||
|           SYS_futex, reinterpret_cast<int *>(&futex_), | ||||
|           FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, 0, | ||||
|           &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); | ||||
|     } else { | ||||
|       // Atomically check that the futex value is still 0, and if it
 | ||||
|       // is, sleep until woken by FUTEX_WAKE.
 | ||||
|       err = syscall(SYS_futex, reinterpret_cast<int *>(&futex_), | ||||
|                     FUTEX_WAIT | FUTEX_PRIVATE_FLAG, 0, nullptr); | ||||
|     } | ||||
|     const int err = Futex::WaitUntil(&futex_, 0, t); | ||||
|     if (err != 0) { | ||||
|       if (errno == EINTR || errno == EWOULDBLOCK) { | ||||
|       if (err == -EINTR || err == -EWOULDBLOCK) { | ||||
|         // Do nothing, the loop will retry.
 | ||||
|       } else if (errno == ETIMEDOUT) { | ||||
|         return false;  // Timeout.
 | ||||
|       } else if (err == -ETIMEDOUT) { | ||||
|         return false; | ||||
|       } else { | ||||
|         ABSL_RAW_LOG(FATAL, "Futex operation failed with errno %d\n", errno); | ||||
|         ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); | ||||
|       } | ||||
|     } | ||||
| 
 | ||||
|  | @ -141,10 +163,9 @@ void Waiter::Post() { | |||
| 
 | ||||
| void Waiter::Poke() { | ||||
|   // Wake one thread waiting on the futex.
 | ||||
|   int err = syscall(SYS_futex, reinterpret_cast<int *>(&futex_), | ||||
|                     FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1); | ||||
|   if (err < 0) { | ||||
|     ABSL_RAW_LOG(FATAL, "FUTEX_WAKE failed with errno %d\n", errno); | ||||
|   const int err = Futex::Wake(&futex_, 1); | ||||
|   if (ABSL_PREDICT_FALSE(err < 0)) { | ||||
|     ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); | ||||
|   } | ||||
| } | ||||
| 
 | ||||
|  |  | |||
|  | @ -29,6 +29,7 @@ | |||
| #endif | ||||
| 
 | ||||
| #include <atomic> | ||||
| #include <cstdint> | ||||
| 
 | ||||
| #include "absl/base/internal/thread_identity.h" | ||||
| #include "absl/synchronization/internal/kernel_timeout.h" | ||||
|  | @ -99,10 +100,10 @@ class Waiter { | |||
| 
 | ||||
|  private: | ||||
| #if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX | ||||
|   // Futexes are defined by specification to be ints.
 | ||||
|   // Thus std::atomic<int> must be just an int with lockfree methods.
 | ||||
|   std::atomic<int> futex_; | ||||
|   static_assert(sizeof(int) == sizeof(futex_), "Wrong size for futex"); | ||||
|   // Futexes are defined by specification to be 32-bits.
 | ||||
|   // Thus std::atomic<int32_t> must be just an int32_t with lockfree methods.
 | ||||
|   std::atomic<int32_t> futex_; | ||||
|   static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex"); | ||||
| 
 | ||||
| #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR | ||||
|   pthread_mutex_t mu_; | ||||
|  |  | |||
|  | @ -506,9 +506,7 @@ std::string UnparseFlag(Duration d); | |||
| //
 | ||||
| // `absl::Time` assumes there are 60 seconds in a minute, which means the
 | ||||
| // underlying time scales must be "smeared" to eliminate leap seconds.
 | ||||
| // POSIX, for example, legislates that a `time_t` value of `536457599` shall
 | ||||
| // correspond to "1986-12-31 23:59:59 +0000".
 | ||||
| //
 | ||||
| // See https://developers.google.com/time/smear.
 | ||||
| //
 | ||||
| // Even though `absl::Time` supports a wide range of timestamps, exercise
 | ||||
| // caution when using values in the distant past. `absl::Time` uses the
 | ||||
|  |  | |||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue