30 #ifndef _GLIBCXX_ATOMIC_WAIT_H
31 #define _GLIBCXX_ATOMIC_WAIT_H 1
33 #pragma GCC system_header
36 #if defined _GLIBCXX_HAS_GTHREADS || defined _GLIBCXX_HAVE_LINUX_FUTEX
38 #include <bits/gthr.h>
41 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
51 #define __cpp_lib_atomic_wait 201907L
53 namespace std _GLIBCXX_VISIBILITY(default)
55 _GLIBCXX_BEGIN_NAMESPACE_VERSION
58 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
59 #define _GLIBCXX_HAVE_PLATFORM_WAIT 1
60 using __platform_wait_t = int;
61 static constexpr
size_t __platform_wait_alignment = 4;
67 using __platform_wait_t = uint64_t;
68 static constexpr
size_t __platform_wait_alignment
69 = __alignof__(__platform_wait_t);
73 template<
typename _Tp>
74 inline constexpr
bool __platform_wait_uses_type
75 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
77 && ((
sizeof(_Tp) ==
sizeof(__detail::__platform_wait_t))
78 && (
alignof(_Tp*) >= __detail::__platform_wait_alignment));
85 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
86 enum class __futex_wait_flags : int
88 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE
97 __wait_private = __wait | __private_flag,
98 __wake_private = __wake | __private_flag,
99 __wait_bitset_private = __wait_bitset | __private_flag,
100 __wake_bitset_private = __wake_bitset | __private_flag,
101 __bitset_match_any = -1
104 template<
typename _Tp>
106 __platform_wait(
const _Tp* __addr, __platform_wait_t __val) noexcept
108 auto __e = syscall (SYS_futex,
static_cast<const void*
>(__addr),
109 static_cast<int>(__futex_wait_flags::__wait_private),
111 if (!__e || errno == EAGAIN)
114 __throw_system_error(errno);
117 template<
typename _Tp>
119 __platform_notify(
const _Tp* __addr,
bool __all) noexcept
121 syscall (SYS_futex,
static_cast<const void*
>(__addr),
122 static_cast<int>(__futex_wait_flags::__wake_private),
123 __all ? INT_MAX : 1);
128 __thread_yield() noexcept
130 #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
136 __thread_relax() noexcept
138 #if defined __i386__ || defined __x86_64__
139 __builtin_ia32_pause();
145 constexpr
auto __atomic_spin_count_relax = 12;
146 constexpr
auto __atomic_spin_count = 16;
148 struct __default_spin_policy
151 operator()() const noexcept
155 template<
typename _Pred,
156 typename _Spin = __default_spin_policy>
158 __atomic_spin(_Pred& __pred, _Spin __spin = _Spin{ }) noexcept
160 for (
auto __i = 0; __i < __atomic_spin_count; ++__i)
165 if (__i < __atomic_spin_count_relax)
166 __detail::__thread_relax();
168 __detail::__thread_yield();
181 template<
typename _Tp>
182 bool __atomic_compare(
const _Tp& __a,
const _Tp& __b)
185 return __builtin_memcmp(&__a, &__b,
sizeof(_Tp)) == 0;
188 struct __waiter_pool_base
190 #ifdef __cpp_lib_hardware_interference_size
191 static constexpr
auto _S_align = hardware_destructive_interference_size;
193 static constexpr
auto _S_align = 64;
196 alignas(_S_align) __platform_wait_t _M_wait = 0;
198 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
202 alignas(_S_align) __platform_wait_t _M_ver = 0;
204 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
207 __waiter_pool_base() =
default;
210 _M_enter_wait() noexcept
211 { __atomic_fetch_add(&_M_wait, 1, __ATOMIC_SEQ_CST); }
214 _M_leave_wait() noexcept
215 { __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_RELEASE); }
218 _M_waiting() const noexcept
220 __platform_wait_t __res;
221 __atomic_load(&_M_wait, &__res, __ATOMIC_SEQ_CST);
226 _M_notify(
const __platform_wait_t* __addr,
bool __all,
bool __bare) noexcept
228 if (!(__bare || _M_waiting()))
231 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
232 __platform_notify(__addr, __all);
241 static __waiter_pool_base&
242 _S_for(
const void* __addr) noexcept
244 constexpr uintptr_t __ct = 16;
245 static __waiter_pool_base __w[__ct];
246 auto __key = (uintptr_t(__addr) >> 2) % __ct;
251 struct __waiter_pool : __waiter_pool_base
254 _M_do_wait(
const __platform_wait_t* __addr, __platform_wait_t __old) noexcept
256 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
257 __platform_wait(__addr, __old);
259 __platform_wait_t __val;
260 __atomic_load(__addr, &__val, __ATOMIC_SEQ_CST);
263 lock_guard<mutex> __l(_M_mtx);
270 template<
typename _Tp>
273 using __waiter_type = _Tp;
276 __platform_wait_t* _M_addr;
278 template<
typename _Up>
279 static __platform_wait_t*
280 _S_wait_addr(
const _Up* __a, __platform_wait_t* __b)
282 if constexpr (__platform_wait_uses_type<_Up>)
283 return reinterpret_cast<__platform_wait_t*
>(
const_cast<_Up*
>(__a));
288 static __waiter_type&
289 _S_for(
const void* __addr) noexcept
291 static_assert(
sizeof(__waiter_type) ==
sizeof(__waiter_pool_base));
292 auto& res = __waiter_pool_base::_S_for(__addr);
293 return reinterpret_cast<__waiter_type&
>(res);
296 template<
typename _Up>
297 explicit __waiter_base(
const _Up* __addr) noexcept
298 : _M_w(_S_for(__addr))
299 , _M_addr(_S_wait_addr(__addr, &_M_w._M_ver))
304 {
return _M_addr == &_M_w._M_ver; }
307 _M_notify(
bool __all,
bool __bare =
false)
311 __atomic_fetch_add(_M_addr, 1, __ATOMIC_SEQ_CST);
314 _M_w._M_notify(_M_addr, __all, __bare);
317 template<
typename _Up,
typename _ValFn,
318 typename _Spin = __default_spin_policy>
320 _S_do_spin_v(__platform_wait_t* __addr,
321 const _Up& __old, _ValFn __vfn,
322 __platform_wait_t& __val,
323 _Spin __spin = _Spin{ })
325 auto const __pred = [=]
326 {
return !__detail::__atomic_compare(__old, __vfn()); };
328 if constexpr (__platform_wait_uses_type<_Up>)
330 __builtin_memcpy(&__val, &__old,
sizeof(__val));
334 __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE);
336 return __atomic_spin(__pred, __spin);
339 template<
typename _Up,
typename _ValFn,
340 typename _Spin = __default_spin_policy>
342 _M_do_spin_v(
const _Up& __old, _ValFn __vfn,
343 __platform_wait_t& __val,
344 _Spin __spin = _Spin{ })
345 {
return _S_do_spin_v(_M_addr, __old, __vfn, __val, __spin); }
347 template<
typename _Pred,
348 typename _Spin = __default_spin_policy>
350 _S_do_spin(
const __platform_wait_t* __addr,
352 __platform_wait_t& __val,
353 _Spin __spin = _Spin{ })
355 __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE);
356 return __atomic_spin(__pred, __spin);
359 template<
typename _Pred,
360 typename _Spin = __default_spin_policy>
362 _M_do_spin(_Pred __pred, __platform_wait_t& __val,
363 _Spin __spin = _Spin{ })
364 {
return _S_do_spin(_M_addr, __pred, __val, __spin); }
367 template<
typename _EntersWait>
368 struct __waiter : __waiter_base<__waiter_pool>
370 using __base_type = __waiter_base<__waiter_pool>;
372 template<
typename _Tp>
373 explicit __waiter(
const _Tp* __addr) noexcept
374 : __base_type(__addr)
376 if constexpr (_EntersWait::value)
377 _M_w._M_enter_wait();
382 if constexpr (_EntersWait::value)
383 _M_w._M_leave_wait();
386 template<
typename _Tp,
typename _ValFn>
388 _M_do_wait_v(_Tp __old, _ValFn __vfn)
392 __platform_wait_t __val;
393 if (__base_type::_M_do_spin_v(__old, __vfn, __val))
395 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
397 while (__detail::__atomic_compare(__old, __vfn()));
400 template<
typename _Pred>
402 _M_do_wait(_Pred __pred) noexcept
406 __platform_wait_t __val;
407 if (__base_type::_M_do_spin(__pred, __val))
409 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
415 using __enters_wait = __waiter<std::true_type>;
416 using __bare_wait = __waiter<std::false_type>;
419 template<
typename _Tp,
typename _ValFn>
421 __atomic_wait_address_v(
const _Tp* __addr, _Tp __old,
422 _ValFn __vfn) noexcept
424 __detail::__enters_wait __w(__addr);
425 __w._M_do_wait_v(__old, __vfn);
428 template<
typename _Tp,
typename _Pred>
430 __atomic_wait_address(
const _Tp* __addr, _Pred __pred) noexcept
432 __detail::__enters_wait __w(__addr);
433 __w._M_do_wait(__pred);
437 template<
typename _Pred>
439 __atomic_wait_address_bare(
const __detail::__platform_wait_t* __addr,
440 _Pred __pred) noexcept
442 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
445 __detail::__platform_wait_t __val;
446 if (__detail::__bare_wait::_S_do_spin(__addr, __pred, __val))
448 __detail::__platform_wait(__addr, __val);
452 __detail::__bare_wait __w(__addr);
453 __w._M_do_wait(__pred);
457 template<
typename _Tp>
459 __atomic_notify_address(
const _Tp* __addr,
bool __all) noexcept
461 __detail::__bare_wait __w(__addr);
462 __w._M_notify(__all);
467 __atomic_notify_address_bare(
const __detail::__platform_wait_t* __addr,
470 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
471 __detail::__platform_notify(__addr, __all);
473 __detail::__bare_wait __w(__addr);
474 __w._M_notify(__all,
true);
477 _GLIBCXX_END_NAMESPACE_VERSION
ISO C++ entities toplevel namespace is std.