Improve/add atomics
- Remove use of non-builtin overloaded `InterlockedExhangeSubtract()` for Windows. Moreover, it was wrongly being used as if it hadn't a 64-bit version instead of being overloaded. - Implement exchange-if-greater (as a CAS loop).
This commit is contained in:
parent
e15a0c5243
commit
27637b9e7f
2 changed files with 50 additions and 8 deletions
|
@ -78,6 +78,15 @@ static _ALWAYS_INLINE_ T _atomic_add_impl(register T *pw, register T val) {
|
||||||
return *pw;
|
return *pw;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <class T>
|
||||||
|
static _ALWAYS_INLINE_ T _atomic_exchange_if_greater_impl(register T *pw, register T val) {
|
||||||
|
|
||||||
|
if (val > *pw)
|
||||||
|
*pw = val;
|
||||||
|
|
||||||
|
return *pw;
|
||||||
|
}
|
||||||
|
|
||||||
#elif defined(__GNUC__)
|
#elif defined(__GNUC__)
|
||||||
|
|
||||||
/* Implementation for GCC & Clang */
|
/* Implementation for GCC & Clang */
|
||||||
|
@ -121,6 +130,18 @@ static _ALWAYS_INLINE_ T _atomic_add_impl(register T *pw, register T val) {
|
||||||
return __sync_add_and_fetch(pw, val);
|
return __sync_add_and_fetch(pw, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <class T>
|
||||||
|
static _ALWAYS_INLINE_ T _atomic_exchange_if_greater_impl(register T *pw, register T val) {
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
T tmp = static_cast<T const volatile &>(*pw);
|
||||||
|
if (tmp >= val)
|
||||||
|
return tmp; // already greater, or equal
|
||||||
|
if (__sync_val_compare_and_swap(pw, tmp, val) == tmp)
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#elif defined(_MSC_VER)
|
#elif defined(_MSC_VER)
|
||||||
|
|
||||||
/* Implementation for MSVC-Windows */
|
/* Implementation for MSVC-Windows */
|
||||||
|
@ -139,6 +160,15 @@ static _ALWAYS_INLINE_ T _atomic_add_impl(register T *pw, register T val) {
|
||||||
return tmp + 1; \
|
return tmp + 1; \
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define ATOMIC_EXCHANGE_IF_GREATER_BODY(m_pw, m_val, m_win_type, m_win_cmpxchg, m_cpp_type) \
|
||||||
|
while (true) { \
|
||||||
|
m_cpp_type tmp = static_cast<m_cpp_type const volatile &>(*(m_pw)); \
|
||||||
|
if (tmp >= m_val) \
|
||||||
|
return tmp; /* already greater, or equal */ \
|
||||||
|
if (m_win_cmpxchg((m_win_type volatile *)(m_pw), m_val, tmp) == tmp) \
|
||||||
|
return m_val; \
|
||||||
|
}
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint32_t _atomic_conditional_increment_impl(register uint32_t *pw) {
|
static _ALWAYS_INLINE_ uint32_t _atomic_conditional_increment_impl(register uint32_t *pw) {
|
||||||
|
|
||||||
ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONG, InterlockedCompareExchange, uint32_t)
|
ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONG, InterlockedCompareExchange, uint32_t)
|
||||||
|
@ -156,11 +186,7 @@ static _ALWAYS_INLINE_ uint32_t _atomic_increment_impl(register uint32_t *pw) {
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint32_t _atomic_sub_impl(register uint32_t *pw, register uint32_t val) {
|
static _ALWAYS_INLINE_ uint32_t _atomic_sub_impl(register uint32_t *pw, register uint32_t val) {
|
||||||
|
|
||||||
#if _WIN32_WINNT >= 0x0601 // Windows 7+
|
|
||||||
return InterlockedExchangeSubtract(pw, val) - val;
|
|
||||||
#else
|
|
||||||
return InterlockedExchangeAdd((LONG volatile *)pw, -(int32_t)val) - val;
|
return InterlockedExchangeAdd((LONG volatile *)pw, -(int32_t)val) - val;
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint32_t _atomic_add_impl(register uint32_t *pw, register uint32_t val) {
|
static _ALWAYS_INLINE_ uint32_t _atomic_add_impl(register uint32_t *pw, register uint32_t val) {
|
||||||
|
@ -168,6 +194,11 @@ static _ALWAYS_INLINE_ uint32_t _atomic_add_impl(register uint32_t *pw, register
|
||||||
return InterlockedAdd((LONG volatile *)pw, val);
|
return InterlockedAdd((LONG volatile *)pw, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static _ALWAYS_INLINE_ uint32_t _atomic_exchange_if_greater_impl(register uint32_t *pw, register uint32_t val) {
|
||||||
|
|
||||||
|
ATOMIC_EXCHANGE_IF_GREATER_BODY(pw, val, LONG, InterlockedCompareExchange, uint32_t)
|
||||||
|
}
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint64_t _atomic_conditional_increment_impl(register uint64_t *pw) {
|
static _ALWAYS_INLINE_ uint64_t _atomic_conditional_increment_impl(register uint64_t *pw) {
|
||||||
|
|
||||||
ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONGLONG, InterlockedCompareExchange64, uint64_t)
|
ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONGLONG, InterlockedCompareExchange64, uint64_t)
|
||||||
|
@ -185,11 +216,7 @@ static _ALWAYS_INLINE_ uint64_t _atomic_increment_impl(register uint64_t *pw) {
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint64_t _atomic_sub_impl(register uint64_t *pw, register uint64_t val) {
|
static _ALWAYS_INLINE_ uint64_t _atomic_sub_impl(register uint64_t *pw, register uint64_t val) {
|
||||||
|
|
||||||
#if _WIN32_WINNT >= 0x0601 && !defined(UWP_ENABLED) // Windows 7+ except UWP
|
|
||||||
return InterlockedExchangeSubtract64(pw, val) - val;
|
|
||||||
#else
|
|
||||||
return InterlockedExchangeAdd64((LONGLONG volatile *)pw, -(int64_t)val) - val;
|
return InterlockedExchangeAdd64((LONGLONG volatile *)pw, -(int64_t)val) - val;
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static _ALWAYS_INLINE_ uint64_t _atomic_add_impl(register uint64_t *pw, register uint64_t val) {
|
static _ALWAYS_INLINE_ uint64_t _atomic_add_impl(register uint64_t *pw, register uint64_t val) {
|
||||||
|
@ -197,6 +224,11 @@ static _ALWAYS_INLINE_ uint64_t _atomic_add_impl(register uint64_t *pw, register
|
||||||
return InterlockedAdd64((LONGLONG volatile *)pw, val);
|
return InterlockedAdd64((LONGLONG volatile *)pw, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static _ALWAYS_INLINE_ uint64_t _atomic_exchange_if_greater_impl(register uint64_t *pw, register uint64_t val) {
|
||||||
|
|
||||||
|
ATOMIC_EXCHANGE_IF_GREATER_BODY(pw, val, LONGLONG, InterlockedCompareExchange64, uint64_t)
|
||||||
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
//no threads supported?
|
//no threads supported?
|
||||||
|
@ -226,6 +258,10 @@ uint32_t atomic_add(register uint32_t *pw, register uint32_t val) {
|
||||||
return _atomic_add_impl(pw, val);
|
return _atomic_add_impl(pw, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint32_t atomic_exchange_if_greater(register uint32_t *pw, register uint32_t val) {
|
||||||
|
return _atomic_exchange_if_greater_impl(pw, val);
|
||||||
|
}
|
||||||
|
|
||||||
uint64_t atomic_conditional_increment(register uint64_t *counter) {
|
uint64_t atomic_conditional_increment(register uint64_t *counter) {
|
||||||
return _atomic_conditional_increment_impl(counter);
|
return _atomic_conditional_increment_impl(counter);
|
||||||
}
|
}
|
||||||
|
@ -245,3 +281,7 @@ uint64_t atomic_sub(register uint64_t *pw, register uint64_t val) {
|
||||||
uint64_t atomic_add(register uint64_t *pw, register uint64_t val) {
|
uint64_t atomic_add(register uint64_t *pw, register uint64_t val) {
|
||||||
return _atomic_add_impl(pw, val);
|
return _atomic_add_impl(pw, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint64_t atomic_exchange_if_greater(register uint64_t *pw, register uint64_t val) {
|
||||||
|
return _atomic_exchange_if_greater_impl(pw, val);
|
||||||
|
}
|
||||||
|
|
|
@ -41,12 +41,14 @@ uint32_t atomic_decrement(register uint32_t *pw);
|
||||||
uint32_t atomic_increment(register uint32_t *pw);
|
uint32_t atomic_increment(register uint32_t *pw);
|
||||||
uint32_t atomic_sub(register uint32_t *pw, register uint32_t val);
|
uint32_t atomic_sub(register uint32_t *pw, register uint32_t val);
|
||||||
uint32_t atomic_add(register uint32_t *pw, register uint32_t val);
|
uint32_t atomic_add(register uint32_t *pw, register uint32_t val);
|
||||||
|
uint32_t atomic_exchange_if_greater(register uint32_t *pw, register uint32_t val);
|
||||||
|
|
||||||
uint64_t atomic_conditional_increment(register uint64_t *counter);
|
uint64_t atomic_conditional_increment(register uint64_t *counter);
|
||||||
uint64_t atomic_decrement(register uint64_t *pw);
|
uint64_t atomic_decrement(register uint64_t *pw);
|
||||||
uint64_t atomic_increment(register uint64_t *pw);
|
uint64_t atomic_increment(register uint64_t *pw);
|
||||||
uint64_t atomic_sub(register uint64_t *pw, register uint64_t val);
|
uint64_t atomic_sub(register uint64_t *pw, register uint64_t val);
|
||||||
uint64_t atomic_add(register uint64_t *pw, register uint64_t val);
|
uint64_t atomic_add(register uint64_t *pw, register uint64_t val);
|
||||||
|
uint64_t atomic_exchange_if_greater(register uint64_t *pw, register uint64_t val);
|
||||||
|
|
||||||
struct SafeRefCount {
|
struct SafeRefCount {
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue