| //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is distributed under the University of Illinois Open Source |
| // License. See LICENSE.TXT for details. |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file is a part of ThreadSanitizer/AddressSanitizer runtime. |
| // Not intended for direct inclusion. Include sanitizer_atomic.h. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef SANITIZER_ATOMIC_MSVC_H |
| #define SANITIZER_ATOMIC_MSVC_H |
| |
| extern "C" void _ReadWriteBarrier(); |
| #pragma intrinsic(_ReadWriteBarrier) |
| extern "C" void _mm_mfence(); |
| #pragma intrinsic(_mm_mfence) |
| extern "C" void _mm_pause(); |
| #pragma intrinsic(_mm_pause) |
| extern "C" long _InterlockedExchangeAdd( // NOLINT |
| long volatile * Addend, long Value); // NOLINT |
| #pragma intrinsic(_InterlockedExchangeAdd) |
| extern "C" short _InterlockedCompareExchange16( // NOLINT |
| short volatile *Destination, // NOLINT |
| short Exchange, short Comparand); // NOLINT |
| #pragma intrinsic(_InterlockedCompareExchange16) |
| extern "C" |
| long long _InterlockedCompareExchange64( // NOLINT |
| long long volatile *Destination, // NOLINT |
| long long Exchange, long long Comparand); // NOLINT |
| #pragma intrinsic(_InterlockedCompareExchange64) |
| extern "C" void *_InterlockedCompareExchangePointer( |
| void *volatile *Destination, |
| void *Exchange, void *Comparand); |
| #pragma intrinsic(_InterlockedCompareExchangePointer) |
| extern "C" |
| long __cdecl _InterlockedCompareExchange( // NOLINT |
| long volatile *Destination, // NOLINT |
| long Exchange, long Comparand); // NOLINT |
| #pragma intrinsic(_InterlockedCompareExchange) |
| |
| #ifdef _WIN64 |
| extern "C" long long _InterlockedExchangeAdd64( // NOLINT |
| long long volatile * Addend, long long Value); // NOLINT |
| #pragma intrinsic(_InterlockedExchangeAdd64) |
| #endif |
| |
| namespace __sanitizer { |
| |
| INLINE void atomic_signal_fence(memory_order) { |
| _ReadWriteBarrier(); |
| } |
| |
| INLINE void atomic_thread_fence(memory_order) { |
| _mm_mfence(); |
| } |
| |
| INLINE void proc_yield(int cnt) { |
| for (int i = 0; i < cnt; i++) |
| _mm_pause(); |
| } |
| |
| template<typename T> |
| INLINE typename T::Type atomic_load( |
| const volatile T *a, memory_order mo) { |
| DCHECK(mo & (memory_order_relaxed | memory_order_consume |
| | memory_order_acquire | memory_order_seq_cst)); |
| DCHECK(!((uptr)a % sizeof(*a))); |
| typename T::Type v; |
| // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. |
| if (mo == memory_order_relaxed) { |
| v = a->val_dont_use; |
| } else { |
| atomic_signal_fence(memory_order_seq_cst); |
| v = a->val_dont_use; |
| atomic_signal_fence(memory_order_seq_cst); |
| } |
| return v; |
| } |
| |
| template<typename T> |
| INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { |
| DCHECK(mo & (memory_order_relaxed | memory_order_release |
| | memory_order_seq_cst)); |
| DCHECK(!((uptr)a % sizeof(*a))); |
| // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. |
| if (mo == memory_order_relaxed) { |
| a->val_dont_use = v; |
| } else { |
| atomic_signal_fence(memory_order_seq_cst); |
| a->val_dont_use = v; |
| atomic_signal_fence(memory_order_seq_cst); |
| } |
| if (mo == memory_order_seq_cst) |
| atomic_thread_fence(memory_order_seq_cst); |
| } |
| |
| INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, |
| u32 v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| return (u32)_InterlockedExchangeAdd( |
| (volatile long*)&a->val_dont_use, (long)v); // NOLINT |
| } |
| |
| INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, |
| uptr v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| #ifdef _WIN64 |
| return (uptr)_InterlockedExchangeAdd64( |
| (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT |
| #else |
| return (uptr)_InterlockedExchangeAdd( |
| (volatile long*)&a->val_dont_use, (long)v); // NOLINT |
| #endif |
| } |
| |
| INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, |
| u32 v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| return (u32)_InterlockedExchangeAdd( |
| (volatile long*)&a->val_dont_use, -(long)v); // NOLINT |
| } |
| |
| INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, |
| uptr v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| #ifdef _WIN64 |
| return (uptr)_InterlockedExchangeAdd64( |
| (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT |
| #else |
| return (uptr)_InterlockedExchangeAdd( |
| (volatile long*)&a->val_dont_use, -(long)v); // NOLINT |
| #endif |
| } |
| |
| INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, |
| u8 v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| __asm { |
| mov eax, a |
| mov cl, v |
| xchg [eax], cl // NOLINT |
| mov v, cl |
| } |
| return v; |
| } |
| |
| INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, |
| u16 v, memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| __asm { |
| mov eax, a |
| mov cx, v |
| xchg [eax], cx // NOLINT |
| mov v, cx |
| } |
| return v; |
| } |
| |
| INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, |
| u8 *cmp, |
| u8 xchgv, |
| memory_order mo) { |
| (void)mo; |
| DCHECK(!((uptr)a % sizeof(*a))); |
| u8 cmpv = *cmp; |
| u8 prev; |
| __asm { |
| mov al, cmpv |
| mov ecx, a |
| mov dl, xchgv |
| lock cmpxchg [ecx], dl |
| mov prev, al |
| } |
| if (prev == cmpv) |
| return true; |
| *cmp = prev; |
| return false; |
| } |
| |
| INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, |
| uptr *cmp, |
| uptr xchg, |
| memory_order mo) { |
| uptr cmpv = *cmp; |
| uptr prev = (uptr)_InterlockedCompareExchangePointer( |
| (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); |
| if (prev == cmpv) |
| return true; |
| *cmp = prev; |
| return false; |
| } |
| |
| INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, |
| u16 *cmp, |
| u16 xchg, |
| memory_order mo) { |
| u16 cmpv = *cmp; |
| u16 prev = (u16)_InterlockedCompareExchange16( |
| (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); |
| if (prev == cmpv) |
| return true; |
| *cmp = prev; |
| return false; |
| } |
| |
| INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, |
| u32 *cmp, |
| u32 xchg, |
| memory_order mo) { |
| u32 cmpv = *cmp; |
| u32 prev = (u32)_InterlockedCompareExchange( |
| (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); |
| if (prev == cmpv) |
| return true; |
| *cmp = prev; |
| return false; |
| } |
| |
| INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, |
| u64 *cmp, |
| u64 xchg, |
| memory_order mo) { |
| u64 cmpv = *cmp; |
| u64 prev = (u64)_InterlockedCompareExchange64( |
| (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); |
| if (prev == cmpv) |
| return true; |
| *cmp = prev; |
| return false; |
| } |
| |
| template<typename T> |
| INLINE bool atomic_compare_exchange_weak(volatile T *a, |
| typename T::Type *cmp, |
| typename T::Type xchg, |
| memory_order mo) { |
| return atomic_compare_exchange_strong(a, cmp, xchg, mo); |
| } |
| |
| } // namespace __sanitizer |
| |
| #endif // SANITIZER_ATOMIC_CLANG_H |
