40#ifndef KOKKOS_ATOMIC_HPP
41#define KOKKOS_ATOMIC_HPP
42#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
43#define KOKKOS_IMPL_PUBLIC_INCLUDE
44#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC
47#include <Kokkos_Macros.hpp>
49#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
50#include <Kokkos_Atomics_Desul_Wrapper.hpp>
51#include <Kokkos_Atomics_Desul_Volatile_Wrapper.hpp>
52#include <impl/Kokkos_Utilities.hpp>
59using desul::MemoryOrderSeqCst;
60using desul::MemoryScopeDevice;
63KOKKOS_INLINE_FUNCTION
void desul_atomic_dec(T* dest, MemoryOrderSeqCst,
65 return desul::atomic_dec(
const_cast<T*
>(dest), desul::MemoryOrderSeqCst(),
66 desul::MemoryScopeDevice());
70KOKKOS_INLINE_FUNCTION
void desul_atomic_inc(T* dest, MemoryOrderSeqCst,
72 return desul::atomic_inc(
const_cast<T*
>(dest), desul::MemoryOrderSeqCst(),
73 desul::MemoryScopeDevice());
77KOKKOS_INLINE_FUNCTION T
78desul_atomic_exchange(T* dest,
const Kokkos::Impl::type_identity_t<T> val,
79 MemoryOrderSeqCst, MemoryScopeDevice) {
80 return desul::atomic_exchange(
const_cast<T*
>(dest), val,
81 desul::MemoryOrderSeqCst(),
82 desul::MemoryScopeDevice());
86KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange(
87 T* dest, Kokkos::Impl::type_identity_t<const T> compare,
88 Kokkos::Impl::type_identity_t<const T> val, MemoryOrderSeqCst,
90 return desul::atomic_compare_exchange(dest, compare, val,
91 desul::MemoryOrderSeqCst(),
92 desul::MemoryScopeDevice());
99#include <Kokkos_HostSpace.hpp>
100#include <impl/Kokkos_Traits.hpp>
106#define KOKKOS_ENABLE_WINDOWS_ATOMICS
108#if defined(KOKKOS_ENABLE_CUDA)
109#define KOKKOS_ENABLE_CUDA_ATOMICS
110#if defined(KOKKOS_COMPILER_CLANG)
111#define KOKKOS_ENABLE_GNU_ATOMICS
116#if defined(KOKKOS_ENABLE_CUDA)
120#define KOKKOS_ENABLE_CUDA_ATOMICS
122#elif defined(KOKKOS_ENABLE_HIP)
124#define KOKKOS_ENABLE_HIP_ATOMICS
128#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \
129 !defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \
130 !defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \
131 !defined(KOKKOS_ENABLE_STD_ATOMICS) && \
132 !defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
138#if defined(KOKKOS_INTERNAL_NOT_PARALLEL)
140#define KOKKOS_ENABLE_SERIAL_ATOMICS
142#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \
143 defined(KOKKOS_COMPILER_NVCC)
145#define KOKKOS_ENABLE_GNU_ATOMICS
147#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC)
149#define KOKKOS_ENABLE_INTEL_ATOMICS
151#elif defined(_OPENMP) && (201107 <= _OPENMP)
153#define KOKKOS_ENABLE_OPENMP_ATOMICS
157#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
164#ifdef KOKKOS_ENABLE_CUDA
165#include <Cuda/Kokkos_Cuda_Locks.hpp>
170KOKKOS_INLINE_FUNCTION
void atomic_add(
volatile T*
const dest,
const T src);
174KOKKOS_INLINE_FUNCTION
void atomic_increment(
volatile T* a);
177KOKKOS_INLINE_FUNCTION
void atomic_decrement(
volatile T* a);
182inline const char* atomic_query_version() {
183#if defined(KOKKOS_ENABLE_CUDA_ATOMICS)
184 return "KOKKOS_ENABLE_CUDA_ATOMICS";
185#elif defined(KOKKOS_ENABLE_GNU_ATOMICS)
186 return "KOKKOS_ENABLE_GNU_ATOMICS";
187#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS)
188 return "KOKKOS_ENABLE_INTEL_ATOMICS";
189#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS)
190 return "KOKKOS_ENABLE_OPENMP_ATOMICS";
191#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
192 return "KOKKOS_ENABLE_WINDOWS_ATOMICS";
193#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
194 return "KOKKOS_ENABLE_SERIAL_ATOMICS";
196#error "No valid response for atomic_query_version!"
206#include "impl/Kokkos_Atomic_Memory_Order.hpp"
208#if defined(KOKKOS_ENABLE_HIP)
209#include <HIP/Kokkos_HIP_Atomic.hpp>
212#if defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
213#include "impl/Kokkos_Atomic_Windows.hpp"
220#include "impl/Kokkos_Atomic_Assembly.hpp"
229#include "impl/Kokkos_Memory_Fence.hpp"
238#include "impl/Kokkos_Atomic_Exchange.hpp"
248#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
250#include "impl/Kokkos_Atomic_Generic.hpp"
259#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
268#include "impl/Kokkos_Atomic_Increment.hpp"
277#include "impl/Kokkos_Atomic_Decrement.hpp"
286#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
295#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
304#include "impl/Kokkos_Atomic_Fetch_And.hpp"
316#include "impl/Kokkos_Atomic_MinMax.hpp"
327#include "impl/Kokkos_Volatile_Load.hpp"
332#include "impl/Kokkos_Atomic_Load.hpp"
333#include "impl/Kokkos_Atomic_Store.hpp"
336#include "impl/Kokkos_Atomic_Generic_Secondary.hpp"
340#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
341 !defined(__CUDA_ARCH__)
343#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
344#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
348#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
349#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
359struct MemoryOrderSeqCst {};
360struct MemoryScopeDevice {};
363KOKKOS_INLINE_FUNCTION
void desul_atomic_dec(T* dest, MemoryOrderSeqCst,
365 return Kokkos::atomic_decrement(dest);
369KOKKOS_INLINE_FUNCTION
void desul_atomic_inc(T* dest, MemoryOrderSeqCst,
371 return Kokkos::atomic_increment(dest);
375KOKKOS_INLINE_FUNCTION T
376desul_atomic_exchange(T* dest, Kokkos::Impl::type_identity_t<const T> val,
377 MemoryOrderSeqCst, MemoryScopeDevice) {
378 return Kokkos::atomic_exchange(dest, val);
382KOKKOS_INLINE_FUNCTION T desul_atomic_compare_exchange(
383 T* dest, Kokkos::Impl::type_identity_t<const T> compare,
384 Kokkos::Impl::type_identity_t<const T> val, MemoryOrderSeqCst,
386 return Kokkos::atomic_compare_exchange(dest, compare, val);
393#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC
394#undef KOKKOS_IMPL_PUBLIC_INCLUDE
395#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC