Halide 16.0.0
Halide compiler and libraries
runtime_atomics.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
3
4// This file provides an abstraction layer over the __sync/__atomic builtins
5// in Clang; for various reasons, we use __sync for 32-bit targets, and
6// __atomic for 64-bit. At some point it may be desirable/necessary to
7// migrate 32-bit to __atomic as well, at which time this file can
8// likely go away. See https://github.com/halide/Halide/issues/7431 for
9// a discussion of the history and issues as to why we work this way.
10
11#include "HalideRuntime.h"
12
13namespace Halide {
14namespace Runtime {
15namespace Internal {
16namespace Synchronization {
17
18namespace {
19
20// TODO: most of these wrappers should do the remove_volatile for secondary arguments;
21// I've only put it in place for the locations necessary at this time.
22template<class T>
23struct remove_volatile { typedef T type; };
24template<class T>
25struct remove_volatile<volatile T> { typedef T type; };
26
27#ifdef BITS_32
28ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
29 return __sync_and_and_fetch(addr, val);
30}
31
32template<typename T>
33ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
34 return __sync_fetch_and_add(addr, val);
35}
36
37template<typename T, typename TV = typename remove_volatile<T>::type>
38ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
39 return __sync_fetch_and_add(addr, val);
40}
41
42template<typename T, typename TV = typename remove_volatile<T>::type>
43ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
44 return __sync_fetch_and_sub(addr, val);
45}
46
47template<typename T, typename TV = typename remove_volatile<T>::type>
48ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
49 return __sync_fetch_and_or(addr, val);
50}
51
52template<typename T>
53ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
54 return __sync_add_and_fetch(addr, val);
55}
56
57template<typename T>
58ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
59 return __sync_sub_and_fetch(addr, val);
60}
61
62template<typename T, typename TV = typename remove_volatile<T>::type>
63ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
64 TV oldval = *expected;
65 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
66 *expected = gotval;
67 return oldval == gotval;
68}
69
70ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
71 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
72}
73
74template<typename T, typename TV = typename remove_volatile<T>::type>
75ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
76 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
77}
78
79ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
81}
82
83template<typename T>
84ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
85 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
86}
87
88ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
90}
91
92ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
94}
95
96template<typename T>
97ALWAYS_INLINE T atomic_fetch_and_release(T *addr, T val) {
98 return __sync_fetch_and_and(addr, val);
99}
100
101template<typename T, typename TV = typename remove_volatile<T>::type>
102ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
103 return __sync_fetch_and_and(addr, val);
104}
105
106template<typename T>
107ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
108 *val = *addr;
109}
110
111template<typename T>
112ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
113 __sync_synchronize();
114 *val = *addr;
115}
116
117template<typename T>
118ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
119 // Despite the name, this is really just an exchange operation with acquire ordering.
120 return __sync_lock_test_and_set(addr, val);
121}
122
123ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
124 return __sync_or_and_fetch(addr, val);
125}
126
127ALWAYS_INLINE void atomic_store_relaxed(uintptr_t *addr, uintptr_t *val) {
128 *addr = *val;
129}
130
131template<typename T>
132ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
133 *addr = *val;
134 __sync_synchronize();
135}
136
137template<typename T, typename TV = typename remove_volatile<T>::type>
138ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
139 *addr = *val;
140 __sync_synchronize();
141}
142
143ALWAYS_INLINE void atomic_thread_fence_acquire() {
144 __sync_synchronize();
145}
146
147ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
148 __sync_synchronize();
149}
150
151#else
152
153ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
154 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
155}
156
157template<typename T>
158ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
159 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
160}
161
162template<typename T, typename TV = typename remove_volatile<T>::type>
163ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
164 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
165}
166
167template<typename T, typename TV = typename remove_volatile<T>::type>
168ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
169 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
170}
171
172template<typename T, typename TV = typename remove_volatile<T>::type>
173ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
174 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
175}
176
177template<typename T>
178ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
179 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
180}
181
182template<typename T>
183ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
184 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
185}
186
187ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
188 return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
189}
190
191template<typename T, typename TV = typename remove_volatile<T>::type>
192ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
193 return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
194}
195
196template<typename T>
197ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
198 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
199}
200
201ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
202 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
203}
204
205ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
206 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
207}
208
209ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
210 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
211}
212
213template<typename T>
214ALWAYS_INLINE uintptr_t atomic_fetch_and_release(T *addr, T val) {
215 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
216}
217
218template<typename T, typename TV = typename remove_volatile<T>::type>
219ALWAYS_INLINE uintptr_t atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
220 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
221}
222
223template<typename T>
224ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
225 __atomic_load(addr, val, __ATOMIC_RELAXED);
226}
227
228template<typename T>
229ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
230 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
231 __sync_synchronize();
232 *val = *addr;
233}
234
235template<typename T>
236ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
237 T result;
238 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
239 return result;
240}
241
242ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
243 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
244}
245
246ALWAYS_INLINE void atomic_store_relaxed(uintptr_t *addr, uintptr_t *val) {
247 __atomic_store(addr, val, __ATOMIC_RELAXED);
248}
249
250template<typename T>
251ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
252 __atomic_store(addr, val, __ATOMIC_RELEASE);
253}
254
255template<typename T, typename TV = typename remove_volatile<T>::type>
256ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
257 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
258}
259
260ALWAYS_INLINE void atomic_thread_fence_acquire() {
261 __atomic_thread_fence(__ATOMIC_ACQUIRE);
262}
263
264ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
265 __atomic_thread_fence(__ATOMIC_SEQ_CST);
266}
267
268#endif
269
270} // namespace
271
272} // namespace Synchronization
273} // namespace Internal
274} // namespace Runtime
275} // namespace Halide
276
277#endif // HALIDE_RUNTIME_RUNTIME_ATOMICS_H
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t
#define ALWAYS_INLINE