1 /*
2 * Copyright (c) 2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * This header provides some gory details to implement the <os/atomic_private.h>
31 * interfaces. Nothing in this header should be called directly, no promise is
32 * made to keep this interface stable.
33 */
34
35 #ifndef __OS_ATOMIC_PRIVATE_H__
36 #error "Do not include <os/atomic_private_impl.h> directly, use <os/atomic_private.h>"
37 #endif
38
39 #ifndef __OS_ATOMIC_PRIVATE_IMPL_H__
40 #define __OS_ATOMIC_PRIVATE_IMPL_H__
41
42 #pragma mark - implementation details
43
44 static inline int
_os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord)45 _os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord)
46 {
47 switch (ord) {
48 case os_atomic_std(memory_order_consume):
49 case os_atomic_std(memory_order_acquire):
50 case os_atomic_std(memory_order_acq_rel):
51 case os_atomic_std(memory_order_seq_cst):
52 return 1;
53 default:
54 return 0;
55 }
56 }
57
58 static inline int
_os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord)59 _os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord)
60 {
61 switch (ord) {
62 case os_atomic_std(memory_order_release):
63 case os_atomic_std(memory_order_acq_rel):
64 case os_atomic_std(memory_order_seq_cst):
65 return 1;
66 default:
67 return 0;
68 }
69 }
70
71 #define _os_atomic_mo_relaxed os_atomic_std(memory_order_relaxed)
72 #define _os_atomic_mo_compiler_acquire os_atomic_std(memory_order_relaxed)
73 #define _os_atomic_mo_compiler_release os_atomic_std(memory_order_relaxed)
74 #define _os_atomic_mo_compiler_acq_rel os_atomic_std(memory_order_relaxed)
75 #define _os_atomic_mo_consume os_atomic_std(memory_order_consume)
76 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
77 #define _os_atomic_mo_dependency os_atomic_std(memory_order_acquire)
78 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
79 #define _os_atomic_mo_acquire os_atomic_std(memory_order_acquire)
80 #define _os_atomic_mo_release os_atomic_std(memory_order_release)
81 #define _os_atomic_mo_acq_rel os_atomic_std(memory_order_acq_rel)
82 #define _os_atomic_mo_seq_cst os_atomic_std(memory_order_seq_cst)
83
84 /*
85 * Mapping between symbolic memory orderings and actual ones
86 * to take SMP into account.
87 */
88 #if OS_ATOMIC_CONFIG_SMP
89 #define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
90 #define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
91 #define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
92 #define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
93 #define _os_atomic_mo_consume_smp _os_atomic_mo_consume
94 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
95 #define _os_atomic_mo_dependency_smp _os_atomic_mo_dependency
96 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
97 #define _os_atomic_mo_acquire_smp _os_atomic_mo_acquire
98 #define _os_atomic_mo_release_smp _os_atomic_mo_release
99 #define _os_atomic_mo_acq_rel_smp _os_atomic_mo_acq_rel
100 #define _os_atomic_mo_seq_cst_smp _os_atomic_mo_seq_cst
101 #else
102 #define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
103 #define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
104 #define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
105 #define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
106 #define _os_atomic_mo_consume_smp _os_atomic_mo_relaxed
107 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
108 #define _os_atomic_mo_dependency_smp _os_atomic_mo_relaxed
109 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
110 #define _os_atomic_mo_acquire_smp _os_atomic_mo_relaxed
111 #define _os_atomic_mo_release_smp _os_atomic_mo_relaxed
112 #define _os_atomic_mo_acq_rel_smp _os_atomic_mo_relaxed
113 #define _os_atomic_mo_seq_cst_smp _os_atomic_mo_relaxed
114 #endif
115
116 #if KERNEL_PRIVATE
117 #define memory_order_relaxed_smp _os_atomic_mo_relaxed_smp
118 #define memory_order_compiler_acquire_smp _os_atomic_mo_compiler_acquire_smp
119 #define memory_order_compiler_release_smp _os_atomic_mo_compiler_release_smp
120 #define memory_order_compiler_acq_rel_smp _os_atomic_mo_compiler_acq_rel_smp
121 #define memory_order_consume_smp _os_atomic_mo_consume_smp
122 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
123 #define memory_order_dependency _os_atomic_mo_dependency
124 #define memory_order_dependency_smp _os_atomic_mo_dependency_smp
125 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
126 #define memory_order_acquire_smp _os_atomic_mo_acquire_smp
127 #define memory_order_release_smp _os_atomic_mo_release_smp
128 #define memory_order_acq_rel_smp _os_atomic_mo_acq_rel_smp
129 #define memory_order_seq_cst_smp _os_atomic_mo_seq_cst_smp
130 #endif
131
132 /*
133 * Hack needed for os_compiler_barrier() to work (including with empty argument)
134 */
135 #define _os_compiler_barrier_relaxed _os_atomic_mo_relaxed
136 #define _os_compiler_barrier_acquire _os_atomic_mo_acquire
137 #define _os_compiler_barrier_release _os_atomic_mo_release
138 #define _os_compiler_barrier_acq_rel _os_atomic_mo_acq_rel
139 #define _os_compiler_barrier_ _os_atomic_mo_acq_rel
140
141 /*
142 * Mapping between compiler barrier/memory orders and:
143 * - compiler barriers before atomics ("rel_barrier")
144 * - compiler barriers after atomics ("acq_barrier")
145 */
146 #define _os_rel_barrier_relaxed _os_atomic_mo_relaxed
147 #define _os_rel_barrier_compiler_acquire _os_atomic_mo_relaxed
148 #define _os_rel_barrier_compiler_release _os_atomic_mo_release
149 #define _os_rel_barrier_compiler_acq_rel _os_atomic_mo_release
150 #define _os_rel_barrier_consume _os_atomic_mo_relaxed
151 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
152 #define _os_rel_barrier_dependency _os_atomic_mo_relaxed
153 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
154 #define _os_rel_barrier_acquire _os_atomic_mo_relaxed
155 #define _os_rel_barrier_release _os_atomic_mo_release
156 #define _os_rel_barrier_acq_rel _os_atomic_mo_release
157 #define _os_rel_barrier_seq_cst _os_atomic_mo_release
158
159 #define _os_acq_barrier_relaxed _os_atomic_mo_relaxed
160 #define _os_acq_barrier_compiler_acquire _os_atomic_mo_acquire
161 #define _os_acq_barrier_compiler_release _os_atomic_mo_relaxed
162 #define _os_acq_barrier_compiler_acq_rel _os_atomic_mo_acquire
163 #define _os_acq_barrier_consume _os_atomic_mo_acquire
164 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
165 #define _os_acq_barrier_dependency _os_atomic_mo_acquire
166 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
167 #define _os_acq_barrier_acquire _os_atomic_mo_acquire
168 #define _os_acq_barrier_release _os_atomic_mo_relaxed
169 #define _os_acq_barrier_acq_rel _os_atomic_mo_acquire
170 #define _os_acq_barrier_seq_cst _os_atomic_mo_acquire
171
172 #define _os_compiler_barrier_before_atomic(m) \
173 os_atomic_std(atomic_signal_fence)(_os_rel_barrier_##m)
174 #define _os_compiler_barrier_after_atomic(m) \
175 os_atomic_std(atomic_signal_fence)(_os_acq_barrier_##m)
176
177 /*
178 * Mapping between compiler barrier/memmory orders and:
179 * - memory fences before atomics ("rel_fence")
180 * - memory fences after atomics ("acq_fence")
181 */
182 #define _os_rel_fence_relaxed _os_atomic_mo_relaxed
183 #define _os_rel_fence_compiler_acquire _os_atomic_mo_relaxed
184 #define _os_rel_fence_compiler_release _os_atomic_mo_release
185 #define _os_rel_fence_compiler_acq_rel _os_atomic_mo_release
186 #define _os_rel_fence_consume _os_atomic_mo_relaxed_smp
187 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
188 #define _os_rel_fence_dependency _os_atomic_mo_relaxed_smp
189 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
190 #define _os_rel_fence_acquire _os_atomic_mo_relaxed_smp
191 #define _os_rel_fence_release _os_atomic_mo_release_smp
192 #define _os_rel_fence_acq_rel _os_atomic_mo_release_smp
193 #define _os_rel_fence_seq_cst _os_atomic_mo_release_smp
194
195 #define _os_acq_fence_relaxed _os_atomic_mo_relaxed
196 #define _os_acq_fence_compiler_acquire _os_atomic_mo_relaxed
197 #define _os_acq_fence_compiler_release _os_atomic_mo_relaxed
198 #define _os_acq_fence_compiler_acq_rel _os_atomic_mo_relaxed
199 #define _os_acq_fence_consume _os_atomic_mo_acquire_smp
200 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
201 #define _os_acq_fence_dependency _os_atomic_mo_dependency_smp
202 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
203 #define _os_acq_fence_acquire _os_atomic_mo_acquire_smp
204 #define _os_acq_fence_release _os_atomic_mo_relaxed_smp
205 #define _os_acq_fence_acq_rel _os_atomic_mo_acquire_smp
206 #define _os_acq_fence_seq_cst _os_atomic_mo_acquire_smp
207
208 #define _os_memory_fence_before_atomic(m) \
209 os_atomic_std(atomic_thread_fence)(_os_rel_fence_##m)
210 #define _os_memory_fence_after_atomic(m) \
211 os_atomic_std(atomic_thread_fence)(_os_acq_fence_##m)
212
213 /*
214 * Misc. helpers
215 */
216
217 #define _os_atomic_value_cast(p, v) \
218 ({ __typeof__(*os_cast_to_nonatomic_pointer(p)) ___v = (v); ___v; })
219
220 #define _os_atomic_c11_op_orig(p, v, m, o) ({ \
221 _os_compiler_barrier_before_atomic(m); \
222 __auto_type _r = os_atomic_std(atomic_##o##_explicit)(\
223 os_cast_to_atomic_pointer(p), \
224 _os_atomic_value_cast(p, v), \
225 _os_atomic_mo_##m##_smp); \
226 _os_compiler_barrier_after_atomic(m); \
227 _r; \
228 })
229
230 #define _os_atomic_c11_op(p, v, m, o, op) ({ \
231 __auto_type _v = _os_atomic_value_cast(p, v); \
232 _os_atomic_c11_op_orig(p, _v, m, o) op _v; \
233 })
234
235 #define _os_atomic_clang_op_orig(p, v, m, o) ({ \
236 _os_compiler_barrier_before_atomic(m); \
237 __auto_type _r = __atomic_##o(os_cast_to_nonatomic_pointer(p), \
238 _os_atomic_value_cast(p, v), \
239 _os_atomic_mo_##m##_smp); \
240 _os_compiler_barrier_after_atomic(m); \
241 _r; \
242 })
243
244 #define _os_atomic_clang_op(p, v, m, o, op) ({ \
245 __auto_type _v = _os_atomic_value_cast(p, v); \
246 __auto_type _s = _os_atomic_clang_op_orig(p, _v, m, o); \
247 op(_s, _v); \
248 })
249
250 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
251 #define _os_atomic_auto_dependency(e) \
252 _Generic(e, \
253 os_atomic_dependency_t: (e), \
254 default: os_atomic_make_dependency(e))
255 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
256
257 #endif /* __OS_ATOMIC_PRIVATE_IMPL_H__ */
258