1 /* 2 * Copyright (c) 2019 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 /* 30 * This header provides some gory details to implement the <os/atomic_private.h> 31 * interfaces. Nothing in this header should be called directly, no promise is 32 * made to keep this interface stable. 33 * 34 * Architecture overrides. 35 */ 36 37 #ifndef __OS_ATOMIC_PRIVATE_H__ 38 #error "Do not include <os/atomic_private_arch.h> directly, use <os/atomic_private.h>" 39 #endif 40 41 #ifndef __OS_ATOMIC_PRIVATE_ARCH_H__ 42 #define __OS_ATOMIC_PRIVATE_ARCH_H__ 43 44 #pragma mark - arm v7 45 46 #if defined(__arm__) 47 48 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 49 /* 50 * On armv7, we do provide fine grained dependency injection, so 51 * memory_order_dependency maps to relaxed as far as thread fences are concerned 52 */ 53 #undef _os_atomic_mo_dependency 54 #define _os_atomic_mo_dependency memory_order_relaxed 55 56 #undef os_atomic_make_dependency 57 #define os_atomic_make_dependency(v) ({ \ 58 os_atomic_dependency_t _dep; \ 59 __asm__ __volatile__("and %[_dep], %[_v], #0" \ 60 : [_dep] "=r" (_dep.__opaque_zero) \ 61 : [_v] "r" (v)); \ 62 os_compiler_barrier(acquire); \ 63 _dep; \ 64 }) 65 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 66 67 #define os_atomic_clear_exclusive() __builtin_arm_clrex() 68 69 #define os_atomic_load_exclusive(p, m) ({ \ 70 os_atomic_basetypeof(p) _r = __builtin_arm_ldrex(os_cast_to_nonatomic_pointer(p)); \ 71 _os_memory_fence_after_atomic(m); \ 72 _os_compiler_barrier_after_atomic(m); \ 73 _r; \ 74 }) 75 76 #define os_atomic_store_exclusive(p, v, m) ({ \ 77 _os_compiler_barrier_before_atomic(m); \ 78 _os_memory_fence_before_atomic(m); \ 79 !__builtin_arm_strex(v, os_cast_to_nonatomic_pointer(p)); \ 80 }) 81 82 /* 83 * armv7 override of os_atomic_rmw_loop 84 * documentation for os_atomic_rmw_loop is in <os/atomic_private.h> 85 */ 86 #undef os_atomic_rmw_loop 87 #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ 88 int _result = 0; uint32_t _err = 0; \ 89 __auto_type *_p = os_cast_to_nonatomic_pointer(p); \ 90 for (;;) { \ 91 ov = __builtin_arm_ldrex(_p); \ 92 __VA_ARGS__; \ 93 if (!_err) { \ 94 /* release barrier only done for the first loop iteration */ \ 95 _os_memory_fence_before_atomic(m); \ 96 } \ 97 _err = __builtin_arm_strex(nv, _p); \ 98 if (__builtin_expect(!_err, 1)) { \ 99 _os_memory_fence_after_atomic(m); \ 100 _result = 1; \ 101 break; \ 102 } \ 103 } \ 104 _os_compiler_barrier_after_atomic(m); \ 105 _result; \ 106 }) 107 108 /* 109 * armv7 override of os_atomic_rmw_loop_give_up 110 * documentation for os_atomic_rmw_loop_give_up is in <os/atomic_private.h> 111 */ 112 #undef os_atomic_rmw_loop_give_up 113 #define os_atomic_rmw_loop_give_up(...) \ 114 ({ os_atomic_clear_exclusive(); __VA_ARGS__; break; }) 115 116 #endif // __arm__ 117 118 #pragma mark - arm64 119 120 #if defined(__arm64__) 121 122 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 123 /* 124 * On arm64, we do provide fine grained dependency injection, so 125 * memory_order_dependency maps to relaxed as far as thread fences are concerned 126 */ 127 #undef _os_atomic_mo_dependency 128 #define _os_atomic_mo_dependency memory_order_relaxed 129 130 #undef os_atomic_make_dependency 131 #if __ARM64_ARCH_8_32__ 132 #define os_atomic_make_dependency(v) ({ \ 133 os_atomic_dependency_t _dep; \ 134 __asm__ __volatile__("and %w[_dep], %w[_v], wzr" \ 135 : [_dep] "=r" (_dep.__opaque_zero) \ 136 : [_v] "r" (v)); \ 137 os_compiler_barrier(acquire); \ 138 _dep; \ 139 }) 140 #else 141 #define os_atomic_make_dependency(v) ({ \ 142 os_atomic_dependency_t _dep; \ 143 __asm__ __volatile__("and %[_dep], %[_v], xzr" \ 144 : [_dep] "=r" (_dep.__opaque_zero) \ 145 : [_v] "r" (v)); \ 146 os_compiler_barrier(acquire); \ 147 _dep; \ 148 }) 149 #endif 150 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 151 152 #if defined(__ARM_ARCH_8_4__) 153 /* on armv8.4 16-byte aligned load/store pair is atomic */ 154 #undef os_atomic_load_is_plain 155 #define os_atomic_load_is_plain(p) (sizeof(*(p)) <= 16) 156 #endif 157 158 #define os_atomic_clear_exclusive() __builtin_arm_clrex() 159 160 #define os_atomic_load_exclusive(p, m) ({ \ 161 os_atomic_basetypeof(p) _r = _os_atomic_mo_has_acquire(_os_atomic_mo_##m##_smp) \ 162 ? __builtin_arm_ldaex(os_cast_to_nonatomic_pointer(p)) \ 163 : __builtin_arm_ldrex(os_cast_to_nonatomic_pointer(p)); \ 164 _os_compiler_barrier_after_atomic(m); \ 165 _r; \ 166 }) 167 168 #define os_atomic_store_exclusive(p, v, m) ({ \ 169 _os_compiler_barrier_before_atomic(m); \ 170 (_os_atomic_mo_has_release(_os_atomic_mo_##m##_smp) \ 171 ? !__builtin_arm_stlex(v, os_cast_to_nonatomic_pointer(p)) \ 172 : !__builtin_arm_strex(v, os_cast_to_nonatomic_pointer(p))); \ 173 }) 174 175 #if OS_ATOMIC_USE_LLSC 176 177 /* 178 * arm64 (without armv81 atomics) override of os_atomic_rmw_loop 179 * documentation for os_atomic_rmw_loop is in <os/atomic_private.h> 180 */ 181 #undef os_atomic_rmw_loop 182 #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ 183 int _result = 0; \ 184 __auto_type *_p = os_cast_to_nonatomic_pointer(p); \ 185 _os_compiler_barrier_before_atomic(m); \ 186 do { \ 187 if (_os_atomic_mo_has_acquire(_os_atomic_mo_##m##_smp)) { \ 188 ov = __builtin_arm_ldaex(_p); \ 189 } else { \ 190 ov = __builtin_arm_ldrex(_p); \ 191 } \ 192 __VA_ARGS__; \ 193 if (_os_atomic_mo_has_release(_os_atomic_mo_##m##_smp)) { \ 194 _result = !__builtin_arm_stlex(nv, _p); \ 195 } else { \ 196 _result = !__builtin_arm_strex(nv, _p); \ 197 } \ 198 } while (__builtin_expect(!_result, 0)); \ 199 _os_compiler_barrier_after_atomic(m); \ 200 _result; \ 201 }) 202 203 /* 204 * arm64 override of os_atomic_rmw_loop_give_up 205 * documentation for os_atomic_rmw_loop_give_up is in <os/atomic_private.h> 206 */ 207 #undef os_atomic_rmw_loop_give_up 208 #define os_atomic_rmw_loop_give_up(...) \ 209 ({ os_atomic_clear_exclusive(); __VA_ARGS__; break; }) 210 211 #endif // OS_ATOMIC_USE_LLSC 212 213 #endif // __arm64__ 214 215 #endif /* __OS_ATOMIC_PRIVATE_ARCH_H__ */ 216