1 /* 2 * Copyright (c) 2018 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef __OS_ATOMIC_PRIVATE_H__ 30 #define __OS_ATOMIC_PRIVATE_H__ 31 32 /*! 33 * @file <os/atomic_private.h> 34 * 35 * @brief 36 * This file defines nicer (terser and safer) wrappers for C11's <stdatomic.h>. 37 * 38 * @discussion 39 * @see xnu.git::doc/atomics.md which provides more extensive documentation 40 * about this header. 41 * 42 * Note that some of the macros defined in this file may be overridden by 43 * architecture specific headers. 44 * 45 * All the os_atomic* functions take an operation ordering argument that can be: 46 * - C11 memory orders: relaxed, acquire, release, acq_rel or seq_cst which 47 * imply a memory fence on SMP machines, and always carry the matching 48 * compiler barrier semantics. 49 * 50 * - the os_atomic-specific `dependency` memory ordering that is used to 51 * document intent to a carry a data or address dependency. 52 * See doc/atomics.md for more information. 53 * 54 * - a compiler barrier: compiler_acquire, compiler_release, compiler_acq_rel 55 * without a corresponding memory fence. 56 */ 57 58 #include <os/atomic.h> 59 60 /*! 61 * @group <os/atomic_private.h> tunables. 62 * 63 * @{ 64 * 65 * @brief 66 * @c OS_ATOMIC_CONFIG_* macros provide tunables for clients. 67 */ 68 69 /*! 70 * @macro OS_ATOMIC_CONFIG_SMP 71 * 72 * @brief 73 * Whether this is used on an SMP system, defaults to 1. 74 */ 75 #ifndef OS_ATOMIC_CONFIG_SMP 76 #define OS_ATOMIC_CONFIG_SMP 1 77 #endif // OS_ATOMIC_CONFIG_SMP 78 79 /*! 80 * @macro OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 81 * 82 * @brief 83 * Hide interfaces that can lead to starvation on certain hardware/build 84 * configurations. 85 * 86 * @discussion 87 * The following ABIs are currently supported by os_atomic: 88 * - i386 and x86_64: Intel atomics 89 * - armv7: load/store exclusive 90 * - armv8: load/store exclusive 91 * - armv8.1: armv8.1 style atomics 92 * 93 * On armv8 hardware with asymmetric cores, using load/store exclusive based 94 * atomics can lead to starvation in very hot code or non-preemptible context, 95 * and code that is sensitive to such must not use these interfaces. 96 * 97 * When OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY is set, any os_atomic_* interface 98 * that may cause starvation will be made unavailable to avoid accidental use. 99 * 100 * Defaults: 101 * - XNU: builds per SoC, already safe 102 * - Kexts: default to avoid starvable interfaces by default 103 * - User: default to allow starvable interfaces by default 104 * 105 * Note: at this time, on Apple supported platforms, the only configuration 106 * that is affected by this would be for the "arm64" slices. 107 * 108 * Intel, armv7 variants, and the arm64e slice always are unaffected. 109 */ 110 #ifndef OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 111 #if XNU_KERNEL_PRIVATE 112 #define OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 0 113 #elif KERNEL 114 #define OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 1 115 #else 116 #define OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 0 117 #endif 118 #endif // OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 119 120 /*! 121 * @macro OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 122 * 123 * @brief 124 * Expose the os_atomic-specific fake `dependency` memory ordering. 125 * 126 * @discussion 127 * The dependency ordering can be used to try to "repair" C11's consume ordering 128 * and should be limited to extremely complex algorithms where every cycle counts. 129 * 130 * Due to the inherent risks (no compiler support) for this feature, it is 131 * reserved for expert and very domain-specific code only and is off by default. 132 * 133 * Default: 0 134 */ 135 #ifndef OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 136 #if XNU_KERNEL_PRIVATE 137 #define OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 1 138 #else 139 #define OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 0 140 #endif 141 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 142 143 /*! @} */ 144 145 /*! 146 * @group <os/atomic_private.h> features (arch specific). 147 * 148 * @{ 149 * 150 * @brief 151 * The @c OS_ATOMIC_USE_* and @c OS_ATOMIC_HAS_* defines expose some 152 * specificities of <os/atomic_private.h> implementation that are relevant to 153 * certain clients and can be used to conditionalize code. 154 */ 155 156 /*! 157 * @const OS_ATOMIC_HAS_LLSC 158 * 159 * @brief 160 * Whether the platform has LL/SC features. 161 * 162 * @discussion 163 * When set, the os_atomic_*_exclusive() macros are defined. 164 */ 165 #if defined(__i386__) || defined(__x86_64__) 166 #define OS_ATOMIC_HAS_LLSC 0 167 #elif defined(__arm__) || defined(__arm64__) 168 #define OS_ATOMIC_HAS_LLSC 1 169 #else 170 #error unsupported architecture 171 #endif 172 173 /*! 174 * @const OS_ATOMIC_USE_LLSC 175 * 176 * @brief 177 * Whether os_atomic* use LL/SC internally. 178 * 179 * @discussion 180 * OS_ATOMIC_USE_LLSC implies OS_ATOMIC_HAS_LLSC. 181 */ 182 #if defined(__arm64__) && defined(__ARM_ARCH_8_2__) 183 #define OS_ATOMIC_USE_LLSC 0 184 #else 185 #define OS_ATOMIC_USE_LLSC OS_ATOMIC_HAS_LLSC 186 #endif 187 188 /*! 189 * @const OS_ATOMIC_HAS_STARVATION_FREE_RMW 190 * 191 * @brief 192 * Whether os_atomic* Read-Modify-Write operations are starvation free 193 * in the current configuration. 194 */ 195 #define OS_ATOMIC_HAS_STARVATION_FREE_RMW (!OS_ATOMIC_USE_LLSC) 196 197 /*! @} */ 198 199 #include "atomic_private_impl.h" // Internal implementation details 200 201 /*! 202 * @function os_compiler_barrier 203 * 204 * @brief 205 * Provide a compiler barrier according to the specified ordering. 206 * 207 * @param m 208 * An optional ordering among `acquire`, `release` or `acq_rel` which defaults 209 * to `acq_rel` when not specified. 210 * These are equivalent to the `compiler_acquire`, `compiler_release` and 211 * `compiler_acq_rel` orderings taken by the os_atomic* functions 212 */ 213 #undef os_compiler_barrier 214 #define os_compiler_barrier(b...) \ 215 os_atomic_std(atomic_signal_fence)(_os_compiler_barrier_##b) 216 217 /*! 218 * @function os_atomic_thread_fence 219 * 220 * @brief 221 * Memory fence which is elided in non-SMP mode, but always carries the 222 * corresponding compiler barrier. 223 * 224 * @param m 225 * The ordering for this fence. 226 */ 227 #define os_atomic_thread_fence(m) ({ \ 228 os_atomic_std(atomic_thread_fence)(_os_atomic_mo_##m##_smp); \ 229 os_atomic_std(atomic_signal_fence)(_os_atomic_mo_##m); \ 230 }) 231 232 /*! 233 * @function os_atomic_barrier_before_lock_acquire() 234 * 235 * @brief 236 * Appropriate barrier so that a lock acquire is fully ordered with 237 * any lock-release. 238 * 239 * @discussion 240 * Note: On all currently supported architecture, 241 * this is a no-op given how locks are implemented. 242 * 243 * If armv8 ever comes up with an ldapr-like kind of acquire semantics 244 * for RMW, then this would have to change. 245 */ 246 #define os_atomic_barrier_before_lock_acquire() ((void)0) 247 248 /*! 249 * @function os_atomic_init 250 * 251 * @brief 252 * Wrapper for C11 atomic_init() 253 * 254 * @discussion 255 * This initialization is not performed atomically, and so must only be used as 256 * part of object initialization before the object is made visible to other 257 * threads/cores. 258 * 259 * @param p 260 * A pointer to an atomic variable. 261 * 262 * @param v 263 * The value to initialize the variable with. 264 * 265 * @returns 266 * The value loaded from @a p. 267 */ 268 #define os_atomic_init(p, v) \ 269 os_atomic_std(atomic_init)(os_cast_to_atomic_pointer(p), v) 270 271 /*! 272 * @function os_atomic_load_is_plain, os_atomic_store_is_plain 273 * 274 * @brief 275 * Return whether a relaxed atomic load (resp. store) to an atomic variable 276 * is implemented as a single plain load (resp. store) instruction. 277 * 278 * @discussion 279 * Non-relaxed loads/stores may involve additional memory fence instructions 280 * or more complex atomic instructions. 281 * 282 * This is a construct that can safely be used in static asserts. 283 * 284 * This doesn't check for alignment and it is assumed that `p` is 285 * "aligned enough". 286 * 287 * @param p 288 * A pointer to an atomic variable. 289 * 290 * @returns 291 * True when relaxed atomic loads (resp. stores) compile to a plain load 292 * (resp. store) instruction, false otherwise. 293 */ 294 #define os_atomic_load_is_plain(p) (sizeof(*(p)) <= sizeof(void *)) 295 #define os_atomic_store_is_plain(p) os_atomic_load_is_plain(p) 296 297 /*! 298 * @function os_atomic_load 299 * 300 * @brief 301 * Wrapper for C11 atomic_load_explicit(), guaranteed to compile to a single 302 * plain load instruction (when @a m is `relaxed`). 303 * 304 * @param p 305 * A pointer to an atomic variable. 306 * 307 * @param m 308 * The ordering to use. 309 * 310 * @returns 311 * The value loaded from @a p. 312 */ 313 #define os_atomic_load(p, m) ({ \ 314 _Static_assert(os_atomic_load_is_plain(p), "Load is wide"); \ 315 _os_compiler_barrier_before_atomic(m); \ 316 __auto_type _r = os_atomic_std(atomic_load_explicit)( \ 317 os_cast_to_atomic_pointer(p), _os_atomic_mo_##m##_smp); \ 318 _os_compiler_barrier_after_atomic(m); \ 319 _r; \ 320 }) 321 322 /*! 323 * @function os_atomic_store 324 * 325 * @brief 326 * Wrapper for C11 atomic_store_explicit(), guaranteed to compile to a single 327 * plain store instruction (when @a m is `relaxed`). 328 * 329 * @param p 330 * A pointer to an atomic variable. 331 * 332 * @param v 333 * The value to store. 334 * 335 * @param m 336 * The ordering to use. 337 * 338 * @returns 339 * The value stored at @a p. 340 */ 341 #define os_atomic_store(p, v, m) ({ \ 342 _Static_assert(os_atomic_store_is_plain(p), "Store is wide"); \ 343 __auto_type _v = (v); \ 344 _os_compiler_barrier_before_atomic(m); \ 345 os_atomic_std(atomic_store_explicit)(os_cast_to_atomic_pointer(p), _v, \ 346 _os_atomic_mo_##m##_smp); \ 347 _os_compiler_barrier_after_atomic(m); \ 348 _v; \ 349 }) 350 351 #if OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 352 353 /*! 354 * @function os_atomic_load_wide 355 * 356 * @brief 357 * Wrapper for C11 atomic_load_explicit(), which may be implemented by a 358 * compare-exchange loop for double-wide variables. 359 * 360 * @param p 361 * A pointer to an atomic variable. 362 * 363 * @param m 364 * The ordering to use. 365 * 366 * @returns 367 * The value loaded from @a p. 368 */ 369 #define os_atomic_load_wide(p, m) ({ \ 370 _os_compiler_barrier_before_atomic(m); \ 371 __auto_type _r = os_atomic_std(atomic_load_explicit)( \ 372 os_cast_to_atomic_pointer(p), _os_atomic_mo_##m##_smp); \ 373 _os_compiler_barrier_after_atomic(m); \ 374 _r; \ 375 }) 376 377 /*! 378 * @function os_atomic_store_wide 379 * 380 * @brief 381 * Wrapper for C11 atomic_store_explicit(), which may be implemented by a 382 * compare-exchange loop for double-wide variables. 383 * 384 * @param p 385 * A pointer to an atomic variable. 386 * 387 * @param v 388 * The value to store. 389 * 390 * @param m 391 * The ordering to use. 392 * 393 * @returns 394 * The value stored at @a p. 395 */ 396 #define os_atomic_store_wide(p, v, m) ({ \ 397 __auto_type _v = (v); \ 398 _os_compiler_barrier_before_atomic(m); \ 399 os_atomic_std(atomic_store_explicit)(os_cast_to_atomic_pointer(p), _v, \ 400 _os_atomic_mo_##m##_smp); \ 401 _os_compiler_barrier_after_atomic(m); \ 402 _v; \ 403 }) 404 405 /*! 406 * @function os_atomic_add, os_atomic_add_orig 407 * 408 * @brief 409 * Wrappers for C11 atomic_fetch_add_explicit(). 410 * 411 * @param p 412 * A pointer to an atomic variable. 413 * 414 * @param v 415 * The value to add. 416 * 417 * @param m 418 * The ordering to use. 419 * 420 * @returns 421 * os_atomic_add_orig returns the value of the variable before the atomic add, 422 * os_atomic_add returns the value of the variable after the atomic add. 423 */ 424 #define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add) 425 #define os_atomic_add(p, v, m) _os_atomic_c11_op(p, v, m, fetch_add, +) 426 427 /*! 428 * @function os_atomic_inc, os_atomic_inc_orig 429 * 430 * @brief 431 * Perform an atomic increment. 432 * 433 * @param p 434 * A pointer to an atomic variable. 435 * 436 * @param m 437 * The ordering to use. 438 * 439 * @returns 440 * os_atomic_inc_orig returns the value of the variable before the atomic increment, 441 * os_atomic_inc returns the value of the variable after the atomic increment. 442 */ 443 #define os_atomic_inc_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_add) 444 #define os_atomic_inc(p, m) _os_atomic_c11_op(p, 1, m, fetch_add, +) 445 446 /*! 447 * @function os_atomic_sub, os_atomic_sub_orig 448 * 449 * @brief 450 * Wrappers for C11 atomic_fetch_sub_explicit(). 451 * 452 * @param p 453 * A pointer to an atomic variable. 454 * 455 * @param v 456 * The value to subtract. 457 * 458 * @param m 459 * The ordering to use. 460 * 461 * @returns 462 * os_atomic_sub_orig returns the value of the variable before the atomic subtract, 463 * os_atomic_sub returns the value of the variable after the atomic subtract. 464 */ 465 #define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub) 466 #define os_atomic_sub(p, v, m) _os_atomic_c11_op(p, v, m, fetch_sub, -) 467 468 /*! 469 * @function os_atomic_dec, os_atomic_dec_orig 470 * 471 * @brief 472 * Perform an atomic decrement. 473 * 474 * @param p 475 * A pointer to an atomic variable. 476 * 477 * @param m 478 * The ordering to use. 479 * 480 * @returns 481 * os_atomic_dec_orig returns the value of the variable before the atomic decrement, 482 * os_atomic_dec returns the value of the variable after the atomic decrement. 483 */ 484 #define os_atomic_dec_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_sub) 485 #define os_atomic_dec(p, m) _os_atomic_c11_op(p, 1, m, fetch_sub, -) 486 487 /*! 488 * @function os_atomic_and, os_atomic_and_orig 489 * 490 * @brief 491 * Wrappers for C11 atomic_fetch_and_explicit(). 492 * 493 * @param p 494 * A pointer to an atomic variable. 495 * 496 * @param v 497 * The value to and. 498 * 499 * @param m 500 * The ordering to use. 501 * 502 * @returns 503 * os_atomic_and_orig returns the value of the variable before the atomic and, 504 * os_atomic_and returns the value of the variable after the atomic and. 505 */ 506 #define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and) 507 #define os_atomic_and(p, v, m) _os_atomic_c11_op(p, v, m, fetch_and, &) 508 509 /*! 510 * @function os_atomic_andnot, os_atomic_andnot_orig 511 * 512 * @brief 513 * Wrappers for C11 atomic_fetch_and_explicit(p, ~value). 514 * 515 * @param p 516 * A pointer to an atomic variable. 517 * 518 * @param v 519 * The value whose complement to and. 520 * 521 * @param m 522 * The ordering to use. 523 * 524 * @returns 525 * os_atomic_andnot_orig returns the value of the variable before the atomic andnot, 526 * os_atomic_andnot returns the value of the variable after the atomic andnot. 527 */ 528 #define os_atomic_andnot_orig(p, v, m) _os_atomic_c11_op_orig(p, (typeof(v))~(v), m, fetch_and) 529 #define os_atomic_andnot(p, v, m) _os_atomic_c11_op(p, (typeof(v))~(v), m, fetch_and, &) 530 531 /*! 532 * @function os_atomic_or, os_atomic_or_orig 533 * 534 * @brief 535 * Wrappers for C11 atomic_fetch_or_explicit(). 536 * 537 * @param p 538 * A pointer to an atomic variable. 539 * 540 * @param v 541 * The value to or. 542 * 543 * @param m 544 * The ordering to use. 545 * 546 * @returns 547 * os_atomic_or_orig returns the value of the variable before the atomic or, 548 * os_atomic_or returns the value of the variable after the atomic or. 549 */ 550 #define os_atomic_or_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_or) 551 #define os_atomic_or(p, v, m) _os_atomic_c11_op(p, v, m, fetch_or, |) 552 553 /*! 554 * @function os_atomic_xor, os_atomic_xor_orig 555 * 556 * @brief 557 * Wrappers for C11 atomic_fetch_xor_explicit(). 558 * 559 * @param p 560 * A pointer to an atomic variable. 561 * 562 * @param v 563 * The value to xor. 564 * 565 * @param m 566 * The ordering to use. 567 * 568 * @returns 569 * os_atomic_xor_orig returns the value of the variable before the atomic xor, 570 * os_atomic_xor returns the value of the variable after the atomic xor. 571 */ 572 #define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor) 573 #define os_atomic_xor(p, v, m) _os_atomic_c11_op(p, v, m, fetch_xor, ^) 574 575 /*! 576 * @function os_atomic_min, os_atomic_min_orig 577 * 578 * @brief 579 * Wrappers for Clang's __atomic_fetch_min() 580 * 581 * @param p 582 * A pointer to an atomic variable. 583 * 584 * @param v 585 * The value to minimize. 586 * 587 * @param m 588 * The ordering to use. 589 * 590 * @returns 591 * os_atomic_min_orig returns the value of the variable before the atomic min, 592 * os_atomic_min returns the value of the variable after the atomic min. 593 */ 594 #define os_atomic_min_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_min) 595 #define os_atomic_min(p, v, m) _os_atomic_clang_op(p, v, m, fetch_min, MIN) 596 597 /*! 598 * @function os_atomic_max, os_atomic_max_orig 599 * 600 * @brief 601 * Wrappers for Clang's __atomic_fetch_max() 602 * 603 * @param p 604 * A pointer to an atomic variable. 605 * 606 * @param v 607 * The value to maximize. 608 * 609 * @param m 610 * The ordering to use. 611 * 612 * @returns 613 * os_atomic_max_orig returns the value of the variable before the atomic max, 614 * os_atomic_max returns the value of the variable after the atomic max. 615 */ 616 #define os_atomic_max_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_max) 617 #define os_atomic_max(p, v, m) _os_atomic_clang_op(p, v, m, fetch_max, MAX) 618 619 /*! 620 * @function os_atomic_xchg 621 * 622 * @brief 623 * Wrapper for C11 atomic_exchange_explicit(). 624 * 625 * @param p 626 * A pointer to an atomic variable. 627 * 628 * @param v 629 * The value to exchange with. 630 * 631 * @param m 632 * The ordering to use. 633 * 634 * @returns 635 * The value of the variable before the exchange. 636 */ 637 #define os_atomic_xchg(p, v, m) _os_atomic_c11_op_orig(p, v, m, exchange) 638 639 /*! 640 * @function os_atomic_cmpxchg 641 * 642 * @brief 643 * Wrapper for C11 atomic_compare_exchange_strong_explicit(). 644 * 645 * @discussion 646 * Loops around os_atomic_cmpxchg() may want to consider using the 647 * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak 648 * compare-exchange operation. 649 * 650 * @param p 651 * A pointer to an atomic variable. 652 * 653 * @param e 654 * The value expected in the atomic variable. 655 * 656 * @param v 657 * The value to store if the atomic variable has the expected value @a e. 658 * 659 * @param m 660 * The ordering to use in case of success. 661 * The ordering in case of failure is always `relaxed`. 662 * 663 * @returns 664 * 0 if the compare-exchange failed. 665 * 1 if the compare-exchange succeeded. 666 */ 667 #define os_atomic_cmpxchg(p, e, v, m) ({ \ 668 os_atomic_basetypeof(p) _r = (e); int _b; \ 669 _os_compiler_barrier_before_atomic(m); \ 670 _b = os_atomic_std(atomic_compare_exchange_strong_explicit)( \ 671 os_cast_to_atomic_pointer(p), &_r, \ 672 _os_atomic_value_cast(p, v), \ 673 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 674 _os_compiler_barrier_after_atomic(m); \ 675 _b; \ 676 }) 677 678 /*! 679 * @function os_atomic_cmpxchgv 680 * 681 * @brief 682 * Wrapper for C11 atomic_compare_exchange_strong_explicit(). 683 * 684 * @discussion 685 * Loops around os_atomic_cmpxchgv() may want to consider using the 686 * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak 687 * compare-exchange operation. 688 * 689 * @param p 690 * A pointer to an atomic variable. 691 * 692 * @param e 693 * The value expected in the atomic variable. 694 * 695 * @param v 696 * The value to store if the atomic variable has the expected value @a e. 697 * 698 * @param g 699 * A pointer to a location that is filled with the value that was present in 700 * the atomic variable before the compare-exchange (whether successful or not). 701 * This can be used to redrive compare-exchange loops. 702 * 703 * @param m 704 * The ordering to use in case of success. 705 * The ordering in case of failure is always `relaxed`. 706 * 707 * @returns 708 * 0 if the compare-exchange failed. 709 * 1 if the compare-exchange succeeded. 710 */ 711 #define os_atomic_cmpxchgv(p, e, v, g, m) ({ \ 712 os_atomic_basetypeof(p) _r = (e); int _b; \ 713 _os_compiler_barrier_before_atomic(m); \ 714 _b = os_atomic_std(atomic_compare_exchange_strong_explicit)( \ 715 os_cast_to_atomic_pointer(p), &_r, \ 716 _os_atomic_value_cast(p, v), \ 717 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 718 _os_compiler_barrier_after_atomic(m); \ 719 *(g) = _r; _b; \ 720 }) 721 722 /*! 723 * @function os_atomic_rmw_loop 724 * 725 * @brief 726 * Advanced read-modify-write construct to wrap compare-exchange loops. 727 * 728 * @param p 729 * A pointer to an atomic variable to be modified. 730 * 731 * @param ov 732 * The name of the variable that will contain the original value of the atomic 733 * variable (reloaded every iteration of the loop). 734 * 735 * @param nv 736 * The name of the variable that will contain the new value to compare-exchange 737 * the atomic variable to (typically computed from @a ov every iteration of the 738 * loop). 739 * 740 * @param m 741 * The ordering to use in case of success. 742 * The ordering in case of failure is always `relaxed`. 743 * 744 * @param ... 745 * Code block that validates the value of @p ov and computes the new value of 746 * @p nv that the atomic variable will be compare-exchanged to in an iteration 747 * of the loop. 748 * 749 * The loop can be aborted using os_atomic_rmw_loop_give_up(), e.g. when the 750 * value of @p ov is found to be "invalid" for the ovarall operation. 751 * `continue` cannot be used in this context. 752 * 753 * No stores to memory should be performed within the code block as it may cause 754 * LL/SC transactions used to implement compare-exchange to fail persistently. 755 * 756 * @returns 757 * 0 if the loop was aborted with os_atomic_rmw_loop_give_up(). 758 * 1 if the loop completed. 759 */ 760 #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ 761 int _result = 0; \ 762 __auto_type _p = os_cast_to_nonatomic_pointer(p); \ 763 _os_compiler_barrier_before_atomic(m); \ 764 ov = *_p; \ 765 do { \ 766 __VA_ARGS__; \ 767 _result = os_atomic_std(atomic_compare_exchange_weak_explicit)( \ 768 os_cast_to_atomic_pointer(_p), &ov, nv, \ 769 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 770 } while (__builtin_expect(!_result, 0)); \ 771 _os_compiler_barrier_after_atomic(m); \ 772 _result; \ 773 }) 774 775 /*! 776 * @function os_atomic_rmw_loop_give_up 777 * 778 * @brief 779 * Abort an os_atomic_rmw_loop() loop. 780 * 781 * @param ... 782 * Optional code block to execute before the `break` out of the loop. May 783 * further alter the control flow (e.g. using `return`, `goto`, ...). 784 */ 785 #define os_atomic_rmw_loop_give_up(...) ({ __VA_ARGS__; break; }) 786 787 #else // !OS_ATOMIC_HAS_STARVATION_FREE_RMW && OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 788 789 #define _os_atomic_error_is_starvable(name) \ 790 _Static_assert(0, #name " is not starvation-free and isn't available in this configuration") 791 #define os_atomic_load_wide(p, m) _os_atomic_error_is_starvable(os_atomic_load_wide) 792 #define os_atomic_store_wide(p, v, m) _os_atomic_error_is_starvable(os_atomic_store_wide) 793 #define os_atomic_add_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_add_orig) 794 #define os_atomic_add(p, v, m) _os_atomic_error_is_starvable(os_atomic_add) 795 #define os_atomic_inc_orig(p, m) _os_atomic_error_is_starvable(os_atomic_inc_orig) 796 #define os_atomic_inc(p, m) _os_atomic_error_is_starvable(os_atomic_inc) 797 #define os_atomic_sub_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_sub_orig) 798 #define os_atomic_sub(p, v, m) _os_atomic_error_is_starvable(os_atomic_sub) 799 #define os_atomic_dec_orig(p, m) _os_atomic_error_is_starvable(os_atomic_dec_orig) 800 #define os_atomic_dec(p, m) _os_atomic_error_is_starvable(os_atomic_dec) 801 #define os_atomic_and_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_and_orig) 802 #define os_atomic_and(p, v, m) _os_atomic_error_is_starvable(os_atomic_and) 803 #define os_atomic_andnot_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_andnot_orig) 804 #define os_atomic_andnot(p, v, m) _os_atomic_error_is_starvable(os_atomic_andnot) 805 #define os_atomic_or_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_or_orig) 806 #define os_atomic_or(p, v, m) _os_atomic_error_is_starvable(os_atomic_or) 807 #define os_atomic_xor_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_xor_orig) 808 #define os_atomic_xor(p, v, m) _os_atomic_error_is_starvable(os_atomic_xor) 809 #define os_atomic_min_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_min_orig) 810 #define os_atomic_min(p, v, m) _os_atomic_error_is_starvable(os_atomic_min) 811 #define os_atomic_max_orig(p, v, m) _os_atomic_error_is_starvable(os_atomic_max_orig) 812 #define os_atomic_max(p, v, m) _os_atomic_error_is_starvable(os_atomic_max) 813 #define os_atomic_xchg(p, v, m) _os_atomic_error_is_starvable(os_atomic_xchg) 814 #define os_atomic_cmpxchg(p, e, v, m) _os_atomic_error_is_starvable(os_atomic_cmpxchg) 815 #define os_atomic_cmpxchgv(p, e, v, g, m) _os_atomic_error_is_starvable(os_atomic_cmpxchgv) 816 #define os_atomic_rmw_loop(p, ov, nv, m, ...) _os_atomic_error_is_starvable(os_atomic_rmw_loop) 817 #define os_atomic_rmw_loop_give_up(...) _os_atomic_error_is_starvable(os_atomic_rmw_loop_give_up) 818 819 #endif // !OS_ATOMIC_HAS_STARVATION_FREE_RMW && OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 820 821 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 822 823 /*! 824 * @typedef os_atomic_dependency_t 825 * 826 * @brief 827 * Type for dependency tokens that can be derived from loads with dependency 828 * and injected into various expressions. 829 * 830 * @warning 831 * The implementation of atomic dependencies makes painstakingly sure that the 832 * compiler doesn't know that os_atomic_dependency_t::__opaque_zero is always 0. 833 * 834 * Users of os_atomic_dependency_t MUST NOT test its value (even with an 835 * assert), as doing so would allow the compiler to reason about the value and 836 * elide its use to inject hardware dependencies (thwarting the entire purpose 837 * of the construct). 838 */ 839 typedef struct { unsigned long __opaque_zero; } os_atomic_dependency_t; 840 841 /*! 842 * @const OS_ATOMIC_DEPENDENCY_NONE 843 * 844 * @brief 845 * A value to pass to functions that can carry dependencies, to indicate that 846 * no dependency should be carried. 847 */ 848 #define OS_ATOMIC_DEPENDENCY_NONE \ 849 ((os_atomic_dependency_t){ 0UL }) 850 851 /*! 852 * @function os_atomic_make_dependency 853 * 854 * @brief 855 * Create a dependency token that can be injected into expressions to force a 856 * hardware dependency. 857 * 858 * @discussion 859 * This function is only useful for cases where the dependency needs to be used 860 * several times. 861 * 862 * os_atomic_load_with_dependency_on() and os_atomic_inject_dependency() are 863 * otherwise capable of automatically creating dependency tokens. 864 * 865 * @param v 866 * The result of: 867 * - an os_atomic_load(..., dependency), 868 * - an os_atomic_inject_dependency(), 869 * - an os_atomic_load_with_dependency_on(). 870 * 871 * Note that due to implementation limitations, the type of @p v must be 872 * register-sized, if necessary an explicit cast is required. 873 * 874 * @returns 875 * An os_atomic_dependency_t token that can be used to prolongate dependency 876 * chains. 877 * 878 * The token value is always 0, but the compiler must never be able to reason 879 * about that fact (c.f. os_atomic_dependency_t) 880 */ 881 #define os_atomic_make_dependency(v) \ 882 ((void)(v), OS_ATOMIC_DEPENDENCY_NONE) 883 884 /*! 885 * @function os_atomic_inject_dependency 886 * 887 * @brief 888 * Inject a hardware dependency resulting from a `dependency` load into a 889 * specified pointer. 890 * 891 * @param p 892 * A pointer to inject the dependency into. 893 * 894 * @param e 895 * - a dependency token returned from os_atomic_make_dependency(), 896 * 897 * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op, 898 * 899 * - any value accepted by os_atomic_make_dependency(). 900 * 901 * @returns 902 * A value equal to @a p but that prolongates the dependency chain rooted at 903 * @a e. 904 */ 905 #define os_atomic_inject_dependency(p, e) \ 906 ((typeof(*(p)) *)((p) + _os_atomic_auto_dependency(e).__opaque_zero)) 907 908 /*! 909 * @function os_atomic_load_with_dependency_on 910 * 911 * @brief 912 * Load that prolongates the dependency chain rooted at `v`. 913 * 914 * @discussion 915 * This is shorthand for: 916 * 917 * <code> 918 * os_atomic_load(os_atomic_inject_dependency(p, e), dependency) 919 * </code> 920 * 921 * @param p 922 * A pointer to an atomic variable. 923 * 924 * @param e 925 * - a dependency token returned from os_atomic_make_dependency(), 926 * 927 * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op, 928 * 929 * - any value accepted by os_atomic_make_dependency(). 930 * 931 * @returns 932 * The value loaded from @a p. 933 */ 934 #define os_atomic_load_with_dependency_on(p, e) \ 935 os_atomic_load(os_atomic_inject_dependency(p, e), dependency) 936 937 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 938 939 #include "atomic_private_arch.h" // Per architecture overrides 940 941 #endif /* __OS_ATOMIC_PRIVATE_H__ */ 942