1 /* 2 * Copyright (c) 2018 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef __OS_ATOMIC_PRIVATE_H__ 30 #define __OS_ATOMIC_PRIVATE_H__ 31 32 /*! 33 * @file <os/atomic_private.h> 34 * 35 * @brief 36 * This file defines nicer (terser and safer) wrappers for C11's <stdatomic.h>. 37 * 38 * @discussion 39 * @see xnu.git::doc/atomics.md which provides more extensive documentation 40 * about this header. 41 * 42 * Note that some of the macros defined in this file may be overridden by 43 * architecture specific headers. 44 * 45 * All the os_atomic* functions take an operation ordering argument that can be: 46 * - C11 memory orders: relaxed, acquire, release, acq_rel or seq_cst which 47 * imply a memory fence on SMP machines, and always carry the matching 48 * compiler barrier semantics. 49 * 50 * - the os_atomic-specific `dependency` memory ordering that is used to 51 * document intent to a carry a data or address dependency. 52 * See doc/atomics.md for more information. 53 * 54 * - a compiler barrier: compiler_acquire, compiler_release, compiler_acq_rel 55 * without a corresponding memory fence. 56 */ 57 58 #include <os/atomic.h> 59 60 /*! 61 * @group <os/atomic_private.h> tunables. 62 * 63 * @{ 64 * 65 * @brief 66 * @c OS_ATOMIC_CONFIG_* macros provide tunables for clients. 67 */ 68 69 /*! 70 * @macro OS_ATOMIC_CONFIG_SMP 71 * 72 * @brief 73 * Whether this is used on an SMP system, defaults to 1. 74 */ 75 #ifndef OS_ATOMIC_CONFIG_SMP 76 #define OS_ATOMIC_CONFIG_SMP 1 77 #endif // OS_ATOMIC_CONFIG_SMP 78 79 /*! 80 * @macro OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 81 * 82 * @brief 83 * Obsolete, kept for backward compatibility 84 */ 85 #ifndef OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 86 #define OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 0 87 #endif // OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY 88 89 /*! 90 * @macro OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 91 * 92 * @brief 93 * Expose the os_atomic-specific fake `dependency` memory ordering. 94 * 95 * @discussion 96 * The dependency ordering can be used to try to "repair" C11's consume ordering 97 * and should be limited to extremely complex algorithms where every cycle counts. 98 * 99 * Due to the inherent risks (no compiler support) for this feature, it is 100 * reserved for expert and very domain-specific code only and is off by default. 101 * 102 * Default: 0 103 */ 104 #ifndef OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 105 #if XNU_KERNEL_PRIVATE 106 #define OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 1 107 #else 108 #define OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 0 109 #endif 110 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 111 112 /*! @} */ 113 114 /*! 115 * @group <os/atomic_private.h> features (arch specific). 116 * 117 * @{ 118 * 119 * @brief 120 * The @c OS_ATOMIC_USE_* and @c OS_ATOMIC_HAS_* defines expose some 121 * specificities of <os/atomic_private.h> implementation that are relevant to 122 * certain clients and can be used to conditionalize code. 123 */ 124 125 /*! 126 * @const OS_ATOMIC_HAS_LLSC 127 * 128 * @brief 129 * Whether the platform has LL/SC features. 130 * 131 * @discussion 132 * When set, the os_atomic_*_exclusive() macros are defined. 133 */ 134 #if defined(__i386__) || defined(__x86_64__) 135 #define OS_ATOMIC_HAS_LLSC 0 136 #elif defined(__arm__) || defined(__arm64__) 137 #define OS_ATOMIC_HAS_LLSC 1 138 #else 139 #error unsupported architecture 140 #endif 141 142 /*! 143 * @const OS_ATOMIC_USE_LLSC 144 * 145 * @brief 146 * Whether os_atomic* use LL/SC internally. 147 * 148 * @discussion 149 * OS_ATOMIC_USE_LLSC implies OS_ATOMIC_HAS_LLSC. 150 */ 151 #if defined(__arm64__) && defined(__ARM_ARCH_8_2__) 152 #define OS_ATOMIC_USE_LLSC 0 153 #else 154 #define OS_ATOMIC_USE_LLSC OS_ATOMIC_HAS_LLSC 155 #endif 156 157 /*! 158 * @const OS_ATOMIC_HAS_STARVATION_FREE_RMW 159 * 160 * @brief 161 * Whether os_atomic* Read-Modify-Write operations are starvation free 162 * in the current configuration. 163 */ 164 #define OS_ATOMIC_HAS_STARVATION_FREE_RMW (!OS_ATOMIC_USE_LLSC) 165 166 /*! @} */ 167 168 #include "atomic_private_impl.h" // Internal implementation details 169 170 /*! 171 * @function os_compiler_barrier 172 * 173 * @brief 174 * Provide a compiler barrier according to the specified ordering. 175 * 176 * @param m 177 * An optional ordering among `acquire`, `release` or `acq_rel` which defaults 178 * to `acq_rel` when not specified. 179 * These are equivalent to the `compiler_acquire`, `compiler_release` and 180 * `compiler_acq_rel` orderings taken by the os_atomic* functions 181 */ 182 #undef os_compiler_barrier 183 #define os_compiler_barrier(b...) \ 184 os_atomic_std(atomic_signal_fence)(_os_compiler_barrier_##b) 185 186 /*! 187 * @function os_atomic_thread_fence 188 * 189 * @brief 190 * Memory fence which is elided in non-SMP mode, but always carries the 191 * corresponding compiler barrier. 192 * 193 * @param m 194 * The ordering for this fence. 195 */ 196 #define os_atomic_thread_fence(m) ({ \ 197 os_atomic_std(atomic_thread_fence)(_os_atomic_mo_##m##_smp); \ 198 os_atomic_std(atomic_signal_fence)(_os_atomic_mo_##m); \ 199 }) 200 201 /*! 202 * @function os_atomic_barrier_before_lock_acquire() 203 * 204 * @brief 205 * Appropriate barrier so that a lock acquire is fully ordered with 206 * any lock-release. 207 * 208 * @discussion 209 * Note: On all currently supported architecture, 210 * this is a no-op given how locks are implemented. 211 * 212 * If armv8 ever comes up with an ldapr-like kind of acquire semantics 213 * for RMW, then this would have to change. 214 */ 215 #define os_atomic_barrier_before_lock_acquire() ((void)0) 216 217 /*! 218 * @function os_atomic_init 219 * 220 * @brief 221 * Wrapper for C11 atomic_init() 222 * 223 * @discussion 224 * This initialization is not performed atomically, and so must only be used as 225 * part of object initialization before the object is made visible to other 226 * threads/cores. 227 * 228 * @param p 229 * A pointer to an atomic variable. 230 * 231 * @param v 232 * The value to initialize the variable with. 233 * 234 * @returns 235 * The value loaded from @a p. 236 */ 237 #define os_atomic_init(p, v) \ 238 os_atomic_std(atomic_init)(os_cast_to_atomic_pointer(p), v) 239 240 /*! 241 * @function os_atomic_load_is_plain, os_atomic_store_is_plain 242 * 243 * @brief 244 * Return whether a relaxed atomic load (resp. store) to an atomic variable 245 * is implemented as a single plain load (resp. store) instruction. 246 * 247 * @discussion 248 * Non-relaxed loads/stores may involve additional memory fence instructions 249 * or more complex atomic instructions. 250 * 251 * This is a construct that can safely be used in static asserts. 252 * 253 * This doesn't check for alignment and it is assumed that `p` is 254 * "aligned enough". 255 * 256 * @param p 257 * A pointer to an atomic variable. 258 * 259 * @returns 260 * True when relaxed atomic loads (resp. stores) compile to a plain load 261 * (resp. store) instruction, false otherwise. 262 */ 263 #define os_atomic_load_is_plain(p) (sizeof(*(p)) <= sizeof(void *)) 264 #define os_atomic_store_is_plain(p) os_atomic_load_is_plain(p) 265 266 /*! 267 * @function os_atomic_load 268 * 269 * @brief 270 * Wrapper for C11 atomic_load_explicit(), guaranteed to compile to a single 271 * plain load instruction (when @a m is `relaxed`). 272 * 273 * @param p 274 * A pointer to an atomic variable. 275 * 276 * @param m 277 * The ordering to use. 278 * 279 * @returns 280 * The value loaded from @a p. 281 */ 282 #define os_atomic_load(p, m) ({ \ 283 _Static_assert(os_atomic_load_is_plain(p), "Load is wide"); \ 284 _os_compiler_barrier_before_atomic(m); \ 285 __auto_type _r = os_atomic_std(atomic_load_explicit)( \ 286 os_cast_to_atomic_pointer(p), _os_atomic_mo_##m##_smp); \ 287 _os_compiler_barrier_after_atomic(m); \ 288 _r; \ 289 }) 290 291 /*! 292 * @function os_atomic_store 293 * 294 * @brief 295 * Wrapper for C11 atomic_store_explicit(), guaranteed to compile to a single 296 * plain store instruction (when @a m is `relaxed`). 297 * 298 * @param p 299 * A pointer to an atomic variable. 300 * 301 * @param v 302 * The value to store. 303 * 304 * @param m 305 * The ordering to use. 306 * 307 * @returns 308 * The value stored at @a p. 309 */ 310 #define os_atomic_store(p, v, m) ({ \ 311 _Static_assert(os_atomic_store_is_plain(p), "Store is wide"); \ 312 __auto_type _v = (v); \ 313 _os_compiler_barrier_before_atomic(m); \ 314 os_atomic_std(atomic_store_explicit)(os_cast_to_atomic_pointer(p), _v, \ 315 _os_atomic_mo_##m##_smp); \ 316 _os_compiler_barrier_after_atomic(m); \ 317 _v; \ 318 }) 319 320 /*! 321 * @function os_atomic_load_wide 322 * 323 * @brief 324 * Wrapper for C11 atomic_load_explicit(), which may be implemented by a 325 * compare-exchange loop for double-wide variables. 326 * 327 * @param p 328 * A pointer to an atomic variable. 329 * 330 * @param m 331 * The ordering to use. 332 * 333 * @returns 334 * The value loaded from @a p. 335 */ 336 #define os_atomic_load_wide(p, m) ({ \ 337 _os_compiler_barrier_before_atomic(m); \ 338 __auto_type _r = os_atomic_std(atomic_load_explicit)( \ 339 os_cast_to_atomic_pointer(p), _os_atomic_mo_##m##_smp); \ 340 _os_compiler_barrier_after_atomic(m); \ 341 _r; \ 342 }) 343 344 /*! 345 * @function os_atomic_store_wide 346 * 347 * @brief 348 * Wrapper for C11 atomic_store_explicit(), which may be implemented by a 349 * compare-exchange loop for double-wide variables. 350 * 351 * @param p 352 * A pointer to an atomic variable. 353 * 354 * @param v 355 * The value to store. 356 * 357 * @param m 358 * The ordering to use. 359 * 360 * @returns 361 * The value stored at @a p. 362 */ 363 #define os_atomic_store_wide(p, v, m) ({ \ 364 __auto_type _v = (v); \ 365 _os_compiler_barrier_before_atomic(m); \ 366 os_atomic_std(atomic_store_explicit)(os_cast_to_atomic_pointer(p), _v, \ 367 _os_atomic_mo_##m##_smp); \ 368 _os_compiler_barrier_after_atomic(m); \ 369 _v; \ 370 }) 371 372 /*! 373 * @function os_atomic_add, os_atomic_add_orig 374 * 375 * @brief 376 * Wrappers for C11 atomic_fetch_add_explicit(). 377 * 378 * @param p 379 * A pointer to an atomic variable. 380 * 381 * @param v 382 * The value to add. 383 * 384 * @param m 385 * The ordering to use. 386 * 387 * @returns 388 * os_atomic_add_orig returns the value of the variable before the atomic add, 389 * os_atomic_add returns the value of the variable after the atomic add. 390 */ 391 #define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add) 392 #define os_atomic_add(p, v, m) _os_atomic_c11_op(p, v, m, fetch_add, +) 393 394 /*! 395 * @function os_atomic_inc, os_atomic_inc_orig 396 * 397 * @brief 398 * Perform an atomic increment. 399 * 400 * @param p 401 * A pointer to an atomic variable. 402 * 403 * @param m 404 * The ordering to use. 405 * 406 * @returns 407 * os_atomic_inc_orig returns the value of the variable before the atomic increment, 408 * os_atomic_inc returns the value of the variable after the atomic increment. 409 */ 410 #define os_atomic_inc_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_add) 411 #define os_atomic_inc(p, m) _os_atomic_c11_op(p, 1, m, fetch_add, +) 412 413 /*! 414 * @function os_atomic_sub, os_atomic_sub_orig 415 * 416 * @brief 417 * Wrappers for C11 atomic_fetch_sub_explicit(). 418 * 419 * @param p 420 * A pointer to an atomic variable. 421 * 422 * @param v 423 * The value to subtract. 424 * 425 * @param m 426 * The ordering to use. 427 * 428 * @returns 429 * os_atomic_sub_orig returns the value of the variable before the atomic subtract, 430 * os_atomic_sub returns the value of the variable after the atomic subtract. 431 */ 432 #define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub) 433 #define os_atomic_sub(p, v, m) _os_atomic_c11_op(p, v, m, fetch_sub, -) 434 435 /*! 436 * @function os_atomic_dec, os_atomic_dec_orig 437 * 438 * @brief 439 * Perform an atomic decrement. 440 * 441 * @param p 442 * A pointer to an atomic variable. 443 * 444 * @param m 445 * The ordering to use. 446 * 447 * @returns 448 * os_atomic_dec_orig returns the value of the variable before the atomic decrement, 449 * os_atomic_dec returns the value of the variable after the atomic decrement. 450 */ 451 #define os_atomic_dec_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_sub) 452 #define os_atomic_dec(p, m) _os_atomic_c11_op(p, 1, m, fetch_sub, -) 453 454 /*! 455 * @function os_atomic_and, os_atomic_and_orig 456 * 457 * @brief 458 * Wrappers for C11 atomic_fetch_and_explicit(). 459 * 460 * @param p 461 * A pointer to an atomic variable. 462 * 463 * @param v 464 * The value to and. 465 * 466 * @param m 467 * The ordering to use. 468 * 469 * @returns 470 * os_atomic_and_orig returns the value of the variable before the atomic and, 471 * os_atomic_and returns the value of the variable after the atomic and. 472 */ 473 #define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and) 474 #define os_atomic_and(p, v, m) _os_atomic_c11_op(p, v, m, fetch_and, &) 475 476 /*! 477 * @function os_atomic_andnot, os_atomic_andnot_orig 478 * 479 * @brief 480 * Wrappers for C11 atomic_fetch_and_explicit(p, ~value). 481 * 482 * @param p 483 * A pointer to an atomic variable. 484 * 485 * @param v 486 * The value whose complement to and. 487 * 488 * @param m 489 * The ordering to use. 490 * 491 * @returns 492 * os_atomic_andnot_orig returns the value of the variable before the atomic andnot, 493 * os_atomic_andnot returns the value of the variable after the atomic andnot. 494 */ 495 #define os_atomic_andnot_orig(p, v, m) _os_atomic_c11_op_orig(p, (typeof(v))~(v), m, fetch_and) 496 #define os_atomic_andnot(p, v, m) _os_atomic_c11_op(p, (typeof(v))~(v), m, fetch_and, &) 497 498 /*! 499 * @function os_atomic_or, os_atomic_or_orig 500 * 501 * @brief 502 * Wrappers for C11 atomic_fetch_or_explicit(). 503 * 504 * @param p 505 * A pointer to an atomic variable. 506 * 507 * @param v 508 * The value to or. 509 * 510 * @param m 511 * The ordering to use. 512 * 513 * @returns 514 * os_atomic_or_orig returns the value of the variable before the atomic or, 515 * os_atomic_or returns the value of the variable after the atomic or. 516 */ 517 #define os_atomic_or_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_or) 518 #define os_atomic_or(p, v, m) _os_atomic_c11_op(p, v, m, fetch_or, |) 519 520 /*! 521 * @function os_atomic_xor, os_atomic_xor_orig 522 * 523 * @brief 524 * Wrappers for C11 atomic_fetch_xor_explicit(). 525 * 526 * @param p 527 * A pointer to an atomic variable. 528 * 529 * @param v 530 * The value to xor. 531 * 532 * @param m 533 * The ordering to use. 534 * 535 * @returns 536 * os_atomic_xor_orig returns the value of the variable before the atomic xor, 537 * os_atomic_xor returns the value of the variable after the atomic xor. 538 */ 539 #define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor) 540 #define os_atomic_xor(p, v, m) _os_atomic_c11_op(p, v, m, fetch_xor, ^) 541 542 /*! 543 * @function os_atomic_min, os_atomic_min_orig 544 * 545 * @brief 546 * Wrappers for Clang's __atomic_fetch_min() 547 * 548 * @param p 549 * A pointer to an atomic variable. 550 * 551 * @param v 552 * The value to minimize. 553 * 554 * @param m 555 * The ordering to use. 556 * 557 * @returns 558 * os_atomic_min_orig returns the value of the variable before the atomic min, 559 * os_atomic_min returns the value of the variable after the atomic min. 560 */ 561 #define os_atomic_min_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_min) 562 #define os_atomic_min(p, v, m) _os_atomic_clang_op(p, v, m, fetch_min, MIN) 563 564 /*! 565 * @function os_atomic_max, os_atomic_max_orig 566 * 567 * @brief 568 * Wrappers for Clang's __atomic_fetch_max() 569 * 570 * @param p 571 * A pointer to an atomic variable. 572 * 573 * @param v 574 * The value to maximize. 575 * 576 * @param m 577 * The ordering to use. 578 * 579 * @returns 580 * os_atomic_max_orig returns the value of the variable before the atomic max, 581 * os_atomic_max returns the value of the variable after the atomic max. 582 */ 583 #define os_atomic_max_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_max) 584 #define os_atomic_max(p, v, m) _os_atomic_clang_op(p, v, m, fetch_max, MAX) 585 586 /*! 587 * @function os_atomic_xchg 588 * 589 * @brief 590 * Wrapper for C11 atomic_exchange_explicit(). 591 * 592 * @param p 593 * A pointer to an atomic variable. 594 * 595 * @param v 596 * The value to exchange with. 597 * 598 * @param m 599 * The ordering to use. 600 * 601 * @returns 602 * The value of the variable before the exchange. 603 */ 604 #define os_atomic_xchg(p, v, m) _os_atomic_c11_op_orig(p, v, m, exchange) 605 606 /*! 607 * @function os_atomic_cmpxchg 608 * 609 * @brief 610 * Wrapper for C11 atomic_compare_exchange_strong_explicit(). 611 * 612 * @discussion 613 * Loops around os_atomic_cmpxchg() may want to consider using the 614 * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak 615 * compare-exchange operation. 616 * 617 * @param p 618 * A pointer to an atomic variable. 619 * 620 * @param e 621 * The value expected in the atomic variable. 622 * 623 * @param v 624 * The value to store if the atomic variable has the expected value @a e. 625 * 626 * @param m 627 * The ordering to use in case of success. 628 * The ordering in case of failure is always `relaxed`. 629 * 630 * @returns 631 * 0 if the compare-exchange failed. 632 * 1 if the compare-exchange succeeded. 633 */ 634 #define os_atomic_cmpxchg(p, e, v, m) ({ \ 635 os_atomic_basetypeof(p) _r = (e); int _b; \ 636 _os_compiler_barrier_before_atomic(m); \ 637 _b = os_atomic_std(atomic_compare_exchange_strong_explicit)( \ 638 os_cast_to_atomic_pointer(p), &_r, \ 639 _os_atomic_value_cast(p, v), \ 640 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 641 _os_compiler_barrier_after_atomic(m); \ 642 _b; \ 643 }) 644 645 /*! 646 * @function os_atomic_cmpxchgv 647 * 648 * @brief 649 * Wrapper for C11 atomic_compare_exchange_strong_explicit(). 650 * 651 * @discussion 652 * Loops around os_atomic_cmpxchgv() may want to consider using the 653 * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak 654 * compare-exchange operation. 655 * 656 * @param p 657 * A pointer to an atomic variable. 658 * 659 * @param e 660 * The value expected in the atomic variable. 661 * 662 * @param v 663 * The value to store if the atomic variable has the expected value @a e. 664 * 665 * @param g 666 * A pointer to a location that is filled with the value that was present in 667 * the atomic variable before the compare-exchange (whether successful or not). 668 * This can be used to redrive compare-exchange loops. 669 * 670 * @param m 671 * The ordering to use in case of success. 672 * The ordering in case of failure is always `relaxed`. 673 * 674 * @returns 675 * 0 if the compare-exchange failed. 676 * 1 if the compare-exchange succeeded. 677 */ 678 #define os_atomic_cmpxchgv(p, e, v, g, m) ({ \ 679 os_atomic_basetypeof(p) _r = (e); int _b; \ 680 _os_compiler_barrier_before_atomic(m); \ 681 _b = os_atomic_std(atomic_compare_exchange_strong_explicit)( \ 682 os_cast_to_atomic_pointer(p), &_r, \ 683 _os_atomic_value_cast(p, v), \ 684 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 685 _os_compiler_barrier_after_atomic(m); \ 686 *(g) = _r; _b; \ 687 }) 688 689 /*! 690 * @function os_atomic_rmw_loop 691 * 692 * @brief 693 * Advanced read-modify-write construct to wrap compare-exchange loops. 694 * 695 * @param p 696 * A pointer to an atomic variable to be modified. 697 * 698 * @param ov 699 * The name of the variable that will contain the original value of the atomic 700 * variable (reloaded every iteration of the loop). 701 * 702 * @param nv 703 * The name of the variable that will contain the new value to compare-exchange 704 * the atomic variable to (typically computed from @a ov every iteration of the 705 * loop). 706 * 707 * @param m 708 * The ordering to use in case of success. 709 * The ordering in case of failure is always `relaxed`. 710 * 711 * @param ... 712 * Code block that validates the value of @p ov and computes the new value of 713 * @p nv that the atomic variable will be compare-exchanged to in an iteration 714 * of the loop. 715 * 716 * The loop can be aborted using os_atomic_rmw_loop_give_up(), e.g. when the 717 * value of @p ov is found to be "invalid" for the ovarall operation. 718 * `continue` cannot be used in this context. 719 * 720 * No stores to memory should be performed within the code block as it may cause 721 * LL/SC transactions used to implement compare-exchange to fail persistently. 722 * 723 * @returns 724 * 0 if the loop was aborted with os_atomic_rmw_loop_give_up(). 725 * 1 if the loop completed. 726 */ 727 #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ 728 int _result = 0; \ 729 __auto_type _p = os_cast_to_nonatomic_pointer(p); \ 730 _os_compiler_barrier_before_atomic(m); \ 731 ov = *_p; \ 732 do { \ 733 __VA_ARGS__; \ 734 _result = os_atomic_std(atomic_compare_exchange_weak_explicit)( \ 735 os_cast_to_atomic_pointer(_p), &ov, nv, \ 736 _os_atomic_mo_##m##_smp, _os_atomic_mo_relaxed); \ 737 } while (__builtin_expect(!_result, 0)); \ 738 _os_compiler_barrier_after_atomic(m); \ 739 _result; \ 740 }) 741 742 /*! 743 * @function os_atomic_rmw_loop_give_up 744 * 745 * @brief 746 * Abort an os_atomic_rmw_loop() loop. 747 * 748 * @param ... 749 * Optional code block to execute before the `break` out of the loop. May 750 * further alter the control flow (e.g. using `return`, `goto`, ...). 751 */ 752 #define os_atomic_rmw_loop_give_up(...) ({ __VA_ARGS__; break; }) 753 754 #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 755 756 /*! 757 * @typedef os_atomic_dependency_t 758 * 759 * @brief 760 * Type for dependency tokens that can be derived from loads with dependency 761 * and injected into various expressions. 762 * 763 * @warning 764 * The implementation of atomic dependencies makes painstakingly sure that the 765 * compiler doesn't know that os_atomic_dependency_t::__opaque_zero is always 0. 766 * 767 * Users of os_atomic_dependency_t MUST NOT test its value (even with an 768 * assert), as doing so would allow the compiler to reason about the value and 769 * elide its use to inject hardware dependencies (thwarting the entire purpose 770 * of the construct). 771 */ 772 typedef struct { unsigned long __opaque_zero; } os_atomic_dependency_t; 773 774 /*! 775 * @const OS_ATOMIC_DEPENDENCY_NONE 776 * 777 * @brief 778 * A value to pass to functions that can carry dependencies, to indicate that 779 * no dependency should be carried. 780 */ 781 #define OS_ATOMIC_DEPENDENCY_NONE \ 782 ((os_atomic_dependency_t){ 0UL }) 783 784 /*! 785 * @function os_atomic_make_dependency 786 * 787 * @brief 788 * Create a dependency token that can be injected into expressions to force a 789 * hardware dependency. 790 * 791 * @discussion 792 * This function is only useful for cases where the dependency needs to be used 793 * several times. 794 * 795 * os_atomic_load_with_dependency_on() and os_atomic_inject_dependency() are 796 * otherwise capable of automatically creating dependency tokens. 797 * 798 * @param v 799 * The result of: 800 * - an os_atomic_load(..., dependency), 801 * - an os_atomic_inject_dependency(), 802 * - an os_atomic_load_with_dependency_on(). 803 * 804 * Note that due to implementation limitations, the type of @p v must be 805 * register-sized, if necessary an explicit cast is required. 806 * 807 * @returns 808 * An os_atomic_dependency_t token that can be used to prolongate dependency 809 * chains. 810 * 811 * The token value is always 0, but the compiler must never be able to reason 812 * about that fact (c.f. os_atomic_dependency_t) 813 */ 814 #define os_atomic_make_dependency(v) \ 815 ((void)(v), OS_ATOMIC_DEPENDENCY_NONE) 816 817 /*! 818 * @function os_atomic_inject_dependency 819 * 820 * @brief 821 * Inject a hardware dependency resulting from a `dependency` load into a 822 * specified pointer. 823 * 824 * @param p 825 * A pointer to inject the dependency into. 826 * 827 * @param e 828 * - a dependency token returned from os_atomic_make_dependency(), 829 * 830 * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op, 831 * 832 * - any value accepted by os_atomic_make_dependency(). 833 * 834 * @returns 835 * A value equal to @a p but that prolongates the dependency chain rooted at 836 * @a e. 837 */ 838 #define os_atomic_inject_dependency(p, e) \ 839 ((typeof(*(p)) *)((p) + _os_atomic_auto_dependency(e).__opaque_zero)) 840 841 /*! 842 * @function os_atomic_load_with_dependency_on 843 * 844 * @brief 845 * Load that prolongates the dependency chain rooted at `v`. 846 * 847 * @discussion 848 * This is shorthand for: 849 * 850 * <code> 851 * os_atomic_load(os_atomic_inject_dependency(p, e), dependency) 852 * </code> 853 * 854 * @param p 855 * A pointer to an atomic variable. 856 * 857 * @param e 858 * - a dependency token returned from os_atomic_make_dependency(), 859 * 860 * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op, 861 * 862 * - any value accepted by os_atomic_make_dependency(). 863 * 864 * @returns 865 * The value loaded from @a p. 866 */ 867 #define os_atomic_load_with_dependency_on(p, e) \ 868 os_atomic_load(os_atomic_inject_dependency(p, e), dependency) 869 870 #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY 871 872 #include "atomic_private_arch.h" // Per architecture overrides 873 874 #endif /* __OS_ATOMIC_PRIVATE_H__ */ 875