1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _TURNSTILE_H_ 30 #define _TURNSTILE_H_ 31 32 #include <mach/mach_types.h> 33 #include <mach/kern_return.h> 34 #include <sys/cdefs.h> 35 36 #if PRIVATE 37 #define TURNSTILE_MAX_HOP_DEFAULT (10) 38 struct turnstile_stats { 39 uint64_t ts_priority_propagation; 40 uint64_t ts_no_inheritor; 41 uint64_t ts_thread_runnable; 42 uint64_t ts_no_priority_change_required; 43 uint64_t ts_above_ui_pri_change; 44 uint64_t ts_no_turnstile; 45 }; 46 #endif 47 48 #ifdef KERNEL_PRIVATE 49 #include <kern/queue.h> 50 #include <sys/queue.h> 51 #include <kern/waitq.h> 52 #include <kern/priority_queue.h> 53 #include <os/refcnt.h> 54 #include <kern/assert.h> 55 #include <kern/kern_types.h> 56 #include <kern/mpsc_queue.h> 57 #include <kern/locks.h> 58 59 /* 60 * turnstile_type_t : Indicates the type of primitive the turnstile is associated with 61 * Please populate turnstile_promote_policy array if a new type is added here. 62 */ 63 typedef enum __attribute__((packed)) turnstile_type { 64 TURNSTILE_NONE = 0, 65 TURNSTILE_KERNEL_MUTEX = 1, 66 TURNSTILE_ULOCK = 2, 67 TURNSTILE_PTHREAD_MUTEX = 3, 68 TURNSTILE_SYNC_IPC = 4, 69 TURNSTILE_WORKLOOPS = 5, 70 TURNSTILE_WORKQS = 6, 71 TURNSTILE_KNOTE = 7, 72 TURNSTILE_SLEEP_INHERITOR = 8, 73 TURNSTILE_TOTAL_TYPES = 9, 74 } turnstile_type_t; 75 76 /* 77 * For each type of turnstile, following are the type of 78 * inheritors passed: 79 * 80 * TURNSTILE_KERNEL_MUTEX 81 * Interlock: kernel mutex interlock. 82 * Inheritor: threads. 83 * Lock order: turnstile lock, thread lock. 84 * 85 * TURNSTILE_ULOCK 86 * Interlock: ulocks interlock. 87 * Inheritor: threads. 88 * Lock order: turnstile lock, thread lock. 89 * 90 * TURNSTILE_PTHREAD_MUTEX 91 * Interlock: pthread mtx interlock. 92 * Inheritor: threads. 93 * Lock order: turnstile lock, thread lock. 94 * 95 * TURNSTILE_SYNC_IPC 96 * Interlock: port's mqueue lock 97 * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. 98 * Lock order: Our turnstile, then turnstile of the port we are enqueued in. 99 * Port circularity will make sure there is never a cycle formation 100 * and lock order is maintained. 101 * 102 * TURNSTILE_WORKLOOPS 103 * Interlock: 104 * - kq req lock 105 * - wq lock when "filt_wlworkq_interlock_needed() is true" 106 * Inheritor: thread, turnstile (of workq) 107 * Lock order: turnstile lock, thread lock 108 * WL turnstile lock, Workq turnstile lock 109 * 110 * TURNSTILE_WORKQS 111 * Interlock: workqueue lock 112 * Inheritor: thread 113 * Lock order: turnstile lock, thread lock. 114 * 115 * TURNSTILE_KNOTE 116 * Interlock: the knote lock 117 * Inheritor: WL turnstile 118 * 119 * TURNSTILE_SLEEP_INHERITOR 120 * Interlock: turnstile_htable bucket spinlock. 121 * Inheritor: threads. 122 * Lock order: turnstile lock, thread lock. 123 * 124 */ 125 126 typedef enum __attribute__((flag_enum)) turnstile_promote_policy { 127 TURNSTILE_PROMOTE_NONE = 0, 128 TURNSTILE_KERNEL_PROMOTE = 0x1, 129 TURNSTILE_USER_PROMOTE = 0x2, 130 TURNSTILE_USER_IPC_PROMOTE = 0x4, 131 } turnstile_promote_policy_t; 132 133 typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy { 134 TURNSTILE_HASH_LOCK_POLICY_NONE = 0, 135 TURNSTILE_IRQ_UNSAFE_HASH = 0x1, 136 TURNSTILE_LOCKED_HASH = 0x2, 137 } turnstile_hash_lock_policy_t; 138 139 /* 140 * Turnstile state flags 141 * 142 * The turnstile state flags represent the current ownership of a turnstile. 143 * The supported flags are: 144 * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread 145 * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile 146 * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive 147 * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor 148 * 149 * The flag updates are done while holding the primitive interlock. 150 * */ 151 152 #define TURNSTILE_STATE_THREAD 0x1 153 #define TURNSTILE_STATE_FREELIST 0x2 154 #define TURNSTILE_STATE_HASHTABLE 0x4 155 #define TURNSTILE_STATE_PROPRIETOR 0x8 156 157 /* Helper macros to set/unset turnstile state flags */ 158 #if DEVELOPMENT || DEBUG 159 160 #define turnstile_state_init(ts, state) \ 161 MACRO_BEGIN \ 162 ts->ts_state = state; \ 163 MACRO_END 164 165 #define turnstile_state_add(ts, state) \ 166 MACRO_BEGIN \ 167 assert((ts->ts_state & (state)) == 0); \ 168 ts->ts_state |= state; \ 169 MACRO_END 170 171 #define turnstile_state_remove(ts, state) \ 172 MACRO_BEGIN \ 173 assert(ts->ts_state & (state)); \ 174 ts->ts_state &= ~(state); \ 175 MACRO_END 176 177 #else /* DEVELOPMENT || DEBUG */ 178 179 #define turnstile_state_init(ts, state) \ 180 MACRO_BEGIN \ 181 (void)ts; \ 182 MACRO_END 183 184 #define turnstile_state_add(ts, state) \ 185 MACRO_BEGIN \ 186 (void)ts; \ 187 MACRO_END 188 189 #define turnstile_state_remove(ts, state) \ 190 MACRO_BEGIN \ 191 (void)ts; \ 192 MACRO_END 193 194 #endif /* DEVELOPMENT || DEBUG */ 195 196 struct knote; 197 struct turnstile; 198 199 /* 200 * Turnstile update flags 201 * 202 * TURNSTILE_IMMEDIATE_UPDATE 203 * When passed to turnstile_update_inheritor 204 * update the inheritor of the turnstile in 205 * the same call. 206 * 207 * TURNSTILE_DELAYED_UPDATE 208 * When passed to turnstile_update_inheritor 209 * it stashed the inheritor on the thread and 210 * turnstile's inheritor is updated in 211 * assert wait. 212 * 213 * TURNSTILE_INHERITOR_THREAD 214 * The turnstile inheritor is of type thread. 215 * 216 * TURNSTILE_INHERITOR_TURNSTILE 217 * The turnstile inheritor is of type turnstile. 218 * 219 * TURNSTILE_INHERITOR_WORKQ 220 * The turnstile inheritor is of type workqueue 221 * 222 * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE 223 * The inheritor needs a chain priority update. 224 * 225 * TURNSTILE_NEEDS_PRI_UPDATE 226 * Current turnstile needs a chain priority update. 227 * 228 * Locking order for passing thread and turnstile as inheritor 229 * 230 * Thread as an inheritor: 231 * When thread is passed as an inheritor of a turnstile 232 * turnstile lock is taken and then thread lock. 233 * 234 * Turnstile as in inheritor: 235 * When turnstile (T1) is passed as an inheritor of 236 * a turnstile (T2), turnstile lock of T2 is taken 237 * and then turnstile lock of T1 is taken. 238 * 239 * Caution: While passing turnstile as an inheritor, its 240 * job of the adopter to make sure that there is no 241 * lock inversion. 242 */ 243 typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { 244 TURNSTILE_UPDATE_FLAGS_NONE = 0, 245 TURNSTILE_IMMEDIATE_UPDATE = 0x1, 246 TURNSTILE_DELAYED_UPDATE = 0x2, 247 TURNSTILE_INHERITOR_THREAD = 0x4, 248 TURNSTILE_INHERITOR_TURNSTILE = 0x8, 249 TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, 250 TURNSTILE_NEEDS_PRI_UPDATE = 0x20, 251 TURNSTILE_INHERITOR_WORKQ = 0x40, 252 TURNSTILE_UPDATE_BOOST = 0x80, 253 } turnstile_update_flags_t; 254 255 #define TURNSTILE_NULL ((struct turnstile *)0) 256 257 typedef void * turnstile_inheritor_t; 258 259 #define TURNSTILE_INHERITOR_NULL NULL 260 261 #ifdef XNU_KERNEL_PRIVATE 262 263 /* Turnstile stats update flags 264 * 265 * TSU_TURNSTILE_BLOCK_COUNT 266 * thread blocking on turnstile waitq, increment global 267 * thread block on turnstile count. 268 * 269 * TSU_REGULAR_WAITQ_BLOCK_COUNT 270 * thread blocking on regular waitq, increment global 271 * thread block on regular waitq count. 272 * 273 * TSU_PRI_PROPAGATION 274 * turnstile propagation update stopped at nth hop, update 275 * priority change count for nth element in stats array. 276 * 277 * TSU_NO_INHERITOR 278 * turnstile propagation update stopped due to turnstile 279 * not having an inheritor after nth hop, update the no 280 * inheritor count for nth element in the stats array. 281 * 282 * TSU_NO_TURNSTILE 283 * turnstile propagation update stopped due to thread 284 * not blocked on a turnstile waitq after nth hop, update 285 * the no turnstile count for the nth element in the stats 286 * array. 287 * 288 * TSU_NO_PRI_CHANGE_NEEDED 289 * turnstile propagation update stopped due to thread or 290 * turnstile having the correct priority or not blocked. 291 * update the no priority change count for the nth element 292 * in the stats array. 293 * 294 * TSU_THREAD_RUNNABLE 295 * turnstile propagation update stopped due to thread 296 * being runnable, update the thread runnable count for 297 * the nth element in the stats array. 298 * 299 * TSU_ABOVE_UI_PRI_CHANGE 300 * turnstile propagation caused an above UI priority change. 301 */ 302 typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { 303 TSU_FLAGS_NONE = 0, 304 TSU_TURNSTILE_BLOCK_COUNT = 0x1, 305 TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, 306 TSU_PRI_PROPAGATION = 0x4, 307 TSU_NO_INHERITOR = 0x8, 308 TSU_NO_TURNSTILE = 0x10, 309 TSU_NO_PRI_CHANGE_NEEDED = 0x20, 310 TSU_THREAD_RUNNABLE = 0x40, 311 TSU_ABOVE_UI_PRI_CHANGE = 0x80, 312 TSU_THREAD_ARG = 0x100, 313 TSU_TURNSTILE_ARG = 0x200, 314 TSU_BOOST_ARG = 0x400, 315 } turnstile_stats_update_flags_t; 316 317 SLIST_HEAD(turnstile_list, turnstile); 318 319 struct turnstile { 320 struct waitq ts_waitq; /* waitq embedded in turnstile */ 321 turnstile_inheritor_t ts_inheritor; /* thread/turnstile inheriting the priority (IL, WL) */ 322 union { 323 struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ 324 SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ 325 }; 326 struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ 327 union { 328 struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */ 329 struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */ 330 }; 331 SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ 332 uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ 333 os_refcnt_t ts_refcount; /* reference count for turnstiles */ 334 _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ 335 uint32_t ts_port_ref; /* number of explicit refs from ports on send turnstile */ 336 turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ 337 uint8_t ts_priority; /* priority of turnstile (WL) */ 338 339 #if DEVELOPMENT || DEBUG 340 uint8_t ts_state; /* current state of turnstile (IL) */ 341 queue_chain_t ts_global_elm; /* global turnstile chain */ 342 thread_t ts_thread; /* thread the turnstile is attached to */ 343 thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ 344 #endif 345 }; 346 347 #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) 348 349 /* IL - interlock, WL - turnstile lock i.e. waitq lock */ 350 351 #define TURNSTILE_PROPRIETOR_NULL 0ul 352 353 /* 354 * Name: turnstiles_init 355 * 356 * Description: Initialize turnstile sub system. 357 * 358 * Args: None. 359 * 360 * Returns: None. 361 */ 362 void 363 turnstiles_init(void); 364 365 /* 366 * Name: turnstile_alloc 367 * 368 * Description: Allocate a turnstile. 369 * 370 * Args: None. 371 * 372 * Returns: 373 * turnstile on Success. 374 */ 375 struct turnstile * 376 turnstile_alloc(void); 377 378 /* 379 * Name: turnstile_destroy 380 * 381 * Description: Deallocates the turnstile. 382 * 383 * Args: 384 * Arg1: turnstile 385 * 386 * Returns: None. 387 */ 388 void 389 turnstile_destroy(struct turnstile *turnstile); 390 391 /* 392 * Name: turnstile_reference 393 * 394 * Description: Take a reference on the turnstile. 395 * 396 * Arg1: turnstile 397 * 398 * Returns: None. 399 */ 400 void 401 turnstile_reference(struct turnstile *turnstile); 402 403 /* 404 * Name: turnstile_deallocate 405 * 406 * Description: Drop a reference on the turnstile. 407 * Destroy the turnstile if the last ref. 408 * 409 * Arg1: turnstile 410 * 411 * Returns: None. 412 */ 413 void 414 turnstile_deallocate(struct turnstile *turnstile); 415 416 /* 417 * Name: turnstile_waitq_add_thread_priority_queue 418 * 419 * Description: add thread to the turnstile waitq 420 * 421 * Arg1: waitq 422 * Arg2: thread 423 * 424 * Conditions: waitq locked 425 */ 426 void 427 turnstile_waitq_add_thread_priority_queue( 428 struct waitq* wq, 429 thread_t thread); 430 431 /* 432 * Name: turnstile_deallocate_safe 433 * 434 * Description: Drop a reference on the turnstile safely without triggering zfree. 435 * 436 * Arg1: turnstile 437 * 438 * Returns: None. 439 */ 440 void 441 turnstile_deallocate_safe(struct turnstile *turnstile); 442 443 /* 444 * Name: turnstile_recompute_priority_locked 445 * 446 * Description: Update turnstile priority based 447 * on highest waiter thread and highest blocking 448 * turnstile. 449 * 450 * Args: turnstile 451 * 452 * Returns: TRUE: if the turnstile priority changed and needs propagation. 453 * FALSE: if the turnstile priority did not change or it does not need propagation. 454 * 455 * Condition: turnstile locked 456 */ 457 boolean_t 458 turnstile_recompute_priority_locked( 459 struct turnstile *turnstile); 460 461 /* 462 * Name: turnstile_recompute_priority 463 * 464 * Description: Update turnstile priority based 465 * on highest waiter thread and highest blocking 466 * turnstile. 467 * 468 * Args: turnstile 469 * 470 * Returns: TRUE: if the turnstile priority changed and needs propagation. 471 * FALSE: if the turnstile priority did not change or it does not need propagation. 472 */ 473 boolean_t 474 turnstile_recompute_priority( 475 struct turnstile *turnstile); 476 477 /* 478 * Name: turnstile_workq_proprietor_of_max_turnstile 479 * 480 * Description: Returns the highest priority and proprietor of a turnstile 481 * pushing on a workqueue turnstile. 482 * 483 * This will not return waiters that are at priority 484 * MAXPRI_THROTTLE or lower. 485 * 486 * Args: turnstile 487 * 488 * Returns: 489 * Priority of the max entry, or 0 490 * Pointer to the max entry proprietor 491 */ 492 int 493 turnstile_workq_proprietor_of_max_turnstile( 494 struct turnstile *turnstile, 495 uintptr_t *proprietor); 496 497 /* 498 * Name: turnstile_workloop_pusher_info 499 * 500 * Description: Returns the priority of the turnstile push for a workloop, 501 * and the thread or knote responsible for this push. 502 * 503 * Args: workloop turnstile 504 * 505 * Returns: 506 * Priority of the push or 0 507 * Thread (with a +1 reference) with that push or THREAD_NULL. 508 * Port (with a +1 reference) with that push, or IP_NULL. 509 * Sync IPC knote with the highest push (or NULL) 510 */ 511 int 512 turnstile_workloop_pusher_info( 513 struct turnstile *turnstile, 514 thread_t *thread, 515 ipc_port_t *port, 516 struct knote **knote_out); 517 518 /* 519 * Name: turnstile_cleanup 520 * 521 * Description: Update priority of a turnstile inheritor 522 * if needed. 523 * 524 * Args: inheritor and flags passed on thread struct. 525 * 526 * Returns: None. 527 */ 528 void 529 turnstile_cleanup(void); 530 531 /* 532 * Name: turnstile_update_thread_priority_chain 533 * 534 * Description: Priority of a thread blocked on a turnstile 535 * has changed, update the turnstile priority. 536 * 537 * Arg1: thread: thread whose priority has changed. 538 * 539 * Returns: None. 540 */ 541 void 542 turnstile_update_thread_priority_chain(thread_t thread); 543 544 /* 545 * Name: turnstile_update_inheritor_locked 546 * 547 * Description: Update the inheritor of the turnstile and boost the 548 * inheritor, called with turnstile locked. 549 * 550 * Args: 551 * Arg1: turnstile 552 * Implicit arg: new inheritor value is stashed in current thread's struct 553 * 554 * Returns: 555 * old inheritor reference is returned on current thread's struct. 556 */ 557 void 558 turnstile_update_inheritor_locked(struct turnstile *turnstile); 559 560 /* 561 * Name: thread_get_inheritor_turnstile_base_priority 562 * 563 * Description: Get the max base priority of all the inheritor turnstiles 564 * 565 * Arg1: thread 566 * 567 * Returns: Max base priority of all the inheritor turnstiles. 568 * 569 * Condition: thread locked 570 */ 571 int 572 thread_get_inheritor_turnstile_base_priority(thread_t thread); 573 574 /* 575 * Name: thread_get_inheritor_turnstile_sched_priority 576 * 577 * Description: Get the max sched priority of all the inheritor turnstiles 578 * 579 * Arg1: thread 580 * 581 * Returns: Max sched priority of all the inheritor turnstiles. 582 * 583 * Condition: thread locked 584 */ 585 int 586 thread_get_inheritor_turnstile_sched_priority(thread_t thread); 587 588 /* 589 * Name: thread_get_waiting_turnstile 590 * 591 * Description: Get the turnstile if the thread is waiting on a turnstile. 592 * 593 * Arg1: thread 594 * 595 * Returns: turnstile: if the thread is blocked on a turnstile. 596 * TURNSTILE_NULL: otherwise. 597 * 598 * Condition: thread locked. 599 */ 600 struct turnstile * 601 thread_get_waiting_turnstile(thread_t thread); 602 603 /* 604 * Name: turnstile_lookup_by_proprietor 605 * 606 * Description: Get turnstile for a proprietor from global 607 * turnstile hash. 608 * 609 * Arg1: port 610 * Arg2: turnstile_type_t type 611 * 612 * Returns: turnstile: if the proprietor has a turnstile. 613 * TURNSTILE_NULL: otherwise. 614 * 615 * Condition: proprietor interlock held. 616 */ 617 struct turnstile * 618 turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type); 619 620 /* 621 * Name: turnstile_has_waiters 622 * 623 * Description: returns if there are waiters on the turnstile 624 * 625 * Arg1: turnstile: turnstile 626 * 627 * Returns: TRUE if there are waiters, FALSE otherwise. 628 */ 629 630 boolean_t 631 turnstile_has_waiters(struct turnstile *turnstile); 632 633 /* 634 * Name: turnstile_stats_update 635 * 636 * Description: Function to update turnstile stats for dev kernel. 637 * 638 * Arg1: hops : number of thread hops in priority propagation 639 * Arg2: flags : turnstile stats update flags 640 * Arg3: inheritor: inheritor 641 * 642 * Returns: Nothing 643 */ 644 void 645 turnstile_stats_update( 646 int hop __assert_only, 647 turnstile_stats_update_flags_t flags __assert_only, 648 turnstile_inheritor_t inheritor __assert_only); 649 650 #if DEVELOPMENT || DEBUG 651 652 #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1 653 #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2 654 #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3 655 #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4 656 657 /* Functions used by debug test primitive exported by sysctls */ 658 int 659 tstile_test_prim_lock(int val); 660 661 int 662 tstile_test_prim_unlock(int val); 663 664 int 665 turnstile_get_boost_stats_sysctl(void *req); 666 int 667 turnstile_get_unboost_stats_sysctl(void *req); 668 #endif /* DEVELOPMENT || DEBUG */ 669 #endif /* XNU_KERNEL_PRIVATE */ 670 671 /* Interface */ 672 673 /* 674 * Name: turnstile_hash_bucket_lock 675 * 676 * Description: locks the spinlock associated with proprietor's bucket. 677 * if proprietor is specified the index for the hash will be 678 * recomputed and returned in index_proprietor, 679 * otherwise the value save in index_proprietor is used as index. 680 * 681 * Args: 682 * Arg1: proprietor (key) for hashing 683 * Arg2: index for proprietor in the hash 684 * Arg3: turnstile type 685 * 686 * Returns: old value of irq if irq were disabled before acquiring the lock. 687 */ 688 unsigned 689 turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type); 690 691 /* 692 * Name: turnstile_hash_bucket_unlock 693 * 694 * Description: unlocks the spinlock associated with proprietor's bucket. 695 * if proprietor is specified the index for the hash will be 696 * recomputed and returned in index_proprietor, 697 * otherwise the value save in index_proprietor is used as index. 698 * 699 * Args: 700 * Arg1: proprietor (key) for hashing 701 * Arg2: index for proprietor in the hash 702 * Arg3: turnstile type 703 * Arg4: irq value returned by turnstile_hash_bucket_lock 704 * 705 */ 706 void 707 turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s); 708 709 /* 710 * Name: turnstile_prepare 711 * 712 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 713 * Function is called holding the interlock (spinlock) of the primitive. 714 * The turnstile returned by this function is safe to use untill the thread calls turnstile_complete. 715 * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to 716 * it untill it calls turnstile_complete. 717 * 718 * Args: 719 * Arg1: proprietor 720 * Arg2: pointer in primitive struct to store turnstile 721 * Arg3: turnstile to use instead of taking it from thread. 722 * Arg4: type of primitive 723 * 724 * Returns: 725 * turnstile. 726 */ 727 struct turnstile * 728 turnstile_prepare( 729 uintptr_t proprietor, 730 struct turnstile **tstore, 731 struct turnstile *turnstile, 732 turnstile_type_t type); 733 734 /* 735 * Name: turnstile_complete 736 * 737 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 738 * Function is called holding the interlock (spinlock) of the primitive. 739 * Current thread will have a turnstile attached to it after this call. 740 * 741 * Args: 742 * Arg1: proprietor 743 * Arg2: pointer in primitive struct to update turnstile 744 * Arg3: pointer to store the returned turnstile instead of attaching it to thread 745 * Arg4: type of primitive 746 * 747 * Returns: 748 * None. 749 */ 750 void 751 turnstile_complete( 752 uintptr_t proprietor, 753 struct turnstile **tstore, 754 struct turnstile **turnstile, 755 turnstile_type_t type); 756 757 /* 758 * Name: turnstile_update_inheritor 759 * 760 * Description: Update the inheritor of the turnstile and boost the 761 * inheritor. It will take a thread reference on the inheritor. 762 * Called with the interlock of the primitive held. 763 * 764 * Args: 765 * Arg1: turnstile 766 * Arg2: inheritor 767 * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait 768 * 769 * Returns: 770 * old inheritor reference is stashed on current thread's struct. 771 */ 772 void 773 turnstile_update_inheritor( 774 struct turnstile *turnstile, 775 turnstile_inheritor_t new_inheritor, 776 turnstile_update_flags_t flags); 777 778 typedef enum turnstile_update_complete_flags { 779 TURNSTILE_INTERLOCK_NOT_HELD = 0x1, 780 TURNSTILE_INTERLOCK_HELD = 0x2, 781 } turnstile_update_complete_flags_t; 782 783 /* 784 * Name: turnstile_update_inheritor_complete 785 * 786 * Description: Update turnstile inheritor's priority and propagate the 787 * priority if the inheritor is blocked on a turnstile. 788 * Consumes thread ref of old inheritor returned by 789 * turnstile_update_inheritor. Recursive priority update 790 * will only happen when called with interlock dropped. 791 * 792 * Args: 793 * Arg1: turnstile 794 * Arg2: interlock held 795 * 796 * Returns: None. 797 */ 798 void 799 turnstile_update_inheritor_complete( 800 struct turnstile *turnstile, 801 turnstile_update_complete_flags_t flags); 802 803 804 /* 805 * Name: turnstile_kernel_update_inheritor_on_wake_locked 806 * 807 * Description: Set thread as the inheritor of the turnstile and 808 * boost the inheritor. 809 * Args: 810 * Arg1: turnstile 811 * Arg2: new_inheritor 812 * Arg3: flags 813 * 814 * Called with turnstile locked 815 */ 816 void 817 turnstile_kernel_update_inheritor_on_wake_locked( 818 struct turnstile *turnstile, 819 turnstile_inheritor_t new_inheritor, 820 turnstile_update_flags_t flags); 821 822 /* 823 * Internal KPI for sleep_with_inheritor, wakeup_with_inheritor, change_sleep_inheritor 824 * meant to allow specifing the turnstile type to use to have different policy 825 * on how to push on the inheritor. 826 * 827 * Differently from the "standard" KPI in locks.h these are meant to be used only 828 * if you know what you are doing with turnstile. 829 */ 830 831 extern wait_result_t 832 lck_mtx_sleep_with_inheritor_and_turnstile_type(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type); 833 834 extern wait_result_t 835 lck_rw_sleep_with_inheritor_and_turnstile_type(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type); 836 837 extern kern_return_t 838 wakeup_with_inheritor_and_turnstile_type(event_t event, turnstile_type_t type, wait_result_t result, bool wake_one, lck_wake_action_t action, thread_t *thread_wokenup); 839 840 extern kern_return_t 841 change_sleep_inheritor_and_turnstile_type(event_t event, thread_t inheritor, turnstile_type_t type); 842 843 #endif /* KERNEL_PRIVATE */ 844 #if XNU_KERNEL_PRIVATE 845 846 struct workqueue; 847 848 /* pthread_workqueue.c */ 849 extern void workq_reference(struct workqueue *wq); 850 extern void workq_deallocate_safe(struct workqueue *wq); 851 extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); 852 extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, 853 bool locked); 854 855 #endif /* XNU_KERNEL_PRIVATE */ 856 857 #endif /* _TURNSTILE_H_ */ 858