1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _TURNSTILE_H_ 30 #define _TURNSTILE_H_ 31 32 #include <mach/mach_types.h> 33 #include <mach/kern_return.h> 34 #include <sys/cdefs.h> 35 36 __BEGIN_DECLS 37 38 #if PRIVATE 39 #define TURNSTILE_MAX_HOP_DEFAULT (10) 40 struct turnstile_stats { 41 uint64_t ts_priority_propagation; 42 uint64_t ts_no_inheritor; 43 uint64_t ts_thread_runnable; 44 uint64_t ts_no_priority_change_required; 45 uint64_t ts_above_ui_pri_change; 46 uint64_t ts_no_turnstile; 47 }; 48 #endif 49 50 #ifdef KERNEL_PRIVATE 51 #include <kern/queue.h> 52 #include <sys/queue.h> 53 #include <kern/waitq.h> 54 #include <kern/priority_queue.h> 55 #include <os/refcnt.h> 56 #include <kern/assert.h> 57 #include <kern/kern_types.h> 58 #include <kern/mpsc_queue.h> 59 #include <kern/locks.h> 60 61 /* 62 * turnstile_type_t : Indicates the type of primitive the turnstile is associated with 63 * Please populate turnstile_promote_policy array if a new type is added here. 64 */ 65 typedef enum __attribute__((packed)) turnstile_type { 66 TURNSTILE_NONE = 0, 67 TURNSTILE_KERNEL_MUTEX = 1, 68 TURNSTILE_ULOCK = 2, 69 TURNSTILE_PTHREAD_MUTEX = 3, 70 TURNSTILE_SYNC_IPC = 4, 71 TURNSTILE_WORKLOOPS = 5, 72 TURNSTILE_WORKQS = 6, 73 TURNSTILE_KNOTE = 7, 74 TURNSTILE_SLEEP_INHERITOR = 8, 75 TURNSTILE_TOTAL_TYPES = 9, 76 } turnstile_type_t; 77 78 /* 79 * For each type of turnstile, following are the type of 80 * inheritors passed: 81 * 82 * TURNSTILE_KERNEL_MUTEX 83 * Interlock: kernel mutex interlock. 84 * Inheritor: threads. 85 * Lock order: turnstile lock, thread lock. 86 * 87 * TURNSTILE_ULOCK 88 * Interlock: ulocks interlock. 89 * Inheritor: threads. 90 * Lock order: turnstile lock, thread lock. 91 * 92 * TURNSTILE_PTHREAD_MUTEX 93 * Interlock: pthread mtx interlock. 94 * Inheritor: threads. 95 * Lock order: turnstile lock, thread lock. 96 * 97 * TURNSTILE_SYNC_IPC 98 * Interlock: port's mqueue lock 99 * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. 100 * Lock order: Our turnstile, then turnstile of the port we are enqueued in. 101 * Port circularity will make sure there is never a cycle formation 102 * and lock order is maintained. 103 * 104 * TURNSTILE_WORKLOOPS 105 * Interlock: 106 * - kq req lock 107 * - wq lock when "filt_wlworkq_interlock_needed() is true" 108 * Inheritor: thread, turnstile (of workq) 109 * Lock order: turnstile lock, thread lock 110 * WL turnstile lock, Workq turnstile lock 111 * 112 * TURNSTILE_WORKQS 113 * Interlock: workqueue lock 114 * Inheritor: thread 115 * Lock order: turnstile lock, thread lock. 116 * 117 * TURNSTILE_KNOTE 118 * Interlock: the knote lock 119 * Inheritor: WL turnstile 120 * 121 * TURNSTILE_SLEEP_INHERITOR 122 * Interlock: turnstile_htable bucket spinlock. 123 * Inheritor: threads. 124 * Lock order: turnstile lock, thread lock. 125 * 126 */ 127 128 typedef enum __attribute__((flag_enum)) turnstile_promote_policy { 129 TURNSTILE_PROMOTE_NONE = 0, 130 TURNSTILE_KERNEL_PROMOTE = 0x1, 131 TURNSTILE_USER_PROMOTE = 0x2, 132 TURNSTILE_USER_IPC_PROMOTE = 0x4, 133 } turnstile_promote_policy_t; 134 135 typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy { 136 TURNSTILE_HASH_LOCK_POLICY_NONE = 0, 137 TURNSTILE_IRQ_UNSAFE_HASH = 0x1, 138 TURNSTILE_LOCKED_HASH = 0x2, 139 } turnstile_hash_lock_policy_t; 140 141 /* 142 * Turnstile state flags 143 * 144 * The turnstile state flags represent the current ownership of a turnstile. 145 * The supported flags are: 146 * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread 147 * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile 148 * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive 149 * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor 150 * 151 * The flag updates are done while holding the primitive interlock. 152 * */ 153 154 #define TURNSTILE_STATE_THREAD 0x1 155 #define TURNSTILE_STATE_FREELIST 0x2 156 #define TURNSTILE_STATE_HASHTABLE 0x4 157 #define TURNSTILE_STATE_PROPRIETOR 0x8 158 159 /* Helper macros to set/unset turnstile state flags */ 160 #if DEVELOPMENT || DEBUG 161 162 #define turnstile_state_init(ts, state) \ 163 MACRO_BEGIN \ 164 ts->ts_state = state; \ 165 MACRO_END 166 167 #define turnstile_state_add(ts, state) \ 168 MACRO_BEGIN \ 169 assert((ts->ts_state & (state)) == 0); \ 170 ts->ts_state |= state; \ 171 MACRO_END 172 173 #define turnstile_state_remove(ts, state) \ 174 MACRO_BEGIN \ 175 assert(ts->ts_state & (state)); \ 176 ts->ts_state &= ~(state); \ 177 MACRO_END 178 179 #else /* DEVELOPMENT || DEBUG */ 180 181 #define turnstile_state_init(ts, state) \ 182 MACRO_BEGIN \ 183 (void)ts; \ 184 MACRO_END 185 186 #define turnstile_state_add(ts, state) \ 187 MACRO_BEGIN \ 188 (void)ts; \ 189 MACRO_END 190 191 #define turnstile_state_remove(ts, state) \ 192 MACRO_BEGIN \ 193 (void)ts; \ 194 MACRO_END 195 196 #endif /* DEVELOPMENT || DEBUG */ 197 198 struct knote; 199 struct turnstile; 200 201 /* 202 * Turnstile update flags 203 * 204 * TURNSTILE_IMMEDIATE_UPDATE 205 * When passed to turnstile_update_inheritor 206 * update the inheritor of the turnstile in 207 * the same call. 208 * 209 * TURNSTILE_DELAYED_UPDATE 210 * When passed to turnstile_update_inheritor 211 * it stashed the inheritor on the thread and 212 * turnstile's inheritor is updated in 213 * assert wait. 214 * 215 * TURNSTILE_INHERITOR_THREAD 216 * The turnstile inheritor is of type thread. 217 * 218 * TURNSTILE_INHERITOR_TURNSTILE 219 * The turnstile inheritor is of type turnstile. 220 * 221 * TURNSTILE_INHERITOR_WORKQ 222 * The turnstile inheritor is of type workqueue 223 * 224 * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE 225 * The inheritor needs a chain priority update. 226 * 227 * TURNSTILE_NEEDS_PRI_UPDATE 228 * Current turnstile needs a chain priority update. 229 * 230 * Locking order for passing thread and turnstile as inheritor 231 * 232 * Thread as an inheritor: 233 * When thread is passed as an inheritor of a turnstile 234 * turnstile lock is taken and then thread lock. 235 * 236 * Turnstile as in inheritor: 237 * When turnstile (T1) is passed as an inheritor of 238 * a turnstile (T2), turnstile lock of T2 is taken 239 * and then turnstile lock of T1 is taken. 240 * 241 * Caution: While passing turnstile as an inheritor, its 242 * job of the adopter to make sure that there is no 243 * lock inversion. 244 */ 245 typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { 246 TURNSTILE_UPDATE_FLAGS_NONE = 0, 247 TURNSTILE_IMMEDIATE_UPDATE = 0x1, 248 TURNSTILE_DELAYED_UPDATE = 0x2, 249 TURNSTILE_INHERITOR_THREAD = 0x4, 250 TURNSTILE_INHERITOR_TURNSTILE = 0x8, 251 TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, 252 TURNSTILE_NEEDS_PRI_UPDATE = 0x20, 253 TURNSTILE_INHERITOR_WORKQ = 0x40, 254 TURNSTILE_UPDATE_BOOST = 0x80, 255 } turnstile_update_flags_t; 256 257 #define TURNSTILE_NULL ((struct turnstile *)0) 258 259 typedef void * turnstile_inheritor_t; 260 261 #define TURNSTILE_INHERITOR_NULL NULL 262 263 #ifdef XNU_KERNEL_PRIVATE 264 #pragma GCC visibility push(hidden) 265 266 /* Turnstile stats update flags 267 * 268 * TSU_TURNSTILE_BLOCK_COUNT 269 * thread blocking on turnstile waitq, increment global 270 * thread block on turnstile count. 271 * 272 * TSU_REGULAR_WAITQ_BLOCK_COUNT 273 * thread blocking on regular waitq, increment global 274 * thread block on regular waitq count. 275 * 276 * TSU_PRI_PROPAGATION 277 * turnstile propagation update stopped at nth hop, update 278 * priority change count for nth element in stats array. 279 * 280 * TSU_NO_INHERITOR 281 * turnstile propagation update stopped due to turnstile 282 * not having an inheritor after nth hop, update the no 283 * inheritor count for nth element in the stats array. 284 * 285 * TSU_NO_TURNSTILE 286 * turnstile propagation update stopped due to thread 287 * not blocked on a turnstile waitq after nth hop, update 288 * the no turnstile count for the nth element in the stats 289 * array. 290 * 291 * TSU_NO_PRI_CHANGE_NEEDED 292 * turnstile propagation update stopped due to thread or 293 * turnstile having the correct priority or not blocked. 294 * update the no priority change count for the nth element 295 * in the stats array. 296 * 297 * TSU_THREAD_RUNNABLE 298 * turnstile propagation update stopped due to thread 299 * being runnable, update the thread runnable count for 300 * the nth element in the stats array. 301 * 302 * TSU_ABOVE_UI_PRI_CHANGE 303 * turnstile propagation caused an above UI priority change. 304 */ 305 typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { 306 TSU_FLAGS_NONE = 0, 307 TSU_TURNSTILE_BLOCK_COUNT = 0x1, 308 TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, 309 TSU_PRI_PROPAGATION = 0x4, 310 TSU_NO_INHERITOR = 0x8, 311 TSU_NO_TURNSTILE = 0x10, 312 TSU_NO_PRI_CHANGE_NEEDED = 0x20, 313 TSU_THREAD_RUNNABLE = 0x40, 314 TSU_ABOVE_UI_PRI_CHANGE = 0x80, 315 TSU_THREAD_ARG = 0x100, 316 TSU_TURNSTILE_ARG = 0x200, 317 TSU_BOOST_ARG = 0x400, 318 } turnstile_stats_update_flags_t; 319 320 SLIST_HEAD(turnstile_list, turnstile); 321 322 #define CTSID_BITS 20 323 #define CTSID_MASK ((1u << CTSID_BITS) - 1) 324 #define CTSID_MAX (CTSID_MASK - 1) 325 326 struct turnstile { 327 union { 328 /* 329 * The waitq_eventmask field is only used on the global queues. 330 * We hence repurpose all those bits for our own use. 331 */ 332 #if MACH_KERNEL_PRIVATE 333 WAITQ_FLAGS(ts_waitq 334 , __ts_unused_bits: 7 335 , ts_compact_id: CTSID_BITS); 336 #endif 337 struct waitq ts_waitq; /* waitq embedded in turnstile */ 338 }; 339 #define ts_inheritor ts_waitq.waitq_inheritor /* thread/turnstile inheriting the priority (IL, WL) */ 340 union { 341 struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ 342 SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ 343 struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */ 344 }; 345 struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ 346 struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */ 347 SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ 348 uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ 349 os_ref_atomic_t ts_refcount; /* reference count for turnstiles */ 350 _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ 351 uint32_t ts_prim_count; /* counter used by the primitive */ 352 turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ 353 uint8_t ts_priority; /* priority of turnstile (WL) */ 354 355 #if DEVELOPMENT || DEBUG 356 uint8_t ts_state; /* current state of turnstile (IL) */ 357 thread_t ts_thread; /* thread the turnstile is attached to */ 358 thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ 359 #endif 360 }; 361 362 #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) 363 364 /* IL - interlock, WL - turnstile lock i.e. waitq lock */ 365 366 #define TURNSTILE_PROPRIETOR_NULL 0ul 367 368 /* 369 * Name: turnstiles_init 370 * 371 * Description: Initialize turnstile sub system. 372 * 373 * Args: None. 374 * 375 * Returns: None. 376 */ 377 void 378 turnstiles_init(void); 379 380 /* 381 * Name: turnstile_alloc 382 * 383 * Description: Allocate a turnstile. 384 * 385 * Args: None. 386 * 387 * Returns: 388 * turnstile on Success. 389 */ 390 struct turnstile * 391 turnstile_alloc(void); 392 393 /* 394 * Name: turnstile_compact_id_get() 395 * 396 * Description: Allocate a compact turnstile ID slot. 397 * 398 * Args: None. 399 * 400 * Returns: 401 * A non 0 compact compact turnstile ID. 402 */ 403 uint32_t 404 turnstile_compact_id_get(void); 405 406 /* 407 * Name: turnstile_compact_id_put() 408 * 409 * Description: Frees a compact turnstile ID slot. 410 * 411 * Args: 412 * Args1: the compact ID to free. 413 */ 414 void 415 turnstile_compact_id_put(uint32_t cid); 416 417 /* 418 * Name: turnstile_get_by_id 419 * 420 * Description: Resolve a turnstile by compact ID 421 * 422 * Args: 423 * Arg1: turnstile compact ID 424 * 425 * Returns: a turnstile 426 */ 427 struct turnstile * 428 turnstile_get_by_id(uint32_t tsid); 429 430 /* 431 * Name: turnstile_reference 432 * 433 * Description: Take a reference on the turnstile. 434 * 435 * Arg1: turnstile 436 * 437 * Returns: None. 438 */ 439 void 440 turnstile_reference(struct turnstile *turnstile); 441 442 /* 443 * Name: turnstile_deallocate 444 * 445 * Description: Drop a reference on the turnstile. 446 * Destroy the turnstile if the last ref. 447 * 448 * Arg1: turnstile 449 * 450 * Returns: None. 451 */ 452 void 453 turnstile_deallocate(struct turnstile *turnstile); 454 455 /* 456 * Name: turnstile_waitq_add_thread_priority_queue 457 * 458 * Description: add thread to the turnstile waitq 459 * 460 * Arg1: waitq 461 * Arg2: thread 462 * 463 * Conditions: waitq locked 464 */ 465 void 466 turnstile_waitq_add_thread_priority_queue( 467 struct waitq* wq, 468 thread_t thread); 469 470 /* 471 * Name: turnstile_deallocate_safe 472 * 473 * Description: Drop a reference on the turnstile safely without triggering zfree. 474 * 475 * Arg1: turnstile 476 * 477 * Returns: None. 478 */ 479 void 480 turnstile_deallocate_safe(struct turnstile *turnstile); 481 482 /* 483 * Name: turnstile_recompute_priority_locked 484 * 485 * Description: Update turnstile priority based 486 * on highest waiter thread and highest blocking 487 * turnstile. 488 * 489 * Args: turnstile 490 * 491 * Returns: TRUE: if the turnstile priority changed and needs propagation. 492 * FALSE: if the turnstile priority did not change or it does not need propagation. 493 * 494 * Condition: turnstile locked 495 */ 496 boolean_t 497 turnstile_recompute_priority_locked( 498 struct turnstile *turnstile); 499 500 /* 501 * Name: turnstile_recompute_priority 502 * 503 * Description: Update turnstile priority based 504 * on highest waiter thread and highest blocking 505 * turnstile. 506 * 507 * Args: turnstile 508 * 509 * Returns: TRUE: if the turnstile priority changed and needs propagation. 510 * FALSE: if the turnstile priority did not change or it does not need propagation. 511 */ 512 boolean_t 513 turnstile_recompute_priority( 514 struct turnstile *turnstile); 515 516 /* 517 * Name: turnstile_workq_proprietor_of_max_turnstile 518 * 519 * Description: Returns the highest priority and proprietor of a turnstile 520 * pushing on a workqueue turnstile. 521 * 522 * This will not return waiters that are at priority 523 * MAXPRI_THROTTLE or lower. 524 * 525 * Args: turnstile 526 * 527 * Returns: 528 * Priority of the max entry, or 0 529 * Pointer to the max entry proprietor 530 */ 531 int 532 turnstile_workq_proprietor_of_max_turnstile( 533 struct turnstile *turnstile, 534 uintptr_t *proprietor); 535 536 /* 537 * Name: turnstile_workloop_pusher_info 538 * 539 * Description: Returns the priority of the turnstile push for a workloop, 540 * and the thread or knote responsible for this push. 541 * 542 * Args: workloop turnstile 543 * 544 * Returns: 545 * Priority of the push or 0 546 * Thread (with a +1 reference) with that push or THREAD_NULL. 547 * Port (with a +1 reference) with that push, or IP_NULL. 548 * Sync IPC knote with the highest push (or NULL) 549 */ 550 int 551 turnstile_workloop_pusher_info( 552 struct turnstile *turnstile, 553 thread_t *thread, 554 ipc_port_t *port, 555 struct knote **knote_out); 556 557 /* 558 * Name: turnstile_cleanup 559 * 560 * Description: Update priority of a turnstile inheritor 561 * if needed. 562 * 563 * Args: inheritor and flags passed on thread struct. 564 * 565 * Returns: None. 566 */ 567 void 568 turnstile_cleanup(void); 569 570 /* 571 * Name: turnstile_update_thread_priority_chain 572 * 573 * Description: Priority of a thread blocked on a turnstile 574 * has changed, update the turnstile priority. 575 * 576 * Arg1: thread: thread whose priority has changed. 577 * 578 * Returns: None. 579 */ 580 void 581 turnstile_update_thread_priority_chain(thread_t thread); 582 583 /* 584 * Name: turnstile_update_inheritor_locked 585 * 586 * Description: Update the inheritor of the turnstile and boost the 587 * inheritor, called with turnstile locked. 588 * 589 * Args: 590 * Arg1: turnstile 591 * Implicit arg: new inheritor value is stashed in current thread's struct 592 * 593 * Returns: 594 * old inheritor reference is returned on current thread's struct. 595 */ 596 void 597 turnstile_update_inheritor_locked(struct turnstile *turnstile); 598 599 /* 600 * Name: thread_get_inheritor_turnstile_base_priority 601 * 602 * Description: Get the max base priority of all the inheritor turnstiles 603 * 604 * Arg1: thread 605 * 606 * Returns: Max base priority of all the inheritor turnstiles. 607 * 608 * Condition: thread locked 609 */ 610 int 611 thread_get_inheritor_turnstile_base_priority(thread_t thread); 612 613 /* 614 * Name: thread_get_inheritor_turnstile_sched_priority 615 * 616 * Description: Get the max sched priority of all the inheritor turnstiles 617 * 618 * Arg1: thread 619 * 620 * Returns: Max sched priority of all the inheritor turnstiles. 621 * 622 * Condition: thread locked 623 */ 624 int 625 thread_get_inheritor_turnstile_sched_priority(thread_t thread); 626 627 /* 628 * Name: thread_get_waiting_turnstile 629 * 630 * Description: Get the turnstile if the thread is waiting on a turnstile. 631 * 632 * Arg1: thread 633 * 634 * Returns: turnstile: if the thread is blocked on a turnstile. 635 * TURNSTILE_NULL: otherwise. 636 * 637 * Condition: thread locked. 638 */ 639 struct turnstile * 640 thread_get_waiting_turnstile(thread_t thread); 641 642 /* 643 * Name: turnstile_lookup_by_proprietor 644 * 645 * Description: Get turnstile for a proprietor from global 646 * turnstile hash. 647 * 648 * Arg1: port 649 * Arg2: turnstile_type_t type 650 * 651 * Returns: turnstile: if the proprietor has a turnstile. 652 * TURNSTILE_NULL: otherwise. 653 * 654 * Condition: proprietor interlock held. 655 */ 656 struct turnstile * 657 turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type); 658 659 /* 660 * Name: turnstile_has_waiters 661 * 662 * Description: returns if there are waiters on the turnstile 663 * 664 * Arg1: turnstile: turnstile 665 * 666 * Returns: TRUE if there are waiters, FALSE otherwise. 667 */ 668 669 boolean_t 670 turnstile_has_waiters(struct turnstile *turnstile); 671 672 /* 673 * Name: turnstile_stats_update 674 * 675 * Description: Function to update turnstile stats for dev kernel. 676 * 677 * Arg1: hops : number of thread hops in priority propagation 678 * Arg2: flags : turnstile stats update flags 679 * Arg3: inheritor: inheritor 680 * 681 * Returns: Nothing 682 */ 683 void 684 turnstile_stats_update( 685 int hop __assert_only, 686 turnstile_stats_update_flags_t flags __assert_only, 687 turnstile_inheritor_t inheritor __assert_only); 688 689 #if DEVELOPMENT || DEBUG 690 691 #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1 692 #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2 693 #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3 694 #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4 695 696 /* Functions used by debug test primitive exported by sysctls */ 697 int 698 tstile_test_prim_lock(int val); 699 700 int 701 tstile_test_prim_unlock(int val); 702 703 int 704 turnstile_get_boost_stats_sysctl(void *req); 705 int 706 turnstile_get_unboost_stats_sysctl(void *req); 707 #endif /* DEVELOPMENT || DEBUG */ 708 709 #pragma GCC visibility pop 710 #endif /* XNU_KERNEL_PRIVATE */ 711 712 /* Interface */ 713 714 /* 715 * Name: turnstile_hash_bucket_lock 716 * 717 * Description: locks the spinlock associated with proprietor's bucket. 718 * if proprietor is specified the index for the hash will be 719 * recomputed and returned in index_proprietor, 720 * otherwise the value save in index_proprietor is used as index. 721 * 722 * Args: 723 * Arg1: proprietor (key) for hashing 724 * Arg2: index for proprietor in the hash 725 * Arg3: turnstile type 726 * 727 * Returns: old value of irq if irq were disabled before acquiring the lock. 728 */ 729 unsigned 730 turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type); 731 732 /* 733 * Name: turnstile_hash_bucket_unlock 734 * 735 * Description: unlocks the spinlock associated with proprietor's bucket. 736 * if proprietor is specified the index for the hash will be 737 * recomputed and returned in index_proprietor, 738 * otherwise the value save in index_proprietor is used as index. 739 * 740 * Args: 741 * Arg1: proprietor (key) for hashing 742 * Arg2: index for proprietor in the hash 743 * Arg3: turnstile type 744 * Arg4: irq value returned by turnstile_hash_bucket_lock 745 * 746 */ 747 void 748 turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s); 749 750 /* 751 * Name: turnstile_prepare 752 * 753 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 754 * Function is called holding the interlock (spinlock) of the primitive. 755 * The turnstile returned by this function is safe to use until 756 * the thread calls turnstile_complete. 757 * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to 758 * it until it calls turnstile_complete. 759 * 760 * Args: 761 * Arg1: proprietor 762 * Arg2: pointer in primitive struct to store turnstile 763 * Arg3: turnstile to use instead of taking it from thread. 764 * Arg4: type of primitive 765 * 766 * Returns: 767 * turnstile. 768 */ 769 struct turnstile * 770 turnstile_prepare( 771 uintptr_t proprietor, 772 struct turnstile **tstore, 773 struct turnstile *turnstile, 774 turnstile_type_t type); 775 776 /* 777 * Name: turnstile_complete 778 * 779 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 780 * Function is called holding the interlock (spinlock) of the primitive. 781 * Current thread will have a turnstile attached to it after this call. 782 * 783 * Args: 784 * Arg1: proprietor 785 * Arg2: pointer in primitive struct to update turnstile 786 * Arg3: pointer to store the returned turnstile instead of attaching it to thread 787 * Arg4: type of primitive 788 * 789 * Returns: 790 * None. 791 */ 792 void 793 turnstile_complete( 794 uintptr_t proprietor, 795 struct turnstile **tstore, 796 struct turnstile **turnstile, 797 turnstile_type_t type); 798 799 /* 800 * Name: turnstile_prepare_compact_id 801 * 802 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 803 * Function is called holding the interlock (spinlock) of the primitive. 804 * The turnstile returned by this function is safe to use until 805 * the thread calls turnstile_complete_compact_id. 806 * The calling thread will not have a turnstile attached to 807 * it until it calls turnstile_complete_compact_id. 808 * 809 * Args: 810 * Arg1: proprietor 811 * Arg2: the current compact ID for the turnstile head 812 * Arg3: type of primitive 813 * 814 * Returns: 815 * turnstile. 816 */ 817 struct turnstile * 818 turnstile_prepare_compact_id( 819 uintptr_t proprietor, 820 uint32_t compact_id, 821 turnstile_type_t type); 822 823 /* 824 * Name: turnstile_complete_compact_id 825 * 826 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 827 * Function is called holding the interlock (spinlock) of the primitive. 828 * Current thread will have a turnstile attached to it after this call. 829 * 830 * Args: 831 * Arg1: proprietor 832 * Arg2: the turnstile pointer that was returned by turnstile_prepare_compact_id() 833 * Arg3: type of primitive 834 * 835 * Returns: 836 * Whether the primitive no longer has a turnstile. 837 */ 838 bool 839 turnstile_complete_compact_id( 840 uintptr_t proprietor, 841 struct turnstile *turnstile, 842 turnstile_type_t type); 843 844 /* 845 * Name: turnstile_prepare_hash 846 * 847 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 848 * Function is called holding the interlock (spinlock) of the primitive. 849 * The turnstile returned by this function is safe to use until 850 * the thread calls turnstile_complete_hash. 851 * The calling thread will not have a turnstile attached to 852 * it until it calls turnstile_complete_hash. 853 * 854 * The turnstile used for this proprietor will be stored in 855 * a global hash table. 856 * 857 * Args: 858 * Arg1: proprietor 859 * Arg3: type of primitive 860 * 861 * Returns: 862 * turnstile. 863 */ 864 struct turnstile * 865 turnstile_prepare_hash( 866 uintptr_t proprietor, 867 turnstile_type_t type); 868 869 /* 870 * Name: turnstile_complete_hash 871 * 872 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 873 * Function is called holding the interlock (spinlock) of the primitive. 874 * Current thread will have a turnstile attached to it after this call. 875 * 876 * Args: 877 * Arg1: proprietor 878 * Arg3: type of primitive 879 * 880 * Returns: 881 * None. 882 */ 883 void 884 turnstile_complete_hash( 885 uintptr_t proprietor, 886 turnstile_type_t type); 887 888 /* 889 * Name: turnstile_update_inheritor 890 * 891 * Description: Update the inheritor of the turnstile and boost the 892 * inheritor. It will take a thread reference on the inheritor. 893 * Called with the interlock of the primitive held. 894 * 895 * Args: 896 * Arg1: turnstile 897 * Arg2: inheritor 898 * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait 899 * 900 * Returns: 901 * old inheritor reference is stashed on current thread's struct. 902 */ 903 void 904 turnstile_update_inheritor( 905 struct turnstile *turnstile, 906 turnstile_inheritor_t new_inheritor, 907 turnstile_update_flags_t flags); 908 909 typedef enum turnstile_update_complete_flags { 910 TURNSTILE_INTERLOCK_NOT_HELD = 0x1, 911 TURNSTILE_INTERLOCK_HELD = 0x2, 912 } turnstile_update_complete_flags_t; 913 914 /* 915 * Name: turnstile_update_inheritor_complete 916 * 917 * Description: Update turnstile inheritor's priority and propagate the 918 * priority if the inheritor is blocked on a turnstile. 919 * Consumes thread ref of old inheritor returned by 920 * turnstile_update_inheritor. Recursive priority update 921 * will only happen when called with interlock dropped. 922 * 923 * Args: 924 * Arg1: turnstile 925 * Arg2: interlock held 926 * 927 * Returns: None. 928 */ 929 void 930 turnstile_update_inheritor_complete( 931 struct turnstile *turnstile, 932 turnstile_update_complete_flags_t flags); 933 934 935 /* 936 * Name: turnstile_kernel_update_inheritor_on_wake_locked 937 * 938 * Description: Set thread as the inheritor of the turnstile and 939 * boost the inheritor. 940 * Args: 941 * Arg1: turnstile 942 * Arg2: new_inheritor 943 * Arg3: flags 944 * 945 * Called with turnstile locked 946 */ 947 void 948 turnstile_kernel_update_inheritor_on_wake_locked( 949 struct turnstile *turnstile, 950 turnstile_inheritor_t new_inheritor, 951 turnstile_update_flags_t flags); 952 953 #endif /* KERNEL_PRIVATE */ 954 #if XNU_KERNEL_PRIVATE 955 956 struct workqueue; 957 958 /* pthread_workqueue.c */ 959 extern void workq_reference(struct workqueue *wq); 960 extern void workq_deallocate_safe(struct workqueue *wq); 961 extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); 962 extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, 963 bool locked); 964 965 #endif /* XNU_KERNEL_PRIVATE */ 966 967 __END_DECLS 968 969 #endif /* _TURNSTILE_H_ */ 970