1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _TURNSTILE_H_ 30 #define _TURNSTILE_H_ 31 32 #include <mach/mach_types.h> 33 #include <mach/kern_return.h> 34 #include <sys/cdefs.h> 35 36 __BEGIN_DECLS 37 38 #if PRIVATE 39 #define TURNSTILE_MAX_HOP_DEFAULT (10) 40 struct turnstile_stats { 41 uint64_t ts_priority_propagation; 42 uint64_t ts_no_inheritor; 43 uint64_t ts_thread_runnable; 44 uint64_t ts_no_priority_change_required; 45 uint64_t ts_above_ui_pri_change; 46 uint64_t ts_no_turnstile; 47 }; 48 #endif 49 50 #ifdef KERNEL_PRIVATE 51 #include <kern/queue.h> 52 #include <sys/queue.h> 53 #include <kern/waitq.h> 54 #include <kern/priority_queue.h> 55 #include <os/refcnt.h> 56 #include <kern/assert.h> 57 #include <kern/kern_types.h> 58 #include <kern/mpsc_queue.h> 59 #include <kern/locks.h> 60 61 /* 62 * turnstile_type_t : Indicates the type of primitive the turnstile is associated with 63 * Please populate turnstile_promote_policy array if a new type is added here. 64 */ 65 typedef enum __attribute__((packed)) turnstile_type { 66 TURNSTILE_NONE = 0, 67 TURNSTILE_KERNEL_MUTEX = 1, 68 TURNSTILE_ULOCK = 2, 69 TURNSTILE_PTHREAD_MUTEX = 3, 70 TURNSTILE_SYNC_IPC = 4, 71 TURNSTILE_WORKLOOPS = 5, 72 TURNSTILE_WORKQS = 6, 73 TURNSTILE_KNOTE = 7, 74 TURNSTILE_SLEEP_INHERITOR = 8, 75 TURNSTILE_EPOCH = 9, 76 TURNSTILE_TOTAL_TYPES = 10, 77 } turnstile_type_t; 78 79 /* 80 * For each type of turnstile, following are the type of 81 * inheritors passed: 82 * 83 * TURNSTILE_KERNEL_MUTEX 84 * Interlock: kernel mutex interlock. 85 * Inheritor: threads. 86 * Lock order: turnstile lock, thread lock. 87 * 88 * TURNSTILE_ULOCK 89 * Interlock: ulocks interlock. 90 * Inheritor: threads. 91 * Lock order: turnstile lock, thread lock. 92 * 93 * TURNSTILE_PTHREAD_MUTEX 94 * Interlock: pthread mtx interlock. 95 * Inheritor: threads. 96 * Lock order: turnstile lock, thread lock. 97 * 98 * TURNSTILE_SYNC_IPC 99 * Interlock: port's mqueue lock 100 * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. 101 * Lock order: Our turnstile, then turnstile of the port we are enqueued in. 102 * Port circularity will make sure there is never a cycle formation 103 * and lock order is maintained. 104 * 105 * TURNSTILE_WORKLOOPS 106 * Interlock: 107 * - kq req lock 108 * - wq lock when "filt_wlworkq_interlock_needed() is true" 109 * Inheritor: thread, turnstile (of workq) 110 * Lock order: turnstile lock, thread lock 111 * WL turnstile lock, Workq turnstile lock 112 * 113 * TURNSTILE_WORKQS 114 * Interlock: workqueue lock 115 * Inheritor: thread 116 * Lock order: turnstile lock, thread lock. 117 * 118 * TURNSTILE_KNOTE 119 * Interlock: the knote lock 120 * Inheritor: WL turnstile 121 * 122 * TURNSTILE_SLEEP_INHERITOR 123 * Interlock: turnstile_htable bucket spinlock. 124 * Inheritor: threads. 125 * Lock order: turnstile lock, thread lock. 126 * 127 * TURNSTILE_EPOCH 128 * Interlock: the epoch sync interlock. 129 * Inheritor: threads. 130 * Lock order: turnstile lock, thread lock. 131 */ 132 133 typedef enum __attribute__((flag_enum)) turnstile_promote_policy { 134 TURNSTILE_PROMOTE_NONE = 0, 135 TURNSTILE_KERNEL_PROMOTE = 0x1, 136 TURNSTILE_USER_PROMOTE = 0x2, 137 TURNSTILE_USER_IPC_PROMOTE = 0x4, 138 } turnstile_promote_policy_t; 139 140 typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy { 141 TURNSTILE_HASH_LOCK_POLICY_NONE = 0, 142 TURNSTILE_IRQ_UNSAFE_HASH = 0x1, 143 TURNSTILE_LOCKED_HASH = 0x2, 144 } turnstile_hash_lock_policy_t; 145 146 /* 147 * Turnstile state flags 148 * 149 * The turnstile state flags represent the current ownership of a turnstile. 150 * The supported flags are: 151 * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread 152 * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile 153 * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive 154 * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor 155 * 156 * The flag updates are done while holding the primitive interlock. 157 * */ 158 159 #define TURNSTILE_STATE_THREAD 0x1 160 #define TURNSTILE_STATE_FREELIST 0x2 161 #define TURNSTILE_STATE_HASHTABLE 0x4 162 #define TURNSTILE_STATE_PROPRIETOR 0x8 163 164 /* Helper macros to set/unset turnstile state flags */ 165 #if DEVELOPMENT || DEBUG 166 167 #define turnstile_state_init(ts, state) \ 168 MACRO_BEGIN \ 169 ts->ts_state = state; \ 170 MACRO_END 171 172 #define turnstile_state_add(ts, state) \ 173 MACRO_BEGIN \ 174 assert((ts->ts_state & (state)) == 0); \ 175 ts->ts_state |= state; \ 176 MACRO_END 177 178 #define turnstile_state_remove(ts, state) \ 179 MACRO_BEGIN \ 180 assert(ts->ts_state & (state)); \ 181 ts->ts_state &= ~(state); \ 182 MACRO_END 183 184 #else /* DEVELOPMENT || DEBUG */ 185 186 #define turnstile_state_init(ts, state) \ 187 MACRO_BEGIN \ 188 (void)ts; \ 189 MACRO_END 190 191 #define turnstile_state_add(ts, state) \ 192 MACRO_BEGIN \ 193 (void)ts; \ 194 MACRO_END 195 196 #define turnstile_state_remove(ts, state) \ 197 MACRO_BEGIN \ 198 (void)ts; \ 199 MACRO_END 200 201 #endif /* DEVELOPMENT || DEBUG */ 202 203 struct knote; 204 struct turnstile; 205 206 /* 207 * Turnstile update flags 208 * 209 * TURNSTILE_IMMEDIATE_UPDATE 210 * When passed to turnstile_update_inheritor 211 * update the inheritor of the turnstile in 212 * the same call. 213 * 214 * TURNSTILE_DELAYED_UPDATE 215 * When passed to turnstile_update_inheritor 216 * it stashed the inheritor on the thread and 217 * turnstile's inheritor is updated in 218 * assert wait. 219 * 220 * TURNSTILE_INHERITOR_THREAD 221 * The turnstile inheritor is of type thread. 222 * 223 * TURNSTILE_INHERITOR_TURNSTILE 224 * The turnstile inheritor is of type turnstile. 225 * 226 * TURNSTILE_INHERITOR_WORKQ 227 * The turnstile inheritor is of type workqueue 228 * 229 * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE 230 * The inheritor needs a chain priority update. 231 * 232 * TURNSTILE_NEEDS_PRI_UPDATE 233 * Current turnstile needs a chain priority update. 234 * 235 * Locking order for passing thread and turnstile as inheritor 236 * 237 * Thread as an inheritor: 238 * When thread is passed as an inheritor of a turnstile 239 * turnstile lock is taken and then thread lock. 240 * 241 * Turnstile as in inheritor: 242 * When turnstile (T1) is passed as an inheritor of 243 * a turnstile (T2), turnstile lock of T2 is taken 244 * and then turnstile lock of T1 is taken. 245 * 246 * Caution: While passing turnstile as an inheritor, its 247 * job of the adopter to make sure that there is no 248 * lock inversion. 249 */ 250 typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { 251 TURNSTILE_UPDATE_FLAGS_NONE = 0, 252 TURNSTILE_IMMEDIATE_UPDATE = 0x1, 253 TURNSTILE_DELAYED_UPDATE = 0x2, 254 TURNSTILE_INHERITOR_THREAD = 0x4, 255 TURNSTILE_INHERITOR_TURNSTILE = 0x8, 256 TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, 257 TURNSTILE_NEEDS_PRI_UPDATE = 0x20, 258 TURNSTILE_INHERITOR_WORKQ = 0x40, 259 TURNSTILE_UPDATE_BOOST = 0x80, 260 } turnstile_update_flags_t; 261 262 #define TURNSTILE_NULL ((struct turnstile *)0) 263 264 typedef void * turnstile_inheritor_t; 265 266 #define TURNSTILE_INHERITOR_NULL NULL 267 268 #ifdef XNU_KERNEL_PRIVATE 269 #pragma GCC visibility push(hidden) 270 271 /* Turnstile stats update flags 272 * 273 * TSU_TURNSTILE_BLOCK_COUNT 274 * thread blocking on turnstile waitq, increment global 275 * thread block on turnstile count. 276 * 277 * TSU_REGULAR_WAITQ_BLOCK_COUNT 278 * thread blocking on regular waitq, increment global 279 * thread block on regular waitq count. 280 * 281 * TSU_PRI_PROPAGATION 282 * turnstile propagation update stopped at nth hop, update 283 * priority change count for nth element in stats array. 284 * 285 * TSU_NO_INHERITOR 286 * turnstile propagation update stopped due to turnstile 287 * not having an inheritor after nth hop, update the no 288 * inheritor count for nth element in the stats array. 289 * 290 * TSU_NO_TURNSTILE 291 * turnstile propagation update stopped due to thread 292 * not blocked on a turnstile waitq after nth hop, update 293 * the no turnstile count for the nth element in the stats 294 * array. 295 * 296 * TSU_NO_PRI_CHANGE_NEEDED 297 * turnstile propagation update stopped due to thread or 298 * turnstile having the correct priority or not blocked. 299 * update the no priority change count for the nth element 300 * in the stats array. 301 * 302 * TSU_THREAD_RUNNABLE 303 * turnstile propagation update stopped due to thread 304 * being runnable, update the thread runnable count for 305 * the nth element in the stats array. 306 * 307 * TSU_ABOVE_UI_PRI_CHANGE 308 * turnstile propagation caused an above UI priority change. 309 */ 310 typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { 311 TSU_FLAGS_NONE = 0, 312 TSU_TURNSTILE_BLOCK_COUNT = 0x1, 313 TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, 314 TSU_PRI_PROPAGATION = 0x4, 315 TSU_NO_INHERITOR = 0x8, 316 TSU_NO_TURNSTILE = 0x10, 317 TSU_NO_PRI_CHANGE_NEEDED = 0x20, 318 TSU_THREAD_RUNNABLE = 0x40, 319 TSU_ABOVE_UI_PRI_CHANGE = 0x80, 320 TSU_THREAD_ARG = 0x100, 321 TSU_TURNSTILE_ARG = 0x200, 322 TSU_BOOST_ARG = 0x400, 323 } turnstile_stats_update_flags_t; 324 325 SLIST_HEAD(turnstile_list, turnstile); 326 327 #define CTSID_BITS 20 328 #define CTSID_MASK ((1u << CTSID_BITS) - 1) 329 #define CTSID_MAX (CTSID_MASK - 1) 330 331 struct turnstile { 332 union { 333 /* 334 * The waitq_eventmask field is only used on the global queues. 335 * We hence repurpose all those bits for our own use. 336 */ 337 #if MACH_KERNEL_PRIVATE 338 WAITQ_FLAGS(ts_waitq 339 , __ts_unused_bits: 7 340 , ts_compact_id: CTSID_BITS); 341 #endif 342 struct waitq ts_waitq; /* waitq embedded in turnstile */ 343 }; 344 #define ts_inheritor ts_waitq.waitq_inheritor /* thread/turnstile inheriting the priority (IL, WL) */ 345 union { 346 struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ 347 SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ 348 struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */ 349 }; 350 struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ 351 struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */ 352 SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ 353 uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ 354 os_ref_atomic_t ts_refcount; /* reference count for turnstiles */ 355 _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ 356 uint32_t ts_prim_count; /* counter used by the primitive */ 357 turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ 358 uint8_t ts_priority; /* priority of turnstile (WL) */ 359 360 #if DEVELOPMENT || DEBUG 361 uint8_t ts_state; /* current state of turnstile (IL) */ 362 thread_t ts_thread; /* thread the turnstile is attached to */ 363 thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ 364 #endif 365 }; 366 367 #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) 368 369 /* IL - interlock, WL - turnstile lock i.e. waitq lock */ 370 371 #define TURNSTILE_PROPRIETOR_NULL 0ul 372 373 /* 374 * Name: turnstiles_init 375 * 376 * Description: Initialize turnstile sub system. 377 * 378 * Args: None. 379 * 380 * Returns: None. 381 */ 382 void 383 turnstiles_init(void); 384 385 /* 386 * Name: turnstile_alloc 387 * 388 * Description: Allocate a turnstile. 389 * 390 * Args: None. 391 * 392 * Returns: 393 * turnstile on Success. 394 */ 395 struct turnstile * 396 turnstile_alloc(void); 397 398 /* 399 * Name: turnstile_compact_id_get() 400 * 401 * Description: Allocate a compact turnstile ID slot. 402 * 403 * Args: None. 404 * 405 * Returns: 406 * A non 0 compact compact turnstile ID. 407 */ 408 uint32_t 409 turnstile_compact_id_get(void); 410 411 /* 412 * Name: turnstile_compact_id_put() 413 * 414 * Description: Frees a compact turnstile ID slot. 415 * 416 * Args: 417 * Args1: the compact ID to free. 418 */ 419 void 420 turnstile_compact_id_put(uint32_t cid); 421 422 /* 423 * Name: turnstile_get_by_id 424 * 425 * Description: Resolve a turnstile by compact ID 426 * 427 * Args: 428 * Arg1: turnstile compact ID 429 * 430 * Returns: a turnstile 431 */ 432 struct turnstile * 433 turnstile_get_by_id(uint32_t tsid); 434 435 /* 436 * Name: turnstile_reference 437 * 438 * Description: Take a reference on the turnstile. 439 * 440 * Arg1: turnstile 441 * 442 * Returns: None. 443 */ 444 void 445 turnstile_reference(struct turnstile *turnstile); 446 447 /* 448 * Name: turnstile_deallocate 449 * 450 * Description: Drop a reference on the turnstile. 451 * Destroy the turnstile if the last ref. 452 * 453 * Arg1: turnstile 454 * 455 * Returns: None. 456 */ 457 void 458 turnstile_deallocate(struct turnstile *turnstile); 459 460 /* 461 * Name: turnstile_waitq_add_thread_priority_queue 462 * 463 * Description: add thread to the turnstile waitq 464 * 465 * Arg1: waitq 466 * Arg2: thread 467 * 468 * Conditions: waitq locked 469 */ 470 void 471 turnstile_waitq_add_thread_priority_queue( 472 struct waitq* wq, 473 thread_t thread); 474 475 /* 476 * Name: turnstile_deallocate_safe 477 * 478 * Description: Drop a reference on the turnstile safely without triggering zfree. 479 * 480 * Arg1: turnstile 481 * 482 * Returns: None. 483 */ 484 void 485 turnstile_deallocate_safe(struct turnstile *turnstile); 486 487 /* 488 * Name: turnstile_recompute_priority_locked 489 * 490 * Description: Update turnstile priority based 491 * on highest waiter thread and highest blocking 492 * turnstile. 493 * 494 * Args: turnstile 495 * 496 * Returns: TRUE: if the turnstile priority changed and needs propagation. 497 * FALSE: if the turnstile priority did not change or it does not need propagation. 498 * 499 * Condition: turnstile locked 500 */ 501 boolean_t 502 turnstile_recompute_priority_locked( 503 struct turnstile *turnstile); 504 505 /* 506 * Name: turnstile_recompute_priority 507 * 508 * Description: Update turnstile priority based 509 * on highest waiter thread and highest blocking 510 * turnstile. 511 * 512 * Args: turnstile 513 * 514 * Returns: TRUE: if the turnstile priority changed and needs propagation. 515 * FALSE: if the turnstile priority did not change or it does not need propagation. 516 */ 517 boolean_t 518 turnstile_recompute_priority( 519 struct turnstile *turnstile); 520 521 /* 522 * Name: turnstile_workq_proprietor_of_max_turnstile 523 * 524 * Description: Returns the highest priority and proprietor of a turnstile 525 * pushing on a workqueue turnstile. 526 * 527 * This will not return waiters that are at priority 528 * MAXPRI_THROTTLE or lower. 529 * 530 * Args: turnstile 531 * 532 * Returns: 533 * Priority of the max entry, or 0 534 * Pointer to the max entry proprietor 535 */ 536 int 537 turnstile_workq_proprietor_of_max_turnstile( 538 struct turnstile *turnstile, 539 uintptr_t *proprietor); 540 541 /* 542 * Name: turnstile_workloop_pusher_info 543 * 544 * Description: Returns the priority of the turnstile push for a workloop, 545 * and the thread or knote responsible for this push. 546 * 547 * Args: workloop turnstile 548 * 549 * Returns: 550 * Priority of the push or 0 551 * Thread (with a +1 reference) with that push or THREAD_NULL. 552 * Port (with a +1 reference) with that push, or IP_NULL. 553 * Sync IPC knote with the highest push (or NULL) 554 */ 555 int 556 turnstile_workloop_pusher_info( 557 struct turnstile *turnstile, 558 thread_t *thread, 559 ipc_port_t *port, 560 struct knote **knote_out); 561 562 /* 563 * Name: turnstile_cleanup 564 * 565 * Description: Update priority of a turnstile inheritor 566 * if needed. 567 * 568 * Args: inheritor and flags passed on thread struct. 569 * 570 * Returns: None. 571 */ 572 void 573 turnstile_cleanup(void); 574 575 /* 576 * Name: turnstile_update_thread_priority_chain 577 * 578 * Description: Priority of a thread blocked on a turnstile 579 * has changed, update the turnstile priority. 580 * 581 * Arg1: thread: thread whose priority has changed. 582 * 583 * Returns: None. 584 */ 585 void 586 turnstile_update_thread_priority_chain(thread_t thread); 587 588 /* 589 * Name: turnstile_update_inheritor_locked 590 * 591 * Description: Update the inheritor of the turnstile and boost the 592 * inheritor, called with turnstile locked. 593 * 594 * Args: 595 * Arg1: turnstile 596 * Implicit arg: new inheritor value is stashed in current thread's struct 597 * 598 * Returns: 599 * old inheritor reference is returned on current thread's struct. 600 */ 601 void 602 turnstile_update_inheritor_locked(struct turnstile *turnstile); 603 604 /* 605 * Name: thread_get_inheritor_turnstile_base_priority 606 * 607 * Description: Get the max base priority of all the inheritor turnstiles 608 * 609 * Arg1: thread 610 * 611 * Returns: Max base priority of all the inheritor turnstiles. 612 * 613 * Condition: thread locked 614 */ 615 int 616 thread_get_inheritor_turnstile_base_priority(thread_t thread); 617 618 /* 619 * Name: thread_get_inheritor_turnstile_sched_priority 620 * 621 * Description: Get the max sched priority of all the inheritor turnstiles 622 * 623 * Arg1: thread 624 * 625 * Returns: Max sched priority of all the inheritor turnstiles. 626 * 627 * Condition: thread locked 628 */ 629 int 630 thread_get_inheritor_turnstile_sched_priority(thread_t thread); 631 632 /* 633 * Name: thread_get_waiting_turnstile 634 * 635 * Description: Get the turnstile if the thread is waiting on a turnstile. 636 * 637 * Arg1: thread 638 * 639 * Returns: turnstile: if the thread is blocked on a turnstile. 640 * TURNSTILE_NULL: otherwise. 641 * 642 * Condition: thread locked. 643 */ 644 struct turnstile * 645 thread_get_waiting_turnstile(thread_t thread); 646 647 /* 648 * Name: turnstile_lookup_by_proprietor 649 * 650 * Description: Get turnstile for a proprietor from global 651 * turnstile hash. 652 * 653 * Arg1: port 654 * Arg2: turnstile_type_t type 655 * 656 * Returns: turnstile: if the proprietor has a turnstile. 657 * TURNSTILE_NULL: otherwise. 658 * 659 * Condition: proprietor interlock held. 660 */ 661 struct turnstile * 662 turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type); 663 664 /* 665 * Name: turnstile_has_waiters 666 * 667 * Description: returns if there are waiters on the turnstile 668 * 669 * Arg1: turnstile: turnstile 670 * 671 * Returns: TRUE if there are waiters, FALSE otherwise. 672 */ 673 674 boolean_t 675 turnstile_has_waiters(struct turnstile *turnstile); 676 677 /* 678 * Name: turnstile_stats_update 679 * 680 * Description: Function to update turnstile stats for dev kernel. 681 * 682 * Arg1: hops : number of thread hops in priority propagation 683 * Arg2: flags : turnstile stats update flags 684 * Arg3: inheritor: inheritor 685 * 686 * Returns: Nothing 687 */ 688 void 689 turnstile_stats_update( 690 int hop __assert_only, 691 turnstile_stats_update_flags_t flags __assert_only, 692 turnstile_inheritor_t inheritor __assert_only); 693 694 #if DEVELOPMENT || DEBUG 695 696 #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1 697 #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2 698 #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3 699 #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4 700 701 /* Functions used by debug test primitive exported by sysctls */ 702 int 703 tstile_test_prim_lock(int val); 704 705 int 706 tstile_test_prim_unlock(int val); 707 708 int 709 turnstile_get_boost_stats_sysctl(void *req); 710 int 711 turnstile_get_unboost_stats_sysctl(void *req); 712 #endif /* DEVELOPMENT || DEBUG */ 713 714 #pragma GCC visibility pop 715 #endif /* XNU_KERNEL_PRIVATE */ 716 717 /* Interface */ 718 719 /* 720 * Name: turnstile_hash_bucket_lock 721 * 722 * Description: locks the spinlock associated with proprietor's bucket. 723 * if proprietor is specified the index for the hash will be 724 * recomputed and returned in index_proprietor, 725 * otherwise the value save in index_proprietor is used as index. 726 * 727 * Args: 728 * Arg1: proprietor (key) for hashing 729 * Arg2: index for proprietor in the hash 730 * Arg3: turnstile type 731 * 732 * Returns: old value of irq if irq were disabled before acquiring the lock. 733 */ 734 unsigned 735 turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type); 736 737 /* 738 * Name: turnstile_hash_bucket_unlock 739 * 740 * Description: unlocks the spinlock associated with proprietor's bucket. 741 * if proprietor is specified the index for the hash will be 742 * recomputed and returned in index_proprietor, 743 * otherwise the value save in index_proprietor is used as index. 744 * 745 * Args: 746 * Arg1: proprietor (key) for hashing 747 * Arg2: index for proprietor in the hash 748 * Arg3: turnstile type 749 * Arg4: irq value returned by turnstile_hash_bucket_lock 750 * 751 */ 752 void 753 turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s); 754 755 /* 756 * Name: turnstile_prepare 757 * 758 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 759 * Function is called holding the interlock (spinlock) of the primitive. 760 * The turnstile returned by this function is safe to use until 761 * the thread calls turnstile_complete. 762 * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to 763 * it until it calls turnstile_complete. 764 * 765 * Args: 766 * Arg1: proprietor 767 * Arg2: pointer in primitive struct to store turnstile 768 * Arg3: turnstile to use instead of taking it from thread. 769 * Arg4: type of primitive 770 * 771 * Returns: 772 * turnstile. 773 */ 774 struct turnstile * 775 turnstile_prepare( 776 uintptr_t proprietor, 777 struct turnstile **tstore, 778 struct turnstile *turnstile, 779 turnstile_type_t type); 780 781 /* 782 * Name: turnstile_complete 783 * 784 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 785 * Function is called holding the interlock (spinlock) of the primitive. 786 * Current thread will have a turnstile attached to it after this call. 787 * 788 * Args: 789 * Arg1: proprietor 790 * Arg2: pointer in primitive struct to update turnstile 791 * Arg3: pointer to store the returned turnstile instead of attaching it to thread 792 * Arg4: type of primitive 793 * 794 * Returns: 795 * None. 796 */ 797 void 798 turnstile_complete( 799 uintptr_t proprietor, 800 struct turnstile **tstore, 801 struct turnstile **turnstile, 802 turnstile_type_t type); 803 804 /* 805 * Name: turnstile_prepare_compact_id 806 * 807 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 808 * Function is called holding the interlock (spinlock) of the primitive. 809 * The turnstile returned by this function is safe to use until 810 * the thread calls turnstile_complete_compact_id. 811 * The calling thread will not have a turnstile attached to 812 * it until it calls turnstile_complete_compact_id. 813 * 814 * Args: 815 * Arg1: proprietor 816 * Arg2: the current compact ID for the turnstile head 817 * Arg3: type of primitive 818 * 819 * Returns: 820 * turnstile. 821 */ 822 struct turnstile * 823 turnstile_prepare_compact_id( 824 uintptr_t proprietor, 825 uint32_t compact_id, 826 turnstile_type_t type); 827 828 /* 829 * Name: turnstile_complete_compact_id 830 * 831 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 832 * Function is called holding the interlock (spinlock) of the primitive. 833 * Current thread will have a turnstile attached to it after this call. 834 * 835 * Args: 836 * Arg1: proprietor 837 * Arg2: the turnstile pointer that was returned by turnstile_prepare_compact_id() 838 * Arg3: type of primitive 839 * 840 * Returns: 841 * Whether the primitive no longer has a turnstile. 842 */ 843 bool 844 turnstile_complete_compact_id( 845 uintptr_t proprietor, 846 struct turnstile *turnstile, 847 turnstile_type_t type); 848 849 /* 850 * Name: turnstile_prepare_hash 851 * 852 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 853 * Function is called holding the interlock (spinlock) of the primitive. 854 * The turnstile returned by this function is safe to use until 855 * the thread calls turnstile_complete_hash. 856 * The calling thread will not have a turnstile attached to 857 * it until it calls turnstile_complete_hash. 858 * 859 * The turnstile used for this proprietor will be stored in 860 * a global hash table. 861 * 862 * Args: 863 * Arg1: proprietor 864 * Arg3: type of primitive 865 * 866 * Returns: 867 * turnstile. 868 */ 869 struct turnstile * 870 turnstile_prepare_hash( 871 uintptr_t proprietor, 872 turnstile_type_t type); 873 874 /* 875 * Name: turnstile_complete_hash 876 * 877 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 878 * Function is called holding the interlock (spinlock) of the primitive. 879 * Current thread will have a turnstile attached to it after this call. 880 * 881 * Args: 882 * Arg1: proprietor 883 * Arg3: type of primitive 884 * 885 * Returns: 886 * None. 887 */ 888 void 889 turnstile_complete_hash( 890 uintptr_t proprietor, 891 turnstile_type_t type); 892 893 /* 894 * Name: turnstile_update_inheritor 895 * 896 * Description: Update the inheritor of the turnstile and boost the 897 * inheritor. It will take a thread reference on the inheritor. 898 * Called with the interlock of the primitive held. 899 * 900 * Args: 901 * Arg1: turnstile 902 * Arg2: inheritor 903 * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait 904 * 905 * Returns: 906 * old inheritor reference is stashed on current thread's struct. 907 */ 908 void 909 turnstile_update_inheritor( 910 struct turnstile *turnstile, 911 turnstile_inheritor_t new_inheritor, 912 turnstile_update_flags_t flags); 913 914 typedef enum turnstile_update_complete_flags { 915 TURNSTILE_INTERLOCK_NOT_HELD = 0x1, 916 TURNSTILE_INTERLOCK_HELD = 0x2, 917 } turnstile_update_complete_flags_t; 918 919 /* 920 * Name: turnstile_update_inheritor_complete 921 * 922 * Description: Update turnstile inheritor's priority and propagate the 923 * priority if the inheritor is blocked on a turnstile. 924 * Consumes thread ref of old inheritor returned by 925 * turnstile_update_inheritor. Recursive priority update 926 * will only happen when called with interlock dropped. 927 * 928 * Args: 929 * Arg1: turnstile 930 * Arg2: interlock held 931 * 932 * Returns: None. 933 */ 934 void 935 turnstile_update_inheritor_complete( 936 struct turnstile *turnstile, 937 turnstile_update_complete_flags_t flags); 938 939 940 /* 941 * Name: turnstile_kernel_update_inheritor_on_wake_locked 942 * 943 * Description: Set thread as the inheritor of the turnstile and 944 * boost the inheritor. 945 * Args: 946 * Arg1: turnstile 947 * Arg2: new_inheritor 948 * Arg3: flags 949 * 950 * Called with turnstile locked 951 */ 952 void 953 turnstile_kernel_update_inheritor_on_wake_locked( 954 struct turnstile *turnstile, 955 turnstile_inheritor_t new_inheritor, 956 turnstile_update_flags_t flags); 957 958 #endif /* KERNEL_PRIVATE */ 959 #if XNU_KERNEL_PRIVATE 960 961 struct workqueue; 962 963 /* pthread_workqueue.c */ 964 extern void workq_reference(struct workqueue *wq); 965 extern void workq_deallocate_safe(struct workqueue *wq); 966 extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); 967 extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, 968 bool locked); 969 970 #endif /* XNU_KERNEL_PRIVATE */ 971 972 __END_DECLS 973 974 #endif /* _TURNSTILE_H_ */ 975