1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _TURNSTILE_H_ 30 #define _TURNSTILE_H_ 31 32 #include <mach/mach_types.h> 33 #include <mach/kern_return.h> 34 #include <sys/cdefs.h> 35 36 __BEGIN_DECLS 37 38 #if PRIVATE 39 #define TURNSTILE_MAX_HOP_DEFAULT (10) 40 struct turnstile_stats { 41 uint64_t ts_priority_propagation; 42 uint64_t ts_no_inheritor; 43 uint64_t ts_thread_runnable; 44 uint64_t ts_no_priority_change_required; 45 uint64_t ts_above_ui_pri_change; 46 uint64_t ts_no_turnstile; 47 }; 48 #endif 49 50 #ifdef KERNEL_PRIVATE 51 #include <kern/queue.h> 52 #include <sys/queue.h> 53 #include <kern/waitq.h> 54 #include <kern/priority_queue.h> 55 #include <os/refcnt.h> 56 #include <kern/assert.h> 57 #include <kern/kern_types.h> 58 #include <kern/mpsc_queue.h> 59 #include <kern/locks.h> 60 61 /* 62 * turnstile_type_t : Indicates the type of primitive the turnstile is associated with 63 * Please populate turnstile_promote_policy array if a new type is added here. 64 */ 65 typedef enum __attribute__((packed)) turnstile_type { 66 TURNSTILE_NONE = 0, 67 TURNSTILE_KERNEL_MUTEX = 1, 68 TURNSTILE_ULOCK = 2, 69 TURNSTILE_PTHREAD_MUTEX = 3, 70 TURNSTILE_SYNC_IPC = 4, 71 TURNSTILE_WORKLOOPS = 5, 72 TURNSTILE_WORKQS = 6, 73 TURNSTILE_KNOTE = 7, 74 TURNSTILE_SLEEP_INHERITOR = 8, 75 TURNSTILE_EPOCH_KERNEL = 9, 76 TURNSTILE_EPOCH_USER = 10, 77 TURNSTILE_TOTAL_TYPES = 11, 78 } turnstile_type_t; 79 80 /* 81 * For each type of turnstile, following are the type of 82 * inheritors passed: 83 * 84 * TURNSTILE_KERNEL_MUTEX 85 * Interlock: kernel mutex interlock. 86 * Inheritor: threads. 87 * Lock order: turnstile lock, thread lock. 88 * 89 * TURNSTILE_ULOCK 90 * Interlock: ulocks interlock. 91 * Inheritor: threads. 92 * Lock order: turnstile lock, thread lock. 93 * 94 * TURNSTILE_PTHREAD_MUTEX 95 * Interlock: pthread mtx interlock. 96 * Inheritor: threads. 97 * Lock order: turnstile lock, thread lock. 98 * 99 * TURNSTILE_SYNC_IPC 100 * Interlock: port's mqueue lock 101 * Inheritor: turnstile (of port in which we are enqueued or WL turnstile. 102 * Lock order: Our turnstile, then turnstile of the port we are enqueued in. 103 * Port circularity will make sure there is never a cycle formation 104 * and lock order is maintained. 105 * 106 * TURNSTILE_WORKLOOPS 107 * Interlock: 108 * - kq req lock 109 * - wq lock when "filt_wlworkq_interlock_needed() is true" 110 * Inheritor: thread, turnstile (of workq) 111 * Lock order: turnstile lock, thread lock 112 * WL turnstile lock, Workq turnstile lock 113 * 114 * TURNSTILE_WORKQS 115 * Interlock: workqueue lock 116 * Inheritor: thread 117 * Lock order: turnstile lock, thread lock. 118 * 119 * TURNSTILE_KNOTE 120 * Interlock: the knote lock 121 * Inheritor: WL turnstile 122 * 123 * TURNSTILE_SLEEP_INHERITOR 124 * Interlock: turnstile_htable bucket spinlock. 125 * Inheritor: threads. 126 * Lock order: turnstile lock, thread lock. 127 * 128 * TURNSTILE_EPOCH_KERNEL 129 * Interlock: the epoch sync interlock. 130 * Inheritor: threads. 131 * Lock order: turnstile lock, thread lock. 132 * 133 * TURNSTILE_EPOCH_USER 134 * Interlock: the epoch sync interlock. 135 * Inheritor: threads. 136 * Lock order: turnstile lock, thread lock. 137 */ 138 139 typedef enum __attribute__((flag_enum)) turnstile_promote_policy { 140 TURNSTILE_PROMOTE_NONE = 0, 141 TURNSTILE_KERNEL_PROMOTE = 0x1, 142 TURNSTILE_USER_PROMOTE = 0x2, 143 TURNSTILE_USER_IPC_PROMOTE = 0x4, 144 } turnstile_promote_policy_t; 145 146 typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy { 147 TURNSTILE_HASH_LOCK_POLICY_NONE = 0, 148 TURNSTILE_IRQ_UNSAFE_HASH = 0x1, 149 TURNSTILE_LOCKED_HASH = 0x2, 150 } turnstile_hash_lock_policy_t; 151 152 /* 153 * Turnstile state flags 154 * 155 * The turnstile state flags represent the current ownership of a turnstile. 156 * The supported flags are: 157 * - TURNSTILE_STATE_THREAD : Turnstile is attached to a thread 158 * - TURNSTILE_STATE_FREELIST : Turnstile is hanging off the freelist of another turnstile 159 * - TURNSTILE_STATE_HASHTABLE : Turnstile is in the global hash table as the turnstile for a primitive 160 * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor 161 * 162 * The flag updates are done while holding the primitive interlock. 163 * */ 164 165 #define TURNSTILE_STATE_THREAD 0x1 166 #define TURNSTILE_STATE_FREELIST 0x2 167 #define TURNSTILE_STATE_HASHTABLE 0x4 168 #define TURNSTILE_STATE_PROPRIETOR 0x8 169 170 /* Helper macros to set/unset turnstile state flags */ 171 #if DEVELOPMENT || DEBUG 172 173 #define turnstile_state_init(ts, state) \ 174 MACRO_BEGIN \ 175 ts->ts_state = state; \ 176 MACRO_END 177 178 #define turnstile_state_add(ts, state) \ 179 MACRO_BEGIN \ 180 assert((ts->ts_state & (state)) == 0); \ 181 ts->ts_state |= state; \ 182 MACRO_END 183 184 #define turnstile_state_remove(ts, state) \ 185 MACRO_BEGIN \ 186 assert(ts->ts_state & (state)); \ 187 ts->ts_state &= ~(state); \ 188 MACRO_END 189 190 #else /* DEVELOPMENT || DEBUG */ 191 192 #define turnstile_state_init(ts, state) \ 193 MACRO_BEGIN \ 194 (void)ts; \ 195 MACRO_END 196 197 #define turnstile_state_add(ts, state) \ 198 MACRO_BEGIN \ 199 (void)ts; \ 200 MACRO_END 201 202 #define turnstile_state_remove(ts, state) \ 203 MACRO_BEGIN \ 204 (void)ts; \ 205 MACRO_END 206 207 #endif /* DEVELOPMENT || DEBUG */ 208 209 struct knote; 210 struct turnstile; 211 212 /* 213 * Turnstile update flags 214 * 215 * TURNSTILE_IMMEDIATE_UPDATE 216 * When passed to turnstile_update_inheritor 217 * update the inheritor of the turnstile in 218 * the same call. 219 * 220 * TURNSTILE_DELAYED_UPDATE 221 * When passed to turnstile_update_inheritor 222 * it stashed the inheritor on the thread and 223 * turnstile's inheritor is updated in 224 * assert wait. 225 * 226 * TURNSTILE_INHERITOR_THREAD 227 * The turnstile inheritor is of type thread. 228 * 229 * TURNSTILE_INHERITOR_TURNSTILE 230 * The turnstile inheritor is of type turnstile. 231 * 232 * TURNSTILE_INHERITOR_WORKQ 233 * The turnstile inheritor is of type workqueue 234 * 235 * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE 236 * The inheritor needs a chain priority update. 237 * 238 * TURNSTILE_NEEDS_PRI_UPDATE 239 * Current turnstile needs a chain priority update. 240 * 241 * Locking order for passing thread and turnstile as inheritor 242 * 243 * Thread as an inheritor: 244 * When thread is passed as an inheritor of a turnstile 245 * turnstile lock is taken and then thread lock. 246 * 247 * Turnstile as in inheritor: 248 * When turnstile (T1) is passed as an inheritor of 249 * a turnstile (T2), turnstile lock of T2 is taken 250 * and then turnstile lock of T1 is taken. 251 * 252 * Caution: While passing turnstile as an inheritor, its 253 * job of the adopter to make sure that there is no 254 * lock inversion. 255 */ 256 typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags { 257 TURNSTILE_UPDATE_FLAGS_NONE = 0, 258 TURNSTILE_IMMEDIATE_UPDATE = 0x1, 259 TURNSTILE_DELAYED_UPDATE = 0x2, 260 TURNSTILE_INHERITOR_THREAD = 0x4, 261 TURNSTILE_INHERITOR_TURNSTILE = 0x8, 262 TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10, 263 TURNSTILE_NEEDS_PRI_UPDATE = 0x20, 264 TURNSTILE_INHERITOR_WORKQ = 0x40, 265 TURNSTILE_UPDATE_BOOST = 0x80, 266 } turnstile_update_flags_t; 267 268 #define TURNSTILE_NULL ((struct turnstile *)0) 269 270 typedef void * turnstile_inheritor_t; 271 272 #define TURNSTILE_INHERITOR_NULL NULL 273 274 #ifdef XNU_KERNEL_PRIVATE 275 #pragma GCC visibility push(hidden) 276 277 /* Turnstile stats update flags 278 * 279 * TSU_TURNSTILE_BLOCK_COUNT 280 * thread blocking on turnstile waitq, increment global 281 * thread block on turnstile count. 282 * 283 * TSU_REGULAR_WAITQ_BLOCK_COUNT 284 * thread blocking on regular waitq, increment global 285 * thread block on regular waitq count. 286 * 287 * TSU_PRI_PROPAGATION 288 * turnstile propagation update stopped at nth hop, update 289 * priority change count for nth element in stats array. 290 * 291 * TSU_NO_INHERITOR 292 * turnstile propagation update stopped due to turnstile 293 * not having an inheritor after nth hop, update the no 294 * inheritor count for nth element in the stats array. 295 * 296 * TSU_NO_TURNSTILE 297 * turnstile propagation update stopped due to thread 298 * not blocked on a turnstile waitq after nth hop, update 299 * the no turnstile count for the nth element in the stats 300 * array. 301 * 302 * TSU_NO_PRI_CHANGE_NEEDED 303 * turnstile propagation update stopped due to thread or 304 * turnstile having the correct priority or not blocked. 305 * update the no priority change count for the nth element 306 * in the stats array. 307 * 308 * TSU_THREAD_RUNNABLE 309 * turnstile propagation update stopped due to thread 310 * being runnable, update the thread runnable count for 311 * the nth element in the stats array. 312 * 313 * TSU_ABOVE_UI_PRI_CHANGE 314 * turnstile propagation caused an above UI priority change. 315 */ 316 typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags { 317 TSU_FLAGS_NONE = 0, 318 TSU_TURNSTILE_BLOCK_COUNT = 0x1, 319 TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2, 320 TSU_PRI_PROPAGATION = 0x4, 321 TSU_NO_INHERITOR = 0x8, 322 TSU_NO_TURNSTILE = 0x10, 323 TSU_NO_PRI_CHANGE_NEEDED = 0x20, 324 TSU_THREAD_RUNNABLE = 0x40, 325 TSU_ABOVE_UI_PRI_CHANGE = 0x80, 326 TSU_THREAD_ARG = 0x100, 327 TSU_TURNSTILE_ARG = 0x200, 328 TSU_BOOST_ARG = 0x400, 329 } turnstile_stats_update_flags_t; 330 331 SLIST_HEAD(turnstile_list, turnstile); 332 333 #define CTSID_BITS 20 334 #define CTSID_MASK ((1u << CTSID_BITS) - 1) 335 #define CTSID_MAX (CTSID_MASK - 1) 336 337 struct turnstile { 338 union { 339 /* 340 * The waitq_eventmask field is only used on the global queues. 341 * We hence repurpose all those bits for our own use. 342 */ 343 #if MACH_KERNEL_PRIVATE 344 WAITQ_FLAGS(ts_waitq 345 , __ts_unused_bits: 7 346 , ts_compact_id: CTSID_BITS); 347 #endif 348 struct waitq ts_waitq; /* waitq embedded in turnstile */ 349 }; 350 #define ts_inheritor ts_waitq.waitq_inheritor /* thread/turnstile inheriting the priority (IL, WL) */ 351 union { 352 struct turnstile_list ts_free_turnstiles; /* turnstile free list (IL) */ 353 SLIST_ENTRY(turnstile) ts_free_elm; /* turnstile free list element (IL) */ 354 struct mpsc_queue_chain ts_deallocate_link; /* thread deallocate link */ 355 }; 356 struct priority_queue_sched_max ts_inheritor_queue; /* Queue of turnstile with us as an inheritor (WL) */ 357 struct priority_queue_entry_sched ts_inheritor_links; /* Inheritor queue links */ 358 SLIST_ENTRY(turnstile) ts_htable_link; /* linkage for turnstile in global hash table */ 359 uintptr_t ts_proprietor; /* hash key lookup turnstile (IL) */ 360 os_ref_atomic_t ts_refcount; /* reference count for turnstiles */ 361 _Atomic uint32_t ts_type_gencount; /* gen count used for priority chaining (IL), type of turnstile (IL) */ 362 uint32_t ts_prim_count; /* counter used by the primitive */ 363 turnstile_update_flags_t ts_inheritor_flags; /* flags for turnstile inheritor (IL, WL) */ 364 uint8_t ts_priority; /* priority of turnstile (WL) */ 365 366 #if DEVELOPMENT || DEBUG 367 uint8_t ts_state; /* current state of turnstile (IL) */ 368 thread_t ts_thread; /* thread the turnstile is attached to */ 369 thread_t ts_prev_thread; /* thread the turnstile was attached before donation */ 370 #endif 371 }; 372 373 #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq) 374 375 /* IL - interlock, WL - turnstile lock i.e. waitq lock */ 376 377 #define TURNSTILE_PROPRIETOR_NULL 0ul 378 379 /* 380 * Name: turnstiles_init 381 * 382 * Description: Initialize turnstile sub system. 383 * 384 * Args: None. 385 * 386 * Returns: None. 387 */ 388 void 389 turnstiles_init(void); 390 391 /* 392 * Name: turnstile_alloc 393 * 394 * Description: Allocate a turnstile. 395 * 396 * Args: None. 397 * 398 * Returns: 399 * turnstile on Success. 400 */ 401 struct turnstile * 402 turnstile_alloc(void); 403 404 /* 405 * Name: turnstile_compact_id_get() 406 * 407 * Description: Allocate a compact turnstile ID slot. 408 * 409 * Args: None. 410 * 411 * Returns: 412 * A non 0 compact compact turnstile ID. 413 */ 414 uint32_t 415 turnstile_compact_id_get(void); 416 417 /* 418 * Name: turnstile_compact_id_put() 419 * 420 * Description: Frees a compact turnstile ID slot. 421 * 422 * Args: 423 * Args1: the compact ID to free. 424 */ 425 void 426 turnstile_compact_id_put(uint32_t cid); 427 428 /* 429 * Name: turnstile_get_by_id 430 * 431 * Description: Resolve a turnstile by compact ID 432 * 433 * Args: 434 * Arg1: turnstile compact ID 435 * 436 * Returns: a turnstile 437 */ 438 struct turnstile * 439 turnstile_get_by_id(uint32_t tsid); 440 441 /* 442 * Name: turnstile_reference 443 * 444 * Description: Take a reference on the turnstile. 445 * 446 * Arg1: turnstile 447 * 448 * Returns: None. 449 */ 450 void 451 turnstile_reference(struct turnstile *turnstile); 452 453 /* 454 * Name: turnstile_deallocate 455 * 456 * Description: Drop a reference on the turnstile. 457 * Destroy the turnstile if the last ref. 458 * 459 * Arg1: turnstile 460 * 461 * Returns: None. 462 */ 463 void 464 turnstile_deallocate(struct turnstile *turnstile); 465 466 /* 467 * Name: turnstile_waitq_add_thread_priority_queue 468 * 469 * Description: add thread to the turnstile waitq 470 * 471 * Arg1: waitq 472 * Arg2: thread 473 * 474 * Conditions: waitq locked 475 */ 476 void 477 turnstile_waitq_add_thread_priority_queue( 478 struct waitq* wq, 479 thread_t thread); 480 481 /* 482 * Name: turnstile_deallocate_safe 483 * 484 * Description: Drop a reference on the turnstile safely without triggering zfree. 485 * 486 * Arg1: turnstile 487 * 488 * Returns: None. 489 */ 490 void 491 turnstile_deallocate_safe(struct turnstile *turnstile); 492 493 /* 494 * Name: turnstile_recompute_priority_locked 495 * 496 * Description: Update turnstile priority based 497 * on highest waiter thread and highest blocking 498 * turnstile. 499 * 500 * Args: turnstile 501 * 502 * Returns: TRUE: if the turnstile priority changed and needs propagation. 503 * FALSE: if the turnstile priority did not change or it does not need propagation. 504 * 505 * Condition: turnstile locked 506 */ 507 boolean_t 508 turnstile_recompute_priority_locked( 509 struct turnstile *turnstile); 510 511 /* 512 * Name: turnstile_recompute_priority 513 * 514 * Description: Update turnstile priority based 515 * on highest waiter thread and highest blocking 516 * turnstile. 517 * 518 * Args: turnstile 519 * 520 * Returns: TRUE: if the turnstile priority changed and needs propagation. 521 * FALSE: if the turnstile priority did not change or it does not need propagation. 522 */ 523 boolean_t 524 turnstile_recompute_priority( 525 struct turnstile *turnstile); 526 527 /* 528 * Name: turnstile_workq_proprietor_of_max_turnstile 529 * 530 * Description: Returns the highest priority and proprietor of a turnstile 531 * pushing on a workqueue turnstile. 532 * 533 * This will not return waiters that are at priority 534 * MAXPRI_THROTTLE or lower. 535 * 536 * Args: turnstile 537 * 538 * Returns: 539 * Priority of the max entry, or 0 540 * Pointer to the max entry proprietor 541 */ 542 int 543 turnstile_workq_proprietor_of_max_turnstile( 544 struct turnstile *turnstile, 545 uintptr_t *proprietor); 546 547 /* 548 * Name: turnstile_workloop_pusher_info 549 * 550 * Description: Returns the priority of the turnstile push for a workloop, 551 * and the thread or knote responsible for this push. 552 * 553 * Args: workloop turnstile 554 * 555 * Returns: 556 * Priority of the push or 0 557 * Thread (with a +1 reference) with that push or THREAD_NULL. 558 * Port (with a +1 reference) with that push, or IP_NULL. 559 * Sync IPC knote with the highest push (or NULL) 560 */ 561 int 562 turnstile_workloop_pusher_info( 563 struct turnstile *turnstile, 564 thread_t *thread, 565 ipc_port_t *port, 566 struct knote **knote_out); 567 568 /* 569 * Name: turnstile_cleanup 570 * 571 * Description: Update priority of a turnstile inheritor 572 * if needed. 573 * 574 * Args: inheritor and flags passed on thread struct. 575 * 576 * Returns: None. 577 */ 578 void 579 turnstile_cleanup(void); 580 581 /* 582 * Name: turnstile_update_thread_priority_chain 583 * 584 * Description: Priority of a thread blocked on a turnstile 585 * has changed, update the turnstile priority. 586 * 587 * Arg1: thread: thread whose priority has changed. 588 * 589 * Returns: None. 590 */ 591 void 592 turnstile_update_thread_priority_chain(thread_t thread); 593 594 /* 595 * Name: turnstile_update_inheritor_locked 596 * 597 * Description: Update the inheritor of the turnstile and boost the 598 * inheritor, called with turnstile locked. 599 * 600 * Args: 601 * Arg1: turnstile 602 * Implicit arg: new inheritor value is stashed in current thread's struct 603 * 604 * Returns: 605 * old inheritor reference is returned on current thread's struct. 606 */ 607 void 608 turnstile_update_inheritor_locked(struct turnstile *turnstile); 609 610 /* 611 * Name: thread_get_inheritor_turnstile_base_priority 612 * 613 * Description: Get the max base priority of all the inheritor turnstiles 614 * 615 * Arg1: thread 616 * 617 * Returns: Max base priority of all the inheritor turnstiles. 618 * 619 * Condition: thread locked 620 */ 621 int 622 thread_get_inheritor_turnstile_base_priority(thread_t thread); 623 624 /* 625 * Name: thread_get_inheritor_turnstile_sched_priority 626 * 627 * Description: Get the max sched priority of all the inheritor turnstiles 628 * 629 * Arg1: thread 630 * 631 * Returns: Max sched priority of all the inheritor turnstiles. 632 * 633 * Condition: thread locked 634 */ 635 int 636 thread_get_inheritor_turnstile_sched_priority(thread_t thread); 637 638 /* 639 * Name: thread_get_waiting_turnstile 640 * 641 * Description: Get the turnstile if the thread is waiting on a turnstile. 642 * 643 * Arg1: thread 644 * 645 * Returns: turnstile: if the thread is blocked on a turnstile. 646 * TURNSTILE_NULL: otherwise. 647 * 648 * Condition: thread locked. 649 */ 650 struct turnstile * 651 thread_get_waiting_turnstile(thread_t thread); 652 653 /* 654 * Name: turnstile_lookup_by_proprietor 655 * 656 * Description: Get turnstile for a proprietor from global 657 * turnstile hash. 658 * 659 * Arg1: port 660 * Arg2: turnstile_type_t type 661 * 662 * Returns: turnstile: if the proprietor has a turnstile. 663 * TURNSTILE_NULL: otherwise. 664 * 665 * Condition: proprietor interlock held. 666 */ 667 struct turnstile * 668 turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type); 669 670 /* 671 * Name: turnstile_has_waiters 672 * 673 * Description: returns if there are waiters on the turnstile 674 * 675 * Arg1: turnstile: turnstile 676 * 677 * Returns: TRUE if there are waiters, FALSE otherwise. 678 */ 679 680 boolean_t 681 turnstile_has_waiters(struct turnstile *turnstile); 682 683 /* 684 * Name: turnstile_stats_update 685 * 686 * Description: Function to update turnstile stats for dev kernel. 687 * 688 * Arg1: hops : number of thread hops in priority propagation 689 * Arg2: flags : turnstile stats update flags 690 * Arg3: inheritor: inheritor 691 * 692 * Returns: Nothing 693 */ 694 void 695 turnstile_stats_update( 696 int hop __assert_only, 697 turnstile_stats_update_flags_t flags __assert_only, 698 turnstile_inheritor_t inheritor __assert_only); 699 700 #if DEVELOPMENT || DEBUG 701 702 #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1 703 #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2 704 #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3 705 #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4 706 707 /* Functions used by debug test primitive exported by sysctls */ 708 int 709 tstile_test_prim_lock(int val); 710 711 int 712 tstile_test_prim_unlock(int val); 713 714 int 715 turnstile_get_boost_stats_sysctl(void *req); 716 int 717 turnstile_get_unboost_stats_sysctl(void *req); 718 #endif /* DEVELOPMENT || DEBUG */ 719 720 #pragma GCC visibility pop 721 #endif /* XNU_KERNEL_PRIVATE */ 722 723 /* Interface */ 724 725 /* 726 * Name: turnstile_hash_bucket_lock 727 * 728 * Description: locks the spinlock associated with proprietor's bucket. 729 * if proprietor is specified the index for the hash will be 730 * recomputed and returned in index_proprietor, 731 * otherwise the value save in index_proprietor is used as index. 732 * 733 * Args: 734 * Arg1: proprietor (key) for hashing 735 * Arg2: index for proprietor in the hash 736 * Arg3: turnstile type 737 * 738 * Returns: old value of irq if irq were disabled before acquiring the lock. 739 */ 740 unsigned 741 turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type); 742 743 /* 744 * Name: turnstile_hash_bucket_unlock 745 * 746 * Description: unlocks the spinlock associated with proprietor's bucket. 747 * if proprietor is specified the index for the hash will be 748 * recomputed and returned in index_proprietor, 749 * otherwise the value save in index_proprietor is used as index. 750 * 751 * Args: 752 * Arg1: proprietor (key) for hashing 753 * Arg2: index for proprietor in the hash 754 * Arg3: turnstile type 755 * Arg4: irq value returned by turnstile_hash_bucket_lock 756 * 757 */ 758 void 759 turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s); 760 761 /* 762 * Name: turnstile_prepare 763 * 764 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 765 * Function is called holding the interlock (spinlock) of the primitive. 766 * The turnstile returned by this function is safe to use until 767 * the thread calls turnstile_complete. 768 * When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to 769 * it until it calls turnstile_complete. 770 * 771 * Args: 772 * Arg1: proprietor 773 * Arg2: pointer in primitive struct to store turnstile 774 * Arg3: turnstile to use instead of taking it from thread. 775 * Arg4: type of primitive 776 * 777 * Returns: 778 * turnstile. 779 */ 780 struct turnstile * 781 turnstile_prepare( 782 uintptr_t proprietor, 783 struct turnstile **tstore, 784 struct turnstile *turnstile, 785 turnstile_type_t type); 786 787 /* 788 * Name: turnstile_complete 789 * 790 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 791 * Function is called holding the interlock (spinlock) of the primitive. 792 * Current thread will have a turnstile attached to it after this call. 793 * 794 * Args: 795 * Arg1: proprietor 796 * Arg2: pointer in primitive struct to update turnstile 797 * Arg3: pointer to store the returned turnstile instead of attaching it to thread 798 * Arg4: type of primitive 799 * 800 * Returns: 801 * None. 802 */ 803 void 804 turnstile_complete( 805 uintptr_t proprietor, 806 struct turnstile **tstore, 807 struct turnstile **turnstile, 808 turnstile_type_t type); 809 810 /* 811 * Name: turnstile_prepare_compact_id 812 * 813 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 814 * Function is called holding the interlock (spinlock) of the primitive. 815 * The turnstile returned by this function is safe to use until 816 * the thread calls turnstile_complete_compact_id. 817 * The calling thread will not have a turnstile attached to 818 * it until it calls turnstile_complete_compact_id. 819 * 820 * Args: 821 * Arg1: proprietor 822 * Arg2: the current compact ID for the turnstile head 823 * Arg3: type of primitive 824 * 825 * Returns: 826 * turnstile. 827 */ 828 struct turnstile * 829 turnstile_prepare_compact_id( 830 uintptr_t proprietor, 831 uint32_t compact_id, 832 turnstile_type_t type); 833 834 /* 835 * Name: turnstile_complete_compact_id 836 * 837 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 838 * Function is called holding the interlock (spinlock) of the primitive. 839 * Current thread will have a turnstile attached to it after this call. 840 * 841 * Args: 842 * Arg1: proprietor 843 * Arg2: the turnstile pointer that was returned by turnstile_prepare_compact_id() 844 * Arg3: type of primitive 845 * 846 * Returns: 847 * Whether the primitive no longer has a turnstile. 848 */ 849 bool 850 turnstile_complete_compact_id( 851 uintptr_t proprietor, 852 struct turnstile *turnstile, 853 turnstile_type_t type); 854 855 /* 856 * Name: turnstile_prepare_hash 857 * 858 * Description: Transfer current thread's turnstile to primitive or it's free turnstile list. 859 * Function is called holding the interlock (spinlock) of the primitive. 860 * The turnstile returned by this function is safe to use until 861 * the thread calls turnstile_complete_hash. 862 * The calling thread will not have a turnstile attached to 863 * it until it calls turnstile_complete_hash. 864 * 865 * The turnstile used for this proprietor will be stored in 866 * a global hash table. 867 * 868 * Args: 869 * Arg1: proprietor 870 * Arg3: type of primitive 871 * 872 * Returns: 873 * turnstile. 874 */ 875 struct turnstile * 876 turnstile_prepare_hash( 877 uintptr_t proprietor, 878 turnstile_type_t type); 879 880 /* 881 * Name: turnstile_complete_hash 882 * 883 * Description: Transfer the primitive's turnstile or from it's freelist to current thread. 884 * Function is called holding the interlock (spinlock) of the primitive. 885 * Current thread will have a turnstile attached to it after this call. 886 * 887 * Args: 888 * Arg1: proprietor 889 * Arg3: type of primitive 890 * 891 * Returns: 892 * None. 893 */ 894 void 895 turnstile_complete_hash( 896 uintptr_t proprietor, 897 turnstile_type_t type); 898 899 /* 900 * Name: turnstile_update_inheritor 901 * 902 * Description: Update the inheritor of the turnstile and boost the 903 * inheritor. It will take a thread reference on the inheritor. 904 * Called with the interlock of the primitive held. 905 * 906 * Args: 907 * Arg1: turnstile 908 * Arg2: inheritor 909 * Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait 910 * 911 * Returns: 912 * old inheritor reference is stashed on current thread's struct. 913 */ 914 void 915 turnstile_update_inheritor( 916 struct turnstile *turnstile, 917 turnstile_inheritor_t new_inheritor, 918 turnstile_update_flags_t flags); 919 920 typedef enum turnstile_update_complete_flags { 921 TURNSTILE_INTERLOCK_NOT_HELD = 0x1, 922 TURNSTILE_INTERLOCK_HELD = 0x2, 923 } turnstile_update_complete_flags_t; 924 925 /* 926 * Name: turnstile_update_inheritor_complete 927 * 928 * Description: Update turnstile inheritor's priority and propagate the 929 * priority if the inheritor is blocked on a turnstile. 930 * Consumes thread ref of old inheritor returned by 931 * turnstile_update_inheritor. Recursive priority update 932 * will only happen when called with interlock dropped. 933 * 934 * Args: 935 * Arg1: turnstile 936 * Arg2: interlock held 937 * 938 * Returns: None. 939 */ 940 void 941 turnstile_update_inheritor_complete( 942 struct turnstile *turnstile, 943 turnstile_update_complete_flags_t flags); 944 945 946 /* 947 * Name: turnstile_kernel_update_inheritor_on_wake_locked 948 * 949 * Description: Set thread as the inheritor of the turnstile and 950 * boost the inheritor. 951 * Args: 952 * Arg1: turnstile 953 * Arg2: new_inheritor 954 * Arg3: flags 955 * 956 * Called with turnstile locked 957 */ 958 void 959 turnstile_kernel_update_inheritor_on_wake_locked( 960 struct turnstile *turnstile, 961 turnstile_inheritor_t new_inheritor, 962 turnstile_update_flags_t flags); 963 964 #endif /* KERNEL_PRIVATE */ 965 #if XNU_KERNEL_PRIVATE 966 967 struct workqueue; 968 969 /* pthread_workqueue.c */ 970 extern void workq_reference(struct workqueue *wq); 971 extern void workq_deallocate_safe(struct workqueue *wq); 972 extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq); 973 extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq, 974 bool locked); 975 976 #endif /* XNU_KERNEL_PRIVATE */ 977 978 __END_DECLS 979 980 #endif /* _TURNSTILE_H_ */ 981