1 /* 2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifndef _ARM_MACHINE_ROUTINES_H_ 33 #define _ARM_MACHINE_ROUTINES_H_ 34 35 #include <mach/mach_types.h> 36 #include <mach/vm_types.h> 37 #include <mach/boolean.h> 38 #include <kern/kern_types.h> 39 #include <pexpert/pexpert.h> 40 41 #include <sys/cdefs.h> 42 #include <sys/appleapiopts.h> 43 44 #include <stdarg.h> 45 46 #ifdef XNU_KERNEL_PRIVATE 47 #include <kern/sched_hygiene.h> 48 #include <kern/startup.h> 49 #endif /* XNU_KERNEL_PRIVATE */ 50 51 __BEGIN_DECLS 52 #ifdef XNU_KERNEL_PRIVATE 53 #ifdef __arm64__ 54 typedef bool (*expected_fault_handler_t)(arm_saved_state_t *); 55 #endif /* __arm64__ */ 56 #endif /* XNU_KERNEL_PRIVATE */ 57 58 /* Interrupt handling */ 59 60 void ml_cpu_signal(unsigned int cpu_id); 61 void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); 62 uint64_t ml_cpu_signal_deferred_get_timer(void); 63 void ml_cpu_signal_deferred(unsigned int cpu_id); 64 void ml_cpu_signal_retract(unsigned int cpu_id); 65 bool ml_cpu_signal_is_enabled(void); 66 67 /* Initialize Interrupts */ 68 void ml_init_interrupt(void); 69 70 /* Get Interrupts Enabled */ 71 boolean_t ml_get_interrupts_enabled(void); 72 73 /* Set Interrupts Enabled */ 74 #if __has_feature(ptrauth_calls) 75 uint64_t ml_pac_safe_interrupts_disable(void); 76 void ml_pac_safe_interrupts_restore(uint64_t); 77 #endif /* __has_feature(ptrauth_calls) */ 78 boolean_t ml_set_interrupts_enabled_with_debug(boolean_t enable, boolean_t debug); 79 boolean_t ml_set_interrupts_enabled(boolean_t enable); 80 boolean_t ml_early_set_interrupts_enabled(boolean_t enable); 81 82 /* Check if running at interrupt context */ 83 boolean_t ml_at_interrupt_context(void); 84 85 /* Generate a fake interrupt */ 86 void ml_cause_interrupt(void); 87 88 89 #ifdef XNU_KERNEL_PRIVATE 90 /* Clear interrupt spin debug state for thread */ 91 92 #if SCHED_HYGIENE_DEBUG 93 void ml_adjust_preemption_disable_time(thread_t thread, int64_t duration); 94 95 void mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs); 96 97 #if MONOTONIC 98 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) \ 99 if (sched_hygiene_debug_pmc) { \ 100 mt_cur_cpu_cycles_instrs_speculative(&thread->machine.intmask_cycles, \ 101 &thread->machine.intmask_instr); \ 102 } 103 #else 104 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) 105 #endif 106 107 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) \ 108 do { \ 109 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ 110 thread_t thread = current_thread(); \ 111 thread->machine.int_type = type; \ 112 thread->machine.int_handler_addr = (uintptr_t)VM_KERNEL_STRIP_UPTR(handler_addr); \ 113 thread->machine.inthandler_timestamp = ml_get_sched_hygiene_timebase(); \ 114 INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread); \ 115 thread->machine.int_vector = (uintptr_t)NULL; \ 116 } \ 117 } while (0) 118 119 #define INTERRUPT_MASKED_DEBUG_END() \ 120 do { \ 121 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ 122 thread_t thread = current_thread(); \ 123 ml_handle_interrupt_handler_duration(thread); \ 124 thread->machine.inthandler_timestamp = 0; \ 125 thread->machine.inthandler_abandon = false; \ 126 } \ 127 } while (0) 128 129 void ml_irq_debug_start(uintptr_t handler, uintptr_t vector); 130 void ml_irq_debug_end(void); 131 void ml_irq_debug_abandon(void); 132 133 void ml_spin_debug_reset(thread_t thread); 134 void ml_spin_debug_clear(thread_t thread); 135 void ml_spin_debug_clear_self(void); 136 void ml_handle_interrupts_disabled_duration(thread_t thread); 137 void ml_handle_stackshot_interrupt_disabled_duration(thread_t thread); 138 void ml_handle_interrupt_handler_duration(thread_t thread); 139 140 #else /* SCHED_HYGIENE_DEBUG */ 141 142 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) 143 #define INTERRUPT_MASKED_DEBUG_END() 144 145 #endif /* SCHED_HYGIENE_DEBUG */ 146 147 extern bool ml_snoop_thread_is_on_core(thread_t thread); 148 extern boolean_t ml_is_quiescing(void); 149 extern void ml_set_is_quiescing(boolean_t); 150 extern uint64_t ml_get_booter_memory_size(void); 151 #endif 152 153 /* Type for the Time Base Enable function */ 154 typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); 155 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 156 /* Type for the Processor Cache Dispatch function */ 157 typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); 158 159 typedef uint32_t (*get_decrementer_t)(void); 160 typedef void (*set_decrementer_t)(uint32_t); 161 typedef void (*fiq_handler_t)(void); 162 163 #endif 164 165 #define CacheConfig 0x00000000UL 166 #define CacheControl 0x00000001UL 167 #define CacheClean 0x00000002UL 168 #define CacheCleanRegion 0x00000003UL 169 #define CacheCleanFlush 0x00000004UL 170 #define CacheCleanFlushRegion 0x00000005UL 171 #define CacheShutdown 0x00000006UL 172 173 #define CacheControlEnable 0x00000000UL 174 175 #define CacheConfigCCSIDR 0x00000001UL 176 #define CacheConfigSize 0x00000100UL 177 178 /* Type for the Processor Idle function */ 179 typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); 180 181 /* Type for the Idle Tickle function */ 182 typedef void (*idle_tickle_t)(void); 183 184 /* Type for the Idle Timer function */ 185 typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); 186 187 /* Type for the IPI Hander */ 188 typedef void (*ipi_handler_t)(void); 189 190 /* Type for the Lockdown Hander */ 191 typedef void (*lockdown_handler_t)(void *); 192 193 /* Type for the Platform specific Error Handler */ 194 typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); 195 196 /* 197 * The exception callback (ex_cb) module allows kernel drivers to 198 * register and receive callbacks for exceptions, and indicate 199 * actions to be taken by the platform kernel 200 * Currently this is supported for ARM64 but extending support for ARM32 201 * should be straightforward 202 */ 203 204 /* Supported exception classes for callbacks */ 205 typedef enum{ 206 EXCB_CLASS_ILLEGAL_INSTR_SET, 207 #ifdef CONFIG_XNUPOST 208 EXCB_CLASS_TEST1, 209 EXCB_CLASS_TEST2, 210 EXCB_CLASS_TEST3, 211 #endif 212 EXCB_CLASS_MAX // this must be last 213 } 214 ex_cb_class_t; 215 216 /* Actions indicated by callbacks to be taken by platform kernel */ 217 typedef enum{ 218 EXCB_ACTION_RERUN, // re-run the faulting instruction 219 EXCB_ACTION_NONE, // continue normal exception handling 220 #ifdef CONFIG_XNUPOST 221 EXCB_ACTION_TEST_FAIL, 222 #endif 223 } 224 ex_cb_action_t; 225 226 /* 227 * Exception state 228 * We cannot use a private kernel data structure such as arm_saved_state_t 229 * The CPSR and ESR are not clobbered when the callback function is invoked so 230 * those registers can be examined by the callback function; 231 * the same is done in the platform error handlers 232 */ 233 typedef struct{ 234 vm_offset_t far; 235 } 236 ex_cb_state_t; 237 238 /* callback type definition */ 239 typedef ex_cb_action_t (*ex_cb_t) ( 240 ex_cb_class_t cb_class, 241 void *refcon,// provided at registration 242 const ex_cb_state_t *state // exception state 243 ); 244 245 /* 246 * Callback registration 247 * Currently we support only one registered callback per class but 248 * it should be possible to support more callbacks 249 */ 250 kern_return_t ex_cb_register( 251 ex_cb_class_t cb_class, 252 ex_cb_t cb, 253 void *refcon ); 254 255 /* 256 * Called internally by platform kernel to invoke the registered callback for class 257 */ 258 ex_cb_action_t ex_cb_invoke( 259 ex_cb_class_t cb_class, 260 vm_offset_t far); 261 262 typedef enum { 263 CLUSTER_TYPE_SMP, 264 CLUSTER_TYPE_E, 265 CLUSTER_TYPE_P, 266 MAX_CPU_TYPES, 267 } cluster_type_t; 268 269 void ml_parse_cpu_topology(void); 270 271 unsigned int ml_get_cpu_count(void); 272 273 unsigned int ml_get_cpu_number_type(cluster_type_t cluster_type, bool logical, bool available); 274 275 unsigned int ml_get_cluster_number_type(cluster_type_t cluster_type); 276 277 unsigned int ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type, bool include_all_cpu_types); 278 279 unsigned int ml_get_cpu_types(void); 280 281 unsigned int ml_get_cluster_count(void); 282 283 int ml_get_boot_cpu_number(void); 284 285 int ml_get_cpu_number(uint32_t phys_id); 286 287 unsigned int ml_get_cpu_number_local(void); 288 289 int ml_get_cluster_number(uint32_t phys_id); 290 291 int ml_get_max_cpu_number(void); 292 293 int ml_get_max_cluster_number(void); 294 295 unsigned int ml_get_first_cpu_id(unsigned int cluster_id); 296 297 298 #ifdef __arm64__ 299 int ml_get_cluster_number_local(void); 300 #endif /* __arm64__ */ 301 302 /* Struct for ml_cpu_get_info */ 303 struct ml_cpu_info { 304 unsigned long vector_unit; 305 unsigned long cache_line_size; 306 unsigned long l1_icache_size; 307 unsigned long l1_dcache_size; 308 unsigned long l2_settings; 309 unsigned long l2_cache_size; 310 unsigned long l3_settings; 311 unsigned long l3_cache_size; 312 }; 313 typedef struct ml_cpu_info ml_cpu_info_t; 314 315 cluster_type_t ml_get_boot_cluster_type(void); 316 317 typedef struct ml_topology_cpu { 318 unsigned int cpu_id; 319 uint32_t phys_id; 320 unsigned int cluster_id; 321 unsigned int reserved; 322 cluster_type_t cluster_type; 323 uint32_t l2_access_penalty; 324 uint32_t l2_cache_size; 325 uint32_t l2_cache_id; 326 uint32_t l3_cache_size; 327 uint32_t l3_cache_id; 328 vm_offset_t cpu_IMPL_regs; 329 uint64_t cpu_IMPL_pa; 330 uint64_t cpu_IMPL_len; 331 vm_offset_t cpu_UTTDBG_regs; 332 uint64_t cpu_UTTDBG_pa; 333 uint64_t cpu_UTTDBG_len; 334 vm_offset_t coresight_regs; 335 uint64_t coresight_pa; 336 uint64_t coresight_len; 337 unsigned int die_cluster_id; 338 unsigned int cluster_core_id; 339 } ml_topology_cpu_t; 340 341 /*! 342 * @typedef ml_topology_cluster_t 343 * @brief Describes one cluster in the topology. 344 * 345 * @field cluster_id Cluster ID (EDT: cluster-id) 346 * @field cluster_type The type of CPUs found in this cluster. 347 * @field num_cpus Total number of usable CPU cores in this cluster. 348 * @field first_cpu_id The cpu_id of the first CPU in the cluster. 349 * @field cpu_mask A bitmask representing the cpu_id's that belong to the cluster. Example: 350 * If the cluster contains CPU4 and CPU5, cpu_mask will be 0x30. 351 * @field acc_IMPL_regs IO-mapped virtual address of acc_IMPL (implementation-defined) register block. 352 * @field acc_IMPL_pa Physical address of acc_IMPL register block. 353 * @field acc_IMPL_len Length of acc_IMPL register block. 354 * @field cpm_IMPL_regs IO-mapped virtual address of cpm_IMPL (implementation-defined) register block. 355 * @field cpm_IMPL_pa Physical address of cpm_IMPL register block. 356 * @field cpm_IMPL_len Length of cpm_IMPL register block. 357 */ 358 typedef struct ml_topology_cluster { 359 unsigned int cluster_id; 360 cluster_type_t cluster_type; 361 unsigned int num_cpus; 362 unsigned int first_cpu_id; 363 uint64_t cpu_mask; 364 vm_offset_t acc_IMPL_regs; 365 uint64_t acc_IMPL_pa; 366 uint64_t acc_IMPL_len; 367 vm_offset_t cpm_IMPL_regs; 368 uint64_t cpm_IMPL_pa; 369 uint64_t cpm_IMPL_len; 370 } ml_topology_cluster_t; 371 372 // Bump this version number any time any ml_topology_* struct changes, so 373 // that KPI users can check whether their headers are compatible with 374 // the running kernel. 375 #define CPU_TOPOLOGY_VERSION 1 376 377 /*! 378 * @typedef ml_topology_info_t 379 * @brief Describes the CPU topology for all APs in the system. Populated from EDT and read-only at runtime. 380 * @discussion This struct only lists CPU cores that are considered usable by both iBoot and XNU. Some 381 * physically present CPU cores may be considered unusable due to configuration options like 382 * the "cpus=" boot-arg. Cores that are disabled in hardware will not show up in EDT at all, so 383 * they also will not be present in this struct. 384 * 385 * @field version Version of the struct (set to CPU_TOPOLOGY_VERSION). 386 * @field num_cpus Total number of usable CPU cores. 387 * @field max_cpu_id The highest usable logical CPU ID. 388 * @field num_clusters Total number of AP CPU clusters on the system (usable or not). 389 * @field max_cluster_id The highest cluster ID found in EDT. 390 * @field cpus List of |num_cpus| entries. 391 * @field clusters List of |num_clusters| entries. 392 * @field boot_cpu Points to the |cpus| entry for the boot CPU. 393 * @field boot_cluster Points to the |clusters| entry which contains the boot CPU. 394 * @field chip_revision Silicon revision reported by iBoot, which comes from the 395 * SoC-specific fuse bits. See CPU_VERSION_xx macros for definitions. 396 */ 397 typedef struct ml_topology_info { 398 unsigned int version; 399 unsigned int num_cpus; 400 unsigned int max_cpu_id; 401 unsigned int num_clusters; 402 unsigned int max_cluster_id; 403 unsigned int reserved; 404 ml_topology_cpu_t *cpus; 405 ml_topology_cluster_t *clusters; 406 ml_topology_cpu_t *boot_cpu; 407 ml_topology_cluster_t *boot_cluster; 408 unsigned int chip_revision; 409 unsigned int cluster_types; 410 unsigned int cluster_type_num_cpus[MAX_CPU_TYPES]; 411 unsigned int cluster_type_num_clusters[MAX_CPU_TYPES]; 412 } ml_topology_info_t; 413 414 /*! 415 * @function ml_get_topology_info 416 * @result A pointer to the read-only topology struct. Does not need to be freed. Returns NULL 417 * if the struct hasn't been initialized or the feature is unsupported. 418 */ 419 const ml_topology_info_t *ml_get_topology_info(void); 420 421 /*! 422 * @function ml_map_cpu_pio 423 * @brief Maps per-CPU and per-cluster PIO registers found in EDT. This needs to be 424 * called after arm_vm_init() so it can't be part of ml_parse_cpu_topology(). 425 */ 426 void ml_map_cpu_pio(void); 427 428 /* Struct for ml_processor_register */ 429 struct ml_processor_info { 430 cpu_id_t cpu_id; 431 vm_offset_t start_paddr; 432 boolean_t supports_nap; 433 void *platform_cache_dispatch; 434 time_base_enable_t time_base_enable; 435 processor_idle_t processor_idle; 436 idle_tickle_t *idle_tickle; 437 idle_timer_t idle_timer; 438 void *idle_timer_refcon; 439 vm_offset_t powergate_stub_addr; 440 uint32_t powergate_stub_length; 441 uint32_t powergate_latency; 442 platform_error_handler_t platform_error_handler; 443 uint64_t regmap_paddr; 444 uint32_t phys_id; 445 uint32_t log_id; 446 uint32_t l2_access_penalty; 447 uint32_t cluster_id; 448 cluster_type_t cluster_type; 449 uint32_t l2_cache_id; 450 uint32_t l2_cache_size; 451 uint32_t l3_cache_id; 452 uint32_t l3_cache_size; 453 }; 454 typedef struct ml_processor_info ml_processor_info_t; 455 456 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 457 /* Struct for ml_init_timebase */ 458 struct tbd_ops { 459 fiq_handler_t tbd_fiq_handler; 460 get_decrementer_t tbd_get_decrementer; 461 set_decrementer_t tbd_set_decrementer; 462 }; 463 typedef struct tbd_ops *tbd_ops_t; 464 typedef struct tbd_ops tbd_ops_data_t; 465 #endif 466 467 468 /*! 469 * @function ml_processor_register 470 * 471 * @abstract callback from platform kext to register processor 472 * 473 * @discussion This function is called by the platform kext when a processor is 474 * being registered. This is called while running on the CPU itself, as part of 475 * its initialization. 476 * 477 * @param ml_processor_info provides machine-specific information about the 478 * processor to xnu. 479 * 480 * @param processor is set as an out-parameter to an opaque handle that should 481 * be used by the platform kext when referring to this processor in the future. 482 * 483 * @param ipi_handler is set as an out-parameter to the function that should be 484 * registered as the IPI handler. 485 * 486 * @param pmi_handler is set as an out-parameter to the function that should be 487 * registered as the PMI handler. 488 * 489 * @returns KERN_SUCCESS on success and an error code, otherwise. 490 */ 491 kern_return_t ml_processor_register(ml_processor_info_t *ml_processor_info, 492 processor_t *processor, ipi_handler_t *ipi_handler, 493 perfmon_interrupt_handler_func *pmi_handler); 494 495 /* Register a lockdown handler */ 496 kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); 497 498 #if XNU_KERNEL_PRIVATE 499 void ml_lockdown_init(void); 500 501 /* Machine layer routine for intercepting panics */ 502 __printflike(1, 0) 503 void ml_panic_trap_to_debugger(const char *panic_format_str, 504 va_list *panic_args, 505 unsigned int reason, 506 void *ctx, 507 uint64_t panic_options_mask, 508 unsigned long panic_caller); 509 #endif /* XNU_KERNEL_PRIVATE */ 510 511 /* Initialize Interrupts */ 512 void ml_install_interrupt_handler( 513 void *nub, 514 int source, 515 void *target, 516 IOInterruptHandler handler, 517 void *refCon); 518 519 vm_offset_t 520 ml_static_vtop( 521 vm_offset_t); 522 523 kern_return_t 524 ml_static_verify_page_protections( 525 uint64_t base, uint64_t size, vm_prot_t prot); 526 527 vm_offset_t 528 ml_static_ptovirt( 529 vm_offset_t); 530 531 vm_offset_t ml_static_slide( 532 vm_offset_t vaddr); 533 534 vm_offset_t ml_static_unslide( 535 vm_offset_t vaddr); 536 537 /* Offset required to obtain absolute time value from tick counter */ 538 uint64_t ml_get_abstime_offset(void); 539 540 /* Offset required to obtain continuous time value from tick counter */ 541 uint64_t ml_get_conttime_offset(void); 542 543 #ifdef __APPLE_API_UNSTABLE 544 /* PCI config cycle probing */ 545 boolean_t ml_probe_read( 546 vm_offset_t paddr, 547 unsigned int *val); 548 boolean_t ml_probe_read_64( 549 addr64_t paddr, 550 unsigned int *val); 551 552 /* Read physical address byte */ 553 unsigned int ml_phys_read_byte( 554 vm_offset_t paddr); 555 unsigned int ml_phys_read_byte_64( 556 addr64_t paddr); 557 558 /* Read physical address half word */ 559 unsigned int ml_phys_read_half( 560 vm_offset_t paddr); 561 unsigned int ml_phys_read_half_64( 562 addr64_t paddr); 563 564 /* Read physical address word*/ 565 unsigned int ml_phys_read( 566 vm_offset_t paddr); 567 unsigned int ml_phys_read_64( 568 addr64_t paddr); 569 unsigned int ml_phys_read_word( 570 vm_offset_t paddr); 571 unsigned int ml_phys_read_word_64( 572 addr64_t paddr); 573 574 /* Read physical address double word */ 575 unsigned long long ml_phys_read_double( 576 vm_offset_t paddr); 577 unsigned long long ml_phys_read_double_64( 578 addr64_t paddr); 579 580 /* Write physical address byte */ 581 void ml_phys_write_byte( 582 vm_offset_t paddr, unsigned int data); 583 void ml_phys_write_byte_64( 584 addr64_t paddr, unsigned int data); 585 586 /* Write physical address half word */ 587 void ml_phys_write_half( 588 vm_offset_t paddr, unsigned int data); 589 void ml_phys_write_half_64( 590 addr64_t paddr, unsigned int data); 591 592 /* Write physical address word */ 593 void ml_phys_write( 594 vm_offset_t paddr, unsigned int data); 595 void ml_phys_write_64( 596 addr64_t paddr, unsigned int data); 597 void ml_phys_write_word( 598 vm_offset_t paddr, unsigned int data); 599 void ml_phys_write_word_64( 600 addr64_t paddr, unsigned int data); 601 602 /* Write physical address double word */ 603 void ml_phys_write_double( 604 vm_offset_t paddr, unsigned long long data); 605 void ml_phys_write_double_64( 606 addr64_t paddr, unsigned long long data); 607 608 void ml_static_mfree( 609 vm_offset_t, 610 vm_size_t); 611 612 kern_return_t 613 ml_static_protect( 614 vm_offset_t start, 615 vm_size_t size, 616 vm_prot_t new_prot); 617 618 /* virtual to physical on wired pages */ 619 vm_offset_t ml_vtophys( 620 vm_offset_t vaddr); 621 622 /* Get processor cache info */ 623 void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); 624 void ml_cpu_get_info_type(ml_cpu_info_t * ml_cpu_info, cluster_type_t cluster_type); 625 626 #endif /* __APPLE_API_UNSTABLE */ 627 628 #ifdef __APPLE_API_PRIVATE 629 #ifdef XNU_KERNEL_PRIVATE 630 vm_size_t ml_nofault_copy( 631 vm_offset_t virtsrc, 632 vm_offset_t virtdst, 633 vm_size_t size); 634 boolean_t ml_validate_nofault( 635 vm_offset_t virtsrc, vm_size_t size); 636 #endif /* XNU_KERNEL_PRIVATE */ 637 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 638 /* IO memory map services */ 639 640 extern vm_offset_t io_map( 641 vm_map_offset_t phys_addr, 642 vm_size_t size, 643 unsigned int flags, 644 vm_prot_t prot, 645 bool unmappable); 646 647 /* Map memory map IO space */ 648 vm_offset_t ml_io_map( 649 vm_offset_t phys_addr, 650 vm_size_t size); 651 652 vm_offset_t ml_io_map_wcomb( 653 vm_offset_t phys_addr, 654 vm_size_t size); 655 656 vm_offset_t ml_io_map_unmappable( 657 vm_offset_t phys_addr, 658 vm_size_t size, 659 uint32_t flags); 660 661 vm_offset_t ml_io_map_with_prot( 662 vm_offset_t phys_addr, 663 vm_size_t size, 664 vm_prot_t prot); 665 666 void ml_io_unmap( 667 vm_offset_t addr, 668 vm_size_t sz); 669 670 void ml_get_bouncepool_info( 671 vm_offset_t *phys_addr, 672 vm_size_t *size); 673 674 vm_map_address_t ml_map_high_window( 675 vm_offset_t phys_addr, 676 vm_size_t len); 677 678 void ml_init_timebase( 679 void *args, 680 tbd_ops_t tbd_funcs, 681 vm_offset_t int_address, 682 vm_offset_t int_value); 683 684 uint64_t ml_get_timebase(void); 685 686 uint64_t ml_get_speculative_timebase(void); 687 688 uint64_t ml_get_timebase_entropy(void); 689 690 void ml_init_lock_timeout(void); 691 692 boolean_t ml_delay_should_spin(uint64_t interval); 693 694 void ml_delay_on_yield(void); 695 696 uint32_t ml_get_decrementer(void); 697 698 #include <machine/config.h> 699 700 uint64_t ml_get_hwclock(void); 701 702 #ifdef __arm64__ 703 boolean_t ml_get_timer_pending(void); 704 #endif 705 706 void platform_syscall( 707 struct arm_saved_state *); 708 709 void ml_set_decrementer( 710 uint32_t dec_value); 711 712 boolean_t is_user_contex( 713 void); 714 715 void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); 716 717 /* These calls are only valid if __ARM_USER_PROTECT__ is defined */ 718 uintptr_t arm_user_protect_begin( 719 thread_t thread); 720 721 void arm_user_protect_end( 722 thread_t thread, 723 uintptr_t up, 724 boolean_t disable_interrupts); 725 726 #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ 727 728 /* Zero bytes starting at a physical address */ 729 void bzero_phys( 730 addr64_t phys_address, 731 vm_size_t length); 732 733 void bzero_phys_nc(addr64_t src64, vm_size_t bytes); 734 735 #if MACH_KERNEL_PRIVATE 736 #ifdef __arm64__ 737 /* Pattern-fill buffer with zeros or a 32-bit pattern; 738 * target must be 128-byte aligned and sized a multiple of 128 739 * Both variants emit stores with non-temporal properties. 740 */ 741 void fill32_dczva(addr64_t, vm_size_t); 742 void fill32_nt(addr64_t, vm_size_t, uint32_t); 743 bool cpu_interrupt_is_pending(void); 744 #endif 745 #endif 746 747 void ml_thread_policy( 748 thread_t thread, 749 unsigned policy_id, 750 unsigned policy_info); 751 752 #define MACHINE_GROUP 0x00000001 753 #define MACHINE_NETWORK_GROUP 0x10000000 754 #define MACHINE_NETWORK_WORKLOOP 0x00000001 755 #define MACHINE_NETWORK_NETISR 0x00000002 756 757 /* Set the maximum number of CPUs */ 758 void ml_set_max_cpus( 759 unsigned int max_cpus); 760 761 /* Return the maximum number of CPUs set by ml_set_max_cpus(), waiting if necessary */ 762 unsigned int ml_wait_max_cpus( 763 void); 764 765 /* Return the maximum memory size */ 766 unsigned int ml_get_machine_mem(void); 767 768 #ifdef XNU_KERNEL_PRIVATE 769 /* Return max offset */ 770 vm_map_offset_t ml_get_max_offset( 771 boolean_t is64, 772 unsigned int option); 773 #define MACHINE_MAX_OFFSET_DEFAULT 0x01 774 #define MACHINE_MAX_OFFSET_MIN 0x02 775 #define MACHINE_MAX_OFFSET_MAX 0x04 776 #define MACHINE_MAX_OFFSET_DEVICE 0x08 777 #endif 778 779 extern void ml_cpu_init_completed(void); 780 extern void ml_cpu_up(void); 781 extern void ml_cpu_down(void); 782 /* 783 * The update to CPU counts needs to be separate from other actions 784 * in ml_cpu_up() and ml_cpu_down() 785 * because we don't update the counts when CLPC causes temporary 786 * cluster powerdown events, as these must be transparent to the user. 787 */ 788 extern void ml_cpu_up_update_counts(int cpu_id); 789 extern void ml_cpu_down_update_counts(int cpu_id); 790 extern void ml_arm_sleep(void); 791 792 extern uint64_t ml_get_wake_timebase(void); 793 extern uint64_t ml_get_conttime_wake_time(void); 794 795 /* Time since the system was reset (as part of boot/wake) */ 796 uint64_t ml_get_time_since_reset(void); 797 798 /* 799 * Called by ApplePMGR to set wake time. Units and epoch are identical 800 * to mach_continuous_time(). Has no effect on !HAS_CONTINUOUS_HWCLOCK 801 * chips. If wake_time == UINT64_MAX, that means the wake time is 802 * unknown and calls to ml_get_time_since_reset() will return UINT64_MAX. 803 */ 804 void ml_set_reset_time(uint64_t wake_time); 805 806 #ifdef XNU_KERNEL_PRIVATE 807 /* Just a stub on ARM */ 808 extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); 809 #define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) 810 #endif /* XNU_KERNEL_PRIVATE */ 811 812 /* Bytes available on current stack */ 813 vm_offset_t ml_stack_remaining(void); 814 815 #ifdef MACH_KERNEL_PRIVATE 816 uint32_t get_fpscr(void); 817 void set_fpscr(uint32_t); 818 void machine_conf(void); 819 void machine_lockdown(void); 820 821 #ifdef __arm64__ 822 unsigned long update_mdscr(unsigned long clear, unsigned long set); 823 #endif /* __arm64__ */ 824 825 extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); 826 extern void fiq_context_init(boolean_t enable_fiq); 827 828 extern void reenable_async_aborts(void); 829 830 #ifdef __arm64__ 831 uint64_t ml_cluster_wfe_timeout(uint32_t wfe_cluster_id); 832 #endif 833 834 #ifdef MONITOR 835 #define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ 836 #define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ 837 unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, 838 uintptr_t arg2, uintptr_t arg3); 839 #endif /* MONITOR */ 840 841 #if __ARM_KERNEL_PROTECT__ 842 extern void set_vbar_el1(uint64_t); 843 #endif /* __ARM_KERNEL_PROTECT__ */ 844 #endif /* MACH_KERNEL_PRIVATE */ 845 846 extern uint32_t arm_debug_read_dscr(void); 847 848 extern int set_be_bit(void); 849 extern int clr_be_bit(void); 850 extern int be_tracing(void); 851 852 /* Please note that cpu_broadcast_xcall is not as simple is you would like it to be. 853 * It will sometimes put the calling thread to sleep, and it is up to your callback 854 * to wake it up as needed, where "as needed" is defined as "all other CPUs have 855 * called the broadcast func". Look around the kernel for examples, or instead use 856 * cpu_broadcast_xcall_simple() which does indeed act like you would expect, given 857 * the prototype. cpu_broadcast_immediate_xcall has the same caveats and has a similar 858 * _simple() wrapper 859 */ 860 typedef void (*broadcastFunc) (void *); 861 unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); 862 unsigned int cpu_broadcast_xcall_simple(boolean_t, broadcastFunc, void *); 863 __result_use_check kern_return_t cpu_xcall(int, broadcastFunc, void *); 864 unsigned int cpu_broadcast_immediate_xcall(uint32_t *, boolean_t, broadcastFunc, void *); 865 unsigned int cpu_broadcast_immediate_xcall_simple(boolean_t, broadcastFunc, void *); 866 __result_use_check kern_return_t cpu_immediate_xcall(int, broadcastFunc, void *); 867 868 #ifdef KERNEL_PRIVATE 869 870 /* Interface to be used by the perf. controller to register a callback, in a 871 * single-threaded fashion. The callback will receive notifications of 872 * processor performance quality-of-service changes from the scheduler. 873 */ 874 875 #ifdef __arm64__ 876 typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); 877 void cpu_qos_update_register(cpu_qos_update_t); 878 #endif /* __arm64__ */ 879 880 struct going_on_core { 881 uint64_t thread_id; 882 uint16_t qos_class; 883 uint16_t urgency; /* XCPM compatibility */ 884 uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ 885 uint32_t is_kernel_thread : 1; 886 uint64_t thread_group_id; 887 void *thread_group_data; 888 uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ 889 uint64_t start_time; 890 uint64_t scheduling_latency_at_same_basepri; 891 uint32_t energy_estimate_nj; /* return: In nanojoules */ 892 /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ 893 }; 894 typedef struct going_on_core *going_on_core_t; 895 896 struct going_off_core { 897 uint64_t thread_id; 898 uint32_t energy_estimate_nj; /* return: In nanojoules */ 899 uint32_t reserved; 900 uint64_t end_time; 901 uint64_t thread_group_id; 902 void *thread_group_data; 903 }; 904 typedef struct going_off_core *going_off_core_t; 905 906 struct thread_group_data { 907 uint64_t thread_group_id; 908 void *thread_group_data; 909 uint32_t thread_group_size; 910 uint32_t thread_group_flags; 911 }; 912 typedef struct thread_group_data *thread_group_data_t; 913 914 struct perfcontrol_max_runnable_latency { 915 uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; 916 }; 917 typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; 918 919 struct perfcontrol_work_interval { 920 uint64_t thread_id; 921 uint16_t qos_class; 922 uint16_t urgency; 923 uint32_t flags; // notify 924 uint64_t work_interval_id; 925 uint64_t start; 926 uint64_t finish; 927 uint64_t deadline; 928 uint64_t next_start; 929 uint64_t thread_group_id; 930 void *thread_group_data; 931 uint32_t create_flags; 932 }; 933 typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; 934 935 typedef enum { 936 WORK_INTERVAL_START, 937 WORK_INTERVAL_UPDATE, 938 WORK_INTERVAL_FINISH 939 } work_interval_ctl_t; 940 941 struct perfcontrol_work_interval_instance { 942 work_interval_ctl_t ctl; 943 uint32_t create_flags; 944 uint64_t complexity; 945 uint64_t thread_id; 946 uint64_t work_interval_id; 947 uint64_t instance_id; /* out: start, in: update/finish */ 948 uint64_t start; 949 uint64_t finish; 950 uint64_t deadline; 951 uint64_t thread_group_id; 952 void *thread_group_data; 953 }; 954 typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t; 955 956 /* 957 * Structure to export per-CPU counters as part of the CLPC callout. 958 * Contains only the fixed CPU counters (instructions and cycles); CLPC 959 * would call back into XNU to get the configurable counters if needed. 960 */ 961 struct perfcontrol_cpu_counters { 962 uint64_t instructions; 963 uint64_t cycles; 964 }; 965 966 __options_decl(perfcontrol_thread_flags_mask_t, uint64_t, { 967 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_RR = 1 << 0, 968 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_NATIVE_FIRST = 1 << 1, 969 }); 970 971 972 /* 973 * Structure used to pass information about a thread to CLPC 974 */ 975 struct perfcontrol_thread_data { 976 /* 977 * Energy estimate (return value) 978 * The field is populated by CLPC and used to update the 979 * energy estimate of the thread 980 */ 981 uint32_t energy_estimate_nj; 982 /* Perfcontrol class for thread */ 983 perfcontrol_class_t perfctl_class; 984 /* Thread ID for the thread */ 985 uint64_t thread_id; 986 /* Thread Group ID */ 987 uint64_t thread_group_id; 988 /* 989 * Scheduling latency for threads at the same base priority. 990 * Calculated by the scheduler and passed into CLPC. The field is 991 * populated only in the thread_data structure for the thread 992 * going on-core. 993 */ 994 uint64_t scheduling_latency_at_same_basepri; 995 /* Thread Group data pointer */ 996 void *thread_group_data; 997 /* perfctl state pointer */ 998 void *perfctl_state; 999 /* Bitmask to indicate which thread flags have been updated as part of the callout */ 1000 perfcontrol_thread_flags_mask_t thread_flags_mask; 1001 /* Actual values for the flags that are getting updated in the callout */ 1002 perfcontrol_thread_flags_mask_t thread_flags; 1003 }; 1004 1005 /* 1006 * All callouts from the scheduler are executed with interrupts 1007 * disabled. Callouts should be implemented in C with minimal 1008 * abstractions, and only use KPI exported by the mach/libkern 1009 * symbolset, restricted to routines like spinlocks and atomic 1010 * operations and scheduler routines as noted below. Spinlocks that 1011 * are used to synchronize data in the perfcontrol_state_t should only 1012 * ever be acquired with interrupts disabled, to avoid deadlocks where 1013 * an quantum expiration timer interrupt attempts to perform a callout 1014 * that attempts to lock a spinlock that is already held. 1015 */ 1016 1017 /* 1018 * When a processor is switching between two threads (after the 1019 * scheduler has chosen a new thread), the low-level platform layer 1020 * will call this routine, which should perform required timestamps, 1021 * MMIO register reads, or other state switching. No scheduler locks 1022 * are held during this callout. 1023 * 1024 * This function is called with interrupts ENABLED. 1025 */ 1026 typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); 1027 1028 /* 1029 * Once the processor has switched to the new thread, the offcore 1030 * callout will indicate the old thread that is no longer being 1031 * run. The thread's scheduler lock is held, so it will not begin 1032 * running on another processor (in the case of preemption where it 1033 * remains runnable) until it completes. If the "thread_terminating" 1034 * boolean is TRUE, this will be the last callout for this thread_id. 1035 */ 1036 typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); 1037 1038 /* 1039 * After the offcore callout and after the old thread can potentially 1040 * start running on another processor, the oncore callout will be 1041 * called with the thread's scheduler lock held. The oncore callout is 1042 * also called any time one of the parameters in the going_on_core_t 1043 * structure changes, like priority/QoS changes, and quantum 1044 * expiration, so the callout must not assume callouts are paired with 1045 * offcore callouts. 1046 */ 1047 typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); 1048 1049 /* 1050 * Periodically (on hundreds of ms scale), the scheduler will perform 1051 * maintenance and report the maximum latency for runnable (but not currently 1052 * running) threads for each urgency class. 1053 */ 1054 typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); 1055 1056 /* 1057 * When the kernel receives information about work intervals from userland, 1058 * it is passed along using this callback. No locks are held, although the state 1059 * object will not go away during the callout. 1060 */ 1061 typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); 1062 1063 /* 1064 * Start, update and finish work interval instance with optional complexity estimate. 1065 */ 1066 typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t); 1067 1068 /* 1069 * These callbacks are used when thread groups are added, removed or properties 1070 * updated. 1071 * No blocking allocations (or anything else blocking) are allowed inside these 1072 * callbacks. No locks allowed in these callbacks as well since the kernel might 1073 * be holding the thread/task locks. 1074 */ 1075 typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); 1076 typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); 1077 typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); 1078 1079 /* 1080 * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, 1081 * this function will be called, passing the timeout deadline that was previously armed as an argument. 1082 * 1083 * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. 1084 */ 1085 typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); 1086 1087 /* 1088 * Context Switch Callout 1089 * 1090 * Parameters: 1091 * event - The perfcontrol_event for this callout 1092 * cpu_id - The CPU doing the context switch 1093 * timestamp - The timestamp for the context switch 1094 * flags - Flags for other relevant information 1095 * offcore - perfcontrol_data structure for thread going off-core 1096 * oncore - perfcontrol_data structure for thread going on-core 1097 * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch 1098 */ 1099 typedef void (*sched_perfcontrol_csw_t)( 1100 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, 1101 struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, 1102 struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); 1103 1104 1105 /* 1106 * Thread State Update Callout 1107 * 1108 * Parameters: 1109 * event - The perfcontrol_event for this callout 1110 * cpu_id - The CPU doing the state update 1111 * timestamp - The timestamp for the state update 1112 * flags - Flags for other relevant information 1113 * thr_data - perfcontrol_data structure for the thread being updated 1114 */ 1115 typedef void (*sched_perfcontrol_state_update_t)( 1116 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, 1117 struct perfcontrol_thread_data *thr_data, __unused void *unused); 1118 1119 /* 1120 * Thread Group Blocking Relationship Callout 1121 * 1122 * Parameters: 1123 * blocked_tg - Thread group blocking on progress of another thread group 1124 * blocking_tg - Thread group blocking progress of another thread group 1125 * flags - Flags for other relevant information 1126 * blocked_thr_state - Per-thread perfcontrol state for blocked thread 1127 */ 1128 typedef void (*sched_perfcontrol_thread_group_blocked_t)( 1129 thread_group_data_t blocked_tg, thread_group_data_t blocking_tg, uint32_t flags, perfcontrol_state_t blocked_thr_state); 1130 1131 /* 1132 * Thread Group Unblocking Callout 1133 * 1134 * Parameters: 1135 * unblocked_tg - Thread group being unblocked from making forward progress 1136 * unblocking_tg - Thread group unblocking progress of another thread group 1137 * flags - Flags for other relevant information 1138 * unblocked_thr_state - Per-thread perfcontrol state for unblocked thread 1139 */ 1140 typedef void (*sched_perfcontrol_thread_group_unblocked_t)( 1141 thread_group_data_t unblocked_tg, thread_group_data_t unblocking_tg, uint32_t flags, perfcontrol_state_t unblocked_thr_state); 1142 1143 /* 1144 * Callers should always use the CURRENT version so that the kernel can detect both older 1145 * and newer structure layouts. New callbacks should always be added at the end of the 1146 * structure, and xnu should expect existing source recompiled against newer headers 1147 * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter 1148 * to reset callbacks to their default in-kernel values. 1149 */ 1150 1151 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ 1152 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ 1153 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ 1154 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ 1155 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ 1156 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ 1157 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ 1158 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */ 1159 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_8 (8) /* up-to thread_group_unblocked */ 1160 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_9 (9) /* allows CLPC to specify resource contention flags */ 1161 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 1162 1163 struct sched_perfcontrol_callbacks { 1164 unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ 1165 sched_perfcontrol_offcore_t offcore; 1166 sched_perfcontrol_context_switch_t context_switch; 1167 sched_perfcontrol_oncore_t oncore; 1168 sched_perfcontrol_max_runnable_latency_t max_runnable_latency; 1169 sched_perfcontrol_work_interval_notify_t work_interval_notify; 1170 sched_perfcontrol_thread_group_init_t thread_group_init; 1171 sched_perfcontrol_thread_group_deinit_t thread_group_deinit; 1172 sched_perfcontrol_deadline_passed_t deadline_passed; 1173 sched_perfcontrol_csw_t csw; 1174 sched_perfcontrol_state_update_t state_update; 1175 sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; 1176 sched_perfcontrol_work_interval_ctl_t work_interval_ctl; 1177 sched_perfcontrol_thread_group_blocked_t thread_group_blocked; 1178 sched_perfcontrol_thread_group_unblocked_t thread_group_unblocked; 1179 }; 1180 typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; 1181 1182 extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); 1183 extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); 1184 extern void sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit); 1185 extern const char* sched_perfcontrol_thread_group_get_name(void *data); 1186 1187 /* 1188 * Edge Scheduler-CLPC Interface 1189 * 1190 * sched_perfcontrol_thread_group_preferred_clusters_set() 1191 * 1192 * The Edge scheduler expects thread group recommendations to be specific clusters rather 1193 * than just E/P. In order to allow more fine grained control, CLPC can specify an override 1194 * preferred cluster per QoS bucket. CLPC passes a common preferred cluster `tg_preferred_cluster` 1195 * and an array of size [PERFCONTROL_CLASS_MAX] with overrides for specific perfctl classes. 1196 * The scheduler translates these preferences into sched_bucket 1197 * preferences and applies the changes. 1198 * 1199 */ 1200 /* Token to indicate a particular perfctl class is not overriden */ 1201 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE ((uint32_t)~0) 1202 1203 /* 1204 * CLPC can also indicate if there should be an immediate rebalancing of threads of this TG as 1205 * part of this preferred cluster change. It does that by specifying the following options. 1206 */ 1207 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING 0x1 1208 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNABLE 0x2 1209 typedef uint64_t sched_perfcontrol_preferred_cluster_options_t; 1210 1211 extern void sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster, 1212 uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options); 1213 1214 /* 1215 * Edge Scheduler-CLPC Interface 1216 * 1217 * sched_perfcontrol_edge_matrix_get()/sched_perfcontrol_edge_matrix_set() 1218 * 1219 * The Edge scheduler uses edges between clusters to define the likelihood of migrating threads 1220 * across clusters. The edge config between any two clusters defines the edge weight and whether 1221 * migation and steal operations are allowed across that edge. The getter and setter allow CLPC 1222 * to query and configure edge properties between various clusters on the platform. 1223 */ 1224 1225 extern void sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order); 1226 extern void sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order); 1227 1228 /* 1229 * Update the deadline after which sched_perfcontrol_deadline_passed will be called. 1230 * Returns TRUE if it successfully canceled a previously set callback, 1231 * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). 1232 * The callback is automatically canceled when it fires, and does not repeat unless rearmed. 1233 * 1234 * This 'timer' executes as the scheduler switches between threads, on a non-idle core 1235 * 1236 * There can be only one outstanding timer globally. 1237 */ 1238 extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); 1239 1240 /* 1241 * SFI configuration. 1242 */ 1243 extern kern_return_t sched_perfcontrol_sfi_set_window(uint64_t window_usecs); 1244 extern kern_return_t sched_perfcontrol_sfi_set_bg_offtime(uint64_t offtime_usecs); 1245 extern kern_return_t sched_perfcontrol_sfi_set_utility_offtime(uint64_t offtime_usecs); 1246 1247 typedef enum perfcontrol_callout_type { 1248 PERFCONTROL_CALLOUT_ON_CORE, 1249 PERFCONTROL_CALLOUT_OFF_CORE, 1250 PERFCONTROL_CALLOUT_CONTEXT, 1251 PERFCONTROL_CALLOUT_STATE_UPDATE, 1252 /* Add other callout types here */ 1253 PERFCONTROL_CALLOUT_MAX 1254 } perfcontrol_callout_type_t; 1255 1256 typedef enum perfcontrol_callout_stat { 1257 PERFCONTROL_STAT_INSTRS, 1258 PERFCONTROL_STAT_CYCLES, 1259 /* Add other stat types here */ 1260 PERFCONTROL_STAT_MAX 1261 } perfcontrol_callout_stat_t; 1262 1263 uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, 1264 perfcontrol_callout_stat_t stat); 1265 1266 #ifdef __arm64__ 1267 /* The performance controller may use this interface to recommend 1268 * that CPUs in the designated cluster employ WFE rather than WFI 1269 * within the idle loop, falling back to WFI after the specified 1270 * timeout. The updates are expected to be serialized by the caller, 1271 * the implementation is not required to perform internal synchronization. 1272 */ 1273 uint32_t ml_update_cluster_wfe_recommendation(uint32_t wfe_cluster_id, uint64_t wfe_timeout_abstime_interval, uint64_t wfe_hint_flags); 1274 #endif /* __arm64__ */ 1275 1276 #if defined(HAS_APPLE_PAC) 1277 #define ONES(x) (BIT((x))-1) 1278 #define PTR_MASK ONES(64-T1SZ_BOOT) 1279 #define PAC_MASK ~PTR_MASK 1280 #define SIGN(p) ((p) & BIT(55)) 1281 #define UNSIGN_PTR(p) \ 1282 SIGN(p) ? ((p) | PAC_MASK) : ((p) & ~PAC_MASK) 1283 1284 uint64_t ml_default_rop_pid(void); 1285 uint64_t ml_default_jop_pid(void); 1286 void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit); 1287 void ml_task_set_jop_pid(task_t task, task_t parent_task, boolean_t inherit); 1288 void ml_task_set_jop_pid_from_shared_region(task_t task); 1289 void ml_task_set_disable_user_jop(task_t task, uint8_t disable_user_jop); 1290 void ml_thread_set_disable_user_jop(thread_t thread, uint8_t disable_user_jop); 1291 void ml_thread_set_jop_pid(thread_t thread, task_t task); 1292 void *ml_auth_ptr_unchecked(void *ptr, unsigned key, uint64_t modifier); 1293 1294 uint64_t ml_enable_user_jop_key(uint64_t user_jop_key); 1295 1296 /** 1297 * Restores the previous JOP key state after a previous ml_enable_user_jop_key() 1298 * call. 1299 * 1300 * @param user_jop_key The userspace JOP key previously passed to 1301 * ml_enable_user_jop_key() 1302 * @param saved_jop_state The saved JOP state returned by 1303 * ml_enable_user_jop_key() 1304 */ 1305 void ml_disable_user_jop_key(uint64_t user_jop_key, uint64_t saved_jop_state); 1306 #endif /* defined(HAS_APPLE_PAC) */ 1307 1308 void ml_enable_monitor(void); 1309 1310 #endif /* KERNEL_PRIVATE */ 1311 1312 boolean_t machine_timeout_suspended(void); 1313 void ml_get_power_state(boolean_t *, boolean_t *); 1314 1315 uint32_t get_arm_cpu_version(void); 1316 boolean_t user_cont_hwclock_allowed(void); 1317 uint8_t user_timebase_type(void); 1318 boolean_t ml_thread_is64bit(thread_t thread); 1319 1320 #ifdef __arm64__ 1321 bool ml_feature_supported(uint32_t feature_bit); 1322 void ml_set_align_checking(void); 1323 extern void wfe_timeout_configure(void); 1324 extern void wfe_timeout_init(void); 1325 #endif /* __arm64__ */ 1326 1327 void ml_timer_evaluate(void); 1328 boolean_t ml_timer_forced_evaluation(void); 1329 void ml_gpu_stat_update(uint64_t); 1330 uint64_t ml_gpu_stat(thread_t); 1331 #endif /* __APPLE_API_PRIVATE */ 1332 1333 1334 1335 #if __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) 1336 extern void ml_expect_fault_begin(expected_fault_handler_t, uintptr_t); 1337 extern void ml_expect_fault_end(void); 1338 #endif /* __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) */ 1339 1340 1341 extern uint32_t phy_read_panic; 1342 extern uint32_t phy_write_panic; 1343 #if DEVELOPMENT || DEBUG 1344 extern uint64_t simulate_stretched_io; 1345 #endif 1346 1347 void ml_hibernate_active_pre(void); 1348 void ml_hibernate_active_post(void); 1349 1350 __END_DECLS 1351 1352 #endif /* _ARM_MACHINE_ROUTINES_H_ */ 1353