1 /* 2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifndef _ARM_MACHINE_ROUTINES_H_ 33 #define _ARM_MACHINE_ROUTINES_H_ 34 35 #include <mach/mach_types.h> 36 #include <mach/vm_types.h> 37 #include <mach/boolean.h> 38 #include <kern/kern_types.h> 39 #include <pexpert/pexpert.h> 40 41 #include <sys/cdefs.h> 42 #include <sys/appleapiopts.h> 43 44 #include <stdarg.h> 45 46 #ifdef XNU_KERNEL_PRIVATE 47 #include <kern/sched_hygiene.h> 48 #include <kern/startup.h> 49 #endif /* XNU_KERNEL_PRIVATE */ 50 51 __BEGIN_DECLS 52 #ifdef XNU_KERNEL_PRIVATE 53 #ifdef __arm64__ 54 typedef bool (*expected_fault_handler_t)(arm_saved_state_t *); 55 #endif /* __arm64__ */ 56 #endif /* XNU_KERNEL_PRIVATE */ 57 58 /* Interrupt handling */ 59 60 void ml_cpu_signal(unsigned int cpu_id); 61 void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); 62 uint64_t ml_cpu_signal_deferred_get_timer(void); 63 void ml_cpu_signal_deferred(unsigned int cpu_id); 64 void ml_cpu_signal_retract(unsigned int cpu_id); 65 bool ml_cpu_signal_is_enabled(void); 66 67 /* Initialize Interrupts */ 68 void ml_init_interrupt(void); 69 70 /* Get Interrupts Enabled */ 71 boolean_t ml_get_interrupts_enabled(void); 72 73 /* Set Interrupts Enabled */ 74 #if __has_feature(ptrauth_calls) 75 uint64_t ml_pac_safe_interrupts_disable(void); 76 void ml_pac_safe_interrupts_restore(uint64_t); 77 #endif /* __has_feature(ptrauth_calls) */ 78 boolean_t ml_set_interrupts_enabled_with_debug(boolean_t enable, boolean_t debug); 79 boolean_t ml_set_interrupts_enabled(boolean_t enable); 80 boolean_t ml_early_set_interrupts_enabled(boolean_t enable); 81 82 /* 83 * Functions for disabling measurements for AppleCLPC only. 84 */ 85 boolean_t sched_perfcontrol_ml_set_interrupts_without_measurement(boolean_t enable); 86 void sched_perfcontrol_abandon_preemption_disable_measurement(void); 87 88 /* Check if running at interrupt context */ 89 boolean_t ml_at_interrupt_context(void); 90 91 92 /* Generate a fake interrupt */ 93 void ml_cause_interrupt(void); 94 95 96 #ifdef XNU_KERNEL_PRIVATE 97 98 /* did this interrupt context interrupt userspace? */ 99 bool ml_did_interrupt_userspace(void); 100 101 /* Clear interrupt spin debug state for thread */ 102 103 #if SCHED_HYGIENE_DEBUG 104 void ml_adjust_preemption_disable_time(thread_t thread, int64_t duration); 105 106 void mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs); 107 108 #if MONOTONIC 109 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) \ 110 if (sched_hygiene_debug_pmc) { \ 111 mt_cur_cpu_cycles_instrs_speculative(&thread->machine.intmask_cycles, \ 112 &thread->machine.intmask_instr); \ 113 } 114 #else 115 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) 116 #endif 117 118 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) \ 119 do { \ 120 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ 121 thread_t thread = current_thread(); \ 122 thread->machine.int_type = type; \ 123 thread->machine.int_handler_addr = (uintptr_t)VM_KERNEL_STRIP_UPTR(handler_addr); \ 124 thread->machine.inthandler_timestamp = ml_get_sched_hygiene_timebase(); \ 125 INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread); \ 126 thread->machine.int_vector = (uintptr_t)NULL; \ 127 } \ 128 } while (0) 129 130 #define INTERRUPT_MASKED_DEBUG_END() \ 131 do { \ 132 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ 133 thread_t thread = current_thread(); \ 134 ml_handle_interrupt_handler_duration(thread); \ 135 thread->machine.inthandler_timestamp = 0; \ 136 thread->machine.inthandler_abandon = false; \ 137 } \ 138 } while (0) 139 140 void ml_irq_debug_start(uintptr_t handler, uintptr_t vector); 141 void ml_irq_debug_end(void); 142 void ml_irq_debug_abandon(void); 143 144 void ml_spin_debug_reset(thread_t thread); 145 void ml_spin_debug_clear(thread_t thread); 146 void ml_spin_debug_clear_self(void); 147 void ml_handle_interrupts_disabled_duration(thread_t thread); 148 void ml_handle_stackshot_interrupt_disabled_duration(thread_t thread); 149 void ml_handle_interrupt_handler_duration(thread_t thread); 150 151 #else /* SCHED_HYGIENE_DEBUG */ 152 153 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) 154 #define INTERRUPT_MASKED_DEBUG_END() 155 156 #endif /* SCHED_HYGIENE_DEBUG */ 157 158 extern bool ml_snoop_thread_is_on_core(thread_t thread); 159 extern boolean_t ml_is_quiescing(void); 160 extern void ml_set_is_quiescing(boolean_t); 161 extern uint64_t ml_get_booter_memory_size(void); 162 #endif 163 164 /* Type for the Time Base Enable function */ 165 typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); 166 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 167 /* Type for the Processor Cache Dispatch function */ 168 typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); 169 170 typedef uint32_t (*get_decrementer_t)(void); 171 typedef void (*set_decrementer_t)(uint32_t); 172 typedef void (*fiq_handler_t)(void); 173 174 #endif 175 176 #define CacheConfig 0x00000000UL 177 #define CacheControl 0x00000001UL 178 #define CacheClean 0x00000002UL 179 #define CacheCleanRegion 0x00000003UL 180 #define CacheCleanFlush 0x00000004UL 181 #define CacheCleanFlushRegion 0x00000005UL 182 #define CacheShutdown 0x00000006UL 183 184 #define CacheControlEnable 0x00000000UL 185 186 #define CacheConfigCCSIDR 0x00000001UL 187 #define CacheConfigSize 0x00000100UL 188 189 /* Type for the Processor Idle function */ 190 typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); 191 192 /* Type for the Idle Tickle function */ 193 typedef void (*idle_tickle_t)(void); 194 195 /* Type for the Idle Timer function */ 196 typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); 197 198 /* Type for the IPI Hander */ 199 typedef void (*ipi_handler_t)(void); 200 201 /* Type for the Lockdown Hander */ 202 typedef void (*lockdown_handler_t)(void *); 203 204 /* Type for the Platform specific Error Handler */ 205 typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); 206 207 /* 208 * The exception callback (ex_cb) module allows kernel drivers to 209 * register and receive callbacks for exceptions, and indicate 210 * actions to be taken by the platform kernel 211 * Currently this is supported for ARM64 but extending support for ARM32 212 * should be straightforward 213 */ 214 215 /* Supported exception classes for callbacks */ 216 typedef enum{ 217 EXCB_CLASS_ILLEGAL_INSTR_SET, 218 #ifdef CONFIG_XNUPOST 219 EXCB_CLASS_TEST1, 220 EXCB_CLASS_TEST2, 221 EXCB_CLASS_TEST3, 222 #endif 223 EXCB_CLASS_MAX // this must be last 224 } 225 ex_cb_class_t; 226 227 /* Actions indicated by callbacks to be taken by platform kernel */ 228 typedef enum{ 229 EXCB_ACTION_RERUN, // re-run the faulting instruction 230 EXCB_ACTION_NONE, // continue normal exception handling 231 #ifdef CONFIG_XNUPOST 232 EXCB_ACTION_TEST_FAIL, 233 #endif 234 } 235 ex_cb_action_t; 236 237 /* 238 * Exception state 239 * We cannot use a private kernel data structure such as arm_saved_state_t 240 * The CPSR and ESR are not clobbered when the callback function is invoked so 241 * those registers can be examined by the callback function; 242 * the same is done in the platform error handlers 243 */ 244 typedef struct{ 245 vm_offset_t far; 246 } 247 ex_cb_state_t; 248 249 /* callback type definition */ 250 typedef ex_cb_action_t (*ex_cb_t) ( 251 ex_cb_class_t cb_class, 252 void *refcon,// provided at registration 253 const ex_cb_state_t *state // exception state 254 ); 255 256 /* 257 * Callback registration 258 * Currently we support only one registered callback per class but 259 * it should be possible to support more callbacks 260 */ 261 kern_return_t ex_cb_register( 262 ex_cb_class_t cb_class, 263 ex_cb_t cb, 264 void *refcon ); 265 266 /* 267 * Called internally by platform kernel to invoke the registered callback for class 268 */ 269 ex_cb_action_t ex_cb_invoke( 270 ex_cb_class_t cb_class, 271 vm_offset_t far); 272 273 typedef enum { 274 CLUSTER_TYPE_SMP, 275 CLUSTER_TYPE_E, 276 CLUSTER_TYPE_P, 277 MAX_CPU_TYPES, 278 } cluster_type_t; 279 280 void ml_parse_cpu_topology(void); 281 282 unsigned int ml_get_cpu_count(void); 283 284 unsigned int ml_get_cpu_number_type(cluster_type_t cluster_type, bool logical, bool available); 285 286 unsigned int ml_get_cluster_number_type(cluster_type_t cluster_type); 287 288 unsigned int ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type, bool include_all_cpu_types); 289 290 unsigned int ml_get_cpu_types(void); 291 292 int ml_get_boot_cpu_number(void); 293 294 int ml_get_cpu_number(uint32_t phys_id); 295 296 unsigned int ml_get_cpu_number_local(void); 297 298 int ml_get_cluster_number(uint32_t phys_id); 299 300 int ml_get_max_cpu_number(void); 301 302 int ml_get_max_cluster_number(void); 303 304 /* 305 * Return the id of a cluster's first cpu. 306 */ 307 unsigned int ml_get_first_cpu_id(unsigned int cluster_id); 308 309 /* 310 * Return the die id of a cluster. 311 */ 312 unsigned int ml_get_die_id(unsigned int cluster_id); 313 314 /* 315 * Return the index of a cluster in its die. 316 */ 317 unsigned int ml_get_die_cluster_id(unsigned int cluster_id); 318 319 /* 320 * Return the highest die id of the system. 321 */ 322 unsigned int ml_get_max_die_id(void); 323 324 #ifdef __arm64__ 325 int ml_get_cluster_number_local(void); 326 #endif /* __arm64__ */ 327 328 /* Struct for ml_cpu_get_info */ 329 struct ml_cpu_info { 330 unsigned long vector_unit; 331 unsigned long cache_line_size; 332 unsigned long l1_icache_size; 333 unsigned long l1_dcache_size; 334 unsigned long l2_settings; 335 unsigned long l2_cache_size; 336 unsigned long l3_settings; 337 unsigned long l3_cache_size; 338 }; 339 typedef struct ml_cpu_info ml_cpu_info_t; 340 341 cluster_type_t ml_get_boot_cluster_type(void); 342 343 /*! 344 * @typedef ml_topology_cpu_t 345 * @brief Describes one CPU core in the topology. 346 * 347 * @field cpu_id Logical CPU ID: 0, 1, 2, 3, 4, ... 348 * Dynamically assigned by XNU so it might not match EDT. No holes. 349 * @field phys_id Physical CPU ID (EDT: reg). Same as MPIDR[15:0], i.e. 350 * (cluster_id << 8) | core_number_within_cluster 351 * @field cluster_id Logical Cluster ID: 0, 1, 2, 3, 4, ... 352 * Dynamically assigned by XNU so it might not match EDT. No holes. 353 * @field die_id Die ID (EDT: die-id) 354 * @field cluster_type The type of CPUs found in this cluster. 355 * @field l2_access_penalty Indicates that the scheduler should try to de-prioritize a core because 356 * L2 accesses are slower than on the boot processor. 357 * @field l2_cache_size Size of the L2 cache, in bytes. 0 if unknown or not present. 358 * @field l2_cache_id l2-cache-id property read from EDT. 359 * @field l3_cache_size Size of the L3 cache, in bytes. 0 if unknown or not present. 360 * @field l3_cache_id l3-cache-id property read from EDT. 361 * @field cpu_IMPL_regs IO-mapped virtual address of cpuX_IMPL (implementation-defined) register block. 362 * @field cpu_IMPL_pa Physical address of cpuX_IMPL register block. 363 * @field cpu_IMPL_len Length of cpuX_IMPL register block. 364 * @field cpu_UTTDBG_regs IO-mapped virtual address of cpuX_UTTDBG register block. 365 * @field cpu_UTTDBG_pa Physical address of cpuX_UTTDBG register block, if set in DT, else zero 366 * @field cpu_UTTDBG_len Length of cpuX_UTTDBG register block, if set in DT, else zero 367 * @field coresight_regs IO-mapped virtual address of CoreSight debug register block. 368 * @field coresight_pa Physical address of CoreSight register block. 369 * @field coresight_len Length of CoreSight register block. 370 * @field die_cluster_id Physical cluster ID within the local die (EDT: die-cluster-id) 371 * @field cluster_core_id Physical core ID within the local cluster (EDT: cluster-core-id) 372 */ 373 typedef struct ml_topology_cpu { 374 unsigned int cpu_id; 375 uint32_t phys_id; 376 unsigned int cluster_id; 377 unsigned int die_id; 378 cluster_type_t cluster_type; 379 uint32_t l2_access_penalty; 380 uint32_t l2_cache_size; 381 uint32_t l2_cache_id; 382 uint32_t l3_cache_size; 383 uint32_t l3_cache_id; 384 vm_offset_t cpu_IMPL_regs; 385 uint64_t cpu_IMPL_pa; 386 uint64_t cpu_IMPL_len; 387 vm_offset_t cpu_UTTDBG_regs; 388 uint64_t cpu_UTTDBG_pa; 389 uint64_t cpu_UTTDBG_len; 390 vm_offset_t coresight_regs; 391 uint64_t coresight_pa; 392 uint64_t coresight_len; 393 unsigned int die_cluster_id; 394 unsigned int cluster_core_id; 395 } ml_topology_cpu_t; 396 397 /*! 398 * @typedef ml_topology_cluster_t 399 * @brief Describes one cluster in the topology. 400 * 401 * @field cluster_id Cluster ID (EDT: cluster-id) 402 * @field cluster_type The type of CPUs found in this cluster. 403 * @field num_cpus Total number of usable CPU cores in this cluster. 404 * @field first_cpu_id The cpu_id of the first CPU in the cluster. 405 * @field cpu_mask A bitmask representing the cpu_id's that belong to the cluster. Example: 406 * If the cluster contains CPU4 and CPU5, cpu_mask will be 0x30. 407 * @field acc_IMPL_regs IO-mapped virtual address of acc_IMPL (implementation-defined) register block. 408 * @field acc_IMPL_pa Physical address of acc_IMPL register block. 409 * @field acc_IMPL_len Length of acc_IMPL register block. 410 * @field cpm_IMPL_regs IO-mapped virtual address of cpm_IMPL (implementation-defined) register block. 411 * @field cpm_IMPL_pa Physical address of cpm_IMPL register block. 412 * @field cpm_IMPL_len Length of cpm_IMPL register block. 413 */ 414 typedef struct ml_topology_cluster { 415 unsigned int cluster_id; 416 cluster_type_t cluster_type; 417 unsigned int num_cpus; 418 unsigned int first_cpu_id; 419 uint64_t cpu_mask; 420 vm_offset_t acc_IMPL_regs; 421 uint64_t acc_IMPL_pa; 422 uint64_t acc_IMPL_len; 423 vm_offset_t cpm_IMPL_regs; 424 uint64_t cpm_IMPL_pa; 425 uint64_t cpm_IMPL_len; 426 } ml_topology_cluster_t; 427 428 // Bump this version number any time any ml_topology_* struct changes, so 429 // that KPI users can check whether their headers are compatible with 430 // the running kernel. 431 #define CPU_TOPOLOGY_VERSION 1 432 433 /*! 434 * @typedef ml_topology_info_t 435 * @brief Describes the CPU topology for all APs in the system. Populated from EDT and read-only at runtime. 436 * @discussion This struct only lists CPU cores that are considered usable by both iBoot and XNU. Some 437 * physically present CPU cores may be considered unusable due to configuration options like 438 * the "cpus=" boot-arg. Cores that are disabled in hardware will not show up in EDT at all, so 439 * they also will not be present in this struct. 440 * 441 * @field version Version of the struct (set to CPU_TOPOLOGY_VERSION). 442 * @field num_cpus Total number of usable CPU cores. 443 * @field max_cpu_id The highest usable logical CPU ID. 444 * @field num_clusters Total number of AP CPU clusters on the system (usable or not). 445 * @field max_cluster_id The highest cluster ID found in EDT. 446 * @field cpus List of |num_cpus| entries. 447 * @field clusters List of |num_clusters| entries. 448 * @field boot_cpu Points to the |cpus| entry for the boot CPU. 449 * @field boot_cluster Points to the |clusters| entry which contains the boot CPU. 450 * @field chip_revision Silicon revision reported by iBoot, which comes from the 451 * SoC-specific fuse bits. See CPU_VERSION_xx macros for definitions. 452 */ 453 typedef struct ml_topology_info { 454 unsigned int version; 455 unsigned int num_cpus; 456 unsigned int max_cpu_id; 457 unsigned int num_clusters; 458 unsigned int max_cluster_id; 459 unsigned int max_die_id; 460 ml_topology_cpu_t *cpus; 461 ml_topology_cluster_t *clusters; 462 ml_topology_cpu_t *boot_cpu; 463 ml_topology_cluster_t *boot_cluster; 464 unsigned int chip_revision; 465 unsigned int cluster_types; 466 unsigned int cluster_type_num_cpus[MAX_CPU_TYPES]; 467 unsigned int cluster_type_num_clusters[MAX_CPU_TYPES]; 468 } ml_topology_info_t; 469 470 /*! 471 * @function ml_get_topology_info 472 * @result A pointer to the read-only topology struct. Does not need to be freed. Returns NULL 473 * if the struct hasn't been initialized or the feature is unsupported. 474 */ 475 const ml_topology_info_t *ml_get_topology_info(void); 476 477 /*! 478 * @function ml_map_cpu_pio 479 * @brief Maps per-CPU and per-cluster PIO registers found in EDT. This needs to be 480 * called after arm_vm_init() so it can't be part of ml_parse_cpu_topology(). 481 */ 482 void ml_map_cpu_pio(void); 483 484 /* Struct for ml_processor_register */ 485 struct ml_processor_info { 486 cpu_id_t cpu_id; 487 vm_offset_t start_paddr; 488 boolean_t supports_nap; 489 void *platform_cache_dispatch; 490 time_base_enable_t time_base_enable; 491 processor_idle_t processor_idle; 492 idle_tickle_t *idle_tickle; 493 idle_timer_t idle_timer; 494 void *idle_timer_refcon; 495 vm_offset_t powergate_stub_addr; 496 uint32_t powergate_stub_length; 497 uint32_t powergate_latency; 498 platform_error_handler_t platform_error_handler; 499 uint64_t regmap_paddr; 500 uint32_t phys_id; 501 uint32_t log_id; 502 uint32_t l2_access_penalty; 503 uint32_t cluster_id; 504 cluster_type_t cluster_type; 505 uint32_t l2_cache_id; 506 uint32_t l2_cache_size; 507 uint32_t l3_cache_id; 508 uint32_t l3_cache_size; 509 }; 510 typedef struct ml_processor_info ml_processor_info_t; 511 512 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 513 /* Struct for ml_init_timebase */ 514 struct tbd_ops { 515 fiq_handler_t tbd_fiq_handler; 516 get_decrementer_t tbd_get_decrementer; 517 set_decrementer_t tbd_set_decrementer; 518 }; 519 typedef struct tbd_ops *tbd_ops_t; 520 typedef struct tbd_ops tbd_ops_data_t; 521 #endif 522 523 524 /*! 525 * @function ml_processor_register 526 * 527 * @abstract callback from platform kext to register processor 528 * 529 * @discussion This function is called by the platform kext when a processor is 530 * being registered. This is called while running on the CPU itself, as part of 531 * its initialization. 532 * 533 * @param ml_processor_info provides machine-specific information about the 534 * processor to xnu. 535 * 536 * @param processor is set as an out-parameter to an opaque handle that should 537 * be used by the platform kext when referring to this processor in the future. 538 * 539 * @param ipi_handler is set as an out-parameter to the function that should be 540 * registered as the IPI handler. 541 * 542 * @param pmi_handler is set as an out-parameter to the function that should be 543 * registered as the PMI handler. 544 * 545 * @returns KERN_SUCCESS on success and an error code, otherwise. 546 */ 547 kern_return_t ml_processor_register(ml_processor_info_t *ml_processor_info, 548 processor_t *processor, ipi_handler_t *ipi_handler, 549 perfmon_interrupt_handler_func *pmi_handler); 550 551 /* Register a lockdown handler */ 552 kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); 553 554 /* Register a M$ flushing */ 555 typedef kern_return_t (*mcache_flush_function)(void *service); 556 kern_return_t ml_mcache_flush_callback_register(mcache_flush_function func, void *service); 557 kern_return_t ml_mcache_flush(void); 558 559 #if XNU_KERNEL_PRIVATE 560 void ml_lockdown_init(void); 561 562 /* Machine layer routine for intercepting panics */ 563 __printflike(1, 0) 564 void ml_panic_trap_to_debugger(const char *panic_format_str, 565 va_list *panic_args, 566 unsigned int reason, 567 void *ctx, 568 uint64_t panic_options_mask, 569 unsigned long panic_caller); 570 #endif /* XNU_KERNEL_PRIVATE */ 571 572 /* Initialize Interrupts */ 573 void ml_install_interrupt_handler( 574 void *nub, 575 int source, 576 void *target, 577 IOInterruptHandler handler, 578 void *refCon); 579 580 vm_offset_t 581 ml_static_vtop( 582 vm_offset_t); 583 584 kern_return_t 585 ml_static_verify_page_protections( 586 uint64_t base, uint64_t size, vm_prot_t prot); 587 588 vm_offset_t 589 ml_static_ptovirt( 590 vm_offset_t); 591 592 /* Offset required to obtain absolute time value from tick counter */ 593 uint64_t ml_get_abstime_offset(void); 594 595 /* Offset required to obtain continuous time value from tick counter */ 596 uint64_t ml_get_conttime_offset(void); 597 598 #ifdef __APPLE_API_UNSTABLE 599 /* PCI config cycle probing */ 600 boolean_t ml_probe_read( 601 vm_offset_t paddr, 602 unsigned int *val); 603 boolean_t ml_probe_read_64( 604 addr64_t paddr, 605 unsigned int *val); 606 607 /* Read physical address byte */ 608 unsigned int ml_phys_read_byte( 609 vm_offset_t paddr); 610 unsigned int ml_phys_read_byte_64( 611 addr64_t paddr); 612 613 /* Read physical address half word */ 614 unsigned int ml_phys_read_half( 615 vm_offset_t paddr); 616 unsigned int ml_phys_read_half_64( 617 addr64_t paddr); 618 619 /* Read physical address word*/ 620 unsigned int ml_phys_read( 621 vm_offset_t paddr); 622 unsigned int ml_phys_read_64( 623 addr64_t paddr); 624 unsigned int ml_phys_read_word( 625 vm_offset_t paddr); 626 unsigned int ml_phys_read_word_64( 627 addr64_t paddr); 628 629 /* Read physical address double word */ 630 unsigned long long ml_phys_read_double( 631 vm_offset_t paddr); 632 unsigned long long ml_phys_read_double_64( 633 addr64_t paddr); 634 635 /* Write physical address byte */ 636 void ml_phys_write_byte( 637 vm_offset_t paddr, unsigned int data); 638 void ml_phys_write_byte_64( 639 addr64_t paddr, unsigned int data); 640 641 /* Write physical address half word */ 642 void ml_phys_write_half( 643 vm_offset_t paddr, unsigned int data); 644 void ml_phys_write_half_64( 645 addr64_t paddr, unsigned int data); 646 647 /* Write physical address word */ 648 void ml_phys_write( 649 vm_offset_t paddr, unsigned int data); 650 void ml_phys_write_64( 651 addr64_t paddr, unsigned int data); 652 void ml_phys_write_word( 653 vm_offset_t paddr, unsigned int data); 654 void ml_phys_write_word_64( 655 addr64_t paddr, unsigned int data); 656 657 /* Write physical address double word */ 658 void ml_phys_write_double( 659 vm_offset_t paddr, unsigned long long data); 660 void ml_phys_write_double_64( 661 addr64_t paddr, unsigned long long data); 662 663 #if defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY 664 /* 665 * Not all dependent projects consuming `machine_routines.h` are built using 666 * toolchains that support 128-bit integers. 667 */ 668 #define BUILD_QUAD_WORD_FUNCS 1 669 #else 670 #define BUILD_QUAD_WORD_FUNCS 0 671 #endif /* defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY */ 672 673 #if BUILD_QUAD_WORD_FUNCS 674 /* 675 * Not all dependent projects have their own typedef of `uint128_t` at the 676 * time they consume `machine_routines.h`. 677 */ 678 typedef unsigned __int128 uint128_t; 679 680 /* Read physical address quad word */ 681 uint128_t ml_phys_read_quad( 682 vm_offset_t paddr); 683 uint128_t ml_phys_read_quad_64( 684 addr64_t paddr); 685 686 /* Write physical address quad word */ 687 void ml_phys_write_quad( 688 vm_offset_t paddr, uint128_t data); 689 void ml_phys_write_quad_64( 690 addr64_t paddr, uint128_t data); 691 #endif /* BUILD_QUAD_WORD_FUNCS */ 692 693 void ml_static_mfree( 694 vm_offset_t, 695 vm_size_t); 696 697 kern_return_t 698 ml_static_protect( 699 vm_offset_t start, 700 vm_size_t size, 701 vm_prot_t new_prot); 702 703 typedef int ml_page_protection_t; 704 705 /* Return the type of page protection supported */ 706 ml_page_protection_t ml_page_protection_type(void); 707 708 /* virtual to physical on wired pages */ 709 vm_offset_t ml_vtophys( 710 vm_offset_t vaddr); 711 712 /* Get processor cache info */ 713 void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); 714 void ml_cpu_get_info_type(ml_cpu_info_t * ml_cpu_info, cluster_type_t cluster_type); 715 716 #endif /* __APPLE_API_UNSTABLE */ 717 718 #ifdef __APPLE_API_PRIVATE 719 #ifdef XNU_KERNEL_PRIVATE 720 vm_size_t ml_nofault_copy( 721 vm_offset_t virtsrc, 722 vm_offset_t virtdst, 723 vm_size_t size); 724 boolean_t ml_validate_nofault( 725 vm_offset_t virtsrc, vm_size_t size); 726 #endif /* XNU_KERNEL_PRIVATE */ 727 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) 728 /* IO memory map services */ 729 730 extern vm_offset_t io_map( 731 vm_map_offset_t phys_addr, 732 vm_size_t size, 733 unsigned int flags, 734 vm_prot_t prot, 735 bool unmappable); 736 737 /* Map memory map IO space */ 738 vm_offset_t ml_io_map( 739 vm_offset_t phys_addr, 740 vm_size_t size); 741 742 vm_offset_t ml_io_map_wcomb( 743 vm_offset_t phys_addr, 744 vm_size_t size); 745 746 vm_offset_t ml_io_map_unmappable( 747 vm_offset_t phys_addr, 748 vm_size_t size, 749 uint32_t flags); 750 751 vm_offset_t ml_io_map_with_prot( 752 vm_offset_t phys_addr, 753 vm_size_t size, 754 vm_prot_t prot); 755 756 void ml_io_unmap( 757 vm_offset_t addr, 758 vm_size_t sz); 759 760 void ml_get_bouncepool_info( 761 vm_offset_t *phys_addr, 762 vm_size_t *size); 763 764 vm_map_address_t ml_map_high_window( 765 vm_offset_t phys_addr, 766 vm_size_t len); 767 768 void ml_init_timebase( 769 void *args, 770 tbd_ops_t tbd_funcs, 771 vm_offset_t int_address, 772 vm_offset_t int_value); 773 774 uint64_t ml_get_timebase(void); 775 776 #if MACH_KERNEL_PRIVATE 777 void ml_memory_to_timebase_fence(void); 778 void ml_timebase_to_memory_fence(void); 779 #endif /* MACH_KERNEL_PRIVATE */ 780 781 uint64_t ml_get_speculative_timebase(void); 782 783 uint64_t ml_get_timebase_entropy(void); 784 785 boolean_t ml_delay_should_spin(uint64_t interval); 786 787 void ml_delay_on_yield(void); 788 789 uint32_t ml_get_decrementer(void); 790 791 #include <machine/config.h> 792 793 uint64_t ml_get_hwclock(void); 794 795 #ifdef __arm64__ 796 boolean_t ml_get_timer_pending(void); 797 #endif 798 799 void platform_syscall( 800 struct arm_saved_state *); 801 802 void ml_set_decrementer( 803 uint32_t dec_value); 804 805 boolean_t is_user_contex( 806 void); 807 808 void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); 809 810 /* These calls are only valid if __ARM_USER_PROTECT__ is defined */ 811 uintptr_t arm_user_protect_begin( 812 thread_t thread); 813 814 void arm_user_protect_end( 815 thread_t thread, 816 uintptr_t up, 817 boolean_t disable_interrupts); 818 819 #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ 820 821 /* Zero bytes starting at a physical address */ 822 void bzero_phys( 823 addr64_t phys_address, 824 vm_size_t length); 825 826 void bzero_phys_nc(addr64_t src64, vm_size_t bytes); 827 828 #if MACH_KERNEL_PRIVATE 829 #ifdef __arm64__ 830 /* Pattern-fill buffer with zeros or a 32-bit pattern; 831 * target must be 128-byte aligned and sized a multiple of 128 832 * Both variants emit stores with non-temporal properties. 833 */ 834 void fill32_dczva(addr64_t, vm_size_t); 835 void fill32_nt(addr64_t, vm_size_t, uint32_t); 836 bool cpu_interrupt_is_pending(void); 837 838 #endif // __arm64__ 839 #endif // MACH_KERNEL_PRIVATE 840 841 void ml_thread_policy( 842 thread_t thread, 843 unsigned policy_id, 844 unsigned policy_info); 845 846 #define MACHINE_GROUP 0x00000001 847 #define MACHINE_NETWORK_GROUP 0x10000000 848 #define MACHINE_NETWORK_WORKLOOP 0x00000001 849 #define MACHINE_NETWORK_NETISR 0x00000002 850 851 /* Set the maximum number of CPUs */ 852 void ml_set_max_cpus( 853 unsigned int max_cpus); 854 855 /* Return the maximum number of CPUs set by ml_set_max_cpus(), waiting if necessary */ 856 unsigned int ml_wait_max_cpus( 857 void); 858 859 /* Return the maximum memory size */ 860 unsigned int ml_get_machine_mem(void); 861 862 #ifdef XNU_KERNEL_PRIVATE 863 /* Return max offset */ 864 vm_map_offset_t ml_get_max_offset( 865 boolean_t is64, 866 unsigned int option); 867 #define MACHINE_MAX_OFFSET_DEFAULT 0x01 868 #define MACHINE_MAX_OFFSET_MIN 0x02 869 #define MACHINE_MAX_OFFSET_MAX 0x04 870 #define MACHINE_MAX_OFFSET_DEVICE 0x08 871 #endif 872 873 extern void ml_cpu_init_completed(void); 874 extern void ml_cpu_up(void); 875 extern void ml_cpu_down(void); 876 /* 877 * The update to CPU counts needs to be separate from other actions 878 * in ml_cpu_up() and ml_cpu_down() 879 * because we don't update the counts when CLPC causes temporary 880 * cluster powerdown events, as these must be transparent to the user. 881 */ 882 extern void ml_cpu_up_update_counts(int cpu_id); 883 extern void ml_cpu_down_update_counts(int cpu_id); 884 extern void ml_arm_sleep(void); 885 886 extern uint64_t ml_get_wake_timebase(void); 887 extern uint64_t ml_get_conttime_wake_time(void); 888 889 /* Time since the system was reset (as part of boot/wake) */ 890 uint64_t ml_get_time_since_reset(void); 891 892 /* 893 * Called by ApplePMGR to set wake time. Units and epoch are identical 894 * to mach_continuous_time(). Has no effect on !HAS_CONTINUOUS_HWCLOCK 895 * chips. If wake_time == UINT64_MAX, that means the wake time is 896 * unknown and calls to ml_get_time_since_reset() will return UINT64_MAX. 897 */ 898 void ml_set_reset_time(uint64_t wake_time); 899 900 #ifdef XNU_KERNEL_PRIVATE 901 /* Just a stub on ARM */ 902 extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); 903 #define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) 904 #endif /* XNU_KERNEL_PRIVATE */ 905 906 /* Bytes available on current stack */ 907 vm_offset_t ml_stack_remaining(void); 908 909 #ifdef MACH_KERNEL_PRIVATE 910 uint32_t get_fpscr(void); 911 void set_fpscr(uint32_t); 912 void machine_conf(void); 913 void machine_lockdown(void); 914 915 #ifdef __arm64__ 916 unsigned long update_mdscr(unsigned long clear, unsigned long set); 917 #endif /* __arm64__ */ 918 919 extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); 920 extern void fiq_context_init(boolean_t enable_fiq); 921 922 extern void reenable_async_aborts(void); 923 924 #ifdef __arm64__ 925 uint64_t ml_cluster_wfe_timeout(uint32_t wfe_cluster_id); 926 #endif 927 928 #ifdef MONITOR 929 #define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ 930 #define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ 931 unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, 932 uintptr_t arg2, uintptr_t arg3); 933 #endif /* MONITOR */ 934 935 #if __ARM_KERNEL_PROTECT__ 936 extern void set_vbar_el1(uint64_t); 937 #endif /* __ARM_KERNEL_PROTECT__ */ 938 #endif /* MACH_KERNEL_PRIVATE */ 939 940 extern uint32_t arm_debug_read_dscr(void); 941 942 extern int set_be_bit(void); 943 extern int clr_be_bit(void); 944 extern int be_tracing(void); 945 946 /* Please note that cpu_broadcast_xcall is not as simple is you would like it to be. 947 * It will sometimes put the calling thread to sleep, and it is up to your callback 948 * to wake it up as needed, where "as needed" is defined as "all other CPUs have 949 * called the broadcast func". Look around the kernel for examples, or instead use 950 * cpu_broadcast_xcall_simple() which does indeed act like you would expect, given 951 * the prototype. cpu_broadcast_immediate_xcall has the same caveats and has a similar 952 * _simple() wrapper 953 */ 954 typedef void (*broadcastFunc) (void *); 955 unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); 956 unsigned int cpu_broadcast_xcall_simple(boolean_t, broadcastFunc, void *); 957 __result_use_check kern_return_t cpu_xcall(int, broadcastFunc, void *); 958 unsigned int cpu_broadcast_immediate_xcall(uint32_t *, boolean_t, broadcastFunc, void *); 959 unsigned int cpu_broadcast_immediate_xcall_simple(boolean_t, broadcastFunc, void *); 960 __result_use_check kern_return_t cpu_immediate_xcall(int, broadcastFunc, void *); 961 962 #ifdef KERNEL_PRIVATE 963 964 /* Interface to be used by the perf. controller to register a callback, in a 965 * single-threaded fashion. The callback will receive notifications of 966 * processor performance quality-of-service changes from the scheduler. 967 */ 968 969 #ifdef __arm64__ 970 typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); 971 void cpu_qos_update_register(cpu_qos_update_t); 972 #endif /* __arm64__ */ 973 974 struct going_on_core { 975 uint64_t thread_id; 976 uint16_t qos_class; 977 uint16_t urgency; /* XCPM compatibility */ 978 uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ 979 uint32_t is_kernel_thread : 1; 980 uint64_t thread_group_id; 981 void *thread_group_data; 982 uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ 983 uint64_t start_time; 984 uint64_t scheduling_latency_at_same_basepri; 985 uint32_t energy_estimate_nj; /* return: In nanojoules */ 986 /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ 987 }; 988 typedef struct going_on_core *going_on_core_t; 989 990 struct going_off_core { 991 uint64_t thread_id; 992 uint32_t energy_estimate_nj; /* return: In nanojoules */ 993 uint32_t reserved; 994 uint64_t end_time; 995 uint64_t thread_group_id; 996 void *thread_group_data; 997 }; 998 typedef struct going_off_core *going_off_core_t; 999 1000 struct thread_group_data { 1001 uint64_t thread_group_id; 1002 void *thread_group_data; 1003 uint32_t thread_group_size; 1004 uint32_t thread_group_flags; 1005 }; 1006 typedef struct thread_group_data *thread_group_data_t; 1007 1008 struct perfcontrol_max_runnable_latency { 1009 uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; 1010 }; 1011 typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; 1012 1013 struct perfcontrol_work_interval { 1014 uint64_t thread_id; 1015 uint16_t qos_class; 1016 uint16_t urgency; 1017 uint32_t flags; // notify 1018 uint64_t work_interval_id; 1019 uint64_t start; 1020 uint64_t finish; 1021 uint64_t deadline; 1022 uint64_t next_start; 1023 uint64_t thread_group_id; 1024 void *thread_group_data; 1025 uint32_t create_flags; 1026 }; 1027 typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; 1028 1029 typedef enum { 1030 WORK_INTERVAL_START, 1031 WORK_INTERVAL_UPDATE, 1032 WORK_INTERVAL_FINISH, 1033 WORK_INTERVAL_CREATE, 1034 WORK_INTERVAL_DEALLOCATE, 1035 } work_interval_ctl_t; 1036 1037 struct perfcontrol_work_interval_instance { 1038 work_interval_ctl_t ctl; 1039 uint32_t create_flags; 1040 uint64_t complexity; 1041 uint64_t thread_id; 1042 uint64_t work_interval_id; 1043 uint64_t instance_id; /* out: start, in: update/finish */ 1044 uint64_t start; 1045 uint64_t finish; 1046 uint64_t deadline; 1047 uint64_t thread_group_id; 1048 void *thread_group_data; 1049 }; 1050 typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t; 1051 1052 /* 1053 * Structure to export per-CPU counters as part of the CLPC callout. 1054 * Contains only the fixed CPU counters (instructions and cycles); CLPC 1055 * would call back into XNU to get the configurable counters if needed. 1056 */ 1057 struct perfcontrol_cpu_counters { 1058 uint64_t instructions; 1059 uint64_t cycles; 1060 }; 1061 1062 __options_decl(perfcontrol_thread_flags_mask_t, uint64_t, { 1063 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_RR = 1 << 0, 1064 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_NATIVE_FIRST = 1 << 1, 1065 }); 1066 1067 1068 /* 1069 * Structure used to pass information about a thread to CLPC 1070 */ 1071 struct perfcontrol_thread_data { 1072 /* 1073 * Energy estimate (return value) 1074 * The field is populated by CLPC and used to update the 1075 * energy estimate of the thread 1076 */ 1077 uint32_t energy_estimate_nj; 1078 /* Perfcontrol class for thread */ 1079 perfcontrol_class_t perfctl_class; 1080 /* Thread ID for the thread */ 1081 uint64_t thread_id; 1082 /* Thread Group ID */ 1083 uint64_t thread_group_id; 1084 /* 1085 * Scheduling latency for threads at the same base priority. 1086 * Calculated by the scheduler and passed into CLPC. The field is 1087 * populated only in the thread_data structure for the thread 1088 * going on-core. 1089 */ 1090 uint64_t scheduling_latency_at_same_basepri; 1091 /* Thread Group data pointer */ 1092 void *thread_group_data; 1093 /* perfctl state pointer */ 1094 void *perfctl_state; 1095 /* Bitmask to indicate which thread flags have been updated as part of the callout */ 1096 perfcontrol_thread_flags_mask_t thread_flags_mask; 1097 /* Actual values for the flags that are getting updated in the callout */ 1098 perfcontrol_thread_flags_mask_t thread_flags; 1099 }; 1100 1101 /* 1102 * All callouts from the scheduler are executed with interrupts 1103 * disabled. Callouts should be implemented in C with minimal 1104 * abstractions, and only use KPI exported by the mach/libkern 1105 * symbolset, restricted to routines like spinlocks and atomic 1106 * operations and scheduler routines as noted below. Spinlocks that 1107 * are used to synchronize data in the perfcontrol_state_t should only 1108 * ever be acquired with interrupts disabled, to avoid deadlocks where 1109 * an quantum expiration timer interrupt attempts to perform a callout 1110 * that attempts to lock a spinlock that is already held. 1111 */ 1112 1113 /* 1114 * When a processor is switching between two threads (after the 1115 * scheduler has chosen a new thread), the low-level platform layer 1116 * will call this routine, which should perform required timestamps, 1117 * MMIO register reads, or other state switching. No scheduler locks 1118 * are held during this callout. 1119 * 1120 * This function is called with interrupts ENABLED. 1121 */ 1122 typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); 1123 1124 /* 1125 * Once the processor has switched to the new thread, the offcore 1126 * callout will indicate the old thread that is no longer being 1127 * run. The thread's scheduler lock is held, so it will not begin 1128 * running on another processor (in the case of preemption where it 1129 * remains runnable) until it completes. If the "thread_terminating" 1130 * boolean is TRUE, this will be the last callout for this thread_id. 1131 */ 1132 typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); 1133 1134 /* 1135 * After the offcore callout and after the old thread can potentially 1136 * start running on another processor, the oncore callout will be 1137 * called with the thread's scheduler lock held. The oncore callout is 1138 * also called any time one of the parameters in the going_on_core_t 1139 * structure changes, like priority/QoS changes, and quantum 1140 * expiration, so the callout must not assume callouts are paired with 1141 * offcore callouts. 1142 */ 1143 typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); 1144 1145 /* 1146 * Periodically (on hundreds of ms scale), the scheduler will perform 1147 * maintenance and report the maximum latency for runnable (but not currently 1148 * running) threads for each urgency class. 1149 */ 1150 typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); 1151 1152 /* 1153 * When the kernel receives information about work intervals from userland, 1154 * it is passed along using this callback. No locks are held, although the state 1155 * object will not go away during the callout. 1156 */ 1157 typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); 1158 1159 /* 1160 * Start, update and finish work interval instance with optional complexity estimate. 1161 */ 1162 typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t); 1163 1164 /* 1165 * These callbacks are used when thread groups are added, removed or properties 1166 * updated. 1167 * No blocking allocations (or anything else blocking) are allowed inside these 1168 * callbacks. No locks allowed in these callbacks as well since the kernel might 1169 * be holding the thread/task locks. 1170 */ 1171 typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); 1172 typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); 1173 typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); 1174 1175 /* 1176 * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, 1177 * this function will be called, passing the timeout deadline that was previously armed as an argument. 1178 * 1179 * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. 1180 */ 1181 typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); 1182 1183 /* 1184 * Context Switch Callout 1185 * 1186 * Parameters: 1187 * event - The perfcontrol_event for this callout 1188 * cpu_id - The CPU doing the context switch 1189 * timestamp - The timestamp for the context switch 1190 * flags - Flags for other relevant information 1191 * offcore - perfcontrol_data structure for thread going off-core 1192 * oncore - perfcontrol_data structure for thread going on-core 1193 * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch 1194 */ 1195 typedef void (*sched_perfcontrol_csw_t)( 1196 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, 1197 struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, 1198 struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); 1199 1200 1201 /* 1202 * Thread State Update Callout 1203 * 1204 * Parameters: 1205 * event - The perfcontrol_event for this callout 1206 * cpu_id - The CPU doing the state update 1207 * timestamp - The timestamp for the state update 1208 * flags - Flags for other relevant information 1209 * thr_data - perfcontrol_data structure for the thread being updated 1210 */ 1211 typedef void (*sched_perfcontrol_state_update_t)( 1212 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, 1213 struct perfcontrol_thread_data *thr_data, __unused void *unused); 1214 1215 /* 1216 * Thread Group Blocking Relationship Callout 1217 * 1218 * Parameters: 1219 * blocked_tg - Thread group blocking on progress of another thread group 1220 * blocking_tg - Thread group blocking progress of another thread group 1221 * flags - Flags for other relevant information 1222 * blocked_thr_state - Per-thread perfcontrol state for blocked thread 1223 */ 1224 typedef void (*sched_perfcontrol_thread_group_blocked_t)( 1225 thread_group_data_t blocked_tg, thread_group_data_t blocking_tg, uint32_t flags, perfcontrol_state_t blocked_thr_state); 1226 1227 /* 1228 * Thread Group Unblocking Callout 1229 * 1230 * Parameters: 1231 * unblocked_tg - Thread group being unblocked from making forward progress 1232 * unblocking_tg - Thread group unblocking progress of another thread group 1233 * flags - Flags for other relevant information 1234 * unblocked_thr_state - Per-thread perfcontrol state for unblocked thread 1235 */ 1236 typedef void (*sched_perfcontrol_thread_group_unblocked_t)( 1237 thread_group_data_t unblocked_tg, thread_group_data_t unblocking_tg, uint32_t flags, perfcontrol_state_t unblocked_thr_state); 1238 1239 /* 1240 * Callers should always use the CURRENT version so that the kernel can detect both older 1241 * and newer structure layouts. New callbacks should always be added at the end of the 1242 * structure, and xnu should expect existing source recompiled against newer headers 1243 * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter 1244 * to reset callbacks to their default in-kernel values. 1245 */ 1246 1247 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ 1248 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ 1249 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ 1250 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ 1251 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ 1252 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ 1253 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ 1254 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */ 1255 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_8 (8) /* up-to thread_group_unblocked */ 1256 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_9 (9) /* allows CLPC to specify resource contention flags */ 1257 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 1258 1259 struct sched_perfcontrol_callbacks { 1260 unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ 1261 sched_perfcontrol_offcore_t offcore; 1262 sched_perfcontrol_context_switch_t context_switch; 1263 sched_perfcontrol_oncore_t oncore; 1264 sched_perfcontrol_max_runnable_latency_t max_runnable_latency; 1265 sched_perfcontrol_work_interval_notify_t work_interval_notify; 1266 sched_perfcontrol_thread_group_init_t thread_group_init; 1267 sched_perfcontrol_thread_group_deinit_t thread_group_deinit; 1268 sched_perfcontrol_deadline_passed_t deadline_passed; 1269 sched_perfcontrol_csw_t csw; 1270 sched_perfcontrol_state_update_t state_update; 1271 sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; 1272 sched_perfcontrol_work_interval_ctl_t work_interval_ctl; 1273 sched_perfcontrol_thread_group_blocked_t thread_group_blocked; 1274 sched_perfcontrol_thread_group_unblocked_t thread_group_unblocked; 1275 }; 1276 typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; 1277 1278 extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); 1279 extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); 1280 extern void sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit); 1281 extern const char* sched_perfcontrol_thread_group_get_name(void *data); 1282 1283 /* 1284 * Edge Scheduler-CLPC Interface 1285 * 1286 * sched_perfcontrol_thread_group_preferred_clusters_set() 1287 * 1288 * The Edge scheduler expects thread group recommendations to be specific clusters rather 1289 * than just E/P. In order to allow more fine grained control, CLPC can specify an override 1290 * preferred cluster per QoS bucket. CLPC passes a common preferred cluster `tg_preferred_cluster` 1291 * and an array of size [PERFCONTROL_CLASS_MAX] with overrides for specific perfctl classes. 1292 * The scheduler translates these preferences into sched_bucket 1293 * preferences and applies the changes. 1294 * 1295 */ 1296 /* Token to indicate a particular perfctl class is not overriden */ 1297 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE ((uint32_t)~0) 1298 1299 /* 1300 * CLPC can also indicate if there should be an immediate rebalancing of threads of this TG as 1301 * part of this preferred cluster change. It does that by specifying the following options. 1302 */ 1303 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING 0x1 1304 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNABLE 0x2 1305 typedef uint64_t sched_perfcontrol_preferred_cluster_options_t; 1306 1307 extern void sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster, 1308 uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options); 1309 1310 /* 1311 * Edge Scheduler-CLPC Interface 1312 * 1313 * sched_perfcontrol_edge_matrix_get()/sched_perfcontrol_edge_matrix_set() 1314 * 1315 * The Edge scheduler uses edges between clusters to define the likelihood of migrating threads 1316 * across clusters. The edge config between any two clusters defines the edge weight and whether 1317 * migation and steal operations are allowed across that edge. The getter and setter allow CLPC 1318 * to query and configure edge properties between various clusters on the platform. 1319 */ 1320 1321 extern void sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order); 1322 extern void sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order); 1323 1324 /* 1325 * sched_perfcontrol_edge_cpu_rotation_bitmasks_get()/sched_perfcontrol_edge_cpu_rotation_bitmasks_set() 1326 * 1327 * In order to drive intra-cluster core rotation CLPC supplies the edge scheduler with a pair of 1328 * per-cluster bitmasks. The preferred_bitmask is a bitmask of CPU cores where if a bit is set, 1329 * CLPC would prefer threads to be scheduled on that core if it is idle. The migration_bitmask 1330 * is a bitmask of CPU cores where if a bit is set, CLPC would prefer threads no longer continue 1331 * running on that core if there is an idle core from the preferred_bitmask that is available. 1332 */ 1333 1334 extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_set(uint32_t cluster_id, uint64_t preferred_bitmask, uint64_t migration_bitmask); 1335 extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_get(uint32_t cluster_id, uint64_t *preferred_bitmask, uint64_t *migration_bitmask); 1336 1337 /* 1338 * Update the deadline after which sched_perfcontrol_deadline_passed will be called. 1339 * Returns TRUE if it successfully canceled a previously set callback, 1340 * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). 1341 * The callback is automatically canceled when it fires, and does not repeat unless rearmed. 1342 * 1343 * This 'timer' executes as the scheduler switches between threads, on a non-idle core 1344 * 1345 * There can be only one outstanding timer globally. 1346 */ 1347 extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); 1348 1349 /* 1350 * SFI configuration. 1351 */ 1352 extern kern_return_t sched_perfcontrol_sfi_set_window(uint64_t window_usecs); 1353 extern kern_return_t sched_perfcontrol_sfi_set_bg_offtime(uint64_t offtime_usecs); 1354 extern kern_return_t sched_perfcontrol_sfi_set_utility_offtime(uint64_t offtime_usecs); 1355 1356 typedef enum perfcontrol_callout_type { 1357 PERFCONTROL_CALLOUT_ON_CORE, 1358 PERFCONTROL_CALLOUT_OFF_CORE, 1359 PERFCONTROL_CALLOUT_CONTEXT, 1360 PERFCONTROL_CALLOUT_STATE_UPDATE, 1361 /* Add other callout types here */ 1362 PERFCONTROL_CALLOUT_MAX 1363 } perfcontrol_callout_type_t; 1364 1365 typedef enum perfcontrol_callout_stat { 1366 PERFCONTROL_STAT_INSTRS, 1367 PERFCONTROL_STAT_CYCLES, 1368 /* Add other stat types here */ 1369 PERFCONTROL_STAT_MAX 1370 } perfcontrol_callout_stat_t; 1371 1372 uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, 1373 perfcontrol_callout_stat_t stat); 1374 1375 #ifdef __arm64__ 1376 /* The performance controller may use this interface to recommend 1377 * that CPUs in the designated cluster employ WFE rather than WFI 1378 * within the idle loop, falling back to WFI after the specified 1379 * timeout. The updates are expected to be serialized by the caller, 1380 * the implementation is not required to perform internal synchronization. 1381 */ 1382 uint32_t ml_update_cluster_wfe_recommendation(uint32_t wfe_cluster_id, uint64_t wfe_timeout_abstime_interval, uint64_t wfe_hint_flags); 1383 #endif /* __arm64__ */ 1384 1385 #if defined(HAS_APPLE_PAC) 1386 #define ONES(x) (BIT((x))-1) 1387 #define PTR_MASK ONES(64-T1SZ_BOOT) 1388 #define PAC_MASK ~PTR_MASK 1389 #define SIGN(p) ((p) & BIT(55)) 1390 #define UNSIGN_PTR(p) \ 1391 SIGN(p) ? ((p) | PAC_MASK) : ((p) & ~PAC_MASK) 1392 1393 uint64_t ml_default_rop_pid(void); 1394 uint64_t ml_default_jop_pid(void); 1395 void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit); 1396 void ml_task_set_jop_pid(task_t task, task_t parent_task, boolean_t inherit); 1397 void ml_task_set_jop_pid_from_shared_region(task_t task); 1398 void ml_task_set_disable_user_jop(task_t task, uint8_t disable_user_jop); 1399 void ml_thread_set_disable_user_jop(thread_t thread, uint8_t disable_user_jop); 1400 void ml_thread_set_jop_pid(thread_t thread, task_t task); 1401 void *ml_auth_ptr_unchecked(void *ptr, unsigned key, uint64_t modifier); 1402 1403 uint64_t ml_enable_user_jop_key(uint64_t user_jop_key); 1404 1405 /** 1406 * Restores the previous JOP key state after a previous ml_enable_user_jop_key() 1407 * call. 1408 * 1409 * @param user_jop_key The userspace JOP key previously passed to 1410 * ml_enable_user_jop_key() 1411 * @param saved_jop_state The saved JOP state returned by 1412 * ml_enable_user_jop_key() 1413 */ 1414 void ml_disable_user_jop_key(uint64_t user_jop_key, uint64_t saved_jop_state); 1415 #endif /* defined(HAS_APPLE_PAC) */ 1416 1417 void ml_enable_monitor(void); 1418 1419 #endif /* KERNEL_PRIVATE */ 1420 1421 boolean_t machine_timeout_suspended(void); 1422 void ml_get_power_state(boolean_t *, boolean_t *); 1423 1424 uint32_t get_arm_cpu_version(void); 1425 boolean_t user_cont_hwclock_allowed(void); 1426 uint8_t user_timebase_type(void); 1427 boolean_t ml_thread_is64bit(thread_t thread); 1428 1429 #ifdef __arm64__ 1430 bool ml_feature_supported(uint32_t feature_bit); 1431 void ml_set_align_checking(void); 1432 extern void wfe_timeout_configure(void); 1433 extern void wfe_timeout_init(void); 1434 #endif /* __arm64__ */ 1435 1436 void ml_timer_evaluate(void); 1437 boolean_t ml_timer_forced_evaluation(void); 1438 void ml_gpu_stat_update(uint64_t); 1439 uint64_t ml_gpu_stat(thread_t); 1440 #endif /* __APPLE_API_PRIVATE */ 1441 1442 1443 1444 #if __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) 1445 extern void ml_expect_fault_begin(expected_fault_handler_t, uintptr_t); 1446 extern void ml_expect_fault_end(void); 1447 #endif /* __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) */ 1448 1449 1450 extern uint32_t phy_read_panic; 1451 extern uint32_t phy_write_panic; 1452 #if DEVELOPMENT || DEBUG 1453 extern uint64_t simulate_stretched_io; 1454 #endif 1455 1456 void ml_hibernate_active_pre(void); 1457 void ml_hibernate_active_post(void); 1458 1459 void ml_report_minor_badness(uint32_t badness_id); 1460 #define ML_MINOR_BADNESS_CONSOLE_BUFFER_FULL 0 1461 #define ML_MINOR_BADNESS_MEMFAULT_REPORTING_NOT_ENABLED 1 1462 #define ML_MINOR_BADNESS_PIO_WRITTEN_FROM_USERSPACE 2 1463 1464 #ifdef XNU_KERNEL_PRIVATE 1465 /** 1466 * Depending on the system, by the time a backtracer starts inspecting an 1467 * interrupted CPU's register state, the value of the PC might have been 1468 * modified. In those cases, the original PC value is placed into a different 1469 * register. This function abstracts out those differences for a backtracer 1470 * wanting the PC of an interrupted CPU. 1471 * 1472 * @param state The ARM register state to parse. 1473 * 1474 * @return The original PC of the interrupted CPU. 1475 */ 1476 uint64_t ml_get_backtrace_pc(struct arm_saved_state *state); 1477 #endif /* XNU_KERNEL_PRIVATE */ 1478 1479 __END_DECLS 1480 1481 #endif /* _ARM_MACHINE_ROUTINES_H_ */ 1482