1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #ifndef _ARM_MACHINE_ROUTINES_H_
33 #define _ARM_MACHINE_ROUTINES_H_
34
35 #include <mach/mach_types.h>
36 #include <mach/vm_types.h>
37 #include <mach/boolean.h>
38 #include <kern/kern_types.h>
39 #include <pexpert/pexpert.h>
40
41 #include <sys/cdefs.h>
42 #include <sys/appleapiopts.h>
43
44 #include <stdarg.h>
45
46 #ifdef XNU_KERNEL_PRIVATE
47 #include <kern/sched_hygiene.h>
48 #include <kern/startup.h>
49 #endif /* XNU_KERNEL_PRIVATE */
50
51 __BEGIN_DECLS
52 #ifdef XNU_KERNEL_PRIVATE
53 #ifdef __arm64__
54 typedef bool (*expected_fault_handler_t)(arm_saved_state_t *);
55 #endif /* __arm64__ */
56 #endif /* XNU_KERNEL_PRIVATE */
57
58 /* Interrupt handling */
59
60 void ml_cpu_signal(unsigned int cpu_id);
61 void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs);
62 uint64_t ml_cpu_signal_deferred_get_timer(void);
63 void ml_cpu_signal_deferred(unsigned int cpu_id);
64 void ml_cpu_signal_retract(unsigned int cpu_id);
65
66 #ifdef XNU_KERNEL_PRIVATE
67 extern void ml_wait_for_cpu_signal_to_enable(void);
68 extern void assert_ml_cpu_signal_is_enabled(bool enabled);
69 #endif /* XNU_KERNEL_PRIVATE */
70
71 /* Initialize Interrupts */
72 void ml_init_interrupt(void);
73
74 /* Get Interrupts Enabled */
75 boolean_t ml_get_interrupts_enabled(void);
76
77 /* Set Interrupts Enabled */
78 #if __has_feature(ptrauth_calls)
79 uint64_t ml_pac_safe_interrupts_disable(void);
80 void ml_pac_safe_interrupts_restore(uint64_t);
81 #endif /* __has_feature(ptrauth_calls) */
82 boolean_t ml_set_interrupts_enabled_with_debug(boolean_t enable, boolean_t debug);
83 boolean_t ml_set_interrupts_enabled(boolean_t enable);
84 boolean_t ml_early_set_interrupts_enabled(boolean_t enable);
85
86 /*
87 * Functions for disabling measurements for AppleCLPC only.
88 */
89 boolean_t sched_perfcontrol_ml_set_interrupts_without_measurement(boolean_t enable);
90 void sched_perfcontrol_abandon_preemption_disable_measurement(void);
91
92 /* Check if running at interrupt context */
93 boolean_t ml_at_interrupt_context(void);
94
95
96 /* Generate a fake interrupt */
97 void ml_cause_interrupt(void);
98
99 void siq_init(void);
100 void siq_cpu_init(void);
101
102 #ifdef XNU_KERNEL_PRIVATE
103
104 /* did this interrupt context interrupt userspace? */
105 bool ml_did_interrupt_userspace(void);
106
107 /* Clear interrupt spin debug state for thread */
108
109 #if SCHED_HYGIENE_DEBUG
110 void mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs);
111
112 #if CONFIG_CPU_COUNTERS
113 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) \
114 if (static_if(sched_debug_pmc)) { \
115 mt_cur_cpu_cycles_instrs_speculative(&thread->machine.intmask_cycles, \
116 &thread->machine.intmask_instr); \
117 }
118 #else /* CONFIG_CPU_COUNTERS */
119 #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread)
120 #endif /* !CONFIG_CPU_COUNTERS */
121
122 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) \
123 do { \
124 if (static_if(sched_debug_interrupt_disable) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \
125 thread_t thread = current_thread(); \
126 thread->machine.int_type = type; \
127 thread->machine.int_handler_addr = (uintptr_t)VM_KERNEL_STRIP_UPTR(handler_addr); \
128 thread->machine.inthandler_timestamp = ml_get_sched_hygiene_timebase(); \
129 INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread); \
130 thread->machine.int_vector = (uintptr_t)NULL; \
131 } \
132 } while (0)
133
134 #define INTERRUPT_MASKED_DEBUG_END() \
135 do { \
136 if (static_if(sched_debug_interrupt_disable) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \
137 thread_t thread = current_thread(); \
138 ml_handle_interrupt_handler_duration(thread); \
139 thread->machine.inthandler_timestamp = 0; \
140 thread->machine.inthandler_abandon = false; \
141 } \
142 } while (0)
143
144 void ml_irq_debug_start(uintptr_t handler, uintptr_t vector);
145 void ml_irq_debug_end(void);
146 void ml_irq_debug_abandon(void);
147
148 void ml_spin_debug_reset(thread_t thread);
149 void ml_spin_debug_clear(thread_t thread);
150 void ml_spin_debug_clear_self(void);
151 void ml_handle_interrupts_disabled_duration(thread_t thread);
152 void ml_handle_stackshot_interrupt_disabled_duration(thread_t thread);
153 void ml_handle_interrupt_handler_duration(thread_t thread);
154
155 #else /* SCHED_HYGIENE_DEBUG */
156
157 #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type)
158 #define INTERRUPT_MASKED_DEBUG_END()
159
160 #endif /* SCHED_HYGIENE_DEBUG */
161
162 extern bool ml_snoop_thread_is_on_core(thread_t thread);
163 extern boolean_t ml_is_quiescing(void);
164 extern void ml_set_is_quiescing(boolean_t);
165 extern uint64_t ml_get_booter_memory_size(void);
166 #endif
167
168 /* Type for the Time Base Enable function */
169 typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable);
170 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
171 /* Type for the Processor Cache Dispatch function */
172 typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1);
173
174 typedef uint32_t (*get_decrementer_t)(void);
175 typedef void (*set_decrementer_t)(uint32_t);
176 typedef void (*fiq_handler_t)(void);
177
178 #endif
179
180 #define CacheConfig 0x00000000UL
181 #define CacheControl 0x00000001UL
182 #define CacheClean 0x00000002UL
183 #define CacheCleanRegion 0x00000003UL
184 #define CacheCleanFlush 0x00000004UL
185 #define CacheCleanFlushRegion 0x00000005UL
186 #define CacheShutdown 0x00000006UL
187
188 #define CacheControlEnable 0x00000000UL
189
190 #define CacheConfigCCSIDR 0x00000001UL
191 #define CacheConfigSize 0x00000100UL
192
193 /* Type for the Processor Idle function */
194 typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks);
195
196 /* Type for the Idle Tickle function */
197 typedef void (*idle_tickle_t)(void);
198
199 /* Type for the Idle Timer function */
200 typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks);
201
202 /* Type for the IPI Hander */
203 typedef void (*ipi_handler_t)(void);
204
205 /* Type for the Lockdown Hander */
206 typedef void (*lockdown_handler_t)(void *);
207
208 /* Type for the Platform specific Error Handler */
209 typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr);
210
211 /*
212 * The exception callback (ex_cb) module is obsolete. Some definitions related
213 * to ex_cb were exported through the SDK, and are only left here for historical
214 * reasons.
215 */
216
217 /* Unused. Left for historical reasons. */
218 typedef enum{
219 EXCB_CLASS_ILLEGAL_INSTR_SET,
220 #ifdef CONFIG_XNUPOST
221 EXCB_CLASS_TEST1,
222 EXCB_CLASS_TEST2,
223 EXCB_CLASS_TEST3,
224 #endif
225 EXCB_CLASS_MAX
226 }
227 ex_cb_class_t;
228
229 /* Unused. Left for historical reasons. */
230 typedef enum{
231 EXCB_ACTION_RERUN,
232 EXCB_ACTION_NONE,
233 #ifdef CONFIG_XNUPOST
234 EXCB_ACTION_TEST_FAIL,
235 #endif
236 }
237 ex_cb_action_t;
238
239 /* Unused. Left for historical reasons. */
240 typedef struct{
241 vm_offset_t far;
242 }
243 ex_cb_state_t;
244
245 /* Unused. Left for historical reasons. */
246 typedef ex_cb_action_t (*ex_cb_t) (
247 ex_cb_class_t cb_class,
248 void *refcon,
249 const ex_cb_state_t *state
250 );
251
252 /*
253 * This function is unimplemented. Its definition is left for historical
254 * reasons.
255 */
256 kern_return_t ex_cb_register(
257 ex_cb_class_t cb_class,
258 ex_cb_t cb,
259 void *refcon );
260
261 /*
262 * This function is unimplemented. Its definition is left for historical
263 * reasons.
264 */
265 ex_cb_action_t ex_cb_invoke(
266 ex_cb_class_t cb_class,
267 vm_offset_t far);
268
269 typedef enum {
270 CLUSTER_TYPE_SMP = 0,
271 CLUSTER_TYPE_E = 1,
272 CLUSTER_TYPE_P = 2,
273 MAX_CPU_TYPES,
274 } cluster_type_t;
275
276 #ifdef XNU_KERNEL_PRIVATE
277 void ml_parse_cpu_topology(void);
278 #endif /* XNU_KERNEL_PRIVATE */
279
280 unsigned int ml_get_cpu_count(void);
281
282 unsigned int ml_get_cpu_number_type(cluster_type_t cluster_type, bool logical, bool available);
283
284 unsigned int ml_get_cluster_number_type(cluster_type_t cluster_type);
285
286 unsigned int ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type, bool include_all_cpu_types);
287
288 unsigned int ml_get_cpu_types(void);
289
290 int ml_get_boot_cpu_number(void);
291
292 int ml_get_cpu_number(uint32_t phys_id);
293
294 unsigned int ml_get_cpu_number_local(void);
295
296 int ml_get_cluster_number(uint32_t phys_id);
297
298 int ml_get_max_cpu_number(void);
299
300 int ml_get_max_cluster_number(void);
301
302 /*
303 * Return the id of a cluster's first cpu.
304 */
305 unsigned int ml_get_first_cpu_id(unsigned int cluster_id);
306
307 /*
308 * Return the die id of a cluster.
309 */
310 unsigned int ml_get_die_id(unsigned int cluster_id);
311
312 /*
313 * Return the index of a cluster in its die.
314 */
315 unsigned int ml_get_die_cluster_id(unsigned int cluster_id);
316
317 /*
318 * Return the highest die id of the system.
319 */
320 unsigned int ml_get_max_die_id(void);
321
322 #ifdef __arm64__
323 int ml_get_cluster_number_local(void);
324 #endif /* __arm64__ */
325
326 /* Struct for ml_cpu_get_info */
327 struct ml_cpu_info {
328 unsigned long vector_unit;
329 unsigned long cache_line_size;
330 unsigned long l1_icache_size;
331 unsigned long l1_dcache_size;
332 unsigned long l2_settings;
333 unsigned long l2_cache_size;
334 unsigned long l3_settings;
335 unsigned long l3_cache_size;
336 };
337 typedef struct ml_cpu_info ml_cpu_info_t;
338
339 cluster_type_t ml_get_boot_cluster_type(void);
340
341 #ifdef KERNEL_PRIVATE
342 #include "cpu_topology.h"
343 #endif /* KERNEL_PRIVATE */
344
345 /*!
346 * @function ml_map_cpu_pio
347 * @brief Maps per-CPU and per-cluster PIO registers found in EDT. This needs to be
348 * called after arm_vm_init() so it can't be part of ml_parse_cpu_topology().
349 */
350 void ml_map_cpu_pio(void);
351
352 /* Struct for ml_processor_register */
353 struct ml_processor_info {
354 cpu_id_t cpu_id;
355 vm_offset_t start_paddr;
356 boolean_t supports_nap;
357 void *platform_cache_dispatch;
358 time_base_enable_t time_base_enable;
359 processor_idle_t processor_idle;
360 idle_tickle_t *idle_tickle;
361 idle_timer_t idle_timer;
362 void *idle_timer_refcon;
363 vm_offset_t powergate_stub_addr;
364 uint32_t powergate_stub_length;
365 uint32_t powergate_latency;
366 platform_error_handler_t platform_error_handler;
367 uint64_t regmap_paddr;
368 uint32_t phys_id;
369 uint32_t log_id;
370 uint32_t l2_access_penalty; /* unused */
371 uint32_t cluster_id;
372 cluster_type_t cluster_type;
373 uint32_t l2_cache_id;
374 uint32_t l2_cache_size;
375 uint32_t l3_cache_id;
376 uint32_t l3_cache_size;
377 };
378 typedef struct ml_processor_info ml_processor_info_t;
379
380 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
381 /* Struct for ml_init_timebase */
382 struct tbd_ops {
383 fiq_handler_t tbd_fiq_handler;
384 get_decrementer_t tbd_get_decrementer;
385 set_decrementer_t tbd_set_decrementer;
386 };
387 typedef struct tbd_ops *tbd_ops_t;
388 typedef struct tbd_ops tbd_ops_data_t;
389 #endif
390
391
392 /*!
393 * @function ml_processor_register
394 *
395 * @abstract callback from platform kext to register processor
396 *
397 * @discussion This function is called by the platform kext when a processor is
398 * being registered. This is called while running on the CPU itself, as part of
399 * its initialization.
400 *
401 * @param ml_processor_info provides machine-specific information about the
402 * processor to xnu.
403 *
404 * @param processor is set as an out-parameter to an opaque handle that should
405 * be used by the platform kext when referring to this processor in the future.
406 *
407 * @param ipi_handler is set as an out-parameter to the function that should be
408 * registered as the IPI handler.
409 *
410 * @param pmi_handler is set as an out-parameter to the function that should be
411 * registered as the PMI handler.
412 *
413 * @returns KERN_SUCCESS on success and an error code, otherwise.
414 */
415 kern_return_t ml_processor_register(ml_processor_info_t *ml_processor_info,
416 processor_t *processor, ipi_handler_t *ipi_handler,
417 perfmon_interrupt_handler_func *pmi_handler);
418
419 /* Register a lockdown handler */
420 kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *);
421
422 /* Register a M$ flushing */
423 typedef kern_return_t (*mcache_flush_function)(void *service);
424 kern_return_t ml_mcache_flush_callback_register(mcache_flush_function func, void *service);
425 kern_return_t ml_mcache_flush(void);
426
427 #if XNU_KERNEL_PRIVATE
428
429 void ml_lockdown_init(void);
430
431 /* Machine layer routine for intercepting panics */
432 __printflike(1, 0)
433 void ml_panic_trap_to_debugger(const char *panic_format_str,
434 va_list *panic_args,
435 unsigned int reason,
436 void *ctx,
437 uint64_t panic_options_mask,
438 unsigned long panic_caller,
439 const char *panic_initiator);
440
441 /* Initialize Interrupts */
442 void ml_install_interrupt_handler(
443 void *nub,
444 int source,
445 void *target,
446 IOInterruptHandler handler,
447 void *refCon);
448
449 #endif /* XNU_KERNEL_PRIVATE */
450
451 vm_offset_t
452 ml_static_vtop(
453 vm_offset_t);
454
455 kern_return_t
456 ml_static_verify_page_protections(
457 uint64_t base, uint64_t size, vm_prot_t prot);
458
459 vm_offset_t
460 ml_static_ptovirt(
461 vm_offset_t);
462
463 /* Offset required to obtain absolute time value from tick counter */
464 uint64_t ml_get_abstime_offset(void);
465
466 /* Offset required to obtain continuous time value from tick counter */
467 uint64_t ml_get_conttime_offset(void);
468
469 #ifdef __APPLE_API_UNSTABLE
470 /* PCI config cycle probing */
471 boolean_t ml_probe_read(
472 vm_offset_t paddr,
473 unsigned int *val);
474 boolean_t ml_probe_read_64(
475 addr64_t paddr,
476 unsigned int *val);
477
478 /* Read physical address byte */
479 unsigned int ml_phys_read_byte(
480 vm_offset_t paddr);
481 unsigned int ml_phys_read_byte_64(
482 addr64_t paddr);
483
484 /* Read physical address half word */
485 unsigned int ml_phys_read_half(
486 vm_offset_t paddr);
487 unsigned int ml_phys_read_half_64(
488 addr64_t paddr);
489
490 /* Read physical address word*/
491 unsigned int ml_phys_read(
492 vm_offset_t paddr);
493 unsigned int ml_phys_read_64(
494 addr64_t paddr);
495 unsigned int ml_phys_read_word(
496 vm_offset_t paddr);
497 unsigned int ml_phys_read_word_64(
498 addr64_t paddr);
499
500 /* Read physical address double word */
501 unsigned long long ml_phys_read_double(
502 vm_offset_t paddr);
503 unsigned long long ml_phys_read_double_64(
504 addr64_t paddr);
505
506 /* Write physical address byte */
507 void ml_phys_write_byte(
508 vm_offset_t paddr, unsigned int data);
509 void ml_phys_write_byte_64(
510 addr64_t paddr, unsigned int data);
511
512 /* Write physical address half word */
513 void ml_phys_write_half(
514 vm_offset_t paddr, unsigned int data);
515 void ml_phys_write_half_64(
516 addr64_t paddr, unsigned int data);
517
518 /* Write physical address word */
519 void ml_phys_write(
520 vm_offset_t paddr, unsigned int data);
521 void ml_phys_write_64(
522 addr64_t paddr, unsigned int data);
523 void ml_phys_write_word(
524 vm_offset_t paddr, unsigned int data);
525 void ml_phys_write_word_64(
526 addr64_t paddr, unsigned int data);
527
528 /* Write physical address double word */
529 void ml_phys_write_double(
530 vm_offset_t paddr, unsigned long long data);
531 void ml_phys_write_double_64(
532 addr64_t paddr, unsigned long long data);
533
534 #if defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY
535 /*
536 * Not all dependent projects consuming `machine_routines.h` are built using
537 * toolchains that support 128-bit integers.
538 */
539 #define BUILD_QUAD_WORD_FUNCS 1
540 #else
541 #define BUILD_QUAD_WORD_FUNCS 0
542 #endif /* defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY */
543
544 #if BUILD_QUAD_WORD_FUNCS
545 /*
546 * Not all dependent projects have their own typedef of `uint128_t` at the
547 * time they consume `machine_routines.h`.
548 */
549 typedef unsigned __int128 uint128_t;
550
551 /* Read physical address quad word */
552 uint128_t ml_phys_read_quad(
553 vm_offset_t paddr);
554 uint128_t ml_phys_read_quad_64(
555 addr64_t paddr);
556
557 /* Write physical address quad word */
558 void ml_phys_write_quad(
559 vm_offset_t paddr, uint128_t data);
560 void ml_phys_write_quad_64(
561 addr64_t paddr, uint128_t data);
562 #endif /* BUILD_QUAD_WORD_FUNCS */
563
564 void ml_static_mfree(
565 vm_offset_t,
566 vm_size_t);
567
568 kern_return_t
569 ml_static_protect(
570 vm_offset_t start,
571 vm_size_t size,
572 vm_prot_t new_prot);
573
574 /* virtual to physical on wired pages */
575 vm_offset_t ml_vtophys(
576 vm_offset_t vaddr);
577
578 /* Get processor cache info */
579 void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info);
580 void ml_cpu_get_info_type(ml_cpu_info_t * ml_cpu_info, cluster_type_t cluster_type);
581
582 #endif /* __APPLE_API_UNSTABLE */
583
584 typedef int ml_page_protection_t;
585
586 /* Return the type of page protection supported */
587 ml_page_protection_t ml_page_protection_type(void);
588
589 #ifdef __APPLE_API_PRIVATE
590 #ifdef XNU_KERNEL_PRIVATE
591 vm_size_t ml_nofault_copy(
592 vm_offset_t virtsrc,
593 vm_offset_t virtdst,
594 vm_size_t size);
595 boolean_t ml_validate_nofault(
596 vm_offset_t virtsrc, vm_size_t size);
597 #endif /* XNU_KERNEL_PRIVATE */
598 #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
599 /* IO memory map services */
600
601 extern vm_offset_t io_map(
602 vm_map_offset_t phys_addr,
603 vm_size_t size,
604 unsigned int flags,
605 vm_prot_t prot,
606 bool unmappable);
607
608 /* Map memory map IO space */
609 vm_offset_t ml_io_map(
610 vm_offset_t phys_addr,
611 vm_size_t size);
612
613 vm_offset_t ml_io_map_wcomb(
614 vm_offset_t phys_addr,
615 vm_size_t size);
616
617 vm_offset_t ml_io_map_unmappable(
618 vm_offset_t phys_addr,
619 vm_size_t size,
620 uint32_t flags);
621
622 vm_offset_t ml_io_map_with_prot(
623 vm_offset_t phys_addr,
624 vm_size_t size,
625 vm_prot_t prot);
626
627 void ml_io_unmap(
628 vm_offset_t addr,
629 vm_size_t sz);
630
631 void ml_get_bouncepool_info(
632 vm_offset_t *phys_addr,
633 vm_size_t *size);
634
635 vm_map_address_t ml_map_high_window(
636 vm_offset_t phys_addr,
637 vm_size_t len);
638
639 void ml_init_timebase(
640 void *args,
641 tbd_ops_t tbd_funcs,
642 vm_offset_t int_address,
643 vm_offset_t int_value);
644
645 uint64_t ml_get_timebase(void);
646
647 #if MACH_KERNEL_PRIVATE
648 void ml_memory_to_timebase_fence(void);
649 void ml_timebase_to_memory_fence(void);
650 #endif /* MACH_KERNEL_PRIVATE */
651
652 uint64_t ml_get_speculative_timebase(void);
653
654 uint64_t ml_get_timebase_entropy(void);
655
656 boolean_t ml_delay_should_spin(uint64_t interval);
657
658 void ml_delay_on_yield(void);
659
660 uint32_t ml_get_decrementer(void);
661
662 #include <machine/config.h>
663
664 uint64_t ml_get_hwclock(void);
665
666 uint64_t ml_get_hwclock_speculative(void);
667
668 #ifdef __arm64__
669 boolean_t ml_get_timer_pending(void);
670 #endif
671
672 void platform_syscall(
673 struct arm_saved_state *);
674
675 void ml_set_decrementer(
676 uint32_t dec_value);
677
678 boolean_t is_user_contex(
679 void);
680
681 void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address);
682
683 /* These calls are only valid if __ARM_USER_PROTECT__ is defined */
684 uintptr_t arm_user_protect_begin(
685 thread_t thread);
686
687 void arm_user_protect_end(
688 thread_t thread,
689 uintptr_t up,
690 boolean_t disable_interrupts);
691
692 #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */
693
694 /* Zero bytes starting at a physical address */
695 void bzero_phys(
696 addr64_t phys_address,
697 vm_size_t length);
698
699 void bzero_phys_nc(addr64_t src64, vm_size_t bytes);
700
701 void bzero_phys_with_options(addr64_t src, vm_size_t bytes, int options);
702
703 #if MACH_KERNEL_PRIVATE
704 #ifdef __arm64__
705 /* Pattern-fill buffer with zeros or a 32-bit pattern;
706 * target must be 128-byte aligned and sized a multiple of 128
707 * Both variants emit stores with non-temporal properties.
708 */
709 void fill32_dczva(addr64_t, vm_size_t);
710 void fill32_nt(addr64_t, vm_size_t, uint32_t);
711 bool cpu_interrupt_is_pending(void);
712
713 #endif // __arm64__
714 #endif // MACH_KERNEL_PRIVATE
715
716 void ml_thread_policy(
717 thread_t thread,
718 unsigned policy_id,
719 unsigned policy_info);
720
721 #define MACHINE_GROUP 0x00000001
722 #define MACHINE_NETWORK_GROUP 0x10000000
723 #define MACHINE_NETWORK_WORKLOOP 0x00000001
724 #define MACHINE_NETWORK_NETISR 0x00000002
725
726 /* Set the maximum number of CPUs */
727 void ml_set_max_cpus(
728 unsigned int max_cpus);
729
730 /* Return the maximum number of CPUs set by ml_set_max_cpus(), waiting if necessary */
731 unsigned int ml_wait_max_cpus(
732 void);
733
734 /* Return the maximum memory size */
735 unsigned int ml_get_machine_mem(void);
736
737 #ifdef XNU_KERNEL_PRIVATE
738 /* Return max offset */
739 vm_map_offset_t ml_get_max_offset(
740 boolean_t is64,
741 unsigned int option);
742 #define MACHINE_MAX_OFFSET_DEFAULT 0x01
743 #define MACHINE_MAX_OFFSET_MIN 0x02
744 #define MACHINE_MAX_OFFSET_MAX 0x04
745 #define MACHINE_MAX_OFFSET_DEVICE 0x08
746 #endif
747
748 extern void ml_cpu_init_completed(void);
749 extern void ml_cpu_up(void);
750 extern void ml_cpu_down(void);
751 extern int ml_find_next_up_processor(void);
752
753 /*
754 * The update to CPU counts needs to be separate from other actions
755 * in ml_cpu_up() and ml_cpu_down()
756 * because we don't update the counts when CLPC causes temporary
757 * cluster powerdown events, as these must be transparent to the user.
758 */
759 extern void ml_cpu_up_update_counts(int cpu_id);
760 extern void ml_cpu_down_update_counts(int cpu_id);
761 extern void ml_arm_sleep(void);
762
763 extern uint64_t ml_get_wake_timebase(void);
764 extern uint64_t ml_get_conttime_wake_time(void);
765
766 /* Time since the system was reset (as part of boot/wake) */
767 uint64_t ml_get_time_since_reset(void);
768
769 /*
770 * Called by ApplePMGR to set wake time. Units and epoch are identical
771 * to mach_continuous_time(). Has no effect on !HAS_CONTINUOUS_HWCLOCK
772 * chips. If wake_time == UINT64_MAX, that means the wake time is
773 * unknown and calls to ml_get_time_since_reset() will return UINT64_MAX.
774 */
775 void ml_set_reset_time(uint64_t wake_time);
776
777 #ifdef XNU_KERNEL_PRIVATE
778 /* Just a stub on ARM */
779 extern kern_return_t ml_interrupt_prewarm(uint64_t deadline);
780 #define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0)
781 #endif /* XNU_KERNEL_PRIVATE */
782
783 /* Bytes available on current stack */
784 vm_offset_t ml_stack_remaining(void);
785
786 #ifdef MACH_KERNEL_PRIVATE
787 uint32_t get_fpscr(void);
788 void set_fpscr(uint32_t);
789 void machine_conf(void);
790 void machine_lockdown(void);
791
792 #ifdef __arm64__
793 unsigned long update_mdscr(unsigned long clear, unsigned long set);
794 #endif /* __arm64__ */
795
796 extern void arm_debug_set_cp14(arm_debug_state_t *debug_state);
797 extern void fiq_context_init(boolean_t enable_fiq);
798
799 extern void reenable_async_aborts(void);
800
801 #ifdef __arm64__
802 uint64_t ml_cluster_wfe_timeout(uint32_t wfe_cluster_id);
803 #endif
804
805 #ifdef MONITOR
806 #define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */
807 #define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */
808 unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
809 uintptr_t arg2, uintptr_t arg3);
810 #endif /* MONITOR */
811
812 #if __ARM_KERNEL_PROTECT__
813 extern void set_vbar_el1(uint64_t);
814 #endif /* __ARM_KERNEL_PROTECT__ */
815
816
817 #endif /* MACH_KERNEL_PRIVATE */
818
819 extern uint32_t arm_debug_read_dscr(void);
820
821 extern int set_be_bit(void);
822 extern int clr_be_bit(void);
823 extern int be_tracing(void);
824
825 /* Please note that cpu_broadcast_xcall is not as simple is you would like it to be.
826 * It will sometimes put the calling thread to sleep, and it is up to your callback
827 * to wake it up as needed, where "as needed" is defined as "all other CPUs have
828 * called the broadcast func". Look around the kernel for examples, or instead use
829 * cpu_broadcast_xcall_simple() which does indeed act like you would expect, given
830 * the prototype.
831 */
832 typedef void (*broadcastFunc) (void *);
833 unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *);
834 unsigned int cpu_broadcast_xcall_simple(boolean_t, broadcastFunc, void *);
835 __result_use_check kern_return_t cpu_xcall(int, broadcastFunc, void *);
836 __result_use_check kern_return_t cpu_immediate_xcall(int, broadcastFunc, void *);
837
838 #ifdef KERNEL_PRIVATE
839
840 /* Interface to be used by the perf. controller to register a callback, in a
841 * single-threaded fashion. The callback will receive notifications of
842 * processor performance quality-of-service changes from the scheduler.
843 */
844
845 #ifdef __arm64__
846 typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2);
847 void cpu_qos_update_register(cpu_qos_update_t);
848 #endif /* __arm64__ */
849
850 struct going_on_core {
851 uint64_t thread_id;
852 uint16_t qos_class;
853 uint16_t urgency; /* XCPM compatibility */
854 uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */
855 uint32_t is_kernel_thread : 1;
856 uint64_t thread_group_id;
857 void *thread_group_data;
858 uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */
859 uint64_t start_time;
860 uint64_t scheduling_latency_at_same_basepri;
861 uint32_t energy_estimate_nj; /* return: In nanojoules */
862 /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */
863 };
864 typedef struct going_on_core *going_on_core_t;
865
866 struct going_off_core {
867 uint64_t thread_id;
868 uint32_t energy_estimate_nj; /* return: In nanojoules */
869 uint32_t reserved;
870 uint64_t end_time;
871 uint64_t thread_group_id;
872 void *thread_group_data;
873 };
874 typedef struct going_off_core *going_off_core_t;
875
876 struct thread_group_data {
877 uint64_t thread_group_id;
878 void *thread_group_data;
879 uint32_t thread_group_size;
880 uint32_t thread_group_flags;
881 };
882 typedef struct thread_group_data *thread_group_data_t;
883
884 struct perfcontrol_max_runnable_latency {
885 uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */];
886 };
887 typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t;
888
889 struct perfcontrol_work_interval {
890 uint64_t thread_id;
891 uint16_t qos_class;
892 uint16_t urgency;
893 uint32_t flags; // notify
894 uint64_t work_interval_id;
895 uint64_t start;
896 uint64_t finish;
897 uint64_t deadline;
898 uint64_t next_start;
899 uint64_t thread_group_id;
900 void *thread_group_data;
901 uint32_t create_flags;
902 };
903 typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t;
904
905 typedef enum {
906 WORK_INTERVAL_START,
907 WORK_INTERVAL_UPDATE,
908 WORK_INTERVAL_FINISH,
909 WORK_INTERVAL_CREATE,
910 WORK_INTERVAL_DEALLOCATE,
911 } work_interval_ctl_t;
912
913 struct perfcontrol_work_interval_instance {
914 work_interval_ctl_t ctl;
915 uint32_t create_flags;
916 uint64_t complexity;
917 uint64_t thread_id;
918 uint64_t work_interval_id;
919 uint64_t instance_id; /* out: start, in: update/finish */
920 uint64_t start;
921 uint64_t finish;
922 uint64_t deadline;
923 uint64_t thread_group_id;
924 void *thread_group_data;
925 };
926 typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t;
927
928 /*
929 * Structure to export per-CPU counters as part of the CLPC callout.
930 * Contains only the fixed CPU counters (instructions and cycles); CLPC
931 * would call back into XNU to get the configurable counters if needed.
932 */
933 struct perfcontrol_cpu_counters {
934 uint64_t instructions;
935 uint64_t cycles;
936 };
937
938 __options_decl(perfcontrol_thread_flags_mask_t, uint64_t, {
939 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_RR = 1 << 0,
940 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_NATIVE_FIRST = 1 << 1,
941 });
942
943
944 /*
945 * Structure used to pass information about a thread to CLPC
946 */
947 struct perfcontrol_thread_data {
948 /*
949 * Energy estimate (return value)
950 * The field is populated by CLPC and used to update the
951 * energy estimate of the thread
952 */
953 uint32_t energy_estimate_nj;
954 /* Perfcontrol class for thread */
955 perfcontrol_class_t perfctl_class;
956 /* Thread ID for the thread */
957 uint64_t thread_id;
958 /* Thread Group ID */
959 uint64_t thread_group_id;
960 /*
961 * Scheduling latency for threads at the same base priority.
962 * Calculated by the scheduler and passed into CLPC. The field is
963 * populated only in the thread_data structure for the thread
964 * going on-core.
965 */
966 uint64_t scheduling_latency_at_same_basepri;
967 /* Thread Group data pointer */
968 void *thread_group_data;
969 /* perfctl state pointer */
970 void *perfctl_state;
971 /* Bitmask to indicate which thread flags have been updated as part of the callout */
972 perfcontrol_thread_flags_mask_t thread_flags_mask;
973 /* Actual values for the flags that are getting updated in the callout */
974 perfcontrol_thread_flags_mask_t thread_flags;
975 };
976
977 /*
978 * All callouts from the scheduler are executed with interrupts
979 * disabled. Callouts should be implemented in C with minimal
980 * abstractions, and only use KPI exported by the mach/libkern
981 * symbolset, restricted to routines like spinlocks and atomic
982 * operations and scheduler routines as noted below. Spinlocks that
983 * are used to synchronize data in the perfcontrol_state_t should only
984 * ever be acquired with interrupts disabled, to avoid deadlocks where
985 * an quantum expiration timer interrupt attempts to perform a callout
986 * that attempts to lock a spinlock that is already held.
987 */
988
989 /*
990 * When a processor is switching between two threads (after the
991 * scheduler has chosen a new thread), the low-level platform layer
992 * will call this routine, which should perform required timestamps,
993 * MMIO register reads, or other state switching. No scheduler locks
994 * are held during this callout.
995 *
996 * This function is called with interrupts ENABLED.
997 */
998 typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t);
999
1000 /*
1001 * Once the processor has switched to the new thread, the offcore
1002 * callout will indicate the old thread that is no longer being
1003 * run. The thread's scheduler lock is held, so it will not begin
1004 * running on another processor (in the case of preemption where it
1005 * remains runnable) until it completes. If the "thread_terminating"
1006 * boolean is TRUE, this will be the last callout for this thread_id.
1007 */
1008 typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t);
1009
1010 /*
1011 * After the offcore callout and after the old thread can potentially
1012 * start running on another processor, the oncore callout will be
1013 * called with the thread's scheduler lock held. The oncore callout is
1014 * also called any time one of the parameters in the going_on_core_t
1015 * structure changes, like priority/QoS changes, and quantum
1016 * expiration, so the callout must not assume callouts are paired with
1017 * offcore callouts.
1018 */
1019 typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t);
1020
1021 /*
1022 * Periodically (on hundreds of ms scale), the scheduler will perform
1023 * maintenance and report the maximum latency for runnable (but not currently
1024 * running) threads for each urgency class.
1025 */
1026 typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t);
1027
1028 /*
1029 * When the kernel receives information about work intervals from userland,
1030 * it is passed along using this callback. No locks are held, although the state
1031 * object will not go away during the callout.
1032 */
1033 typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t);
1034
1035 /*
1036 * Start, update and finish work interval instance with optional complexity estimate.
1037 */
1038 typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t);
1039
1040 /*
1041 * These callbacks are used when thread groups are added, removed or properties
1042 * updated.
1043 * No blocking allocations (or anything else blocking) are allowed inside these
1044 * callbacks. No locks allowed in these callbacks as well since the kernel might
1045 * be holding the thread/task locks.
1046 */
1047 typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t);
1048 typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t);
1049 typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t);
1050
1051 /*
1052 * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed,
1053 * this function will be called, passing the timeout deadline that was previously armed as an argument.
1054 *
1055 * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context.
1056 */
1057 typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline);
1058
1059 /*
1060 * Context Switch Callout
1061 *
1062 * Parameters:
1063 * event - The perfcontrol_event for this callout
1064 * cpu_id - The CPU doing the context switch
1065 * timestamp - The timestamp for the context switch
1066 * flags - Flags for other relevant information
1067 * offcore - perfcontrol_data structure for thread going off-core
1068 * oncore - perfcontrol_data structure for thread going on-core
1069 * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch
1070 */
1071 typedef void (*sched_perfcontrol_csw_t)(
1072 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
1073 struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore,
1074 struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused);
1075
1076
1077 /*
1078 * Thread State Update Callout
1079 *
1080 * Parameters:
1081 * event - The perfcontrol_event for this callout
1082 * cpu_id - The CPU doing the state update
1083 * timestamp - The timestamp for the state update
1084 * flags - Flags for other relevant information
1085 * thr_data - perfcontrol_data structure for the thread being updated
1086 */
1087 typedef void (*sched_perfcontrol_state_update_t)(
1088 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
1089 struct perfcontrol_thread_data *thr_data, __unused void *unused);
1090
1091 /*
1092 * Thread Group Blocking Relationship Callout
1093 *
1094 * Parameters:
1095 * blocked_tg - Thread group blocking on progress of another thread group
1096 * blocking_tg - Thread group blocking progress of another thread group
1097 * flags - Flags for other relevant information
1098 * blocked_thr_state - Per-thread perfcontrol state for blocked thread
1099 */
1100 typedef void (*sched_perfcontrol_thread_group_blocked_t)(
1101 thread_group_data_t blocked_tg, thread_group_data_t blocking_tg, uint32_t flags, perfcontrol_state_t blocked_thr_state);
1102
1103 /*
1104 * Thread Group Unblocking Callout
1105 *
1106 * Parameters:
1107 * unblocked_tg - Thread group being unblocked from making forward progress
1108 * unblocking_tg - Thread group unblocking progress of another thread group
1109 * flags - Flags for other relevant information
1110 * unblocked_thr_state - Per-thread perfcontrol state for unblocked thread
1111 */
1112 typedef void (*sched_perfcontrol_thread_group_unblocked_t)(
1113 thread_group_data_t unblocked_tg, thread_group_data_t unblocking_tg, uint32_t flags, perfcontrol_state_t unblocked_thr_state);
1114
1115 /*
1116 * Callers should always use the CURRENT version so that the kernel can detect both older
1117 * and newer structure layouts. New callbacks should always be added at the end of the
1118 * structure, and xnu should expect existing source recompiled against newer headers
1119 * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter
1120 * to reset callbacks to their default in-kernel values.
1121 */
1122
1123 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */
1124 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */
1125 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */
1126 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */
1127 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */
1128 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */
1129 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */
1130 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */
1131 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_8 (8) /* up-to thread_group_unblocked */
1132 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_9 (9) /* allows CLPC to specify resource contention flags */
1133 #define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6
1134
1135 struct sched_perfcontrol_callbacks {
1136 unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */
1137 sched_perfcontrol_offcore_t offcore;
1138 sched_perfcontrol_context_switch_t context_switch;
1139 sched_perfcontrol_oncore_t oncore;
1140 sched_perfcontrol_max_runnable_latency_t max_runnable_latency;
1141 sched_perfcontrol_work_interval_notify_t work_interval_notify;
1142 sched_perfcontrol_thread_group_init_t thread_group_init;
1143 sched_perfcontrol_thread_group_deinit_t thread_group_deinit;
1144 sched_perfcontrol_deadline_passed_t deadline_passed;
1145 sched_perfcontrol_csw_t csw;
1146 sched_perfcontrol_state_update_t state_update;
1147 sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update;
1148 sched_perfcontrol_work_interval_ctl_t work_interval_ctl;
1149 sched_perfcontrol_thread_group_blocked_t thread_group_blocked;
1150 sched_perfcontrol_thread_group_unblocked_t thread_group_unblocked;
1151 };
1152 typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t;
1153
1154 extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state);
1155 extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation);
1156 extern void sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit);
1157 extern const char* sched_perfcontrol_thread_group_get_name(void *data);
1158
1159 /*
1160 * Edge Scheduler-CLPC Interface
1161 *
1162 * sched_perfcontrol_thread_group_preferred_clusters_set()
1163 *
1164 * The Edge scheduler expects thread group recommendations to be specific clusters rather
1165 * than just E/P. In order to allow more fine grained control, CLPC can specify an override
1166 * preferred cluster per QoS bucket. CLPC passes a common preferred cluster `tg_preferred_cluster`
1167 * and an array of size [PERFCONTROL_CLASS_MAX] with overrides for specific perfctl classes.
1168 * The scheduler translates these preferences into sched_bucket
1169 * preferences and applies the changes.
1170 *
1171 */
1172 /* Token to indicate a particular perfctl class is not overriden */
1173 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE ((uint32_t)~0)
1174
1175 /*
1176 * CLPC can also indicate if there should be an immediate rebalancing of threads of this TG as
1177 * part of this preferred cluster change. It does that by specifying the following options.
1178 */
1179 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING 0x1
1180 #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNABLE 0x2
1181 typedef uint64_t sched_perfcontrol_preferred_cluster_options_t;
1182
1183 extern void sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster,
1184 uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options);
1185
1186 /*
1187 * Edge Scheduler-CLPC Interface
1188 *
1189 * sched_perfcontrol_edge_matrix_get()/sched_perfcontrol_edge_matrix_set()
1190 *
1191 * The Edge scheduler uses edges between clusters to define the likelihood of migrating threads
1192 * across clusters. The edge config between any two clusters defines the edge weight and whether
1193 * migation and steal operations are allowed across that edge. The getter and setter allow CLPC
1194 * to query and configure edge properties between various clusters on the platform.
1195 */
1196
1197 extern void sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order);
1198 extern void sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order);
1199
1200 /*
1201 * sched_perfcontrol_edge_cpu_rotation_bitmasks_get()/sched_perfcontrol_edge_cpu_rotation_bitmasks_set()
1202 *
1203 * In order to drive intra-cluster core rotation CLPC supplies the edge scheduler with a pair of
1204 * per-cluster bitmasks. The preferred_bitmask is a bitmask of CPU cores where if a bit is set,
1205 * CLPC would prefer threads to be scheduled on that core if it is idle. The migration_bitmask
1206 * is a bitmask of CPU cores where if a bit is set, CLPC would prefer threads no longer continue
1207 * running on that core if there is any other non-avoided idle core in the cluster that is available.
1208 */
1209
1210 extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_set(uint32_t cluster_id, uint64_t preferred_bitmask, uint64_t migration_bitmask);
1211 extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_get(uint32_t cluster_id, uint64_t *preferred_bitmask, uint64_t *migration_bitmask);
1212
1213 /*
1214 * Update the deadline after which sched_perfcontrol_deadline_passed will be called.
1215 * Returns TRUE if it successfully canceled a previously set callback,
1216 * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight).
1217 * The callback is automatically canceled when it fires, and does not repeat unless rearmed.
1218 *
1219 * This 'timer' executes as the scheduler switches between threads, on a non-idle core
1220 *
1221 * There can be only one outstanding timer globally.
1222 */
1223 extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline);
1224
1225 /*
1226 * SFI configuration.
1227 */
1228 extern kern_return_t sched_perfcontrol_sfi_set_window(uint64_t window_usecs);
1229 extern kern_return_t sched_perfcontrol_sfi_set_bg_offtime(uint64_t offtime_usecs);
1230 extern kern_return_t sched_perfcontrol_sfi_set_utility_offtime(uint64_t offtime_usecs);
1231
1232 typedef enum perfcontrol_callout_type {
1233 PERFCONTROL_CALLOUT_ON_CORE,
1234 PERFCONTROL_CALLOUT_OFF_CORE,
1235 PERFCONTROL_CALLOUT_CONTEXT,
1236 PERFCONTROL_CALLOUT_STATE_UPDATE,
1237 /* Add other callout types here */
1238 PERFCONTROL_CALLOUT_MAX
1239 } perfcontrol_callout_type_t;
1240
1241 typedef enum perfcontrol_callout_stat {
1242 PERFCONTROL_STAT_INSTRS,
1243 PERFCONTROL_STAT_CYCLES,
1244 /* Add other stat types here */
1245 PERFCONTROL_STAT_MAX
1246 } perfcontrol_callout_stat_t;
1247
1248 uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type,
1249 perfcontrol_callout_stat_t stat);
1250
1251 #ifdef __arm64__
1252 /* The performance controller may use this interface to recommend
1253 * that CPUs in the designated cluster employ WFE rather than WFI
1254 * within the idle loop, falling back to WFI after the specified
1255 * timeout. The updates are expected to be serialized by the caller,
1256 * the implementation is not required to perform internal synchronization.
1257 */
1258 uint32_t ml_update_cluster_wfe_recommendation(uint32_t wfe_cluster_id, uint64_t wfe_timeout_abstime_interval, uint64_t wfe_hint_flags);
1259 #endif /* __arm64__ */
1260
1261 #if defined(HAS_APPLE_PAC)
1262 #define ONES(x) (BIT((x))-1)
1263 #define PTR_MASK ONES(64-T1SZ_BOOT)
1264 #define PAC_MASK ~PTR_MASK
1265 #define SIGN(p) ((p) & BIT(55))
1266 #define UNSIGN_PTR(p) \
1267 SIGN(p) ? ((p) | PAC_MASK) : ((p) & ~PAC_MASK)
1268
1269 uint64_t ml_default_rop_pid(void);
1270 uint64_t ml_default_jop_pid(void);
1271 uint64_t ml_non_arm64e_user_jop_pid(void);
1272 void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit);
1273 void ml_task_set_jop_pid(task_t task, task_t parent_task, boolean_t inherit, boolean_t disable_user_jop);
1274 void ml_task_set_jop_pid_from_shared_region(task_t task, boolean_t disable_user_jop);
1275 uint8_t ml_task_get_disable_user_jop(task_t task);
1276 void ml_task_set_disable_user_jop(task_t task, uint8_t disable_user_jop);
1277 void ml_thread_set_disable_user_jop(thread_t thread, uint8_t disable_user_jop);
1278 void ml_thread_set_jop_pid(thread_t thread, task_t task);
1279
1280 #if !__has_ptrcheck
1281
1282 /*
1283 * There are two implementations of _ml_auth_ptr_unchecked(). Non-FPAC CPUs
1284 * take a fast path that directly auths the pointer, relying on the CPU to
1285 * poison invald pointers without trapping. FPAC CPUs take a slower path which
1286 * emulates a non-trapping auth using strip + sign + compare, and manually
1287 * poisons the output when necessary.
1288 *
1289 * The FPAC implementation is also safe for non-FPAC CPUs, but less efficient;
1290 * guest kernels need to use it because it does not know at compile time whether
1291 * the host CPU supports FPAC.
1292 */
1293
1294 void *
1295 ml_poison_ptr(void *ptr, ptrauth_key key);
1296
1297 #if __ARM_ARCH_8_6__ || APPLEVIRTUALPLATFORM
1298 /*
1299 * ptrauth_sign_unauthenticated() reimplemented using asm volatile, forcing the
1300 * compiler to assume this operation has side-effects and cannot be reordered
1301 */
1302 #define ptrauth_sign_volatile(__value, __suffix, __data) \
1303 ({ \
1304 void *__ret = __value; \
1305 asm volatile ( \
1306 "pac" #__suffix " %[value], %[data]" \
1307 : [value] "+r"(__ret) \
1308 : [data] "r"(__data) \
1309 ); \
1310 __ret; \
1311 })
1312
1313 #define ml_auth_ptr_unchecked_for_key(_ptr, _suffix, _key, _modifier) \
1314 do { \
1315 void *stripped = ptrauth_strip(_ptr, _key); \
1316 void *reauthed = ptrauth_sign_volatile(stripped, _suffix, _modifier); \
1317 if (__probable(_ptr == reauthed)) { \
1318 _ptr = stripped; \
1319 } else { \
1320 _ptr = ml_poison_ptr(stripped, _key); \
1321 } \
1322 } while (0)
1323
1324 #define _ml_auth_ptr_unchecked(_ptr, _suffix, _modifier) \
1325 ml_auth_ptr_unchecked_for_key(_ptr, _suffix, ptrauth_key_as ## _suffix, _modifier)
1326 #else
1327 #define _ml_auth_ptr_unchecked(_ptr, _suffix, _modifier) \
1328 asm volatile ("aut" #_suffix " %[ptr], %[modifier]" : [ptr] "+r"(_ptr) : [modifier] "r"(_modifier));
1329 #endif /* __ARM_ARCH_8_6__ || APPLEVIRTUALPLATFORM */
1330
1331 /**
1332 * Authenticates a signed pointer without trapping on failure.
1333 *
1334 * @warning This function must be called with interrupts disabled.
1335 *
1336 * @warning Pointer authentication failure should normally be treated as a fatal
1337 * error. This function is intended for a handful of callers that cannot panic
1338 * on failure, and that understand the risks in handling a poisoned return
1339 * value. Other code should generally use the trapping variant
1340 * ptrauth_auth_data() instead.
1341 *
1342 * @param ptr the pointer to authenticate
1343 * @param key which key to use for authentication
1344 * @param modifier a modifier to mix into the key
1345 * @return an authenticated version of ptr, possibly with poison bits set
1346 */
1347 static inline OS_ALWAYS_INLINE void *
ml_auth_ptr_unchecked(void * ptr,ptrauth_key key,uint64_t modifier)1348 ml_auth_ptr_unchecked(void *ptr, ptrauth_key key, uint64_t modifier)
1349 {
1350 switch (key & 0x3) {
1351 case ptrauth_key_asia:
1352 _ml_auth_ptr_unchecked(ptr, ia, modifier);
1353 break;
1354 case ptrauth_key_asib:
1355 _ml_auth_ptr_unchecked(ptr, ib, modifier);
1356 break;
1357 case ptrauth_key_asda:
1358 _ml_auth_ptr_unchecked(ptr, da, modifier);
1359 break;
1360 case ptrauth_key_asdb:
1361 _ml_auth_ptr_unchecked(ptr, db, modifier);
1362 break;
1363 }
1364
1365 return ptr;
1366 }
1367
1368 #endif /* !__has_ptrcheck */
1369
1370 uint64_t ml_enable_user_jop_key(uint64_t user_jop_key);
1371
1372 /**
1373 * Restores the previous JOP key state after a previous ml_enable_user_jop_key()
1374 * call.
1375 *
1376 * @param user_jop_key The userspace JOP key previously passed to
1377 * ml_enable_user_jop_key()
1378 * @param saved_jop_state The saved JOP state returned by
1379 * ml_enable_user_jop_key()
1380 */
1381 void ml_disable_user_jop_key(uint64_t user_jop_key, uint64_t saved_jop_state);
1382 #endif /* defined(HAS_APPLE_PAC) */
1383
1384 void ml_enable_monitor(void);
1385
1386 #endif /* KERNEL_PRIVATE */
1387
1388 boolean_t machine_timeout_suspended(void);
1389 void ml_get_power_state(boolean_t *, boolean_t *);
1390
1391 uint32_t get_arm_cpu_version(void);
1392 boolean_t user_cont_hwclock_allowed(void);
1393 uint8_t user_timebase_type(void);
1394 boolean_t ml_thread_is64bit(thread_t thread);
1395
1396 #ifdef __arm64__
1397 bool ml_feature_supported(uint64_t feature_bit);
1398 void ml_set_align_checking(void);
1399 extern void wfe_timeout_configure(void);
1400 extern void wfe_timeout_init(void);
1401 #endif /* __arm64__ */
1402
1403 void ml_timer_evaluate(void);
1404 boolean_t ml_timer_forced_evaluation(void);
1405 void ml_gpu_stat_update(uint64_t);
1406 uint64_t ml_gpu_stat(thread_t);
1407 #endif /* __APPLE_API_PRIVATE */
1408
1409
1410
1411 #if __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE)
1412 extern void ml_expect_fault_begin(expected_fault_handler_t, uintptr_t);
1413 extern void ml_expect_fault_pc_begin(expected_fault_handler_t, uintptr_t);
1414 extern void ml_expect_fault_end(void);
1415 #endif /* __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) */
1416
1417 #if defined(HAS_OBJC_BP_HELPER) && defined(XNU_KERNEL_PRIVATE)
1418 kern_return_t objc_bp_assist_cfg(uint64_t adr, uint64_t ctl);
1419 #endif /* defined(HAS_OBJC_BP_HELPER) && defined(XNU_KERNEL_PRIVATE) */
1420
1421 extern uint32_t phy_read_panic;
1422 extern uint32_t phy_write_panic;
1423 #if DEVELOPMENT || DEBUG
1424 extern uint64_t simulate_stretched_io;
1425 #endif
1426
1427 void ml_hibernate_active_pre(void);
1428 void ml_hibernate_active_post(void);
1429
1430 void ml_report_minor_badness(uint32_t badness_id);
1431 #define ML_MINOR_BADNESS_CONSOLE_BUFFER_FULL 0
1432 #define ML_MINOR_BADNESS_MEMFAULT_REPORTING_NOT_ENABLED 1
1433 #define ML_MINOR_BADNESS_PIO_WRITTEN_FROM_USERSPACE 2
1434
1435 #ifdef XNU_KERNEL_PRIVATE
1436 /**
1437 * Depending on the system, by the time a backtracer starts inspecting an
1438 * interrupted CPU's register state, the value of the PC might have been
1439 * modified. In those cases, the original PC value is placed into a different
1440 * register. This function abstracts out those differences for a backtracer
1441 * wanting the PC of an interrupted CPU.
1442 *
1443 * @param state The ARM register state to parse.
1444 *
1445 * @return The original PC of the interrupted CPU.
1446 */
1447 uint64_t ml_get_backtrace_pc(struct arm_saved_state *state);
1448
1449 /**
1450 * Returns whether a secure hibernation flow is supported.
1451 *
1452 * @note Hibernation itself might still be supported even if this function
1453 * returns false. This function just denotes whether a hibernation process
1454 * which securely hashes and stores the hibernation image is supported.
1455 *
1456 * @return True if the kernel supports a secure hibernation process, false
1457 * otherwise.
1458 */
1459 bool ml_is_secure_hib_supported(void);
1460
1461 /**
1462 * Returns whether the task should use 1 GHz timebase.
1463 *
1464 * @return True if the task should use 1 GHz timebase, false
1465 * otherwise.
1466 */
1467 bool ml_task_uses_1ghz_timebase(const task_t task);
1468 #endif /* XNU_KERNEL_PRIVATE */
1469
1470 #ifdef KERNEL_PRIVATE
1471 /**
1472 * Given a physical address, return whether that address is owned by the secure
1473 * world.
1474 *
1475 * @note This does not include memory shared between XNU and the secure world.
1476 *
1477 * @param paddr The physical address to check.
1478 *
1479 * @return True if the physical address is owned and being used exclusively by
1480 * the secure world, false otherwise.
1481 */
1482 bool ml_paddr_is_exclaves_owned(vm_offset_t paddr);
1483 #endif /* KERNEL_PRIVATE */
1484
1485
1486
1487 __END_DECLS
1488
1489 #endif /* _ARM_MACHINE_ROUTINES_H_ */
1490