1 /*
2 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <arm/proc_reg.h>
30 #include <arm/machine_cpu.h>
31 #include <arm/cpu_internal.h>
32 #include <arm/cpuid.h>
33 #include <arm/io_map_entries.h>
34 #include <arm/cpu_data.h>
35 #include <arm/cpu_data_internal.h>
36 #include <arm/machine_routines.h>
37 #include <arm/misc_protos.h>
38 #include <arm/rtclock.h>
39 #include <arm/caches_internal.h>
40 #include <console/serial_protos.h>
41 #include <kern/machine.h>
42 #include <prng/random.h>
43 #include <kern/startup.h>
44 #include <kern/sched.h>
45 #include <kern/thread.h>
46 #include <mach/machine.h>
47 #include <machine/atomic.h>
48 #include <vm/pmap.h>
49 #include <vm/vm_page.h>
50 #include <vm/vm_map.h>
51 #include <sys/kdebug.h>
52 #include <kern/coalition.h>
53 #include <pexpert/device_tree.h>
54 #include <arm/cpuid_internal.h>
55 #include <arm/cpu_capabilities.h>
56 #include <san/kcov.h>
57
58 #include <IOKit/IOPlatformExpert.h>
59
60 #if KPC
61 #include <kern/kpc.h>
62 #endif
63
64 /* arm32 only supports a highly simplified topology, fixed at 1 cluster */
65 static ml_topology_cpu_t topology_cpu_array[MAX_CPUS];
66 static ml_topology_cluster_t topology_cluster = {
67 .cluster_id = 0,
68 .cluster_type = CLUSTER_TYPE_SMP,
69 .first_cpu_id = 0,
70 };
71 static ml_topology_info_t topology_info = {
72 .version = CPU_TOPOLOGY_VERSION,
73 .num_clusters = 1,
74 .cluster_types = 0x1,
75 .max_cluster_id = 0,
76 .cpus = topology_cpu_array,
77 .clusters = &topology_cluster,
78 .boot_cpu = &topology_cpu_array[0],
79 .boot_cluster = &topology_cluster,
80 };
81
82 _Atomic unsigned int cluster_type_num_active_cpus[MAX_CPU_TYPES];
83
84 MACHINE_TIMEOUT32_WRITEABLE(LockTimeOut, "lock", 6e6 /* 0.25s */, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
85 machine_timeout32_t LockTimeOutUsec; // computed in ml_init_lock_timeout
86
87 MACHINE_TIMEOUT32_WRITEABLE(TLockTimeOut, "ticket-lock", 6e6 /* 0.25s */, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
88
89 MACHINE_TIMEOUT32_WRITEABLE(MutexSpin, "mutex-spin", 240 /* 10us */, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
90
91 extern uint32_t lockdown_done;
92 uint64_t low_MutexSpin;
93 int64_t high_MutexSpin;
94
95 void
machine_startup(__unused boot_args * args)96 machine_startup(__unused boot_args * args)
97 {
98 machine_conf();
99
100 /*
101 * Kick off the kernel bootstrap.
102 */
103 kernel_bootstrap();
104 /* NOTREACHED */
105 }
106
107 char *
machine_boot_info(__unused char * buf,__unused vm_size_t size)108 machine_boot_info(
109 __unused char *buf,
110 __unused vm_size_t size)
111 {
112 return PE_boot_args();
113 }
114
115 void
slave_machine_init(__unused void * param)116 slave_machine_init(__unused void *param)
117 {
118 cpu_machine_init(); /* Initialize the processor */
119 clock_init(); /* Init the clock */
120 }
121
122 /*
123 * Routine: machine_processor_shutdown
124 * Function:
125 */
126 thread_t
machine_processor_shutdown(__unused thread_t thread,void (* doshutdown)(processor_t),processor_t processor)127 machine_processor_shutdown(
128 __unused thread_t thread,
129 void (*doshutdown)(processor_t),
130 processor_t processor)
131 {
132 return Shutdown_context(doshutdown, processor);
133 }
134
135 /*
136 * Routine: ml_init_lock_timeout
137 * Function:
138 */
139 void
ml_init_lock_timeout(void)140 ml_init_lock_timeout(void)
141 {
142 /*
143 * This function is called after STARUP_SUB_TIMEOUTS
144 * initialization, so using the "legacy" boot-args here overrides
145 * the ml-timeout-... configuration. (Given that these boot-args
146 * here are usually explicitly specified, this makes sense by
147 * overriding ml-timeout-..., which may come from the device tree.
148 */
149
150 uint64_t lto_timeout_ns;
151 uint64_t lto_abstime;
152 uint32_t slto;
153
154 if (PE_parse_boot_argn("slto_us", &slto, sizeof(slto))) {
155 lto_timeout_ns = slto * NSEC_PER_USEC;
156 nanoseconds_to_absolutetime(lto_timeout_ns, <o_abstime);
157 os_atomic_store(&LockTimeOut, (uint32_t)lto_abstime, relaxed);
158 } else {
159 lto_abstime = os_atomic_load(&LockTimeOut, relaxed);
160 os_atomic_store(&TLockTimeOut, (uint32_t)lto_abstime, relaxed);
161 absolutetime_to_nanoseconds(lto_abstime, <o_timeout_ns);
162 }
163
164 os_atomic_store(&LockTimeOutUsec, (uint32_t)(lto_timeout_ns / NSEC_PER_USEC), relaxed);
165
166 uint64_t mtxspin;
167 uint64_t mtx_abstime;
168 if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof(mtxspin))) {
169 if (mtxspin > USEC_PER_SEC >> 4) {
170 mtxspin = USEC_PER_SEC >> 4;
171 }
172 nanoseconds_to_absolutetime(mtxspin * NSEC_PER_USEC, &mtx_abstime);
173 os_atomic_store(&MutexSpin, (uint32_t)mtx_abstime, relaxed);
174 } else {
175 mtx_abstime = os_atomic_load(&MutexSpin, relaxed);
176 }
177 low_MutexSpin = os_atomic_load(&MutexSpin, relaxed);
178 /*
179 * high_MutexSpin should be initialized as low_MutexSpin * real_ncpus, but
180 * real_ncpus is not set at this time
181 *
182 * NOTE: active spinning is disabled in arm. It can be activated
183 * by setting high_MutexSpin through the sysctl.
184 */
185 high_MutexSpin = low_MutexSpin;
186 }
187
188 /*
189 * This is called when all of the ml_processor_info_t structures have been
190 * initialized and all the processors have been started through processor_start().
191 *
192 * Required by the scheduler subsystem.
193 */
194 void
ml_cpu_init_completed(void)195 ml_cpu_init_completed(void)
196 {
197 }
198
199 /*
200 * This is called from the machine-independent routine cpu_up()
201 * to perform machine-dependent info updates.
202 */
203 void
ml_cpu_up(void)204 ml_cpu_up(void)
205 {
206 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[ml_get_cpu_number_local()];
207
208 os_atomic_inc(&cluster_type_num_active_cpus[cpu->cluster_type], relaxed);
209
210 os_atomic_inc(&machine_info.physical_cpu, relaxed);
211 os_atomic_inc(&machine_info.logical_cpu, relaxed);
212 }
213
214 /*
215 * This is called from the machine-independent routine cpu_down()
216 * to perform machine-dependent info updates.
217 */
218 void
ml_cpu_down(void)219 ml_cpu_down(void)
220 {
221 cpu_data_t *cpu_data_ptr;
222 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[ml_get_cpu_number_local()];
223
224 os_atomic_dec(&cluster_type_num_active_cpus[cpu->cluster_type], relaxed);
225
226 os_atomic_dec(&machine_info.physical_cpu, relaxed);
227 os_atomic_dec(&machine_info.logical_cpu, relaxed);
228
229 /*
230 * If we want to deal with outstanding IPIs, we need to
231 * do relatively early in the processor_doshutdown path,
232 * as we pend decrementer interrupts using the IPI
233 * mechanism if we cannot immediately service them (if
234 * IRQ is masked). Do so now.
235 *
236 * We aren't on the interrupt stack here; would it make
237 * more sense to disable signaling and then enable
238 * interrupts? It might be a bit cleaner.
239 */
240 cpu_data_ptr = getCpuDatap();
241 cpu_data_ptr->cpu_running = FALSE;
242
243 cpu_signal_handler_internal(TRUE);
244 }
245
246 unsigned int
ml_get_machine_mem(void)247 ml_get_machine_mem(void)
248 {
249 return machine_info.memory_size;
250 }
251
252 /* Return max offset */
253 vm_map_offset_t
ml_get_max_offset(boolean_t is64,unsigned int option)254 ml_get_max_offset(
255 boolean_t is64,
256 unsigned int option)
257 {
258 unsigned int pmap_max_offset_option = 0;
259
260 switch (option) {
261 case MACHINE_MAX_OFFSET_DEFAULT:
262 pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEFAULT;
263 break;
264 case MACHINE_MAX_OFFSET_MIN:
265 pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MIN;
266 break;
267 case MACHINE_MAX_OFFSET_MAX:
268 pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MAX;
269 break;
270 case MACHINE_MAX_OFFSET_DEVICE:
271 pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEVICE;
272 break;
273 default:
274 panic("ml_get_max_offset(): Illegal option 0x%x", option);
275 break;
276 }
277 return pmap_max_offset(is64, pmap_max_offset_option);
278 }
279
280 void
ml_panic_trap_to_debugger(__unused const char * panic_format_str,__unused va_list * panic_args,__unused unsigned int reason,__unused void * ctx,__unused uint64_t panic_options_mask,__unused unsigned long panic_caller)281 ml_panic_trap_to_debugger(__unused const char *panic_format_str,
282 __unused va_list *panic_args,
283 __unused unsigned int reason,
284 __unused void *ctx,
285 __unused uint64_t panic_options_mask,
286 __unused unsigned long panic_caller)
287 {
288 return;
289 }
290
291 __attribute__((noreturn))
292 void
halt_all_cpus(boolean_t reboot)293 halt_all_cpus(boolean_t reboot)
294 {
295 if (reboot) {
296 printf("MACH Reboot\n");
297 PEHaltRestart(kPERestartCPU);
298 } else {
299 printf("CPU halted\n");
300 PEHaltRestart(kPEHaltCPU);
301 }
302 while (1) {
303 ;
304 }
305 }
306
307 __attribute__((noreturn))
308 void
halt_cpu(void)309 halt_cpu(void)
310 {
311 halt_all_cpus(FALSE);
312 }
313
314 /*
315 * Routine: machine_signal_idle
316 * Function:
317 */
318 void
machine_signal_idle(processor_t processor)319 machine_signal_idle(
320 processor_t processor)
321 {
322 cpu_signal(processor_to_cpu_datap(processor), SIGPnop, (void *)NULL, (void *)NULL);
323 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0);
324 }
325
326 void
machine_signal_idle_deferred(processor_t processor)327 machine_signal_idle_deferred(
328 processor_t processor)
329 {
330 cpu_signal_deferred(processor_to_cpu_datap(processor));
331 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_DEFERRED_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0);
332 }
333
334 void
machine_signal_idle_cancel(processor_t processor)335 machine_signal_idle_cancel(
336 processor_t processor)
337 {
338 cpu_signal_cancel(processor_to_cpu_datap(processor));
339 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_CANCEL_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0);
340 }
341
342 /*
343 * Routine: ml_install_interrupt_handler
344 * Function: Initialize Interrupt Handler
345 */
346 void
ml_install_interrupt_handler(void * nub,int source,void * target,IOInterruptHandler handler,void * refCon)347 ml_install_interrupt_handler(
348 void *nub,
349 int source,
350 void *target,
351 IOInterruptHandler handler,
352 void *refCon)
353 {
354 cpu_data_t *cpu_data_ptr;
355 boolean_t current_state;
356
357 current_state = ml_set_interrupts_enabled(FALSE);
358 cpu_data_ptr = getCpuDatap();
359
360 cpu_data_ptr->interrupt_nub = nub;
361 cpu_data_ptr->interrupt_source = source;
362 cpu_data_ptr->interrupt_target = target;
363 cpu_data_ptr->interrupt_handler = handler;
364 cpu_data_ptr->interrupt_refCon = refCon;
365
366 (void) ml_set_interrupts_enabled(current_state);
367 }
368
369 /*
370 * Routine: ml_init_interrupt
371 * Function: Initialize Interrupts
372 */
373 void
ml_init_interrupt(void)374 ml_init_interrupt(void)
375 {
376 }
377
378 /*
379 * Routine: ml_init_timebase
380 * Function: register and setup Timebase, Decremeter services
381 */
382 void
ml_init_timebase(void * args,tbd_ops_t tbd_funcs,vm_offset_t int_address,vm_offset_t int_value)383 ml_init_timebase(
384 void *args,
385 tbd_ops_t tbd_funcs,
386 vm_offset_t int_address,
387 vm_offset_t int_value)
388 {
389 cpu_data_t *cpu_data_ptr;
390
391 cpu_data_ptr = (cpu_data_t *)args;
392
393 if ((cpu_data_ptr == &BootCpuData)
394 && (rtclock_timebase_func.tbd_fiq_handler == (void *)NULL)) {
395 rtclock_timebase_func = *tbd_funcs;
396 rtclock_timebase_addr = int_address;
397 rtclock_timebase_val = int_value;
398 }
399 }
400
401 void
ml_parse_cpu_topology(void)402 ml_parse_cpu_topology(void)
403 {
404 DTEntry entry, child;
405 OpaqueDTEntryIterator iter;
406 uint32_t cpu_boot_arg;
407 int err;
408
409 err = SecureDTLookupEntry(NULL, "/cpus", &entry);
410 assert(err == kSuccess);
411
412 err = SecureDTInitEntryIterator(entry, &iter);
413 assert(err == kSuccess);
414
415 cpu_boot_arg = MAX_CPUS;
416 PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg));
417
418 ml_topology_cluster_t *cluster = &topology_info.clusters[0];
419 unsigned int cpu_id = 0;
420 while (kSuccess == SecureDTIterateEntries(&iter, &child)) {
421 #if MACH_ASSERT
422 unsigned int propSize;
423 void const *prop = NULL;
424 if (cpu_id == 0) {
425 if (kSuccess != SecureDTGetProperty(child, "state", &prop, &propSize)) {
426 panic("unable to retrieve state for cpu %u", cpu_id);
427 }
428
429 if (strncmp((char const *)prop, "running", propSize) != 0) {
430 panic("cpu 0 has not been marked as running!");
431 }
432 }
433 assert(kSuccess == SecureDTGetProperty(child, "reg", &prop, &propSize));
434 assert(cpu_id == *((uint32_t const *)prop));
435 #endif
436 if (cpu_id >= cpu_boot_arg) {
437 break;
438 }
439
440 ml_topology_cpu_t *cpu = &topology_info.cpus[cpu_id];
441
442 cpu->cpu_id = cpu_id;
443 cpu->phys_id = cpu_id;
444 cpu->cluster_type = cluster->cluster_type;
445
446 cluster->num_cpus++;
447 cluster->cpu_mask |= 1ULL << cpu_id;
448
449 topology_info.num_cpus++;
450 topology_info.max_cpu_id = cpu_id;
451 topology_info.cluster_type_num_cpus[cpu->cluster_type]++;
452
453 cpu_id++;
454 }
455
456 if (cpu_id == 0) {
457 panic("No cpus found!");
458 }
459 }
460
461 const ml_topology_info_t *
ml_get_topology_info(void)462 ml_get_topology_info(void)
463 {
464 return &topology_info;
465 }
466
467 unsigned int
ml_get_cpu_count(void)468 ml_get_cpu_count(void)
469 {
470 return topology_info.num_cpus;
471 }
472
473 unsigned int
ml_get_cluster_count(void)474 ml_get_cluster_count(void)
475 {
476 return topology_info.num_clusters;
477 }
478
479 int
ml_get_boot_cpu_number(void)480 ml_get_boot_cpu_number(void)
481 {
482 return 0;
483 }
484
485 cluster_type_t
ml_get_boot_cluster_type(void)486 ml_get_boot_cluster_type(void)
487 {
488 return CLUSTER_TYPE_SMP;
489 }
490
491 int
ml_get_cpu_number(uint32_t phys_id)492 ml_get_cpu_number(uint32_t phys_id)
493 {
494 if (phys_id > (uint32_t)ml_get_max_cpu_number()) {
495 return -1;
496 }
497
498 return (int)phys_id;
499 }
500
501 unsigned int
ml_get_cpu_number_local(void)502 ml_get_cpu_number_local(void)
503 {
504 uint32_t mpidr_value = 0;
505 unsigned int cpu_id;
506
507 /* We identify the CPU based on the constant bits of MPIDR_EL1. */
508 asm volatile ("mrc p15, 0, %0, c0, c0, 5" : "=r" (mpidr_value));
509 cpu_id = mpidr_value & 0xFF;
510
511 assert(cpu_id <= (unsigned int)ml_get_max_cpu_number());
512
513 return cpu_id;
514 }
515
516 int
ml_get_cluster_number(__unused uint32_t phys_id)517 ml_get_cluster_number(__unused uint32_t phys_id)
518 {
519 return 0;
520 }
521
522 int
ml_get_max_cpu_number(void)523 ml_get_max_cpu_number(void)
524 {
525 return topology_info.num_cpus - 1;
526 }
527
528 int
ml_get_max_cluster_number(void)529 ml_get_max_cluster_number(void)
530 {
531 return topology_info.max_cluster_id;
532 }
533
534 unsigned int
ml_get_first_cpu_id(unsigned int cluster_id)535 ml_get_first_cpu_id(unsigned int cluster_id)
536 {
537 return topology_info.clusters[cluster_id].first_cpu_id;
538 }
539
540
541 kern_return_t
ml_processor_register(ml_processor_info_t * in_processor_info,processor_t * processor_out,ipi_handler_t * ipi_handler_out,perfmon_interrupt_handler_func * pmi_handler_out)542 ml_processor_register(ml_processor_info_t *in_processor_info,
543 processor_t * processor_out, ipi_handler_t *ipi_handler_out,
544 perfmon_interrupt_handler_func *pmi_handler_out)
545 {
546 cpu_data_t *this_cpu_datap;
547 boolean_t is_boot_cpu;
548
549 const unsigned int max_cpu_id = ml_get_max_cpu_number();
550 if (in_processor_info->phys_id > max_cpu_id) {
551 /*
552 * The physical CPU ID indicates that we have more CPUs than
553 * this xnu build support. This probably means we have an
554 * incorrect board configuration.
555 *
556 * TODO: Should this just return a failure instead? A panic
557 * is simply a convenient way to catch bugs in the pexpert
558 * headers.
559 */
560 panic("phys_id %u is too large for max_cpu_id (%u)", in_processor_info->phys_id, max_cpu_id);
561 }
562
563 /* Fail the registration if the number of CPUs has been limited by boot-arg. */
564 if ((in_processor_info->phys_id >= topology_info.num_cpus) ||
565 (in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number())) {
566 return KERN_FAILURE;
567 }
568
569 if (in_processor_info->log_id != (uint32_t)ml_get_boot_cpu_number()) {
570 is_boot_cpu = FALSE;
571 this_cpu_datap = cpu_data_alloc(FALSE);
572 cpu_data_init(this_cpu_datap);
573 } else {
574 this_cpu_datap = &BootCpuData;
575 is_boot_cpu = TRUE;
576 }
577
578 this_cpu_datap->cpu_id = in_processor_info->cpu_id;
579
580 if (!is_boot_cpu) {
581 if (cpu_data_register(this_cpu_datap) != KERN_SUCCESS) {
582 goto processor_register_error;
583 }
584 }
585
586 this_cpu_datap->cpu_idle_notify = in_processor_info->processor_idle;
587 this_cpu_datap->cpu_cache_dispatch = (cache_dispatch_t) in_processor_info->platform_cache_dispatch;
588 nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency);
589 this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr);
590
591 this_cpu_datap->idle_timer_notify = in_processor_info->idle_timer;
592 this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon;
593
594 this_cpu_datap->platform_error_handler = in_processor_info->platform_error_handler;
595 this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr;
596 this_cpu_datap->cpu_phys_id = in_processor_info->phys_id;
597 this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty;
598
599 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, this_cpu_datap);
600 if (!is_boot_cpu) {
601 processor_init(processor, this_cpu_datap->cpu_number,
602 processor_pset(master_processor));
603
604 if (this_cpu_datap->cpu_l2_access_penalty) {
605 /*
606 * Cores that have a non-zero L2 access penalty compared
607 * to the boot processor should be de-prioritized by the
608 * scheduler, so that threads use the cores with better L2
609 * preferentially.
610 */
611 processor_set_primary(processor, master_processor);
612 }
613 }
614
615 *processor_out = processor;
616 *ipi_handler_out = cpu_signal_handler;
617 *pmi_handler_out = NULL;
618 if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL) {
619 *in_processor_info->idle_tickle = (idle_tickle_t) cpu_idle_tickle;
620 }
621
622 #if KPC
623 if (kpc_register_cpu(this_cpu_datap) != TRUE) {
624 goto processor_register_error;
625 }
626 #endif
627
628 if (!is_boot_cpu) {
629 random_cpu_init(this_cpu_datap->cpu_number);
630 }
631
632 return KERN_SUCCESS;
633
634 processor_register_error:
635 #if KPC
636 kpc_unregister_cpu(this_cpu_datap);
637 #endif
638 if (!is_boot_cpu) {
639 cpu_data_free(this_cpu_datap);
640 }
641 return KERN_FAILURE;
642 }
643
644 void
ml_init_arm_debug_interface(void * in_cpu_datap,vm_offset_t virt_address)645 ml_init_arm_debug_interface(
646 void * in_cpu_datap,
647 vm_offset_t virt_address)
648 {
649 ((cpu_data_t *)in_cpu_datap)->cpu_debug_interface_map = virt_address;
650 do_debugid();
651 }
652
653 /*
654 * Routine: init_ast_check
655 * Function:
656 */
657 void
init_ast_check(__unused processor_t processor)658 init_ast_check(
659 __unused processor_t processor)
660 {
661 }
662
663 /*
664 * Routine: cause_ast_check
665 * Function:
666 */
667 void
cause_ast_check(processor_t processor)668 cause_ast_check(
669 processor_t processor)
670 {
671 if (current_processor() != processor) {
672 cpu_signal(processor_to_cpu_datap(processor), SIGPast, (void *)NULL, (void *)NULL);
673 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 1 /* ast */, 0, 0, 0);
674 }
675 }
676
677 extern uint32_t cpu_idle_count;
678
679 void
ml_get_power_state(boolean_t * icp,boolean_t * pidlep)680 ml_get_power_state(boolean_t *icp, boolean_t *pidlep)
681 {
682 *icp = ml_at_interrupt_context();
683 *pidlep = (cpu_idle_count == real_ncpus);
684 }
685
686 /*
687 * Routine: ml_cause_interrupt
688 * Function: Generate a fake interrupt
689 */
690 void
ml_cause_interrupt(void)691 ml_cause_interrupt(void)
692 {
693 return; /* BS_XXX */
694 }
695
696 /* Map memory map IO space */
697 vm_offset_t
ml_io_map(vm_offset_t phys_addr,vm_size_t size)698 ml_io_map(
699 vm_offset_t phys_addr,
700 vm_size_t size)
701 {
702 return io_map(phys_addr, size, VM_WIMG_IO);
703 }
704
705 /* Map memory map IO space (with protections specified) */
706 vm_offset_t
ml_io_map_with_prot(vm_offset_t phys_addr,vm_size_t size,vm_prot_t prot)707 ml_io_map_with_prot(
708 vm_offset_t phys_addr,
709 vm_size_t size,
710 vm_prot_t prot)
711 {
712 return io_map_with_prot(phys_addr, size, VM_WIMG_IO, prot);
713 }
714
715 vm_offset_t
ml_io_map_wcomb(vm_offset_t phys_addr,vm_size_t size)716 ml_io_map_wcomb(
717 vm_offset_t phys_addr,
718 vm_size_t size)
719 {
720 return io_map(phys_addr, size, VM_WIMG_WCOMB);
721 }
722
723 void
ml_io_unmap(vm_offset_t addr,vm_size_t sz)724 ml_io_unmap(vm_offset_t addr, vm_size_t sz)
725 {
726 pmap_remove(kernel_pmap, addr, addr + sz);
727 kmem_free(kernel_map, addr, sz);
728 }
729
730 /* boot memory allocation */
731 vm_offset_t
ml_static_malloc(__unused vm_size_t size)732 ml_static_malloc(
733 __unused vm_size_t size)
734 {
735 return (vm_offset_t) NULL;
736 }
737
738 vm_map_address_t
ml_map_high_window(vm_offset_t phys_addr,vm_size_t len)739 ml_map_high_window(
740 vm_offset_t phys_addr,
741 vm_size_t len)
742 {
743 return pmap_map_high_window_bd(phys_addr, len, VM_PROT_READ | VM_PROT_WRITE);
744 }
745
746 vm_offset_t
ml_static_ptovirt(vm_offset_t paddr)747 ml_static_ptovirt(
748 vm_offset_t paddr)
749 {
750 return phystokv(paddr);
751 }
752
753 vm_offset_t
ml_static_vtop(vm_offset_t vaddr)754 ml_static_vtop(
755 vm_offset_t vaddr)
756 {
757 assertf(((vm_address_t)(vaddr) - gVirtBase) < gPhysSize, "%s: illegal vaddr: %p", __func__, (void*)vaddr);
758 return (vm_address_t)(vaddr) - gVirtBase + gPhysBase;
759 }
760
761 /*
762 * Return the maximum contiguous KVA range that can be accessed from this
763 * physical address. For arm64, we employ a segmented physical aperture
764 * relocation table which can limit the available range for a given PA to
765 * something less than the extent of physical memory. But here, we still
766 * have a flat physical aperture, so no such requirement exists.
767 */
768 vm_map_address_t
phystokv_range(pmap_paddr_t pa,vm_size_t * max_len)769 phystokv_range(pmap_paddr_t pa, vm_size_t *max_len)
770 {
771 vm_size_t len = gPhysSize - (pa - gPhysBase);
772 if (*max_len > len) {
773 *max_len = len;
774 }
775 assertf((pa - gPhysBase) < gPhysSize, "%s: illegal PA: 0x%lx", __func__, (unsigned long)pa);
776 return pa - gPhysBase + gVirtBase;
777 }
778
779 vm_offset_t
ml_static_slide(vm_offset_t vaddr)780 ml_static_slide(
781 vm_offset_t vaddr)
782 {
783 return VM_KERNEL_SLIDE(vaddr);
784 }
785
786 kern_return_t
ml_static_verify_page_protections(uint64_t base,uint64_t size,vm_prot_t prot)787 ml_static_verify_page_protections(
788 uint64_t base, uint64_t size, vm_prot_t prot)
789 {
790 /* XXX Implement Me */
791 (void)base;
792 (void)size;
793 (void)prot;
794 return KERN_FAILURE;
795 }
796
797
798 vm_offset_t
ml_static_unslide(vm_offset_t vaddr)799 ml_static_unslide(
800 vm_offset_t vaddr)
801 {
802 return VM_KERNEL_UNSLIDE(vaddr);
803 }
804
805 kern_return_t
ml_static_protect(vm_offset_t vaddr,vm_size_t size,vm_prot_t new_prot)806 ml_static_protect(
807 vm_offset_t vaddr, /* kernel virtual address */
808 vm_size_t size,
809 vm_prot_t new_prot)
810 {
811 pt_entry_t arm_prot = 0;
812 pt_entry_t arm_block_prot = 0;
813 vm_offset_t vaddr_cur;
814 ppnum_t ppn;
815 kern_return_t result = KERN_SUCCESS;
816
817 if (vaddr < VM_MIN_KERNEL_ADDRESS) {
818 return KERN_FAILURE;
819 }
820
821 assert((vaddr & (ARM_PGBYTES - 1)) == 0); /* must be page aligned */
822
823 if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) {
824 panic("ml_static_protect(): WX request on %p", (void *) vaddr);
825 }
826 if (lockdown_done && (new_prot & VM_PROT_EXECUTE)) {
827 panic("ml_static_protect(): attempt to inject executable mapping on %p", (void *) vaddr);
828 }
829
830 /* Set up the protection bits, and block bits so we can validate block mappings. */
831 if (new_prot & VM_PROT_WRITE) {
832 arm_prot |= ARM_PTE_AP(AP_RWNA);
833 arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RWNA);
834 } else {
835 arm_prot |= ARM_PTE_AP(AP_RONA);
836 arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RONA);
837 }
838
839 if (!(new_prot & VM_PROT_EXECUTE)) {
840 arm_prot |= ARM_PTE_NX;
841 arm_block_prot |= ARM_TTE_BLOCK_NX;
842 }
843
844 for (vaddr_cur = vaddr;
845 vaddr_cur < ((vaddr + size) & ~ARM_PGMASK);
846 vaddr_cur += ARM_PGBYTES) {
847 ppn = pmap_find_phys(kernel_pmap, vaddr_cur);
848 if (ppn != (vm_offset_t) NULL) {
849 tt_entry_t *ttp = &kernel_pmap->tte[ttenum(vaddr_cur)];
850 tt_entry_t tte = *ttp;
851
852 if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) {
853 if (((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) &&
854 ((tte & (ARM_TTE_BLOCK_APMASK | ARM_TTE_BLOCK_NX_MASK)) == arm_block_prot)) {
855 /*
856 * We can support ml_static_protect on a block mapping if the mapping already has
857 * the desired protections. We still want to run checks on a per-page basis.
858 */
859 continue;
860 }
861
862 result = KERN_FAILURE;
863 break;
864 }
865
866 pt_entry_t *pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(vaddr_cur);
867 pt_entry_t ptmp = *pte_p;
868
869 ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_NX_MASK)) | arm_prot;
870 *pte_p = ptmp;
871 }
872 }
873
874 if (vaddr_cur > vaddr) {
875 flush_mmu_tlb_region(vaddr, (vm_size_t)(vaddr_cur - vaddr));
876 }
877
878 return result;
879 }
880
881 /*
882 * Routine: ml_static_mfree
883 * Function:
884 */
885 void
ml_static_mfree(vm_offset_t vaddr,vm_size_t size)886 ml_static_mfree(
887 vm_offset_t vaddr,
888 vm_size_t size)
889 {
890 vm_offset_t vaddr_cur;
891 ppnum_t ppn;
892 uint32_t freed_pages = 0;
893 uint32_t freed_kernelcache_pages = 0;
894
895 /* It is acceptable (if bad) to fail to free. */
896 if (vaddr < VM_MIN_KERNEL_ADDRESS) {
897 return;
898 }
899
900 assert((vaddr & (PAGE_SIZE - 1)) == 0); /* must be page aligned */
901
902 for (vaddr_cur = vaddr;
903 vaddr_cur < trunc_page_32(vaddr + size);
904 vaddr_cur += PAGE_SIZE) {
905 ppn = pmap_find_phys(kernel_pmap, vaddr_cur);
906 if (ppn != (vm_offset_t) NULL) {
907 /*
908 * It is not acceptable to fail to update the protections on a page
909 * we will release to the VM. We need to either panic or continue.
910 * For now, we'll panic (to help flag if there is memory we can
911 * reclaim).
912 */
913 if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) {
914 panic("Failed ml_static_mfree on %p", (void *) vaddr_cur);
915 }
916 vm_page_create(ppn, (ppn + 1));
917 freed_pages++;
918 if (vaddr_cur >= segLOWEST && vaddr_cur < end_kern) {
919 freed_kernelcache_pages++;
920 }
921 }
922 }
923 vm_page_lockspin_queues();
924 vm_page_wire_count -= freed_pages;
925 vm_page_wire_count_initial -= freed_pages;
926 vm_page_kernelcache_count -= freed_kernelcache_pages;
927 vm_page_unlock_queues();
928 #if DEBUG
929 kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn);
930 #endif
931 }
932
933
934 /* virtual to physical on wired pages */
935 vm_offset_t
ml_vtophys(vm_offset_t vaddr)936 ml_vtophys(vm_offset_t vaddr)
937 {
938 return kvtophys(vaddr);
939 }
940
941 /*
942 * Routine: ml_nofault_copy
943 * Function: Perform a physical mode copy if the source and destination have
944 * valid translations in the kernel pmap. If translations are present, they are
945 * assumed to be wired; e.g., no attempt is made to guarantee that the
946 * translations obtained remain valid for the duration of the copy process.
947 */
948 vm_size_t
ml_nofault_copy(vm_offset_t virtsrc,vm_offset_t virtdst,vm_size_t size)949 ml_nofault_copy(vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size)
950 {
951 addr64_t cur_phys_dst, cur_phys_src;
952 uint32_t count, nbytes = 0;
953
954 while (size > 0) {
955 if (!(cur_phys_src = kvtophys(virtsrc))) {
956 break;
957 }
958 if (!(cur_phys_dst = kvtophys(virtdst))) {
959 break;
960 }
961 if (!pmap_valid_address(trunc_page_64(cur_phys_dst)) ||
962 !pmap_valid_address(trunc_page_64(cur_phys_src))) {
963 break;
964 }
965 count = PAGE_SIZE - (cur_phys_src & PAGE_MASK);
966 if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK))) {
967 count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK);
968 }
969 if (count > size) {
970 count = size;
971 }
972
973 bcopy_phys(cur_phys_src, cur_phys_dst, count);
974
975 nbytes += count;
976 virtsrc += count;
977 virtdst += count;
978 size -= count;
979 }
980
981 return nbytes;
982 }
983
984 /*
985 * Routine: ml_validate_nofault
986 * Function: Validate that ths address range has a valid translations
987 * in the kernel pmap. If translations are present, they are
988 * assumed to be wired; i.e. no attempt is made to guarantee
989 * that the translation persist after the check.
990 * Returns: TRUE if the range is mapped and will not cause a fault,
991 * FALSE otherwise.
992 */
993
994 boolean_t
ml_validate_nofault(vm_offset_t virtsrc,vm_size_t size)995 ml_validate_nofault(
996 vm_offset_t virtsrc, vm_size_t size)
997 {
998 addr64_t cur_phys_src;
999 uint32_t count;
1000
1001 while (size > 0) {
1002 if (!(cur_phys_src = kvtophys(virtsrc))) {
1003 return FALSE;
1004 }
1005 if (!pmap_valid_address(trunc_page_64(cur_phys_src))) {
1006 return FALSE;
1007 }
1008 count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK));
1009 if (count > size) {
1010 count = (uint32_t)size;
1011 }
1012
1013 virtsrc += count;
1014 size -= count;
1015 }
1016
1017 return TRUE;
1018 }
1019
1020 void
ml_get_bouncepool_info(vm_offset_t * phys_addr,vm_size_t * size)1021 ml_get_bouncepool_info(vm_offset_t * phys_addr, vm_size_t * size)
1022 {
1023 *phys_addr = 0;
1024 *size = 0;
1025 }
1026
1027 /*
1028 * Stubs for CPU Stepper
1029 */
1030 void
active_rt_threads(__unused boolean_t active)1031 active_rt_threads(__unused boolean_t active)
1032 {
1033 }
1034
1035 void
thread_tell_urgency(__unused thread_urgency_t urgency,__unused uint64_t rt_period,__unused uint64_t rt_deadline,__unused uint64_t sched_latency,__unused thread_t nthread)1036 thread_tell_urgency(__unused thread_urgency_t urgency,
1037 __unused uint64_t rt_period,
1038 __unused uint64_t rt_deadline,
1039 __unused uint64_t sched_latency,
1040 __unused thread_t nthread)
1041 {
1042 }
1043
1044 void
machine_run_count(__unused uint32_t count)1045 machine_run_count(__unused uint32_t count)
1046 {
1047 }
1048
1049 processor_t
machine_choose_processor(__unused processor_set_t pset,processor_t processor)1050 machine_choose_processor(__unused processor_set_t pset, processor_t processor)
1051 {
1052 return processor;
1053 }
1054
1055 #ifdef CONFIG_KCOV
1056
1057 kcov_cpu_data_t *
current_kcov_data(void)1058 current_kcov_data(void)
1059 {
1060 return ¤t_cpu_datap()->cpu_kcov_data;
1061 }
1062
1063 kcov_cpu_data_t *
cpu_kcov_data(int cpuid)1064 cpu_kcov_data(int cpuid)
1065 {
1066 return &cpu_datap(cpuid)->cpu_kcov_data;
1067 }
1068
1069 #endif /* CONFIG_KCOV */
1070
1071 boolean_t
machine_timeout_suspended(void)1072 machine_timeout_suspended(void)
1073 {
1074 return FALSE;
1075 }
1076
1077 kern_return_t
ml_interrupt_prewarm(__unused uint64_t deadline)1078 ml_interrupt_prewarm(__unused uint64_t deadline)
1079 {
1080 return KERN_FAILURE;
1081 }
1082
1083 uint64_t
ml_get_hwclock(void)1084 ml_get_hwclock(void)
1085 {
1086 uint64_t high_first = 0;
1087 uint64_t high_second = 0;
1088 uint64_t low = 0;
1089
1090 __builtin_arm_isb(ISB_SY);
1091
1092 do {
1093 high_first = __builtin_arm_mrrc(15, 0, 14) >> 32;
1094 low = __builtin_arm_mrrc(15, 0, 14) & 0xFFFFFFFFULL;
1095 high_second = __builtin_arm_mrrc(15, 0, 14) >> 32;
1096 } while (high_first != high_second);
1097
1098 return (high_first << 32) | (low);
1099 }
1100
1101 boolean_t
ml_delay_should_spin(uint64_t interval)1102 ml_delay_should_spin(uint64_t interval)
1103 {
1104 cpu_data_t *cdp = getCpuDatap();
1105
1106 if (cdp->cpu_idle_latency) {
1107 return (interval < cdp->cpu_idle_latency) ? TRUE : FALSE;
1108 } else {
1109 /*
1110 * Early boot, latency is unknown. Err on the side of blocking,
1111 * which should always be safe, even if slow
1112 */
1113 return FALSE;
1114 }
1115 }
1116
1117 void
ml_delay_on_yield(void)1118 ml_delay_on_yield(void)
1119 {
1120 }
1121
1122 boolean_t
ml_thread_is64bit(thread_t thread)1123 ml_thread_is64bit(thread_t thread)
1124 {
1125 return thread_is_64bit_addr(thread);
1126 }
1127
1128 void
ml_timer_evaluate(void)1129 ml_timer_evaluate(void)
1130 {
1131 }
1132
1133 boolean_t
ml_timer_forced_evaluation(void)1134 ml_timer_forced_evaluation(void)
1135 {
1136 return FALSE;
1137 }
1138
1139 uint64_t
ml_energy_stat(__unused thread_t t)1140 ml_energy_stat(__unused thread_t t)
1141 {
1142 return 0;
1143 }
1144
1145
1146 void
ml_gpu_stat_update(__unused uint64_t gpu_ns_delta)1147 ml_gpu_stat_update(__unused uint64_t gpu_ns_delta)
1148 {
1149 /*
1150 * For now: update the resource coalition stats of the
1151 * current thread's coalition
1152 */
1153 task_coalition_update_gpu_stats(current_task(), gpu_ns_delta);
1154 }
1155
1156 uint64_t
ml_gpu_stat(__unused thread_t t)1157 ml_gpu_stat(__unused thread_t t)
1158 {
1159 return 0;
1160 }
1161
1162 #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
1163 static void
timer_state_event(boolean_t switch_to_kernel)1164 timer_state_event(boolean_t switch_to_kernel)
1165 {
1166 thread_t thread = current_thread();
1167 if (!thread->precise_user_kernel_time) {
1168 return;
1169 }
1170
1171 processor_t pd = current_processor();
1172 uint64_t now = ml_get_timebase();
1173
1174 timer_stop(pd->current_state, now);
1175 pd->current_state = (switch_to_kernel) ? &pd->system_state : &pd->user_state;
1176 timer_start(pd->current_state, now);
1177
1178 timer_stop(pd->thread_timer, now);
1179 pd->thread_timer = (switch_to_kernel) ? &thread->system_timer : &thread->user_timer;
1180 timer_start(pd->thread_timer, now);
1181 }
1182
1183 void
timer_state_event_user_to_kernel(void)1184 timer_state_event_user_to_kernel(void)
1185 {
1186 timer_state_event(TRUE);
1187 }
1188
1189 void
timer_state_event_kernel_to_user(void)1190 timer_state_event_kernel_to_user(void)
1191 {
1192 timer_state_event(FALSE);
1193 }
1194 #endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */
1195
1196 uint32_t
get_arm_cpu_version(void)1197 get_arm_cpu_version(void)
1198 {
1199 uint32_t value = machine_read_midr();
1200
1201 /* Compose the register values into 8 bits; variant[7:4], revision[3:0]. */
1202 return ((value & MIDR_REV_MASK) >> MIDR_REV_SHIFT) | ((value & MIDR_VAR_MASK) >> (MIDR_VAR_SHIFT - 4));
1203 }
1204
1205 boolean_t
user_cont_hwclock_allowed(void)1206 user_cont_hwclock_allowed(void)
1207 {
1208 return FALSE;
1209 }
1210
1211 uint8_t
user_timebase_type(void)1212 user_timebase_type(void)
1213 {
1214 #if __ARM_TIME__
1215 return USER_TIMEBASE_SPEC;
1216 #else
1217 return USER_TIMEBASE_NONE;
1218 #endif
1219 }
1220
1221 thread_t
current_thread(void)1222 current_thread(void)
1223 {
1224 return current_thread_fast();
1225 }
1226
1227 #if __ARM_USER_PROTECT__
1228 uintptr_t
arm_user_protect_begin(thread_t thread)1229 arm_user_protect_begin(thread_t thread)
1230 {
1231 uintptr_t ttbr0, asid = 0; // kernel asid
1232
1233 ttbr0 = __builtin_arm_mrc(15, 0, 2, 0, 0); // Get TTBR0
1234 if (ttbr0 != thread->machine.kptw_ttb) {
1235 __builtin_arm_mcr(15, 0, thread->machine.kptw_ttb, 2, 0, 0); // Set TTBR0
1236 __builtin_arm_mcr(15, 0, asid, 13, 0, 1); // Set CONTEXTIDR
1237 __builtin_arm_isb(ISB_SY);
1238 }
1239 return ttbr0;
1240 }
1241
1242 void
arm_user_protect_end(thread_t thread,uintptr_t ttbr0,boolean_t disable_interrupts)1243 arm_user_protect_end(thread_t thread, uintptr_t ttbr0, boolean_t disable_interrupts)
1244 {
1245 if ((ttbr0 != thread->machine.kptw_ttb) && (thread->machine.uptw_ttb != thread->machine.kptw_ttb)) {
1246 if (disable_interrupts) {
1247 __asm__ volatile ("cpsid if" ::: "memory"); // Disable FIQ/IRQ
1248 }
1249 __builtin_arm_mcr(15, 0, thread->machine.uptw_ttb, 2, 0, 0); // Set TTBR0
1250 __builtin_arm_mcr(15, 0, thread->machine.asid, 13, 0, 1); // Set CONTEXTIDR with thread asid
1251 __builtin_arm_dsb(DSB_ISH);
1252 __builtin_arm_isb(ISB_SY);
1253 }
1254 }
1255 #endif // __ARM_USER_PROTECT__
1256
1257 void
machine_lockdown(void)1258 machine_lockdown(void)
1259 {
1260 arm_vm_prot_finalize(PE_state.bootArgs);
1261 lockdown_done = 1;
1262 }
1263
1264 void
ml_lockdown_init(void)1265 ml_lockdown_init(void)
1266 {
1267 }
1268
1269 void
ml_hibernate_active_pre(void)1270 ml_hibernate_active_pre(void)
1271 {
1272 }
1273
1274 void
ml_hibernate_active_post(void)1275 ml_hibernate_active_post(void)
1276 {
1277 }
1278
1279 size_t
ml_get_vm_reserved_regions(bool vm_is64bit,struct vm_reserved_region ** regions)1280 ml_get_vm_reserved_regions(bool vm_is64bit, struct vm_reserved_region **regions)
1281 {
1282 #pragma unused(vm_is64bit)
1283 assert(regions != NULL);
1284
1285 *regions = NULL;
1286 return 0;
1287 }
1288