1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/machine.c
60 * Author: Avadis Tevanian, Jr.
61 * Date: 1987
62 *
63 * Support for machine independent machine abstraction.
64 */
65
66 #include <string.h>
67
68 #include <mach/mach_types.h>
69 #include <mach/boolean.h>
70 #include <mach/kern_return.h>
71 #include <mach/machine.h>
72 #include <mach/host_info.h>
73 #include <mach/host_reboot.h>
74 #include <mach/host_priv_server.h>
75 #include <mach/processor_server.h>
76
77 #include <kern/kern_types.h>
78 #include <kern/cpu_data.h>
79 #include <kern/cpu_quiesce.h>
80 #include <kern/ipc_host.h>
81 #include <kern/host.h>
82 #include <kern/machine.h>
83 #include <kern/misc_protos.h>
84 #include <kern/processor.h>
85 #include <kern/queue.h>
86 #include <kern/sched.h>
87 #include <kern/startup.h>
88 #include <kern/task.h>
89 #include <kern/thread.h>
90
91 #include <libkern/OSDebug.h>
92
93 #include <pexpert/device_tree.h>
94
95 #include <machine/commpage.h>
96 #include <machine/machine_routines.h>
97
98 #if HIBERNATION
99 #include <IOKit/IOHibernatePrivate.h>
100 #endif
101 #include <IOKit/IOPlatformExpert.h>
102
103 #if CONFIG_DTRACE
104 extern void (*dtrace_cpu_state_changed_hook)(int, boolean_t);
105 #endif
106
107 #if defined(__x86_64__)
108 #include <i386/panic_notify.h>
109 #endif
110
111 /*
112 * Exported variables:
113 */
114
115 struct machine_info machine_info;
116
117 /* Forwards */
118 static void
119 processor_doshutdown(processor_t processor);
120
121 static void
122 processor_offline(void * parameter, __unused wait_result_t result);
123
124 static void
125 processor_offline_intstack(processor_t processor) __dead2;
126
127 /*
128 * processor_up:
129 *
130 * Flag processor as up and running, and available
131 * for scheduling.
132 */
133 void
processor_up(processor_t processor)134 processor_up(
135 processor_t processor)
136 {
137 processor_set_t pset;
138 spl_t s;
139
140 s = splsched();
141 init_ast_check(processor);
142 pset = processor->processor_set;
143 pset_lock(pset);
144
145 ++pset->online_processor_count;
146 pset_update_processor_state(pset, processor, PROCESSOR_RUNNING);
147 os_atomic_inc(&processor_avail_count, relaxed);
148 if (processor->is_recommended) {
149 os_atomic_inc(&processor_avail_count_user, relaxed);
150 SCHED(pset_made_schedulable)(processor, pset, false);
151 }
152 if (processor->processor_primary == processor) {
153 os_atomic_inc(&primary_processor_avail_count, relaxed);
154 if (processor->is_recommended) {
155 os_atomic_inc(&primary_processor_avail_count_user, relaxed);
156 }
157 }
158 commpage_update_active_cpus();
159 pset_unlock(pset);
160 ml_cpu_up();
161 splx(s);
162
163 #if CONFIG_DTRACE
164 if (dtrace_cpu_state_changed_hook) {
165 (*dtrace_cpu_state_changed_hook)(processor->cpu_id, TRUE);
166 }
167 #endif
168 }
169 #include <atm/atm_internal.h>
170
171 kern_return_t
host_reboot(host_priv_t host_priv,int options)172 host_reboot(
173 host_priv_t host_priv,
174 int options)
175 {
176 if (host_priv == HOST_PRIV_NULL) {
177 return KERN_INVALID_HOST;
178 }
179
180 #if DEVELOPMENT || DEBUG
181 if (options & HOST_REBOOT_DEBUGGER) {
182 Debugger("Debugger");
183 return KERN_SUCCESS;
184 }
185 #endif
186
187 if (options & HOST_REBOOT_UPSDELAY) {
188 // UPS power cutoff path
189 PEHaltRestart( kPEUPSDelayHaltCPU );
190 } else {
191 halt_all_cpus(!(options & HOST_REBOOT_HALT));
192 }
193
194 return KERN_SUCCESS;
195 }
196
197 kern_return_t
processor_assign(__unused processor_t processor,__unused processor_set_t new_pset,__unused boolean_t wait)198 processor_assign(
199 __unused processor_t processor,
200 __unused processor_set_t new_pset,
201 __unused boolean_t wait)
202 {
203 return KERN_FAILURE;
204 }
205
206 kern_return_t
processor_shutdown(processor_t processor)207 processor_shutdown(
208 processor_t processor)
209 {
210 processor_set_t pset;
211 spl_t s;
212
213 ml_cpu_begin_state_transition(processor->cpu_id);
214 s = splsched();
215 pset = processor->processor_set;
216 pset_lock(pset);
217 if (processor->state == PROCESSOR_OFF_LINE) {
218 /*
219 * Success if already shutdown.
220 */
221 pset_unlock(pset);
222 splx(s);
223 ml_cpu_end_state_transition(processor->cpu_id);
224
225 return KERN_SUCCESS;
226 }
227
228 if (!ml_cpu_can_exit(processor->cpu_id)) {
229 /*
230 * Failure if disallowed by arch code.
231 */
232 pset_unlock(pset);
233 splx(s);
234 ml_cpu_end_state_transition(processor->cpu_id);
235
236 return KERN_NOT_SUPPORTED;
237 }
238
239 if (processor->state == PROCESSOR_START) {
240 /*
241 * Failure if currently being started.
242 */
243 pset_unlock(pset);
244 splx(s);
245
246 return KERN_FAILURE;
247 }
248
249 /*
250 * If the processor is dispatching, let it finish.
251 */
252 while (processor->state == PROCESSOR_DISPATCHING) {
253 pset_unlock(pset);
254 splx(s);
255 delay(1);
256 s = splsched();
257 pset_lock(pset);
258 }
259
260 /*
261 * Success if already being shutdown.
262 */
263 if (processor->state == PROCESSOR_SHUTDOWN) {
264 pset_unlock(pset);
265 splx(s);
266 ml_cpu_end_state_transition(processor->cpu_id);
267
268 return KERN_SUCCESS;
269 }
270
271 ml_broadcast_cpu_event(CPU_EXIT_REQUESTED, processor->cpu_id);
272 pset_update_processor_state(pset, processor, PROCESSOR_SHUTDOWN);
273 pset_unlock(pset);
274
275 processor_doshutdown(processor);
276 splx(s);
277
278 cpu_exit_wait(processor->cpu_id);
279 ml_cpu_end_state_transition(processor->cpu_id);
280 ml_broadcast_cpu_event(CPU_EXITED, processor->cpu_id);
281
282 return KERN_SUCCESS;
283 }
284
285 /*
286 * Called with interrupts disabled.
287 */
288 static void
processor_doshutdown(processor_t processor)289 processor_doshutdown(
290 processor_t processor)
291 {
292 thread_t self = current_thread();
293
294 /*
295 * Get onto the processor to shutdown
296 */
297 processor_t prev = thread_bind(processor);
298 thread_block(THREAD_CONTINUE_NULL);
299
300 /* interrupts still disabled */
301 assert(ml_get_interrupts_enabled() == FALSE);
302
303 assert(processor == current_processor());
304 assert(processor->state == PROCESSOR_SHUTDOWN);
305
306 #if CONFIG_DTRACE
307 if (dtrace_cpu_state_changed_hook) {
308 (*dtrace_cpu_state_changed_hook)(processor->cpu_id, FALSE);
309 }
310 #endif
311
312 ml_cpu_down();
313
314 #if HIBERNATION
315 if (processor_avail_count < 2) {
316 hibernate_vm_lock();
317 hibernate_vm_unlock();
318 }
319 #endif
320
321 processor_set_t pset = processor->processor_set;
322
323 pset_lock(pset);
324 pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE);
325 --pset->online_processor_count;
326 os_atomic_dec(&processor_avail_count, relaxed);
327 if (processor->is_recommended) {
328 os_atomic_dec(&processor_avail_count_user, relaxed);
329 }
330 if (processor->processor_primary == processor) {
331 os_atomic_dec(&primary_processor_avail_count, relaxed);
332 if (processor->is_recommended) {
333 os_atomic_dec(&primary_processor_avail_count_user, relaxed);
334 }
335 }
336 commpage_update_active_cpus();
337 SCHED(processor_queue_shutdown)(processor);
338 /* pset lock dropped */
339 SCHED(rt_queue_shutdown)(processor);
340
341 thread_bind(prev);
342
343 /* interrupts still disabled */
344
345 /*
346 * Continue processor shutdown on the processor's idle thread.
347 * The handoff won't fail because the idle thread has a reserved stack.
348 * Switching to the idle thread leaves interrupts disabled,
349 * so we can't accidentally take an interrupt after the context switch.
350 */
351 thread_t shutdown_thread = processor->idle_thread;
352 shutdown_thread->continuation = processor_offline;
353 shutdown_thread->parameter = processor;
354
355 thread_run(self, NULL, NULL, shutdown_thread);
356 }
357
358 /*
359 * Called in the context of the idle thread to shut down the processor
360 *
361 * A shut-down processor looks like it's 'running' the idle thread parked
362 * in this routine, but it's actually been powered off and has no hardware state.
363 */
364 static void
processor_offline(void * parameter,__unused wait_result_t result)365 processor_offline(
366 void * parameter,
367 __unused wait_result_t result)
368 {
369 processor_t processor = (processor_t) parameter;
370 thread_t self = current_thread();
371 __assert_only thread_t old_thread = THREAD_NULL;
372
373 assert(processor == current_processor());
374 assert(self->state & TH_IDLE);
375 assert(processor->idle_thread == self);
376 assert(ml_get_interrupts_enabled() == FALSE);
377 assert(self->continuation == NULL);
378 assert(processor->processor_offlined == false);
379 assert(processor->running_timers_active == false);
380
381 bool enforce_quiesce_safety = gEnforceQuiesceSafety;
382
383 /*
384 * Scheduling is now disabled for this processor.
385 * Ensure that primitives that need scheduling (like mutexes) know this.
386 */
387 if (enforce_quiesce_safety) {
388 disable_preemption_without_measurements();
389 }
390
391 /* convince slave_main to come back here */
392 processor->processor_offlined = true;
393
394 /*
395 * Switch to the interrupt stack and shut down the processor.
396 *
397 * When the processor comes back, it will eventually call load_context which
398 * restores the context saved by machine_processor_shutdown, returning here.
399 */
400 old_thread = machine_processor_shutdown(self, processor_offline_intstack, processor);
401
402 /* old_thread should be NULL because we got here through Load_context */
403 assert(old_thread == THREAD_NULL);
404
405 assert(processor == current_processor());
406 assert(processor->idle_thread == current_thread());
407
408 assert(ml_get_interrupts_enabled() == FALSE);
409 assert(self->continuation == NULL);
410
411 /* Extract the machine_param value stashed by slave_main */
412 void * machine_param = self->parameter;
413 self->parameter = NULL;
414
415 /* Re-initialize the processor */
416 slave_machine_init(machine_param);
417
418 assert(processor->processor_offlined == true);
419 processor->processor_offlined = false;
420
421 if (enforce_quiesce_safety) {
422 enable_preemption();
423 }
424
425 /*
426 * Now that the processor is back, invoke the idle thread to find out what to do next.
427 * idle_thread will enable interrupts.
428 */
429 thread_block(idle_thread);
430 /*NOTREACHED*/
431 }
432
433 /*
434 * Complete the shutdown and place the processor offline.
435 *
436 * Called at splsched in the shutdown context
437 * (i.e. on the idle thread, on the interrupt stack)
438 *
439 * The onlining half of this is done in load_context().
440 */
441 static void
processor_offline_intstack(processor_t processor)442 processor_offline_intstack(
443 processor_t processor)
444 {
445 assert(processor == current_processor());
446 assert(processor->active_thread == current_thread());
447
448 timer_stop(processor->current_state, processor->last_dispatch);
449
450 cpu_quiescent_counter_leave(processor->last_dispatch);
451
452 PMAP_DEACTIVATE_KERNEL(processor->cpu_id);
453
454 cpu_sleep();
455 panic("zombie processor");
456 /*NOTREACHED*/
457 }
458
459 kern_return_t
host_get_boot_info(host_priv_t host_priv,kernel_boot_info_t boot_info)460 host_get_boot_info(
461 host_priv_t host_priv,
462 kernel_boot_info_t boot_info)
463 {
464 const char *src = "";
465 if (host_priv == HOST_PRIV_NULL) {
466 return KERN_INVALID_HOST;
467 }
468
469 /*
470 * Copy first operator string terminated by '\0' followed by
471 * standardized strings generated from boot string.
472 */
473 src = machine_boot_info(boot_info, KERNEL_BOOT_INFO_MAX);
474 if (src != boot_info) {
475 (void) strncpy(boot_info, src, KERNEL_BOOT_INFO_MAX);
476 }
477
478 return KERN_SUCCESS;
479 }
480
481 #if CONFIG_DTRACE
482 #include <mach/sdt.h>
483 #endif
484
485
486 // These are configured through sysctls.
487 #if DEVELOPMENT || DEBUG
488 uint32_t phy_read_panic = 1;
489 uint32_t phy_write_panic = 1;
490 uint64_t simulate_stretched_io = 0;
491 #else
492 uint32_t phy_read_panic = 0;
493 uint32_t phy_write_panic = 0;
494 #endif
495
496 #if !defined(__x86_64__)
497 // The MACHINE_TIMEOUT facility only exists on ARM.
498 MACHINE_TIMEOUT32_WRITEABLE(report_phy_read_delay_to, "report-phy-read-delay", 0, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
499 MACHINE_TIMEOUT32_WRITEABLE(report_phy_write_delay_to, "report-phy-write-delay", 0, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
500 MACHINE_TIMEOUT32_WRITEABLE(trace_phy_read_delay_to, "trace-phy-read-delay", 0, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
501 MACHINE_TIMEOUT32_WRITEABLE(trace_phy_write_delay_to, "trace-phy-write-delay", 0, MACHINE_TIMEOUT_UNIT_TIMEBASE, NULL);
502
503 unsigned int report_phy_read_osbt;
504 unsigned int report_phy_write_osbt;
505
506 extern pmap_paddr_t kvtophys(vm_offset_t va);
507 #endif
508
509 unsigned long long
ml_io_read(uintptr_t vaddr,int size)510 ml_io_read(uintptr_t vaddr, int size)
511 {
512 unsigned long long result = 0;
513 unsigned char s1;
514 unsigned short s2;
515
516 #ifdef ML_IO_VERIFY_UNCACHEABLE
517 uintptr_t const paddr = pmap_verify_noncacheable(vaddr);
518 #endif /* ML_IO_VERIFY_UNCACHEABLE */
519
520 #ifdef ML_IO_TIMEOUTS_ENABLED
521 uint64_t sabs, eabs;
522 boolean_t istate, timeread = FALSE;
523
524 #if !defined(__x86_64__)
525 uint32_t const report_phy_read_delay = os_atomic_load(&report_phy_read_delay_to, relaxed);
526 uint32_t const trace_phy_read_delay = os_atomic_load(&trace_phy_read_delay_to, relaxed);
527 #endif /* !defined(__x86_64__) */
528
529 if (__improbable(report_phy_read_delay != 0)) {
530 istate = ml_set_interrupts_enabled(FALSE);
531 sabs = mach_absolute_time();
532 timeread = TRUE;
533 }
534
535 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
536 if (__improbable(timeread && simulate_stretched_io)) {
537 sabs -= simulate_stretched_io;
538 }
539 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
540 #endif /* ML_IO_TIMEOUTS_ENABLED */
541
542 switch (size) {
543 case 1:
544 s1 = *(volatile unsigned char *)vaddr;
545 result = s1;
546 break;
547 case 2:
548 s2 = *(volatile unsigned short *)vaddr;
549 result = s2;
550 break;
551 case 4:
552 result = *(volatile unsigned int *)vaddr;
553 break;
554 case 8:
555 result = *(volatile unsigned long long *)vaddr;
556 break;
557 default:
558 panic("Invalid size %d for ml_io_read(%p)", size, (void *)vaddr);
559 break;
560 }
561
562 #ifdef ML_IO_TIMEOUTS_ENABLED
563 if (__improbable(timeread == TRUE)) {
564 eabs = mach_absolute_time();
565
566 #ifdef ML_IO_IOTRACE_ENABLED
567 iotrace(IOTRACE_IO_READ, vaddr, paddr, size, result, sabs, eabs - sabs);
568 #endif /* ML_IO_IOTRACE_ENABLED */
569
570 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
571 #ifndef ML_IO_VERIFY_UNCACHEABLE
572 uintptr_t const paddr = kvtophys(vaddr);
573 #endif /* ML_IO_VERIFY_UNCACHEABLE */
574
575 if (phy_read_panic && (machine_timeout_suspended() == FALSE)) {
576 #if defined(__x86_64__)
577 panic_notify();
578 #endif /* defined(__x86_64__) */
579 panic("Read from IO vaddr 0x%lx paddr 0x%lx took %llu ns, "
580 "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
581 vaddr, paddr, (eabs - sabs), result, sabs, eabs,
582 (uint64_t)report_phy_read_delay);
583 }
584
585 (void)ml_set_interrupts_enabled(istate);
586
587 if (report_phy_read_osbt) {
588 OSReportWithBacktrace("ml_io_read(v=%p, p=%p) size %d result 0x%llx "
589 "took %lluus",
590 (void *)vaddr, (void *)paddr, size, result,
591 (eabs - sabs) / NSEC_PER_USEC);
592 }
593 #if CONFIG_DTRACE
594 DTRACE_PHYSLAT5(physioread, uint64_t, (eabs - sabs),
595 uint64_t, vaddr, uint32_t, size, uint64_t, paddr, uint64_t, result);
596 #endif /* CONFIG_DTRACE */
597 } else if (__improbable(trace_phy_read_delay > 0 && (eabs - sabs) > trace_phy_read_delay)) {
598 #ifndef ML_IO_VERIFY_UNCACHEABLE
599 uintptr_t const __unused paddr = kvtophys(vaddr);
600 #endif /* ML_IO_VERIFY_UNCACHEABLE */
601
602 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_MMIO_READ),
603 (eabs - sabs), VM_KERNEL_UNSLIDE_OR_PERM(vaddr), paddr, result);
604
605 (void)ml_set_interrupts_enabled(istate);
606 } else {
607 (void)ml_set_interrupts_enabled(istate);
608 }
609 }
610 #endif /* ML_IO_TIMEOUTS_ENABLED */
611 return result;
612 }
613
614 unsigned int
ml_io_read8(uintptr_t vaddr)615 ml_io_read8(uintptr_t vaddr)
616 {
617 return (unsigned) ml_io_read(vaddr, 1);
618 }
619
620 unsigned int
ml_io_read16(uintptr_t vaddr)621 ml_io_read16(uintptr_t vaddr)
622 {
623 return (unsigned) ml_io_read(vaddr, 2);
624 }
625
626 unsigned int
ml_io_read32(uintptr_t vaddr)627 ml_io_read32(uintptr_t vaddr)
628 {
629 return (unsigned) ml_io_read(vaddr, 4);
630 }
631
632 unsigned long long
ml_io_read64(uintptr_t vaddr)633 ml_io_read64(uintptr_t vaddr)
634 {
635 return ml_io_read(vaddr, 8);
636 }
637
638 /* ml_io_write* */
639
640 void
ml_io_write(uintptr_t vaddr,uint64_t val,int size)641 ml_io_write(uintptr_t vaddr, uint64_t val, int size)
642 {
643 #ifdef ML_IO_VERIFY_UNCACHEABLE
644 uintptr_t const paddr = pmap_verify_noncacheable(vaddr);
645 #endif
646 #ifdef ML_IO_TIMEOUTS_ENABLED
647 uint64_t sabs, eabs;
648 boolean_t istate, timewrite = FALSE;
649 #if !defined(__x86_64__)
650 uint32_t report_phy_write_delay = os_atomic_load(&report_phy_write_delay_to, relaxed);
651 uint32_t trace_phy_write_delay = os_atomic_load(&trace_phy_write_delay_to, relaxed);
652 #endif /* !defined(__x86_64__) */
653 if (__improbable(report_phy_write_delay != 0)) {
654 istate = ml_set_interrupts_enabled(FALSE);
655 sabs = mach_absolute_time();
656 timewrite = TRUE;
657 }
658
659 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
660 if (__improbable(timewrite && simulate_stretched_io)) {
661 sabs -= simulate_stretched_io;
662 }
663 #endif /* DEVELOPMENT || DEBUG */
664 #endif /* ML_IO_TIMEOUTS_ENABLED */
665
666 switch (size) {
667 case 1:
668 *(volatile uint8_t *)vaddr = (uint8_t)val;
669 break;
670 case 2:
671 *(volatile uint16_t *)vaddr = (uint16_t)val;
672 break;
673 case 4:
674 *(volatile uint32_t *)vaddr = (uint32_t)val;
675 break;
676 case 8:
677 *(volatile uint64_t *)vaddr = (uint64_t)val;
678 break;
679 default:
680 panic("Invalid size %d for ml_io_write(%p, 0x%llx)", size, (void *)vaddr, val);
681 break;
682 }
683
684 #ifdef ML_IO_TIMEOUTS_ENABLED
685 if (__improbable(timewrite == TRUE)) {
686 eabs = mach_absolute_time();
687
688 #ifdef ML_IO_IOTRACE_ENABLED
689 iotrace(IOTRACE_IO_WRITE, vaddr, paddr, size, val, sabs, eabs - sabs);
690 #endif /* ML_IO_IOTRACE_ENABLED */
691
692 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
693 #ifndef ML_IO_VERIFY_UNCACHEABLE
694 uintptr_t const paddr = kvtophys(vaddr);
695 #endif /* ML_IO_VERIFY_UNCACHEABLE */
696
697 if (phy_write_panic && (machine_timeout_suspended() == FALSE)) {
698 #if defined(__x86_64__)
699 panic_notify();
700 #endif /* defined(__x86_64__) */
701 panic("Write to IO vaddr %p paddr %p val 0x%llx took %llu ns,"
702 " (start: %llu, end: %llu), ceiling: %llu",
703 (void *)vaddr, (void *)paddr, val, (eabs - sabs), sabs, eabs,
704 (uint64_t)report_phy_write_delay);
705 }
706
707 (void)ml_set_interrupts_enabled(istate);
708
709 if (report_phy_write_osbt) {
710 OSReportWithBacktrace("ml_io_write size %d (v=%p, p=%p, 0x%llx) "
711 "took %lluus",
712 size, (void *)vaddr, (void *)paddr, val, (eabs - sabs) / NSEC_PER_USEC);
713 }
714 #if CONFIG_DTRACE
715 DTRACE_PHYSLAT5(physiowrite, uint64_t, (eabs - sabs),
716 uint64_t, vaddr, uint32_t, size, uint64_t, paddr, uint64_t, val);
717 #endif /* CONFIG_DTRACE */
718 } else if (__improbable(trace_phy_write_delay > 0 && (eabs - sabs) > trace_phy_write_delay)) {
719 #ifndef ML_IO_VERIFY_UNCACHEABLE
720 uintptr_t const __unused paddr = kvtophys(vaddr);
721 #endif /* ML_IO_VERIFY_UNCACHEABLE */
722
723 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_MMIO_WRITE),
724 (eabs - sabs), VM_KERNEL_UNSLIDE_OR_PERM(vaddr), paddr, val);
725
726 (void)ml_set_interrupts_enabled(istate);
727 } else {
728 (void)ml_set_interrupts_enabled(istate);
729 }
730 }
731 #endif /* ML_IO_TIMEOUTS_ENABLED */
732 }
733
734 void
ml_io_write8(uintptr_t vaddr,uint8_t val)735 ml_io_write8(uintptr_t vaddr, uint8_t val)
736 {
737 ml_io_write(vaddr, val, 1);
738 }
739
740 void
ml_io_write16(uintptr_t vaddr,uint16_t val)741 ml_io_write16(uintptr_t vaddr, uint16_t val)
742 {
743 ml_io_write(vaddr, val, 2);
744 }
745
746 void
ml_io_write32(uintptr_t vaddr,uint32_t val)747 ml_io_write32(uintptr_t vaddr, uint32_t val)
748 {
749 ml_io_write(vaddr, val, 4);
750 }
751
752 void
ml_io_write64(uintptr_t vaddr,uint64_t val)753 ml_io_write64(uintptr_t vaddr, uint64_t val)
754 {
755 ml_io_write(vaddr, val, 8);
756 }
757
758 struct cpu_callback_chain_elem {
759 cpu_callback_t fn;
760 void *param;
761 struct cpu_callback_chain_elem *next;
762 };
763
764 static struct cpu_callback_chain_elem *cpu_callback_chain;
765 static LCK_GRP_DECLARE(cpu_callback_chain_lock_grp, "cpu_callback_chain");
766 static LCK_SPIN_DECLARE(cpu_callback_chain_lock, &cpu_callback_chain_lock_grp);
767
768 void
cpu_event_register_callback(cpu_callback_t fn,void * param)769 cpu_event_register_callback(cpu_callback_t fn, void *param)
770 {
771 struct cpu_callback_chain_elem *new_elem;
772
773 new_elem = zalloc_permanent_type(struct cpu_callback_chain_elem);
774 if (!new_elem) {
775 panic("can't allocate cpu_callback_chain_elem");
776 }
777
778 lck_spin_lock(&cpu_callback_chain_lock);
779 new_elem->next = cpu_callback_chain;
780 new_elem->fn = fn;
781 new_elem->param = param;
782 os_atomic_store(&cpu_callback_chain, new_elem, release);
783 lck_spin_unlock(&cpu_callback_chain_lock);
784 }
785
786 __attribute__((noreturn))
787 void
cpu_event_unregister_callback(__unused cpu_callback_t fn)788 cpu_event_unregister_callback(__unused cpu_callback_t fn)
789 {
790 panic("Unfortunately, cpu_event_unregister_callback is unimplemented.");
791 }
792
793 void
ml_broadcast_cpu_event(enum cpu_event event,unsigned int cpu_or_cluster)794 ml_broadcast_cpu_event(enum cpu_event event, unsigned int cpu_or_cluster)
795 {
796 struct cpu_callback_chain_elem *cursor;
797
798 cursor = os_atomic_load(&cpu_callback_chain, dependency);
799 for (; cursor != NULL; cursor = cursor->next) {
800 cursor->fn(cursor->param, event, cpu_or_cluster);
801 }
802 }
803
804 // Initialize Machine Timeouts (see the MACHINE_TIMEOUT macro
805 // definition)
806
807 void
machine_timeout_init_with_suffix(const struct machine_timeout_spec * spec,char const * suffix)808 machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *suffix)
809 {
810 if (spec->skip_predicate != NULL && spec->skip_predicate(spec)) {
811 // This timeout should be disabled.
812 if (spec->is32) {
813 os_atomic_store((uint32_t*)spec->ptr, 0, relaxed);
814 } else {
815 os_atomic_store_wide((uint64_t*)spec->ptr, 0, relaxed);
816 }
817 return;
818 }
819
820 assert(suffix != NULL);
821 assert(strlen(spec->name) <= MACHINE_TIMEOUT_MAX_NAME_LEN);
822
823 size_t const suffix_len = strlen(suffix);
824
825 size_t const dt_name_size = MACHINE_TIMEOUT_MAX_NAME_LEN + suffix_len + 1;
826 char dt_name[dt_name_size];
827
828 strlcpy(dt_name, spec->name, dt_name_size);
829 strlcat(dt_name, suffix, dt_name_size);
830
831 size_t const scale_name_size = MACHINE_TIMEOUT_MAX_NAME_LEN + suffix_len + strlen("-scale") + 1;
832 char scale_name[scale_name_size];
833
834 strlcpy(scale_name, spec->name, scale_name_size);
835 strlcat(scale_name, suffix, scale_name_size);
836 strlcat(scale_name, "-scale", scale_name_size);
837
838 size_t const boot_arg_name_size = MACHINE_TIMEOUT_MAX_NAME_LEN + strlen("ml-timeout-") + suffix_len + 1;
839 char boot_arg_name[boot_arg_name_size];
840
841 strlcpy(boot_arg_name, "ml-timeout-", boot_arg_name_size);
842 strlcat(boot_arg_name, spec->name, boot_arg_name_size);
843 strlcat(boot_arg_name, suffix, boot_arg_name_size);
844
845 size_t const boot_arg_scale_name_size = MACHINE_TIMEOUT_MAX_NAME_LEN +
846 strlen("ml-timeout-") + strlen("-scale") + suffix_len + 1;
847 char boot_arg_scale_name[boot_arg_scale_name_size];
848
849 strlcpy(boot_arg_scale_name, "ml-timeout-", boot_arg_scale_name_size);
850 strlcat(boot_arg_scale_name, spec->name, boot_arg_scale_name_size);
851 strlcat(boot_arg_scale_name, suffix, boot_arg_name_size);
852 strlcat(boot_arg_scale_name, "-scale", boot_arg_scale_name_size);
853
854
855 /*
856 * Determine base value from DT and boot-args.
857 */
858
859 DTEntry base, chosen;
860
861 if (SecureDTLookupEntry(NULL, "/machine-timeouts", &base) != kSuccess) {
862 base = NULL;
863 }
864
865 if (SecureDTLookupEntry(NULL, "/chosen/machine-timeouts", &chosen) != kSuccess) {
866 chosen = NULL;
867 }
868
869 uint64_t timeout = spec->default_value;
870 bool found = false;
871
872 uint64_t const *data = NULL;
873 unsigned int data_size = sizeof(*data);
874
875 /* First look in /machine-timeouts/<name> */
876 if (base != NULL && SecureDTGetProperty(base, dt_name, (const void **)&data, &data_size) == kSuccess) {
877 if (data_size != sizeof(*data)) {
878 panic("%s: unexpected machine timeout data_size %u for /machine-timeouts/%s", __func__, data_size, dt_name);
879 }
880
881 timeout = *data;
882 found = true;
883 }
884
885 /* A value in /chosen/machine-timeouts/<name> overrides */
886 if (chosen != NULL && SecureDTGetProperty(chosen, dt_name, (const void **)&data, &data_size) == kSuccess) {
887 if (data_size != sizeof(*data)) {
888 panic("%s: unexpected machine timeout data_size %u for /chosen/machine-timeouts/%s", __func__, data_size, dt_name);
889 }
890
891 timeout = *data;
892 found = true;
893 }
894
895 /* A boot-arg ml-timeout-<name> overrides */
896 uint64_t boot_arg = 0;
897
898 if (PE_parse_boot_argn(boot_arg_name, &boot_arg, sizeof(boot_arg))) {
899 timeout = boot_arg;
900 found = true;
901 }
902
903
904 /*
905 * Determine scale value from DT and boot-args.
906 */
907
908 uint32_t scale = 1;
909 uint32_t const *scale_data;
910 unsigned int scale_size = sizeof(scale_data);
911
912 /* If there is a scale factor /machine-timeouts/<name>-scale,
913 * apply it. */
914 if (base != NULL && SecureDTGetProperty(base, scale_name, (const void **)&scale_data, &scale_size) == kSuccess) {
915 if (scale_size != sizeof(*scale_data)) {
916 panic("%s: unexpected machine timeout data_size %u for /machine-timeouts/%s-scale", __func__, scale_size, dt_name);
917 }
918
919 scale *= *scale_data;
920 }
921
922 /* If there is a scale factor /chosen/machine-timeouts/<name>-scale,
923 * apply it as well. */
924 if (chosen != NULL && SecureDTGetProperty(chosen, scale_name, (const void **)&scale_data, &scale_size) == kSuccess) {
925 if (scale_size != sizeof(*scale_data)) {
926 panic("%s: unexpected machine timeout data_size %u for /chosen/machine-timeouts/%s-scale", __func__,
927 scale_size, dt_name);
928 }
929
930 scale *= *scale_data;
931 }
932
933 /* Finally, a boot-arg ml-timeout-<name>-scale applies as well. */
934 if (PE_parse_boot_argn(boot_arg_scale_name, &boot_arg, sizeof(boot_arg))) {
935 scale *= boot_arg;
936 }
937
938 static bool global_scale_set;
939 static uint32_t global_scale;
940
941 if (!global_scale_set) {
942 /* Apply /machine-timeouts/global-scale if present */
943 if (SecureDTGetProperty(base, "global-scale", (const void **)&scale_data, &scale_size) == kSuccess) {
944 if (scale_size != sizeof(*scale_data)) {
945 panic("%s: unexpected machine timeout data_size %u for /machine-timeouts/global-scale", __func__,
946 scale_size);
947 }
948
949 global_scale *= *scale_data;
950 global_scale_set = true;
951 }
952
953 /* Apply /chosen/machine-timeouts/global-scale if present */
954 if (SecureDTGetProperty(chosen, "global-scale", (const void **)&scale_data, &scale_size) == kSuccess) {
955 if (scale_size != sizeof(*scale_data)) {
956 panic("%s: unexpected machine timeout data_size %u for /chosen/machine-timeouts/global-scale", __func__,
957 scale_size);
958 }
959
960 global_scale *= *scale_data;
961 global_scale_set = true;
962 }
963
964 /* Finally, the boot-arg ml-timeout-global-scale applies */
965 if (PE_parse_boot_argn("ml-timeout-global-scale", &boot_arg, sizeof(boot_arg))) {
966 global_scale *= boot_arg;
967 global_scale_set = true;
968 }
969 }
970
971 if (global_scale_set) {
972 scale *= global_scale;
973 }
974
975 /* Compute the final timeout, and done. */
976 if (found && timeout > 0) {
977 /* Only apply inherent unit scale if the value came in
978 * externally. */
979
980 if (spec->unit_scale == MACHINE_TIMEOUT_UNIT_TIMEBASE) {
981 uint64_t nanoseconds = timeout / 1000;
982 nanoseconds_to_absolutetime(nanoseconds, &timeout);
983 } else {
984 timeout /= spec->unit_scale;
985 }
986
987 if (timeout == 0) {
988 /* Ensure unit scaling did not disable the timeout. */
989 timeout = 1;
990 }
991 }
992
993 if (os_mul_overflow(timeout, scale, &timeout)) {
994 timeout = UINT64_MAX; // clamp
995 }
996
997 if (spec->is32) {
998 os_atomic_store((uint32_t*)spec->ptr, timeout > UINT32_MAX ? UINT32_MAX : (uint32_t)timeout, relaxed);
999 } else {
1000 os_atomic_store_wide((uint64_t*)spec->ptr, timeout, relaxed);
1001 }
1002 }
1003
1004 void
machine_timeout_init(const struct machine_timeout_spec * spec)1005 machine_timeout_init(const struct machine_timeout_spec *spec)
1006 {
1007 machine_timeout_init_with_suffix(spec, "");
1008 }
1009
1010 /*
1011 * Late timeout (re-)initialization, at the end of bsd_init()
1012 */
1013 void
machine_timeout_bsd_init(void)1014 machine_timeout_bsd_init(void)
1015 {
1016 char const * const __unused mt_suffix = "-b";
1017 #if INTERRUPT_MASKED_DEBUG
1018 machine_timeout_init_with_suffix(MACHINE_TIMEOUT_SPEC_REF(interrupt_masked_timeout), mt_suffix);
1019 #endif
1020 #if SCHED_PREEMPTION_DISABLE_DEBUG
1021 machine_timeout_init_with_suffix(MACHINE_TIMEOUT_SPEC_REF(sched_preemption_disable_threshold_mt), mt_suffix);
1022 #endif
1023 }
1024