1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <kern/clock.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <mach/processor_info.h>
50 #include <machine/atomic.h>
51 #include <machine/config.h>
52 #include <vm/vm_kern_xnu.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58 #include <arm64/proc_reg.h>
59 #include <libkern/OSAtomic.h>
60
61 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
62 vm_address_t percpu_base_cur;
63 cpu_data_t PERCPU_DATA(cpu_data);
64 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
65
66 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
67 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
68 static LCK_MTX_DECLARE(cpu_xcall_mtx, &cpu_lck_grp);
69
70 unsigned int real_ncpus = 1;
71 boolean_t idle_enable = FALSE;
72 uint64_t wake_abstime = 0x0ULL;
73
74 extern uint64_t xcall_ack_timeout_abstime;
75
76 #if defined(HAS_IPI)
77 extern unsigned int gFastIPI;
78 #endif /* defined(HAS_IPI) */
79
80 cpu_data_t *
cpu_datap(int cpu)81 cpu_datap(int cpu)
82 {
83 assert(cpu <= ml_get_max_cpu_number());
84 return CpuDataEntries[cpu].cpu_data_vaddr;
85 }
86
87 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)88 cpu_control(int slot_num,
89 processor_info_t info,
90 unsigned int count)
91 {
92 printf("cpu_control(%d,%p,%d) not implemented\n",
93 slot_num, info, count);
94 return KERN_FAILURE;
95 }
96
97 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)98 cpu_info_count(processor_flavor_t flavor,
99 unsigned int *count)
100 {
101 switch (flavor) {
102 case PROCESSOR_CPU_STAT:
103 *count = PROCESSOR_CPU_STAT_COUNT;
104 return KERN_SUCCESS;
105
106 case PROCESSOR_CPU_STAT64:
107 *count = PROCESSOR_CPU_STAT64_COUNT;
108 return KERN_SUCCESS;
109
110 default:
111 *count = 0;
112 return KERN_FAILURE;
113 }
114 }
115
116 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)117 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
118 unsigned int *count)
119 {
120 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
121
122 switch (flavor) {
123 case PROCESSOR_CPU_STAT:
124 {
125 if (*count < PROCESSOR_CPU_STAT_COUNT) {
126 return KERN_FAILURE;
127 }
128
129 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
130 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
131 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
132 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
133 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
134 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
135 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
136 cpu_stat->vfp_shortv_cnt = 0;
137 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
138 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
139
140 *count = PROCESSOR_CPU_STAT_COUNT;
141
142 return KERN_SUCCESS;
143 }
144
145 case PROCESSOR_CPU_STAT64:
146 {
147 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
148 return KERN_FAILURE;
149 }
150
151 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
152 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
153 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
154 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
155 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
156 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
157 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
158 cpu_stat->vfp_shortv_cnt = 0;
159 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
160 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
161 #if CONFIG_CPU_COUNTERS
162 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
163 #endif /* CONFIG_CPU_COUNTERS */
164
165 *count = PROCESSOR_CPU_STAT64_COUNT;
166
167 return KERN_SUCCESS;
168 }
169
170 default:
171 return KERN_FAILURE;
172 }
173 }
174
175 /*
176 * Routine: cpu_doshutdown
177 * Function:
178 */
179 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)180 cpu_doshutdown(void (*doshutdown)(processor_t),
181 processor_t processor)
182 {
183 doshutdown(processor);
184 }
185
186 /*
187 * Routine: cpu_idle_tickle
188 *
189 */
190 void
cpu_idle_tickle(void)191 cpu_idle_tickle(void)
192 {
193 boolean_t intr;
194 cpu_data_t *cpu_data_ptr;
195 uint64_t new_idle_timeout_ticks = 0x0ULL;
196
197 intr = ml_set_interrupts_enabled(FALSE);
198 cpu_data_ptr = getCpuDatap();
199
200 if (cpu_data_ptr->idle_timer_notify != NULL) {
201 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
202 if (new_idle_timeout_ticks != 0x0ULL) {
203 /* if a new idle timeout was requested set the new idle timer deadline */
204 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
205 } else {
206 /* turn off the idle timer */
207 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
208 }
209 timer_resync_deadlines();
210 }
211 (void) ml_set_interrupts_enabled(intr);
212 }
213
214 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)215 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
216 {
217 broadcastFunc xfunc;
218 void *xparam;
219
220 os_atomic_thread_fence(acquire);
221 /* Come back around if cpu_signal_internal is running on another CPU and has just
222 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
223 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
224 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
225 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
226 xparam = cpu_data_ptr->cpu_xcall_p1;
227 cpu_data_ptr->cpu_xcall_p0 = NULL;
228 cpu_data_ptr->cpu_xcall_p1 = NULL;
229 os_atomic_thread_fence(acq_rel);
230 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
231 xfunc(xparam);
232 INTERRUPT_MASKED_DEBUG_END();
233 }
234 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
235 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
236 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
237 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
238 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
239 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
240 os_atomic_thread_fence(acq_rel);
241 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
242 xfunc(xparam);
243 INTERRUPT_MASKED_DEBUG_END();
244 }
245 }
246
247 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)248 cpu_broadcast_xcall_internal(unsigned int signal,
249 uint32_t *synch,
250 boolean_t self_xcall,
251 broadcastFunc func,
252 void *parm)
253 {
254 boolean_t intr;
255 cpu_data_t *cpu_data_ptr;
256 cpu_data_t *target_cpu_datap;
257 unsigned int failsig;
258 int cpu;
259 int max_cpu = ml_get_max_cpu_number() + 1;
260
261 //yes, param ALSO cannot be NULL
262 assert(synch);
263 assert(func);
264 assert(parm);
265
266 lck_mtx_lock(&cpu_xcall_mtx);
267 intr = ml_set_interrupts_enabled(FALSE);
268 cpu_data_ptr = getCpuDatap();
269
270 failsig = 0;
271
272 *synch = max_cpu;
273 assert_wait((event_t)synch, THREAD_UNINT);
274
275 for (cpu = 0; cpu < max_cpu; cpu++) {
276 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
277
278 if (target_cpu_datap == cpu_data_ptr) {
279 continue;
280 }
281
282 if ((target_cpu_datap == NULL) ||
283 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
284 failsig++;
285 }
286 }
287
288 if (self_xcall) {
289 func(parm);
290 }
291
292 (void) ml_set_interrupts_enabled(intr);
293
294 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
295 clear_wait(current_thread(), THREAD_AWAKENED);
296 } else {
297 thread_block(THREAD_CONTINUE_NULL);
298 }
299 lck_mtx_unlock(&cpu_xcall_mtx);
300
301 if (!self_xcall) {
302 return max_cpu - failsig - 1;
303 } else {
304 return max_cpu - failsig;
305 }
306 }
307
308 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)309 cpu_broadcast_xcall(uint32_t *synch,
310 boolean_t self_xcall,
311 broadcastFunc func,
312 void *parm)
313 {
314 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
315 }
316
317 struct cpu_broadcast_xcall_simple_data {
318 broadcastFunc func;
319 void* parm;
320 uint32_t sync;
321 };
322
323 static void
cpu_broadcast_xcall_simple_cbk(void * parm)324 cpu_broadcast_xcall_simple_cbk(void *parm)
325 {
326 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
327
328 data->func(data->parm);
329
330 if (os_atomic_dec(&data->sync, relaxed) == 0) {
331 thread_wakeup((event_t)&data->sync);
332 }
333 }
334
335 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)336 cpu_xcall_simple(boolean_t self_xcall,
337 broadcastFunc func,
338 void *parm,
339 bool immediate)
340 {
341 struct cpu_broadcast_xcall_simple_data data = {};
342
343 data.func = func;
344 data.parm = parm;
345
346 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
347 }
348
349 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)350 cpu_broadcast_xcall_simple(boolean_t self_xcall,
351 broadcastFunc func,
352 void *parm)
353 {
354 return cpu_xcall_simple(self_xcall, func, parm, false);
355 }
356
357 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)358 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
359 {
360 cpu_data_t *target_cpu_datap;
361
362 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
363 panic("cpu_xcall_internal: invalid cpu_number %d", cpu_number);
364 }
365
366 if (func == NULL || param == NULL) {
367 // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready
368 panic("cpu_xcall_internal: cannot have null func/param: %p %p", func, param);
369 }
370
371 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
372 if (target_cpu_datap == NULL) {
373 panic("cpu_xcall_internal: cpu %d not initialized", cpu_number);
374 }
375
376 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
377 }
378
379 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)380 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
381 {
382 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
383 }
384
385 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)386 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
387 {
388 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
389 }
390
391 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1,boolean_t defer)392 cpu_signal_internal(cpu_data_t *target_proc,
393 cpu_signal_t signal,
394 void *p0,
395 void *p1,
396 boolean_t defer)
397 {
398 cpu_signal_t current_signals;
399 bool swap_success;
400 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
401 cpu_data_t *current_proc = getCpuDatap();
402
403 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
404 if (defer) {
405 assert(signal == SIGPnop);
406 }
407
408 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
409 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
410 current_mabs_time = start_mabs_time = mach_absolute_time();
411 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
412 assert(max_mabs_time > current_mabs_time);
413
414 do {
415 current_signals = target_proc->cpu_signal;
416 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
417 ml_set_interrupts_enabled(interruptible);
418 return KERN_FAILURE;
419 }
420 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals & (~signal),
421 current_signals | signal, release);
422
423 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
424 ml_set_interrupts_enabled(interruptible);
425 return KERN_ALREADY_WAITING;
426 }
427
428 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
429 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
430 * so break the deadlock by draining pending xcalls. */
431 if (!swap_success && (current_proc->cpu_signal & signal)) {
432 cpu_handle_xcall(current_proc);
433 }
434 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
435
436 /*
437 * If we time out while waiting for the target CPU to respond, it's possible that no
438 * other CPU is available to handle the watchdog interrupt that would eventually trigger
439 * a panic. To prevent this from happening, we just panic here to flag this condition.
440 */
441 if (__improbable(current_mabs_time >= max_mabs_time)) {
442 uint64_t end_time_ns, xcall_ack_timeout_ns;
443 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
444 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
445 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
446 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
447 }
448
449 if (signal == SIGPxcallImm) {
450 target_proc->cpu_imm_xcall_p0 = p0;
451 target_proc->cpu_imm_xcall_p1 = p1;
452 } else {
453 target_proc->cpu_xcall_p0 = p0;
454 target_proc->cpu_xcall_p1 = p1;
455 }
456 } else {
457 do {
458 current_signals = target_proc->cpu_signal;
459 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
460 if (current_proc == target_proc) {
461 panic("cpu_signal of self while signals are disabled");
462 }
463 ml_set_interrupts_enabled(interruptible);
464 return KERN_FAILURE;
465 }
466
467 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals,
468 current_signals | signal, release);
469 } while (!swap_success);
470 }
471
472 /*
473 * DSB is needed here to ensure prior stores to the pending signal mask and xcall params
474 * will be visible by the time the other cores are signaled. The IPI mechanism on any
475 * given platform will very likely use either an MSR or a non-coherent store that would
476 * not be ordered by a simple DMB.
477 */
478 __builtin_arm_dsb(DSB_ISHST);
479
480 if (!(target_proc->cpu_signal & SIGPdisabled)) {
481 /* Make sure cpu_phys_id is actually initialized */
482 assert3u(os_atomic_load(&target_proc->cpu_flags, relaxed) & (InitState | StartedState),
483 ==, InitState | StartedState);
484
485 if (defer) {
486 #if defined(HAS_IPI)
487 if (gFastIPI) {
488 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
489 } else {
490 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
491 }
492 #else
493 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
494 #endif /* defined(HAS_IPI) */
495 } else {
496 #if defined(HAS_IPI)
497 if (gFastIPI) {
498 ml_cpu_signal(target_proc->cpu_phys_id);
499 } else {
500 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
501 }
502 #else
503 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
504 #endif /* defined(HAS_IPI) */
505 }
506 }
507
508 ml_set_interrupts_enabled(interruptible);
509 return KERN_SUCCESS;
510 }
511
512 kern_return_t
cpu_signal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1)513 cpu_signal(cpu_data_t *target_proc,
514 cpu_signal_t signal,
515 void *p0,
516 void *p1)
517 {
518 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
519 }
520
521 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc)522 cpu_signal_deferred(cpu_data_t *target_proc)
523 {
524 return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE);
525 }
526
527 void
cpu_signal_cancel(cpu_data_t * target_proc)528 cpu_signal_cancel(cpu_data_t *target_proc)
529 {
530 /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */
531 if (!(target_proc->cpu_signal & SIGPdisabled)) {
532 #if defined(HAS_IPI)
533 if (gFastIPI) {
534 ml_cpu_signal_retract(target_proc->cpu_phys_id);
535 } else {
536 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
537 }
538 #else
539 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
540 #endif /* defined(HAS_IPI) */
541 }
542 }
543
544 void
cpu_signal_handler(void)545 cpu_signal_handler(void)
546 {
547 cpu_signal_handler_internal(FALSE);
548 }
549
550 bool
cpu_has_SIGPdebug_pending(void)551 cpu_has_SIGPdebug_pending(void)
552 {
553 cpu_data_t *cpu_data_ptr = getCpuDatap();
554
555 return cpu_data_ptr->cpu_signal & SIGPdebug;
556 }
557
558 void
cpu_signal_handler_internal(boolean_t disable_signal)559 cpu_signal_handler_internal(boolean_t disable_signal)
560 {
561 cpu_data_t *cpu_data_ptr = getCpuDatap();
562
563 cpu_data_ptr->cpu_stat.ipi_cnt++;
564 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
565 SCHED_STATS_INC(ipi_count);
566
567 /*
568 * Employ an acquire barrier when loading cpu_signal to ensure that
569 * loads within individual signal handlers won't be speculated ahead
570 * of the load of cpu_signal. This pairs with the release barrier
571 * in cpu_signal_internal() to ensure that once a flag has been set in
572 * the cpu_signal mask, any prerequisite setup is also visible to signal
573 * handlers.
574 */
575 cpu_signal_t cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
576
577 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
578 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
579 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
580 /* We must not clear SIGPdisabled unless the CPU is properly started */
581 assert3u(os_atomic_load(&cpu_data_ptr->cpu_flags, relaxed) & (InitState | StartedState),
582 ==, InitState | StartedState);
583 cpu_signal = os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
584 }
585
586 while (cpu_signal & ~SIGPdisabled) {
587 if (cpu_signal & SIGPdebug) {
588 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, acquire);
589 INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI);
590 DebuggerXCall(cpu_data_ptr->cpu_int_state);
591 INTERRUPT_MASKED_DEBUG_END();
592 }
593 #if KPERF
594 if (cpu_signal & SIGPkppet) {
595 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, acquire);
596 extern void kperf_signal_handler(void);
597 INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI);
598 kperf_signal_handler();
599 INTERRUPT_MASKED_DEBUG_END();
600 }
601 #endif /* KPERF */
602 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
603 cpu_handle_xcall(cpu_data_ptr);
604 }
605 if (cpu_signal & SIGPast) {
606 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, acquire);
607 INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI);
608 ast_check(current_processor());
609 INTERRUPT_MASKED_DEBUG_END();
610 }
611 if (cpu_signal & SIGPTimerLocal) {
612 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPTimerLocal, acquire);
613 INTERRUPT_MASKED_DEBUG_START(timer_queue_expire_local, DBG_INTR_TYPE_IPI);
614 timer_queue_expire_local(current_processor());
615 INTERRUPT_MASKED_DEBUG_END();
616 }
617
618 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
619 }
620 }
621
622 void
cpu_exit_wait(int cpu_id)623 cpu_exit_wait(int cpu_id)
624 {
625 #if !APPLEVIRTUALPLATFORM /* AVP doesn't provide an equivalent poll-for-powerdown operation */
626 #if USE_APPLEARMSMP
627 if (!ml_is_quiescing()) {
628 // For runtime disable (non S2R) the CPU will shut down immediately.
629 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
630 assert(cpu && cpu->cpu_IMPL_regs);
631 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
632
633 // Poll the "CPU running state" field until it is 0 (off)
634 // This loop typically finishes in about 600ns. Sometimes it takes as long as 10us.
635 // If it takes longer than 10s, assume something went horribly wrong and panic.
636 uint64_t start = mach_absolute_time(), interval;
637 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
638
639 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
640 __builtin_arm_dsb(DSB_ISH);
641 if (mach_absolute_time() > start + interval) {
642 #if NO_CPU_OVRD
643 // On platforms where CPU_OVRD is unavailable, a core can get stuck
644 // in a loop where it tries to enter WFI but is constantly woken up
645 // by an IRQ or FIQ. This condition persists until the cluster-wide
646 // deep sleep bits are set.
647 //
648 // Making this a fatal condition would be a poor UX, but it's good to
649 // print a warning so we know how often it happens.
650 kprintf("CPU%d failed to shut down\n", cpu_id);
651 #else
652 panic("CPU%d failed to shut down", cpu_id);
653 #endif
654 }
655 }
656 }
657 #endif /* USE_APPLEARMSMP */
658 #endif /* !APPLEVIRTUALPLATFORM */
659
660 if (cpu_id != master_cpu || (support_bootcpu_shutdown && !ml_is_quiescing())) {
661 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
662 cpu_data_t *cpu_data_ptr;
663
664 uint64_t start = mach_absolute_time(), interval;
665 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
666
667 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
668 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
669 if (mach_absolute_time() > start + interval) {
670 panic("CPU %d failed to reach ARM_CPU_ON_SLEEP_PATH: %d", cpu_id, cpu_data_ptr->cpu_sleep_token);
671 }
672 }
673 }
674 }
675
676 void
cpu_machine_init(void)677 cpu_machine_init(void)
678 {
679 cpu_data_t *cpu_data_ptr = getCpuDatap();
680
681 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
682 platform_cache_init();
683 }
684
685 bool started = os_atomic_or_orig(&cpu_data_ptr->cpu_flags, StartedState, relaxed) & StartedState;
686
687 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
688 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
689
690 ml_init_interrupt();
691 }
692
693 processor_t
current_processor(void)694 current_processor(void)
695 {
696 return PERCPU_GET(processor);
697 }
698
699 processor_t
cpu_to_processor(int cpu)700 cpu_to_processor(int cpu)
701 {
702 cpu_data_t *cpu_data = cpu_datap(cpu);
703 if (cpu_data != NULL) {
704 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
705 } else {
706 return NULL;
707 }
708 }
709
710 cpu_data_t *
processor_to_cpu_datap(processor_t processor)711 processor_to_cpu_datap(processor_t processor)
712 {
713 assert(processor->cpu_id <= ml_get_max_cpu_number());
714 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
715
716 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
717 }
718
719 __startup_func
720 static void
cpu_data_startup_init(void)721 cpu_data_startup_init(void)
722 {
723 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
724
725 percpu_base.size = percpu_section_size();
726 if (ml_get_cpu_count() == 1) {
727 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
728 return;
729 }
730
731 /*
732 * The memory needs to be physically contiguous because it contains
733 * cpu_data_t structures sometimes accessed during reset
734 * with the MMU off.
735 *
736 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
737 * normally runs, so we instead steal the memory for the PERCPU subsystem
738 * even earlier.
739 */
740 percpu_base.start = (vm_offset_t)pmap_steal_memory(size, PAGE_SIZE);
741 bzero((void *)percpu_base.start, size);
742
743 percpu_base.start -= percpu_section_start();
744 percpu_base.end = percpu_base.start + size - 1;
745 percpu_base_cur = percpu_base.start;
746 }
747 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
748
749 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)750 cpu_data_alloc(boolean_t is_boot_cpu)
751 {
752 cpu_data_t *cpu_data_ptr = NULL;
753 vm_address_t base;
754
755 if (is_boot_cpu) {
756 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
757 } else {
758 base = os_atomic_add_orig(&percpu_base_cur,
759 percpu_section_size(), relaxed);
760
761 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
762 cpu_stack_alloc(cpu_data_ptr);
763 }
764
765 return cpu_data_ptr;
766 }
767
768 ast_t *
ast_pending(void)769 ast_pending(void)
770 {
771 return &getCpuDatap()->cpu_pending_ast;
772 }
773
774 cpu_type_t
slot_type(int slot_num)775 slot_type(int slot_num)
776 {
777 return cpu_datap(slot_num)->cpu_type;
778 }
779
780 cpu_subtype_t
slot_subtype(int slot_num)781 slot_subtype(int slot_num)
782 {
783 return cpu_datap(slot_num)->cpu_subtype;
784 }
785
786 cpu_threadtype_t
slot_threadtype(int slot_num)787 slot_threadtype(int slot_num)
788 {
789 return cpu_datap(slot_num)->cpu_threadtype;
790 }
791
792 cpu_type_t
cpu_type(void)793 cpu_type(void)
794 {
795 return getCpuDatap()->cpu_type;
796 }
797
798 cpu_subtype_t
cpu_subtype(void)799 cpu_subtype(void)
800 {
801 return getCpuDatap()->cpu_subtype;
802 }
803
804 cpu_threadtype_t
cpu_threadtype(void)805 cpu_threadtype(void)
806 {
807 return getCpuDatap()->cpu_threadtype;
808 }
809
810 int
cpu_number(void)811 cpu_number(void)
812 {
813 return current_thread()->machine.cpu_number;
814 }
815
816 vm_offset_t
current_percpu_base(void)817 current_percpu_base(void)
818 {
819 long base = current_thread()->machine.pcpu_data_base_and_cpu_number;
820
821 return (vm_offset_t)(base >> 16);
822 }
823
824 vm_offset_t
other_percpu_base(int cpu)825 other_percpu_base(int cpu)
826 {
827 return (char *)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
828 }
829
830 uint64_t
ml_get_wake_timebase(void)831 ml_get_wake_timebase(void)
832 {
833 return wake_abstime;
834 }
835
836 /*
837 * Called while running on a specific CPU to wait for it to handle the
838 * self-IPI that clears SIGPdisabled.
839 */
840 void
ml_wait_for_cpu_signal_to_enable(void)841 ml_wait_for_cpu_signal_to_enable(void)
842 {
843 assert(ml_get_interrupts_enabled());
844 cpu_data_t *cpu_data_ptr = getCpuDatap();
845
846 hw_wait_while_equals32(__DEVOLATILE(uint32_t*, &cpu_data_ptr->cpu_signal), SIGPdisabled);
847 }
848
849 void
assert_ml_cpu_signal_is_enabled(bool enabled)850 assert_ml_cpu_signal_is_enabled(bool enabled)
851 {
852 if (enabled) {
853 assert((getCpuDatap()->cpu_signal & SIGPdisabled) == 0);
854 } else {
855 assert3u(getCpuDatap()->cpu_signal, ==, SIGPdisabled);
856 }
857 }
858
859 bool
ml_cpu_can_exit(__unused int cpu_id)860 ml_cpu_can_exit(__unused int cpu_id)
861 {
862 #if USE_APPLEARMSMP
863 /*
864 * Cyprus and newer chips can disable individual non-boot CPUs. The
865 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
866 * Until the feature is known to be stable, guard it with a boot-arg.
867 */
868
869 bool cpu_supported = true;
870
871 if (cpu_id == master_cpu && !support_bootcpu_shutdown) {
872 cpu_supported = false;
873 }
874
875 if (enable_processor_exit) {
876 return cpu_supported;
877 }
878
879 #if HAS_CLUSTER
880 if (ml_get_topology_info()->cluster_power_down) {
881 return cpu_supported;
882 }
883 #endif /* HAS_CLUSTER */
884
885 #endif /* USE_APPLEARMSMP */
886
887 return false;
888 }
889
890 #ifdef USE_APPLEARMSMP
891
892 void
ml_cpu_begin_state_transition(int cpu_id)893 ml_cpu_begin_state_transition(int cpu_id)
894 {
895 lck_rw_lock_exclusive(&cpu_state_lock);
896 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
897 lck_rw_unlock_exclusive(&cpu_state_lock);
898 }
899
900 void
ml_cpu_end_state_transition(int cpu_id)901 ml_cpu_end_state_transition(int cpu_id)
902 {
903 lck_rw_lock_exclusive(&cpu_state_lock);
904 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
905 lck_rw_unlock_exclusive(&cpu_state_lock);
906 }
907
908 void
ml_cpu_begin_loop(void)909 ml_cpu_begin_loop(void)
910 {
911 lck_rw_lock_shared(&cpu_state_lock);
912 }
913
914 void
ml_cpu_end_loop(void)915 ml_cpu_end_loop(void)
916 {
917 lck_rw_unlock_shared(&cpu_state_lock);
918 }
919
920 void
ml_cpu_power_enable(int cpu_id)921 ml_cpu_power_enable(int cpu_id)
922 {
923 PE_cpu_power_enable(cpu_id);
924 }
925
926 void
ml_cpu_power_disable(int cpu_id)927 ml_cpu_power_disable(int cpu_id)
928 {
929 PE_cpu_power_disable(cpu_id);
930 }
931
932 #else /* USE_APPLEARMSMP */
933
934 void
ml_cpu_begin_state_transition(__unused int cpu_id)935 ml_cpu_begin_state_transition(__unused int cpu_id)
936 {
937 }
938
939 void
ml_cpu_end_state_transition(__unused int cpu_id)940 ml_cpu_end_state_transition(__unused int cpu_id)
941 {
942 }
943
944 void
ml_cpu_begin_loop(void)945 ml_cpu_begin_loop(void)
946 {
947 }
948
949 void
ml_cpu_end_loop(void)950 ml_cpu_end_loop(void)
951 {
952 }
953
954 void
ml_cpu_power_enable(__unused int cpu_id)955 ml_cpu_power_enable(__unused int cpu_id)
956 {
957 }
958
959 void
ml_cpu_power_disable(__unused int cpu_id)960 ml_cpu_power_disable(__unused int cpu_id)
961 {
962 }
963
964 #endif /* USE_APPLEARMSMP */
965