1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <kern/clock.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <mach/processor_info.h>
50 #include <machine/atomic.h>
51 #include <machine/config.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58 #include <arm64/proc_reg.h>
59 #include <libkern/OSAtomic.h>
60
61 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
62 vm_address_t percpu_base_cur;
63 cpu_data_t PERCPU_DATA(cpu_data);
64 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
65
66 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
67 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
68
69 unsigned int real_ncpus = 1;
70 boolean_t idle_enable = FALSE;
71 uint64_t wake_abstime = 0x0ULL;
72
73 extern uint64_t xcall_ack_timeout_abstime;
74
75 #if defined(HAS_IPI)
76 extern unsigned int gFastIPI;
77 #endif /* defined(HAS_IPI) */
78
79 cpu_data_t *
cpu_datap(int cpu)80 cpu_datap(int cpu)
81 {
82 assert(cpu <= ml_get_max_cpu_number());
83 return CpuDataEntries[cpu].cpu_data_vaddr;
84 }
85
86 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)87 cpu_control(int slot_num,
88 processor_info_t info,
89 unsigned int count)
90 {
91 printf("cpu_control(%d,%p,%d) not implemented\n",
92 slot_num, info, count);
93 return KERN_FAILURE;
94 }
95
96 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)97 cpu_info_count(processor_flavor_t flavor,
98 unsigned int *count)
99 {
100 switch (flavor) {
101 case PROCESSOR_CPU_STAT:
102 *count = PROCESSOR_CPU_STAT_COUNT;
103 return KERN_SUCCESS;
104
105 case PROCESSOR_CPU_STAT64:
106 *count = PROCESSOR_CPU_STAT64_COUNT;
107 return KERN_SUCCESS;
108
109 default:
110 *count = 0;
111 return KERN_FAILURE;
112 }
113 }
114
115 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)116 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
117 unsigned int *count)
118 {
119 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
120
121 switch (flavor) {
122 case PROCESSOR_CPU_STAT:
123 {
124 if (*count < PROCESSOR_CPU_STAT_COUNT) {
125 return KERN_FAILURE;
126 }
127
128 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
129 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
130 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
131 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
132 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
133 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
134 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
135 cpu_stat->vfp_shortv_cnt = 0;
136 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
137 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
138
139 *count = PROCESSOR_CPU_STAT_COUNT;
140
141 return KERN_SUCCESS;
142 }
143
144 case PROCESSOR_CPU_STAT64:
145 {
146 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
147 return KERN_FAILURE;
148 }
149
150 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
151 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
152 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
153 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
154 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
155 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
156 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
157 cpu_stat->vfp_shortv_cnt = 0;
158 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
159 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
160 #if CONFIG_CPU_COUNTERS
161 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
162 #endif /* CONFIG_CPU_COUNTERS */
163
164 *count = PROCESSOR_CPU_STAT64_COUNT;
165
166 return KERN_SUCCESS;
167 }
168
169 default:
170 return KERN_FAILURE;
171 }
172 }
173
174 /*
175 * Routine: cpu_doshutdown
176 * Function:
177 */
178 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)179 cpu_doshutdown(void (*doshutdown)(processor_t),
180 processor_t processor)
181 {
182 doshutdown(processor);
183 }
184
185 /*
186 * Routine: cpu_idle_tickle
187 *
188 */
189 void
cpu_idle_tickle(void)190 cpu_idle_tickle(void)
191 {
192 boolean_t intr;
193 cpu_data_t *cpu_data_ptr;
194 uint64_t new_idle_timeout_ticks = 0x0ULL;
195
196 intr = ml_set_interrupts_enabled(FALSE);
197 cpu_data_ptr = getCpuDatap();
198
199 if (cpu_data_ptr->idle_timer_notify != NULL) {
200 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
201 if (new_idle_timeout_ticks != 0x0ULL) {
202 /* if a new idle timeout was requested set the new idle timer deadline */
203 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
204 } else {
205 /* turn off the idle timer */
206 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
207 }
208 timer_resync_deadlines();
209 }
210 (void) ml_set_interrupts_enabled(intr);
211 }
212
213 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)214 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
215 {
216 broadcastFunc xfunc;
217 void *xparam;
218
219 os_atomic_thread_fence(acquire);
220 /* Come back around if cpu_signal_internal is running on another CPU and has just
221 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
222 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
223 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
224 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
225 xparam = cpu_data_ptr->cpu_xcall_p1;
226 cpu_data_ptr->cpu_xcall_p0 = NULL;
227 cpu_data_ptr->cpu_xcall_p1 = NULL;
228 os_atomic_thread_fence(acq_rel);
229 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
230 xfunc(xparam);
231 INTERRUPT_MASKED_DEBUG_END();
232 }
233 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
234 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
235 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
236 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
237 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
238 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
239 os_atomic_thread_fence(acq_rel);
240 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
241 xfunc(xparam);
242 INTERRUPT_MASKED_DEBUG_END();
243 }
244 }
245
246 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)247 cpu_broadcast_xcall_internal(unsigned int signal,
248 uint32_t *synch,
249 boolean_t self_xcall,
250 broadcastFunc func,
251 void *parm)
252 {
253 boolean_t intr;
254 cpu_data_t *cpu_data_ptr;
255 cpu_data_t *target_cpu_datap;
256 unsigned int failsig;
257 int cpu;
258 int max_cpu = ml_get_max_cpu_number() + 1;
259
260 //yes, param ALSO cannot be NULL
261 assert(func);
262 assert(parm);
263
264 intr = ml_set_interrupts_enabled(FALSE);
265 cpu_data_ptr = getCpuDatap();
266
267 failsig = 0;
268
269 if (synch != NULL) {
270 *synch = max_cpu;
271 assert_wait((event_t)synch, THREAD_UNINT);
272 }
273
274 for (cpu = 0; cpu < max_cpu; cpu++) {
275 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
276
277 if (target_cpu_datap == cpu_data_ptr) {
278 continue;
279 }
280
281 if ((target_cpu_datap == NULL) ||
282 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
283 failsig++;
284 }
285 }
286
287 if (self_xcall) {
288 func(parm);
289 }
290
291 (void) ml_set_interrupts_enabled(intr);
292
293 if (synch != NULL) {
294 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
295 clear_wait(current_thread(), THREAD_AWAKENED);
296 } else {
297 thread_block(THREAD_CONTINUE_NULL);
298 }
299 }
300
301 if (!self_xcall) {
302 return max_cpu - failsig - 1;
303 } else {
304 return max_cpu - failsig;
305 }
306 }
307
308 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)309 cpu_broadcast_xcall(uint32_t *synch,
310 boolean_t self_xcall,
311 broadcastFunc func,
312 void *parm)
313 {
314 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
315 }
316
317 struct cpu_broadcast_xcall_simple_data {
318 broadcastFunc func;
319 void* parm;
320 uint32_t sync;
321 };
322
323 static void
cpu_broadcast_xcall_simple_cbk(void * parm)324 cpu_broadcast_xcall_simple_cbk(void *parm)
325 {
326 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
327
328 data->func(data->parm);
329
330 if (os_atomic_dec(&data->sync, relaxed) == 0) {
331 thread_wakeup((event_t)&data->sync);
332 }
333 }
334
335 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)336 cpu_xcall_simple(boolean_t self_xcall,
337 broadcastFunc func,
338 void *parm,
339 bool immediate)
340 {
341 struct cpu_broadcast_xcall_simple_data data = {};
342
343 data.func = func;
344 data.parm = parm;
345
346 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
347 }
348
349 unsigned int
cpu_broadcast_immediate_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)350 cpu_broadcast_immediate_xcall(uint32_t *synch,
351 boolean_t self_xcall,
352 broadcastFunc func,
353 void *parm)
354 {
355 return cpu_broadcast_xcall_internal(SIGPxcallImm, synch, self_xcall, func, parm);
356 }
357
358 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)359 cpu_broadcast_xcall_simple(boolean_t self_xcall,
360 broadcastFunc func,
361 void *parm)
362 {
363 return cpu_xcall_simple(self_xcall, func, parm, false);
364 }
365
366 unsigned int
cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)367 cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,
368 broadcastFunc func,
369 void *parm)
370 {
371 return cpu_xcall_simple(self_xcall, func, parm, true);
372 }
373
374 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)375 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
376 {
377 cpu_data_t *target_cpu_datap;
378
379 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
380 panic("cpu_xcall_internal: invalid cpu_number %d", cpu_number);
381 }
382
383 if (func == NULL || param == NULL) {
384 // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready
385 panic("cpu_xcall_internal: cannot have null func/param: %p %p", func, param);
386 }
387
388 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
389 if (target_cpu_datap == NULL) {
390 panic("cpu_xcall_internal: cpu %d not initialized", cpu_number);
391 }
392
393 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
394 }
395
396 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)397 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
398 {
399 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
400 }
401
402 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)403 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
404 {
405 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
406 }
407
408 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1,boolean_t defer)409 cpu_signal_internal(cpu_data_t *target_proc,
410 unsigned int signal,
411 void *p0,
412 void *p1,
413 boolean_t defer)
414 {
415 unsigned int Check_SIGPdisabled;
416 int current_signals;
417 bool swap_success;
418 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
419 cpu_data_t *current_proc = getCpuDatap();
420
421 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
422 if (defer) {
423 assert(signal == SIGPnop);
424 }
425
426 if (current_proc != target_proc) {
427 Check_SIGPdisabled = SIGPdisabled;
428 } else {
429 Check_SIGPdisabled = 0;
430 }
431
432 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
433 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
434 current_mabs_time = start_mabs_time = mach_absolute_time();
435 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
436 assert(max_mabs_time > current_mabs_time);
437
438 do {
439 current_signals = target_proc->cpu_signal;
440 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
441 ml_set_interrupts_enabled(interruptible);
442 return KERN_FAILURE;
443 }
444 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals & (~signal),
445 current_signals | signal, release);
446
447 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
448 ml_set_interrupts_enabled(interruptible);
449 return KERN_ALREADY_WAITING;
450 }
451
452 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
453 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
454 * so break the deadlock by draining pending xcalls. */
455 if (!swap_success && (current_proc->cpu_signal & signal)) {
456 cpu_handle_xcall(current_proc);
457 }
458 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
459
460 /*
461 * If we time out while waiting for the target CPU to respond, it's possible that no
462 * other CPU is available to handle the watchdog interrupt that would eventually trigger
463 * a panic. To prevent this from happening, we just panic here to flag this condition.
464 */
465 if (__improbable(current_mabs_time >= max_mabs_time)) {
466 uint64_t end_time_ns, xcall_ack_timeout_ns;
467 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
468 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
469 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
470 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
471 }
472
473 if (signal == SIGPxcallImm) {
474 target_proc->cpu_imm_xcall_p0 = p0;
475 target_proc->cpu_imm_xcall_p1 = p1;
476 } else {
477 target_proc->cpu_xcall_p0 = p0;
478 target_proc->cpu_xcall_p1 = p1;
479 }
480 } else {
481 do {
482 current_signals = target_proc->cpu_signal;
483 if ((Check_SIGPdisabled != 0) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) {
484 ml_set_interrupts_enabled(interruptible);
485 return KERN_FAILURE;
486 }
487
488 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals,
489 current_signals | signal, release);
490 } while (!swap_success);
491 }
492
493 /*
494 * DSB is needed here to ensure prior stores to the pending signal mask and xcall params
495 * will be visible by the time the other cores are signaled. The IPI mechanism on any
496 * given platform will very likely use either an MSR or a non-coherent store that would
497 * not be ordered by a simple DMB.
498 */
499 __builtin_arm_dsb(DSB_ISHST);
500
501 if (!(target_proc->cpu_signal & SIGPdisabled)) {
502 if (defer) {
503 #if defined(HAS_IPI)
504 if (gFastIPI) {
505 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
506 } else {
507 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
508 }
509 #else
510 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
511 #endif /* defined(HAS_IPI) */
512 } else {
513 #if defined(HAS_IPI)
514 if (gFastIPI) {
515 ml_cpu_signal(target_proc->cpu_phys_id);
516 } else {
517 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
518 }
519 #else
520 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
521 #endif /* defined(HAS_IPI) */
522 }
523 }
524
525 ml_set_interrupts_enabled(interruptible);
526 return KERN_SUCCESS;
527 }
528
529 kern_return_t
cpu_signal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1)530 cpu_signal(cpu_data_t *target_proc,
531 unsigned int signal,
532 void *p0,
533 void *p1)
534 {
535 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
536 }
537
538 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc)539 cpu_signal_deferred(cpu_data_t *target_proc)
540 {
541 return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE);
542 }
543
544 void
cpu_signal_cancel(cpu_data_t * target_proc)545 cpu_signal_cancel(cpu_data_t *target_proc)
546 {
547 /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */
548 if (!(target_proc->cpu_signal & SIGPdisabled)) {
549 #if defined(HAS_IPI)
550 if (gFastIPI) {
551 ml_cpu_signal_retract(target_proc->cpu_phys_id);
552 } else {
553 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
554 }
555 #else
556 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
557 #endif /* defined(HAS_IPI) */
558 }
559 }
560
561 void
cpu_signal_handler(void)562 cpu_signal_handler(void)
563 {
564 cpu_signal_handler_internal(FALSE);
565 }
566
567 bool
cpu_has_SIGPdebug_pending(void)568 cpu_has_SIGPdebug_pending(void)
569 {
570 cpu_data_t *cpu_data_ptr = getCpuDatap();
571
572 return cpu_data_ptr->cpu_signal & SIGPdebug;
573 }
574
575 void
cpu_signal_handler_internal(boolean_t disable_signal)576 cpu_signal_handler_internal(boolean_t disable_signal)
577 {
578 cpu_data_t *cpu_data_ptr = getCpuDatap();
579 unsigned int cpu_signal;
580
581 cpu_data_ptr->cpu_stat.ipi_cnt++;
582 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
583 SCHED_STATS_INC(ipi_count);
584
585 /*
586 * Employ an acquire barrier when loading cpu_signal to ensure that
587 * loads within individual signal handlers won't be speculated ahead
588 * of the load of cpu_signal. This pairs with the release barrier
589 * in cpu_signal_internal() to ensure that once a flag has been set in
590 * the cpu_signal mask, any prerequisite setup is also visible to signal
591 * handlers.
592 */
593 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
594
595 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
596 os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
597 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
598 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
599 }
600
601 while (cpu_signal & ~SIGPdisabled) {
602 if (cpu_signal & SIGPdebug) {
603 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, relaxed);
604 INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI);
605 DebuggerXCall(cpu_data_ptr->cpu_int_state);
606 INTERRUPT_MASKED_DEBUG_END();
607 }
608 if (cpu_signal & SIGPdec) {
609 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdec, relaxed);
610 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_IPI);
611 rtclock_intr(FALSE);
612 INTERRUPT_MASKED_DEBUG_END();
613 }
614 #if KPERF
615 if (cpu_signal & SIGPkppet) {
616 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, relaxed);
617 extern void kperf_signal_handler(void);
618 INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI);
619 kperf_signal_handler();
620 INTERRUPT_MASKED_DEBUG_END();
621 }
622 #endif /* KPERF */
623 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
624 cpu_handle_xcall(cpu_data_ptr);
625 }
626 if (cpu_signal & SIGPast) {
627 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, acquire);
628 INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI);
629 ast_check(current_processor());
630 INTERRUPT_MASKED_DEBUG_END();
631 }
632
633 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
634 }
635 }
636
637 void
cpu_exit_wait(int cpu_id)638 cpu_exit_wait(int cpu_id)
639 {
640 #if USE_APPLEARMSMP
641 if (!ml_is_quiescing()) {
642 // For runtime disable (non S2R) the CPU will shut down immediately.
643 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
644 assert(cpu && cpu->cpu_IMPL_regs);
645 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
646
647 // Poll the "CPU running state" field until it is 0 (off)
648 // This loop typically finishes in about 600ns. Sometimes it takes as long as 10us.
649 // If it takes longer than 10s, assume something went horribly wrong and panic.
650 uint64_t start = mach_absolute_time(), interval;
651 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
652
653 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
654 __builtin_arm_dsb(DSB_ISH);
655 if (mach_absolute_time() > start + interval) {
656 #if NO_CPU_OVRD
657 // On platforms where CPU_OVRD is unavailable, a core can get stuck
658 // in a loop where it tries to enter WFI but is constantly woken up
659 // by an IRQ or FIQ. This condition persists until the cluster-wide
660 // deep sleep bits are set.
661 //
662 // Making this a fatal condition would be a poor UX, but it's good to
663 // print a warning so we know how often it happens.
664 kprintf("CPU%d failed to shut down\n", cpu_id);
665 #else
666 panic("CPU%d failed to shut down", cpu_id);
667 #endif
668 }
669 }
670 return;
671 }
672 #endif /* USE_APPLEARMSMP */
673
674 if (cpu_id != master_cpu) {
675 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
676 cpu_data_t *cpu_data_ptr;
677
678 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
679 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
680 }
681 ;
682 }
683 }
684
685 boolean_t
cpu_can_exit(__unused int cpu)686 cpu_can_exit(__unused int cpu)
687 {
688 return TRUE;
689 }
690
691 void
cpu_machine_init(void)692 cpu_machine_init(void)
693 {
694 static boolean_t started = FALSE;
695 cpu_data_t *cpu_data_ptr;
696
697 cpu_data_ptr = getCpuDatap();
698 started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState);
699 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
700 platform_cache_init();
701 }
702
703 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
704 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
705
706 cpu_data_ptr->cpu_flags |= StartedState;
707 ml_init_interrupt();
708 }
709
710 processor_t
current_processor(void)711 current_processor(void)
712 {
713 return PERCPU_GET(processor);
714 }
715
716 processor_t
cpu_to_processor(int cpu)717 cpu_to_processor(int cpu)
718 {
719 cpu_data_t *cpu_data = cpu_datap(cpu);
720 if (cpu_data != NULL) {
721 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
722 } else {
723 return NULL;
724 }
725 }
726
727 cpu_data_t *
processor_to_cpu_datap(processor_t processor)728 processor_to_cpu_datap(processor_t processor)
729 {
730 assert(processor->cpu_id <= ml_get_max_cpu_number());
731 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
732
733 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
734 }
735
736 __startup_func
737 static void
cpu_data_startup_init(void)738 cpu_data_startup_init(void)
739 {
740 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
741
742 percpu_base.size = percpu_section_size();
743 if (ml_get_cpu_count() == 1) {
744 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
745 return;
746 }
747
748 /*
749 * The memory needs to be physically contiguous because it contains
750 * cpu_data_t structures sometimes accessed during reset
751 * with the MMU off.
752 *
753 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
754 * normally runs, so we instead steal the memory for the PERCPU subsystem
755 * even earlier.
756 */
757 percpu_base.start = (vm_offset_t)pmap_steal_memory(size, PAGE_SIZE);
758 bzero((void *)percpu_base.start, size);
759
760 percpu_base.start -= percpu_section_start();
761 percpu_base.end = percpu_base.start + size - 1;
762 percpu_base_cur = percpu_base.start;
763 }
764 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
765
766 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)767 cpu_data_alloc(boolean_t is_boot_cpu)
768 {
769 cpu_data_t *cpu_data_ptr = NULL;
770 vm_address_t base;
771
772 if (is_boot_cpu) {
773 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
774 } else {
775 base = os_atomic_add_orig(&percpu_base_cur,
776 percpu_section_size(), relaxed);
777
778 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
779 cpu_stack_alloc(cpu_data_ptr);
780 }
781
782 return cpu_data_ptr;
783 }
784
785 ast_t *
ast_pending(void)786 ast_pending(void)
787 {
788 return &getCpuDatap()->cpu_pending_ast;
789 }
790
791 cpu_type_t
slot_type(int slot_num)792 slot_type(int slot_num)
793 {
794 return cpu_datap(slot_num)->cpu_type;
795 }
796
797 cpu_subtype_t
slot_subtype(int slot_num)798 slot_subtype(int slot_num)
799 {
800 return cpu_datap(slot_num)->cpu_subtype;
801 }
802
803 cpu_threadtype_t
slot_threadtype(int slot_num)804 slot_threadtype(int slot_num)
805 {
806 return cpu_datap(slot_num)->cpu_threadtype;
807 }
808
809 cpu_type_t
cpu_type(void)810 cpu_type(void)
811 {
812 return getCpuDatap()->cpu_type;
813 }
814
815 cpu_subtype_t
cpu_subtype(void)816 cpu_subtype(void)
817 {
818 return getCpuDatap()->cpu_subtype;
819 }
820
821 cpu_threadtype_t
cpu_threadtype(void)822 cpu_threadtype(void)
823 {
824 return getCpuDatap()->cpu_threadtype;
825 }
826
827 int
cpu_number(void)828 cpu_number(void)
829 {
830 return getCpuDatap()->cpu_number;
831 }
832
833 vm_offset_t
current_percpu_base(void)834 current_percpu_base(void)
835 {
836 return current_thread()->machine.pcpu_data_base;
837 }
838
839 vm_offset_t
other_percpu_base(int cpu)840 other_percpu_base(int cpu)
841 {
842 return (char *)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
843 }
844
845 uint64_t
ml_get_wake_timebase(void)846 ml_get_wake_timebase(void)
847 {
848 return wake_abstime;
849 }
850
851 bool
ml_cpu_signal_is_enabled(void)852 ml_cpu_signal_is_enabled(void)
853 {
854 return !(getCpuDatap()->cpu_signal & SIGPdisabled);
855 }
856
857 bool
ml_cpu_can_exit(__unused int cpu_id,processor_reason_t reason)858 ml_cpu_can_exit(__unused int cpu_id, processor_reason_t reason)
859 {
860 /* processor_exit() is always allowed on the S2R path */
861 if (ml_is_quiescing()) {
862 return true;
863 }
864 #if HAS_CLUSTER && USE_APPLEARMSMP
865 /*
866 * Until the feature is known to be stable, guard it with a boot-arg
867 */
868 extern bool enable_processor_exit;
869 extern bool cluster_power_supported;
870
871 if (!(enable_processor_exit || cluster_power_supported)) {
872 return false;
873 }
874
875 if (!enable_processor_exit) {
876 /* Only cluster_power_supported so disallow any USER reason */
877 if ((reason == REASON_USER) || (reason == REASON_CLPC_USER)) {
878 return false;
879 }
880 }
881
882 /*
883 * Cyprus and newer chips can disable individual non-boot CPUs. The
884 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
885 */
886 if (CpuDataEntries[cpu_id].cpu_data_vaddr != &BootCpuData) {
887 return true;
888 }
889 #else
890 (void)reason;
891 #endif
892 return false;
893 }
894
895 #ifdef USE_APPLEARMSMP
896
897 void
ml_cpu_begin_state_transition(int cpu_id)898 ml_cpu_begin_state_transition(int cpu_id)
899 {
900 lck_rw_lock_exclusive(&cpu_state_lock);
901 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
902 lck_rw_unlock_exclusive(&cpu_state_lock);
903 }
904
905 void
ml_cpu_end_state_transition(int cpu_id)906 ml_cpu_end_state_transition(int cpu_id)
907 {
908 lck_rw_lock_exclusive(&cpu_state_lock);
909 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
910 lck_rw_unlock_exclusive(&cpu_state_lock);
911 }
912
913 void
ml_cpu_begin_loop(void)914 ml_cpu_begin_loop(void)
915 {
916 lck_rw_lock_shared(&cpu_state_lock);
917 }
918
919 void
ml_cpu_end_loop(void)920 ml_cpu_end_loop(void)
921 {
922 lck_rw_unlock_shared(&cpu_state_lock);
923 }
924
925 void
ml_cpu_power_enable(int cpu_id)926 ml_cpu_power_enable(int cpu_id)
927 {
928 PE_cpu_power_enable(cpu_id);
929 }
930
931 void
ml_cpu_power_disable(int cpu_id)932 ml_cpu_power_disable(int cpu_id)
933 {
934 PE_cpu_power_disable(cpu_id);
935 }
936
937 #else /* USE_APPLEARMSMP */
938
939 void
ml_cpu_begin_state_transition(__unused int cpu_id)940 ml_cpu_begin_state_transition(__unused int cpu_id)
941 {
942 }
943
944 void
ml_cpu_end_state_transition(__unused int cpu_id)945 ml_cpu_end_state_transition(__unused int cpu_id)
946 {
947 }
948
949 void
ml_cpu_begin_loop(void)950 ml_cpu_begin_loop(void)
951 {
952 }
953
954 void
ml_cpu_end_loop(void)955 ml_cpu_end_loop(void)
956 {
957 }
958
959 void
ml_cpu_power_enable(__unused int cpu_id)960 ml_cpu_power_enable(__unused int cpu_id)
961 {
962 }
963
964 void
ml_cpu_power_disable(__unused int cpu_id)965 ml_cpu_power_disable(__unused int cpu_id)
966 {
967 }
968
969 #endif /* USE_APPLEARMSMP */
970