1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <kern/clock.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <mach/processor_info.h>
50 #include <machine/atomic.h>
51 #include <machine/config.h>
52 #include <vm/vm_kern_xnu.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58 #include <arm64/proc_reg.h>
59 #include <libkern/OSAtomic.h>
60
61 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
62 vm_address_t percpu_base_cur;
63 cpu_data_t PERCPU_DATA(cpu_data);
64 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
65
66 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
67 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
68 static LCK_MTX_DECLARE(cpu_xcall_mtx, &cpu_lck_grp);
69
70 unsigned int real_ncpus = 1;
71 boolean_t idle_enable = FALSE;
72 uint64_t wake_abstime = 0x0ULL;
73
74 extern uint64_t xcall_ack_timeout_abstime;
75
76 #if defined(HAS_IPI)
77 extern unsigned int gFastIPI;
78 #endif /* defined(HAS_IPI) */
79
80 cpu_data_t *
cpu_datap(int cpu)81 cpu_datap(int cpu)
82 {
83 assert(cpu <= ml_get_max_cpu_number());
84 return CpuDataEntries[cpu].cpu_data_vaddr;
85 }
86
87 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)88 cpu_control(int slot_num,
89 processor_info_t info,
90 unsigned int count)
91 {
92 printf("cpu_control(%d,%p,%d) not implemented\n",
93 slot_num, info, count);
94 return KERN_FAILURE;
95 }
96
97 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)98 cpu_info_count(processor_flavor_t flavor,
99 unsigned int *count)
100 {
101 switch (flavor) {
102 case PROCESSOR_CPU_STAT:
103 *count = PROCESSOR_CPU_STAT_COUNT;
104 return KERN_SUCCESS;
105
106 case PROCESSOR_CPU_STAT64:
107 *count = PROCESSOR_CPU_STAT64_COUNT;
108 return KERN_SUCCESS;
109
110 default:
111 *count = 0;
112 return KERN_FAILURE;
113 }
114 }
115
116 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)117 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
118 unsigned int *count)
119 {
120 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
121
122 switch (flavor) {
123 case PROCESSOR_CPU_STAT:
124 {
125 if (*count < PROCESSOR_CPU_STAT_COUNT) {
126 return KERN_FAILURE;
127 }
128
129 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
130 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
131 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
132 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
133 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
134 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
135 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
136 cpu_stat->vfp_shortv_cnt = 0;
137 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
138 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
139
140 *count = PROCESSOR_CPU_STAT_COUNT;
141
142 return KERN_SUCCESS;
143 }
144
145 case PROCESSOR_CPU_STAT64:
146 {
147 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
148 return KERN_FAILURE;
149 }
150
151 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
152 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
153 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
154 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
155 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
156 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
157 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
158 cpu_stat->vfp_shortv_cnt = 0;
159 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
160 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
161 #if CONFIG_CPU_COUNTERS
162 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
163 #endif /* CONFIG_CPU_COUNTERS */
164
165 *count = PROCESSOR_CPU_STAT64_COUNT;
166
167 return KERN_SUCCESS;
168 }
169
170 default:
171 return KERN_FAILURE;
172 }
173 }
174
175 /*
176 * Routine: cpu_doshutdown
177 * Function:
178 */
179 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)180 cpu_doshutdown(void (*doshutdown)(processor_t),
181 processor_t processor)
182 {
183 doshutdown(processor);
184 }
185
186 /*
187 * Routine: cpu_idle_tickle
188 *
189 */
190 void
cpu_idle_tickle(void)191 cpu_idle_tickle(void)
192 {
193 boolean_t intr;
194 cpu_data_t *cpu_data_ptr;
195 uint64_t new_idle_timeout_ticks = 0x0ULL;
196
197 intr = ml_set_interrupts_enabled(FALSE);
198 cpu_data_ptr = getCpuDatap();
199
200 if (cpu_data_ptr->idle_timer_notify != NULL) {
201 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
202 if (new_idle_timeout_ticks != 0x0ULL) {
203 /* if a new idle timeout was requested set the new idle timer deadline */
204 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
205 } else {
206 /* turn off the idle timer */
207 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
208 }
209 timer_resync_deadlines();
210 }
211 (void) ml_set_interrupts_enabled(intr);
212 }
213
214 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)215 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
216 {
217 broadcastFunc xfunc;
218 void *xparam;
219
220 os_atomic_thread_fence(acquire);
221 /* Come back around if cpu_signal_internal is running on another CPU and has just
222 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
223 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
224 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
225 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
226 xparam = cpu_data_ptr->cpu_xcall_p1;
227 cpu_data_ptr->cpu_xcall_p0 = NULL;
228 cpu_data_ptr->cpu_xcall_p1 = NULL;
229 os_atomic_thread_fence(acq_rel);
230 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
231 xfunc(xparam);
232 INTERRUPT_MASKED_DEBUG_END();
233 }
234 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
235 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
236 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
237 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
238 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
239 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
240 os_atomic_thread_fence(acq_rel);
241 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
242 xfunc(xparam);
243 INTERRUPT_MASKED_DEBUG_END();
244 }
245 }
246
247 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)248 cpu_broadcast_xcall_internal(unsigned int signal,
249 uint32_t *synch,
250 boolean_t self_xcall,
251 broadcastFunc func,
252 void *parm)
253 {
254 boolean_t intr;
255 cpu_data_t *cpu_data_ptr;
256 cpu_data_t *target_cpu_datap;
257 unsigned int failsig;
258 int cpu;
259 int max_cpu = ml_get_max_cpu_number() + 1;
260
261 //yes, param ALSO cannot be NULL
262 assert(synch);
263 assert(func);
264 assert(parm);
265
266 lck_mtx_lock(&cpu_xcall_mtx);
267 intr = ml_set_interrupts_enabled(FALSE);
268 cpu_data_ptr = getCpuDatap();
269
270 failsig = 0;
271
272 *synch = max_cpu;
273 assert_wait((event_t)synch, THREAD_UNINT);
274
275 for (cpu = 0; cpu < max_cpu; cpu++) {
276 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
277
278 if (target_cpu_datap == cpu_data_ptr) {
279 continue;
280 }
281
282 if ((target_cpu_datap == NULL) ||
283 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
284 failsig++;
285 }
286 }
287
288 if (self_xcall) {
289 func(parm);
290 }
291
292 (void) ml_set_interrupts_enabled(intr);
293
294 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
295 clear_wait(current_thread(), THREAD_AWAKENED);
296 } else {
297 thread_block(THREAD_CONTINUE_NULL);
298 }
299 lck_mtx_unlock(&cpu_xcall_mtx);
300
301 if (!self_xcall) {
302 return max_cpu - failsig - 1;
303 } else {
304 return max_cpu - failsig;
305 }
306 }
307
308 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)309 cpu_broadcast_xcall(uint32_t *synch,
310 boolean_t self_xcall,
311 broadcastFunc func,
312 void *parm)
313 {
314 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
315 }
316
317 struct cpu_broadcast_xcall_simple_data {
318 broadcastFunc func;
319 void* parm;
320 uint32_t sync;
321 };
322
323 static void
cpu_broadcast_xcall_simple_cbk(void * parm)324 cpu_broadcast_xcall_simple_cbk(void *parm)
325 {
326 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
327
328 data->func(data->parm);
329
330 if (os_atomic_dec(&data->sync, relaxed) == 0) {
331 thread_wakeup((event_t)&data->sync);
332 }
333 }
334
335 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)336 cpu_xcall_simple(boolean_t self_xcall,
337 broadcastFunc func,
338 void *parm,
339 bool immediate)
340 {
341 struct cpu_broadcast_xcall_simple_data data = {};
342
343 data.func = func;
344 data.parm = parm;
345
346 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
347 }
348
349 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)350 cpu_broadcast_xcall_simple(boolean_t self_xcall,
351 broadcastFunc func,
352 void *parm)
353 {
354 return cpu_xcall_simple(self_xcall, func, parm, false);
355 }
356
357 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)358 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
359 {
360 cpu_data_t *target_cpu_datap;
361
362 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
363 panic("cpu_xcall_internal: invalid cpu_number %d", cpu_number);
364 }
365
366 if (func == NULL || param == NULL) {
367 // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready
368 panic("cpu_xcall_internal: cannot have null func/param: %p %p", func, param);
369 }
370
371 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
372 if (target_cpu_datap == NULL) {
373 panic("cpu_xcall_internal: cpu %d not initialized", cpu_number);
374 }
375
376 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
377 }
378
379 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)380 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
381 {
382 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
383 }
384
385 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)386 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
387 {
388 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
389 }
390
391 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1,boolean_t defer)392 cpu_signal_internal(cpu_data_t *target_proc,
393 cpu_signal_t signal,
394 void *p0,
395 void *p1,
396 boolean_t defer)
397 {
398 cpu_signal_t current_signals;
399 bool swap_success;
400 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
401 cpu_data_t *current_proc = getCpuDatap();
402
403 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
404 if (defer) {
405 assert(signal == SIGPnop || signal == SIGPdeferred);
406 }
407
408 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
409 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
410 current_mabs_time = start_mabs_time = mach_absolute_time();
411 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
412 assert(max_mabs_time > current_mabs_time);
413
414 do {
415 current_signals = target_proc->cpu_signal;
416 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
417 ml_set_interrupts_enabled(interruptible);
418 return KERN_FAILURE;
419 }
420 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals & (~signal),
421 current_signals | signal, release);
422
423 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
424 ml_set_interrupts_enabled(interruptible);
425 return KERN_ALREADY_WAITING;
426 }
427
428 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
429 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
430 * so break the deadlock by draining pending xcalls. */
431 if (!swap_success && (current_proc->cpu_signal & signal)) {
432 cpu_handle_xcall(current_proc);
433 }
434 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
435
436 /*
437 * If we time out while waiting for the target CPU to respond, it's possible that no
438 * other CPU is available to handle the watchdog interrupt that would eventually trigger
439 * a panic. To prevent this from happening, we just panic here to flag this condition.
440 */
441 if (__improbable(current_mabs_time >= max_mabs_time)) {
442 uint64_t end_time_ns, xcall_ack_timeout_ns;
443 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
444 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
445 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
446 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
447 }
448
449 if (signal == SIGPxcallImm) {
450 target_proc->cpu_imm_xcall_p0 = p0;
451 target_proc->cpu_imm_xcall_p1 = p1;
452 } else {
453 target_proc->cpu_xcall_p0 = p0;
454 target_proc->cpu_xcall_p1 = p1;
455 }
456 } else {
457 do {
458 current_signals = target_proc->cpu_signal;
459 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
460 if (current_proc == target_proc) {
461 panic("cpu_signal of self while signals are disabled");
462 }
463 ml_set_interrupts_enabled(interruptible);
464 return KERN_FAILURE;
465 }
466
467 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals,
468 current_signals | signal, release);
469 } while (!swap_success);
470 }
471
472 /*
473 * DSB is needed here to ensure prior stores to the pending signal mask and xcall params
474 * will be visible by the time the other cores are signaled. The IPI mechanism on any
475 * given platform will very likely use either an MSR or a non-coherent store that would
476 * not be ordered by a simple DMB.
477 */
478 __builtin_arm_dsb(DSB_ISHST);
479
480 if (!(target_proc->cpu_signal & SIGPdisabled)) {
481 /* Make sure cpu_phys_id is actually initialized */
482 assert3u(os_atomic_load(&target_proc->cpu_flags, relaxed) & (InitState | StartedState),
483 ==, InitState | StartedState);
484
485 if (defer) {
486 #if defined(HAS_IPI)
487 if (gFastIPI) {
488 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
489 } else {
490 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
491 }
492 #else
493 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
494 #endif /* defined(HAS_IPI) */
495 } else {
496 #if defined(HAS_IPI)
497 if (gFastIPI) {
498 ml_cpu_signal(target_proc->cpu_phys_id);
499 } else {
500 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
501 }
502 #else
503 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
504 #endif /* defined(HAS_IPI) */
505 }
506 }
507
508 ml_set_interrupts_enabled(interruptible);
509 return KERN_SUCCESS;
510 }
511
512 kern_return_t
cpu_signal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1)513 cpu_signal(cpu_data_t *target_proc,
514 cpu_signal_t signal,
515 void *p0,
516 void *p1)
517 {
518 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
519 }
520
521 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc,cpu_signal_t signal)522 cpu_signal_deferred(cpu_data_t *target_proc, cpu_signal_t signal)
523 {
524 return cpu_signal_internal(target_proc, signal, NULL, NULL, TRUE);
525 }
526
527 void
cpu_signal_cancel(cpu_data_t * target_proc,cpu_signal_t signal)528 cpu_signal_cancel(cpu_data_t *target_proc, cpu_signal_t signal)
529 {
530 cpu_signal_t current_signals;
531
532 current_signals = os_atomic_andnot(&target_proc->cpu_signal, signal, acq_rel);
533
534
535 if (!(current_signals & SIGPdisabled)) {
536 #if defined(HAS_IPI)
537 if (gFastIPI) {
538 ml_cpu_signal_retract(target_proc->cpu_phys_id);
539 } else {
540 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
541 }
542 #else
543 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
544 #endif /* defined(HAS_IPI) */
545 }
546 }
547
548 void
cpu_signal_handler(void)549 cpu_signal_handler(void)
550 {
551 cpu_signal_handler_internal(FALSE);
552 }
553
554 bool
cpu_has_SIGPdebug_pending(void)555 cpu_has_SIGPdebug_pending(void)
556 {
557 cpu_data_t *cpu_data_ptr = getCpuDatap();
558
559 return cpu_data_ptr->cpu_signal & SIGPdebug;
560 }
561
562 void
cpu_signal_handler_internal(boolean_t disable_signal)563 cpu_signal_handler_internal(boolean_t disable_signal)
564 {
565 cpu_data_t *cpu_data_ptr = getCpuDatap();
566
567 cpu_data_ptr->cpu_stat.ipi_cnt++;
568 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
569 SCHED_STATS_INC(ipi_count);
570
571 /*
572 * Employ an acquire barrier when loading cpu_signal to ensure that
573 * loads within individual signal handlers won't be speculated ahead
574 * of the load of cpu_signal. This pairs with the release barrier
575 * in cpu_signal_internal() to ensure that once a flag has been set in
576 * the cpu_signal mask, any prerequisite setup is also visible to signal
577 * handlers.
578 */
579 cpu_signal_t cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
580
581 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
582 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
583 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
584 /* We must not clear SIGPdisabled unless the CPU is properly started */
585 assert3u(os_atomic_load(&cpu_data_ptr->cpu_flags, relaxed) & (InitState | StartedState),
586 ==, InitState | StartedState);
587 cpu_signal = os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
588 }
589
590 while (cpu_signal & ~SIGPdisabled) {
591 if (cpu_signal & SIGPdebug) {
592 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, acquire);
593 INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI);
594 DebuggerXCall(cpu_data_ptr->cpu_int_state);
595 INTERRUPT_MASKED_DEBUG_END();
596 }
597 #if KPERF
598 if (cpu_signal & SIGPkppet) {
599 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, acquire);
600 extern void kperf_signal_handler(void);
601 INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI);
602 kperf_signal_handler();
603 INTERRUPT_MASKED_DEBUG_END();
604 }
605 #endif /* KPERF */
606 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
607 cpu_handle_xcall(cpu_data_ptr);
608 }
609 if (cpu_signal & SIGPast) {
610 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, acquire);
611 INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI);
612 ast_check(current_processor());
613 INTERRUPT_MASKED_DEBUG_END();
614 }
615 if (cpu_signal & SIGPTimerLocal) {
616 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPTimerLocal, acquire);
617 INTERRUPT_MASKED_DEBUG_START(timer_queue_expire_local, DBG_INTR_TYPE_IPI);
618 timer_queue_expire_local(current_processor());
619 INTERRUPT_MASKED_DEBUG_END();
620 }
621 if (cpu_signal & SIGPdeferred) {
622 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdeferred, acquire);
623 }
624
625 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
626 }
627 }
628
629 void
cpu_exit_wait(int cpu_id)630 cpu_exit_wait(int cpu_id)
631 {
632 #if !APPLEVIRTUALPLATFORM /* AVP doesn't provide an equivalent poll-for-powerdown operation */
633 #if USE_APPLEARMSMP
634 if (!ml_is_quiescing()) {
635 // For runtime disable (non S2R) the CPU will shut down immediately.
636 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
637 assert(cpu && cpu->cpu_IMPL_regs);
638 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
639
640 // Poll the "CPU running state" field until it is 0 (off)
641 // This loop typically finishes in about 600ns. Sometimes it takes as long as 10us.
642 // If it takes longer than 10s, assume something went horribly wrong and panic.
643 uint64_t start = mach_absolute_time(), interval;
644 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
645
646 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
647 __builtin_arm_dsb(DSB_ISH);
648 if (mach_absolute_time() > start + interval) {
649 #if NO_CPU_OVRD
650 // On platforms where CPU_OVRD is unavailable, a core can get stuck
651 // in a loop where it tries to enter WFI but is constantly woken up
652 // by an IRQ or FIQ. This condition persists until the cluster-wide
653 // deep sleep bits are set.
654 //
655 // Making this a fatal condition would be a poor UX, but it's good to
656 // print a warning so we know how often it happens.
657 kprintf("CPU%d failed to shut down\n", cpu_id);
658 #else
659 panic("CPU%d failed to shut down", cpu_id);
660 #endif
661 }
662 }
663 }
664 #endif /* USE_APPLEARMSMP */
665 #endif /* !APPLEVIRTUALPLATFORM */
666
667 if (cpu_id != master_cpu || (support_bootcpu_shutdown && !ml_is_quiescing())) {
668 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
669 cpu_data_t *cpu_data_ptr;
670
671 uint64_t start = mach_absolute_time(), interval;
672 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
673
674 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
675 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
676 if (mach_absolute_time() > start + interval) {
677 panic("CPU %d failed to reach ARM_CPU_ON_SLEEP_PATH: %d", cpu_id, cpu_data_ptr->cpu_sleep_token);
678 }
679 }
680 }
681 }
682
683 void
cpu_machine_init(void)684 cpu_machine_init(void)
685 {
686 cpu_data_t *cpu_data_ptr = getCpuDatap();
687
688 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
689 platform_cache_init();
690 }
691
692 bool started = os_atomic_or_orig(&cpu_data_ptr->cpu_flags, StartedState, relaxed) & StartedState;
693
694 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
695 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
696
697 ml_init_interrupt();
698 }
699
700 processor_t
current_processor(void)701 current_processor(void)
702 {
703 return PERCPU_GET(processor);
704 }
705
706 processor_t
cpu_to_processor(int cpu)707 cpu_to_processor(int cpu)
708 {
709 cpu_data_t *cpu_data = cpu_datap(cpu);
710 if (cpu_data != NULL) {
711 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
712 } else {
713 return NULL;
714 }
715 }
716
717 cpu_data_t *
processor_to_cpu_datap(processor_t processor)718 processor_to_cpu_datap(processor_t processor)
719 {
720 assert(processor->cpu_id <= ml_get_max_cpu_number());
721 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
722
723 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
724 }
725
726 __startup_func
727 static void
cpu_data_startup_init(void)728 cpu_data_startup_init(void)
729 {
730 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
731
732 percpu_base.size = percpu_section_size();
733 if (ml_get_cpu_count() == 1) {
734 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
735 return;
736 }
737
738 /*
739 * The memory needs to be physically contiguous because it contains
740 * cpu_data_t structures sometimes accessed during reset
741 * with the MMU off.
742 *
743 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
744 * normally runs, so we instead steal the memory for the PERCPU subsystem
745 * even earlier.
746 */
747 percpu_base.start = (vm_offset_t)pmap_steal_memory(size, PAGE_SIZE);
748 bzero((void *)percpu_base.start, size);
749
750 percpu_base.start -= percpu_section_start();
751 percpu_base.end = percpu_base.start + size - 1;
752 percpu_base_cur = percpu_base.start;
753 }
754 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
755
756 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)757 cpu_data_alloc(boolean_t is_boot_cpu)
758 {
759 cpu_data_t *cpu_data_ptr = NULL;
760 vm_address_t base;
761
762 if (is_boot_cpu) {
763 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
764 } else {
765 base = os_atomic_add_orig(&percpu_base_cur,
766 percpu_section_size(), relaxed);
767
768 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
769 cpu_stack_alloc(cpu_data_ptr);
770 }
771
772 return cpu_data_ptr;
773 }
774
775 ast_t *
ast_pending(void)776 ast_pending(void)
777 {
778 return &getCpuDatap()->cpu_pending_ast;
779 }
780
781 cpu_type_t
slot_type(int slot_num)782 slot_type(int slot_num)
783 {
784 return cpu_datap(slot_num)->cpu_type;
785 }
786
787 cpu_subtype_t
slot_subtype(int slot_num)788 slot_subtype(int slot_num)
789 {
790 return cpu_datap(slot_num)->cpu_subtype;
791 }
792
793 cpu_threadtype_t
slot_threadtype(int slot_num)794 slot_threadtype(int slot_num)
795 {
796 return cpu_datap(slot_num)->cpu_threadtype;
797 }
798
799 cpu_type_t
cpu_type(void)800 cpu_type(void)
801 {
802 return getCpuDatap()->cpu_type;
803 }
804
805 cpu_subtype_t
cpu_subtype(void)806 cpu_subtype(void)
807 {
808 return getCpuDatap()->cpu_subtype;
809 }
810
811 cpu_threadtype_t
cpu_threadtype(void)812 cpu_threadtype(void)
813 {
814 return getCpuDatap()->cpu_threadtype;
815 }
816
817 int
cpu_number(void)818 cpu_number(void)
819 {
820 return current_thread()->machine.cpu_number;
821 }
822
823 vm_offset_t
current_percpu_base(void)824 current_percpu_base(void)
825 {
826 long base = current_thread()->machine.pcpu_data_base_and_cpu_number;
827
828 return (vm_offset_t)(base >> 16);
829 }
830
831 vm_offset_t
other_percpu_base(int cpu)832 other_percpu_base(int cpu)
833 {
834 return (vm_address_t)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
835 }
836
837 uint64_t
ml_get_wake_timebase(void)838 ml_get_wake_timebase(void)
839 {
840 return wake_abstime;
841 }
842
843 /*
844 * Called while running on a specific CPU to wait for it to handle the
845 * self-IPI that clears SIGPdisabled.
846 */
847 void
ml_wait_for_cpu_signal_to_enable(void)848 ml_wait_for_cpu_signal_to_enable(void)
849 {
850 assert(ml_get_interrupts_enabled());
851 cpu_data_t *cpu_data_ptr = getCpuDatap();
852
853 hw_wait_while_equals32(__DEVOLATILE(uint32_t*, &cpu_data_ptr->cpu_signal), SIGPdisabled);
854 }
855
856 void
assert_ml_cpu_signal_is_enabled(bool enabled)857 assert_ml_cpu_signal_is_enabled(bool enabled)
858 {
859 if (enabled) {
860 assert((getCpuDatap()->cpu_signal & SIGPdisabled) == 0);
861 } else {
862 assert3u(getCpuDatap()->cpu_signal, ==, SIGPdisabled);
863 }
864 }
865
866 bool
ml_cpu_can_exit(__unused int cpu_id)867 ml_cpu_can_exit(__unused int cpu_id)
868 {
869 #if USE_APPLEARMSMP
870 /*
871 * Cyprus and newer chips can disable individual non-boot CPUs. The
872 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
873 * Until the feature is known to be stable, guard it with a boot-arg.
874 */
875
876 bool cpu_supported = true;
877
878 if (cpu_id == master_cpu && !support_bootcpu_shutdown) {
879 cpu_supported = false;
880 }
881
882 if (enable_processor_exit) {
883 return cpu_supported;
884 }
885
886 #if HAS_CLUSTER
887 if (ml_get_topology_info()->cluster_power_down) {
888 return cpu_supported;
889 }
890 #endif /* HAS_CLUSTER */
891
892 #endif /* USE_APPLEARMSMP */
893
894 return false;
895 }
896
897 #ifdef USE_APPLEARMSMP
898
899 void
ml_cpu_begin_state_transition(int cpu_id)900 ml_cpu_begin_state_transition(int cpu_id)
901 {
902 lck_rw_lock_exclusive(&cpu_state_lock);
903 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
904 lck_rw_unlock_exclusive(&cpu_state_lock);
905 }
906
907 void
ml_cpu_end_state_transition(int cpu_id)908 ml_cpu_end_state_transition(int cpu_id)
909 {
910 lck_rw_lock_exclusive(&cpu_state_lock);
911 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
912 lck_rw_unlock_exclusive(&cpu_state_lock);
913 }
914
915 void
ml_cpu_begin_loop(void)916 ml_cpu_begin_loop(void)
917 {
918 lck_rw_lock_shared(&cpu_state_lock);
919 }
920
921 void
ml_cpu_end_loop(void)922 ml_cpu_end_loop(void)
923 {
924 lck_rw_unlock_shared(&cpu_state_lock);
925 }
926
927 void
ml_cpu_power_enable(int cpu_id)928 ml_cpu_power_enable(int cpu_id)
929 {
930 PE_cpu_power_enable(cpu_id);
931 }
932
933 void
ml_cpu_power_disable(int cpu_id)934 ml_cpu_power_disable(int cpu_id)
935 {
936 PE_cpu_power_disable(cpu_id);
937 }
938
939 #else /* USE_APPLEARMSMP */
940
941 void
ml_cpu_begin_state_transition(__unused int cpu_id)942 ml_cpu_begin_state_transition(__unused int cpu_id)
943 {
944 }
945
946 void
ml_cpu_end_state_transition(__unused int cpu_id)947 ml_cpu_end_state_transition(__unused int cpu_id)
948 {
949 }
950
951 void
ml_cpu_begin_loop(void)952 ml_cpu_begin_loop(void)
953 {
954 }
955
956 void
ml_cpu_end_loop(void)957 ml_cpu_end_loop(void)
958 {
959 }
960
961 void
ml_cpu_power_enable(__unused int cpu_id)962 ml_cpu_power_enable(__unused int cpu_id)
963 {
964 }
965
966 void
ml_cpu_power_disable(__unused int cpu_id)967 ml_cpu_power_disable(__unused int cpu_id)
968 {
969 }
970
971 #endif /* USE_APPLEARMSMP */
972