1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <kern/clock.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <mach/processor_info.h>
50 #include <machine/atomic.h>
51 #include <machine/config.h>
52 #include <vm/vm_kern_xnu.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58 #include <arm64/proc_reg.h>
59 #include <libkern/OSAtomic.h>
60
61 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
62 vm_address_t percpu_base_cur;
63 cpu_data_t PERCPU_DATA(cpu_data);
64 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
65
66 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
67 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
68 static LCK_MTX_DECLARE(cpu_xcall_mtx, &cpu_lck_grp);
69
70 unsigned int real_ncpus = 1;
71 boolean_t idle_enable = FALSE;
72 uint64_t wake_abstime = 0x0ULL;
73
74 extern uint64_t xcall_ack_timeout_abstime;
75
76 #if defined(HAS_IPI)
77 extern unsigned int gFastIPI;
78 #endif /* defined(HAS_IPI) */
79
80 cpu_data_t *
cpu_datap(int cpu)81 cpu_datap(int cpu)
82 {
83 assert(cpu <= ml_get_max_cpu_number());
84 return CpuDataEntries[cpu].cpu_data_vaddr;
85 }
86
87 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)88 cpu_control(int slot_num,
89 processor_info_t info,
90 unsigned int count)
91 {
92 printf("cpu_control(%d,%p,%d) not implemented\n",
93 slot_num, info, count);
94 return KERN_FAILURE;
95 }
96
97 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)98 cpu_info_count(processor_flavor_t flavor,
99 unsigned int *count)
100 {
101 switch (flavor) {
102 case PROCESSOR_CPU_STAT:
103 *count = PROCESSOR_CPU_STAT_COUNT;
104 return KERN_SUCCESS;
105
106 case PROCESSOR_CPU_STAT64:
107 *count = PROCESSOR_CPU_STAT64_COUNT;
108 return KERN_SUCCESS;
109
110 default:
111 *count = 0;
112 return KERN_FAILURE;
113 }
114 }
115
116 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)117 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
118 unsigned int *count)
119 {
120 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
121
122 switch (flavor) {
123 case PROCESSOR_CPU_STAT:
124 {
125 if (*count < PROCESSOR_CPU_STAT_COUNT) {
126 return KERN_FAILURE;
127 }
128
129 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
130 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
131 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
132 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
133 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
134 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
135 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
136 cpu_stat->vfp_shortv_cnt = 0;
137 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
138 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
139
140 *count = PROCESSOR_CPU_STAT_COUNT;
141
142 return KERN_SUCCESS;
143 }
144
145 case PROCESSOR_CPU_STAT64:
146 {
147 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
148 return KERN_FAILURE;
149 }
150
151 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
152 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
153 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
154 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
155 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
156 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
157 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
158 cpu_stat->vfp_shortv_cnt = 0;
159 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
160 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
161 #if CONFIG_CPU_COUNTERS
162 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
163 #endif /* CONFIG_CPU_COUNTERS */
164
165 *count = PROCESSOR_CPU_STAT64_COUNT;
166
167 return KERN_SUCCESS;
168 }
169
170 default:
171 return KERN_FAILURE;
172 }
173 }
174
175 /*
176 * Routine: cpu_doshutdown
177 * Function:
178 */
179 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)180 cpu_doshutdown(void (*doshutdown)(processor_t),
181 processor_t processor)
182 {
183 doshutdown(processor);
184 }
185
186 /*
187 * Routine: cpu_idle_tickle
188 *
189 */
190 void
cpu_idle_tickle(void)191 cpu_idle_tickle(void)
192 {
193 boolean_t intr;
194 cpu_data_t *cpu_data_ptr;
195 uint64_t new_idle_timeout_ticks = 0x0ULL;
196
197 intr = ml_set_interrupts_enabled(FALSE);
198 cpu_data_ptr = getCpuDatap();
199
200 if (cpu_data_ptr->idle_timer_notify != NULL) {
201 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
202 if (new_idle_timeout_ticks != 0x0ULL) {
203 /* if a new idle timeout was requested set the new idle timer deadline */
204 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
205 } else {
206 /* turn off the idle timer */
207 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
208 }
209 timer_resync_deadlines();
210 }
211 (void) ml_set_interrupts_enabled(intr);
212 }
213
214 /*
215 * Routine: cpu_set_perfcontrol_timer
216 *
217 */
218 void
cpu_set_perfcontrol_timer(uint64_t now,uint64_t timeout_ticks)219 cpu_set_perfcontrol_timer(uint64_t now, uint64_t timeout_ticks)
220 {
221 assert(ml_get_interrupts_enabled() == FALSE);
222 processor_t processor = current_processor();
223 if (timeout_ticks == EndOfAllTime) {
224 running_timer_cancel(processor, RUNNING_TIMER_PERFCONTROL);
225 } else {
226 uint64_t deadline = now + timeout_ticks;
227 running_timer_enter(processor, RUNNING_TIMER_PERFCONTROL, NULL, deadline, now);
228 }
229 }
230
231 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)232 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
233 {
234 broadcastFunc xfunc;
235 void *xparam;
236
237 os_atomic_thread_fence(acquire);
238 /* Come back around if cpu_signal_internal is running on another CPU and has just
239 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
240 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
241 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
242 ml_interrupt_masked_debug_start(xfunc, DBG_INTR_TYPE_IPI);
243 xparam = cpu_data_ptr->cpu_xcall_p1;
244 cpu_data_ptr->cpu_xcall_p0 = NULL;
245 cpu_data_ptr->cpu_xcall_p1 = NULL;
246 os_atomic_thread_fence(acq_rel);
247 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
248 xfunc(xparam);
249 ml_interrupt_masked_debug_end();
250 }
251 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
252 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
253 ml_interrupt_masked_debug_start(xfunc, DBG_INTR_TYPE_IPI);
254 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
255 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
256 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
257 os_atomic_thread_fence(acq_rel);
258 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
259 xfunc(xparam);
260 ml_interrupt_masked_debug_end();
261 }
262 }
263
264 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)265 cpu_broadcast_xcall_internal(unsigned int signal,
266 uint32_t *synch,
267 boolean_t self_xcall,
268 broadcastFunc func,
269 void *parm)
270 {
271 boolean_t intr;
272 cpu_data_t *cpu_data_ptr;
273 cpu_data_t *target_cpu_datap;
274 unsigned int failsig;
275 int cpu;
276 int max_cpu = ml_get_max_cpu_number() + 1;
277
278 //yes, param ALSO cannot be NULL
279 assert(synch);
280 assert(func);
281 assert(parm);
282
283 lck_mtx_lock(&cpu_xcall_mtx);
284 intr = ml_set_interrupts_enabled(FALSE);
285 cpu_data_ptr = getCpuDatap();
286
287 failsig = 0;
288
289 *synch = max_cpu;
290 assert_wait((event_t)synch, THREAD_UNINT);
291
292 for (cpu = 0; cpu < max_cpu; cpu++) {
293 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
294
295 if (target_cpu_datap == cpu_data_ptr) {
296 continue;
297 }
298
299 if ((target_cpu_datap == NULL) ||
300 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
301 failsig++;
302 }
303 }
304
305 if (self_xcall) {
306 func(parm);
307 }
308
309 (void) ml_set_interrupts_enabled(intr);
310
311 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
312 clear_wait(current_thread(), THREAD_AWAKENED);
313 } else {
314 thread_block(THREAD_CONTINUE_NULL);
315 }
316 lck_mtx_unlock(&cpu_xcall_mtx);
317
318 if (!self_xcall) {
319 return max_cpu - failsig - 1;
320 } else {
321 return max_cpu - failsig;
322 }
323 }
324
325 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)326 cpu_broadcast_xcall(uint32_t *synch,
327 boolean_t self_xcall,
328 broadcastFunc func,
329 void *parm)
330 {
331 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
332 }
333
334 struct cpu_broadcast_xcall_simple_data {
335 broadcastFunc func;
336 void* parm;
337 uint32_t sync;
338 };
339
340 static void
cpu_broadcast_xcall_simple_cbk(void * parm)341 cpu_broadcast_xcall_simple_cbk(void *parm)
342 {
343 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
344
345 data->func(data->parm);
346
347 if (os_atomic_dec(&data->sync, relaxed) == 0) {
348 thread_wakeup((event_t)&data->sync);
349 }
350 }
351
352 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)353 cpu_xcall_simple(boolean_t self_xcall,
354 broadcastFunc func,
355 void *parm,
356 bool immediate)
357 {
358 struct cpu_broadcast_xcall_simple_data data = {};
359
360 data.func = func;
361 data.parm = parm;
362
363 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
364 }
365
366 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)367 cpu_broadcast_xcall_simple(boolean_t self_xcall,
368 broadcastFunc func,
369 void *parm)
370 {
371 return cpu_xcall_simple(self_xcall, func, parm, false);
372 }
373
374 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)375 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
376 {
377 cpu_data_t *target_cpu_datap;
378
379 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
380 panic("cpu_xcall_internal: invalid cpu_number %d", cpu_number);
381 }
382
383 if (func == NULL || param == NULL) {
384 // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready
385 panic("cpu_xcall_internal: cannot have null func/param: %p %p", func, param);
386 }
387
388 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
389 if (target_cpu_datap == NULL) {
390 panic("cpu_xcall_internal: cpu %d not initialized", cpu_number);
391 }
392
393 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
394 }
395
396 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)397 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
398 {
399 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
400 }
401
402 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)403 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
404 {
405 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
406 }
407
408 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1,boolean_t defer)409 cpu_signal_internal(cpu_data_t *target_proc,
410 cpu_signal_t signal,
411 void *p0,
412 void *p1,
413 boolean_t defer)
414 {
415 cpu_signal_t current_signals;
416 bool swap_success;
417 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
418 cpu_data_t *current_proc = getCpuDatap();
419
420 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
421 if (defer) {
422 assert(signal == SIGPnop || signal == SIGPdeferred);
423 }
424
425 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
426 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
427 current_mabs_time = start_mabs_time = mach_absolute_time();
428 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
429 assert(max_mabs_time > current_mabs_time);
430
431 do {
432 current_signals = target_proc->cpu_signal;
433 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
434 ml_set_interrupts_enabled(interruptible);
435 return KERN_FAILURE;
436 }
437 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals & (~signal),
438 current_signals | signal, release);
439
440 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
441 ml_set_interrupts_enabled(interruptible);
442 return KERN_ALREADY_WAITING;
443 }
444
445 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
446 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
447 * so break the deadlock by draining pending xcalls. */
448 if (!swap_success && (current_proc->cpu_signal & signal)) {
449 cpu_handle_xcall(current_proc);
450 }
451 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
452
453 /*
454 * If we time out while waiting for the target CPU to respond, it's possible that no
455 * other CPU is available to handle the watchdog interrupt that would eventually trigger
456 * a panic. To prevent this from happening, we just panic here to flag this condition.
457 */
458 if (__improbable(current_mabs_time >= max_mabs_time)) {
459 uint64_t end_time_ns, xcall_ack_timeout_ns;
460 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
461 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
462 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
463 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
464 }
465
466 if (signal == SIGPxcallImm) {
467 target_proc->cpu_imm_xcall_p0 = p0;
468 target_proc->cpu_imm_xcall_p1 = p1;
469 } else {
470 target_proc->cpu_xcall_p0 = p0;
471 target_proc->cpu_xcall_p1 = p1;
472 }
473 } else {
474 do {
475 current_signals = target_proc->cpu_signal;
476 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
477 if (current_proc == target_proc) {
478 panic("cpu_signal of self while signals are disabled");
479 }
480 ml_set_interrupts_enabled(interruptible);
481 return KERN_FAILURE;
482 }
483
484 swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals,
485 current_signals | signal, release);
486 } while (!swap_success);
487 }
488
489 /*
490 * DSB is needed here to ensure prior stores to the pending signal mask and xcall params
491 * will be visible by the time the other cores are signaled. The IPI mechanism on any
492 * given platform will very likely use either an MSR or a non-coherent store that would
493 * not be ordered by a simple DMB.
494 */
495 __builtin_arm_dsb(DSB_ISHST);
496
497 if (!(target_proc->cpu_signal & SIGPdisabled)) {
498 /* Make sure cpu_phys_id is actually initialized */
499 assert3u(os_atomic_load(&target_proc->cpu_flags, relaxed) & (InitState | StartedState),
500 ==, InitState | StartedState);
501
502 if (defer) {
503 #if defined(HAS_IPI)
504 if (gFastIPI) {
505 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
506 } else {
507 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
508 }
509 #else
510 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
511 #endif /* defined(HAS_IPI) */
512 } else {
513 #if defined(HAS_IPI)
514 if (gFastIPI) {
515 ml_cpu_signal(target_proc->cpu_phys_id);
516 } else {
517 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
518 }
519 #else
520 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
521 #endif /* defined(HAS_IPI) */
522 }
523 }
524
525 ml_set_interrupts_enabled(interruptible);
526 return KERN_SUCCESS;
527 }
528
529 kern_return_t
cpu_signal(cpu_data_t * target_proc,cpu_signal_t signal,void * p0,void * p1)530 cpu_signal(cpu_data_t *target_proc,
531 cpu_signal_t signal,
532 void *p0,
533 void *p1)
534 {
535 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
536 }
537
538 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc,cpu_signal_t signal)539 cpu_signal_deferred(cpu_data_t *target_proc, cpu_signal_t signal)
540 {
541 return cpu_signal_internal(target_proc, signal, NULL, NULL, TRUE);
542 }
543
544 void
cpu_signal_cancel(cpu_data_t * target_proc,cpu_signal_t signal)545 cpu_signal_cancel(cpu_data_t *target_proc, cpu_signal_t signal)
546 {
547 cpu_signal_t current_signals;
548
549 current_signals = os_atomic_andnot(&target_proc->cpu_signal, signal, acq_rel);
550
551
552 if (!(current_signals & SIGPdisabled)) {
553 #if defined(HAS_IPI)
554 if (gFastIPI) {
555 ml_cpu_signal_retract(target_proc->cpu_phys_id);
556 } else {
557 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
558 }
559 #else
560 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
561 #endif /* defined(HAS_IPI) */
562 }
563 }
564
565 void
cpu_signal_handler(void)566 cpu_signal_handler(void)
567 {
568 cpu_signal_handler_internal(FALSE);
569 }
570
571 bool
cpu_has_SIGPdebug_pending(void)572 cpu_has_SIGPdebug_pending(void)
573 {
574 cpu_data_t *cpu_data_ptr = getCpuDatap();
575
576 return cpu_data_ptr->cpu_signal & SIGPdebug;
577 }
578
579 void
cpu_signal_handler_internal(boolean_t disable_signal)580 cpu_signal_handler_internal(boolean_t disable_signal)
581 {
582 cpu_data_t *cpu_data_ptr = getCpuDatap();
583
584 cpu_data_ptr->cpu_stat.ipi_cnt++;
585 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
586 SCHED_STATS_INC(ipi_count);
587
588 /*
589 * Employ an acquire barrier when loading cpu_signal to ensure that
590 * loads within individual signal handlers won't be speculated ahead
591 * of the load of cpu_signal. This pairs with the release barrier
592 * in cpu_signal_internal() to ensure that once a flag has been set in
593 * the cpu_signal mask, any prerequisite setup is also visible to signal
594 * handlers.
595 */
596 cpu_signal_t cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
597
598 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
599 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
600 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
601 /* We must not clear SIGPdisabled unless the CPU is properly started */
602 assert3u(os_atomic_load(&cpu_data_ptr->cpu_flags, relaxed) & (InitState | StartedState),
603 ==, InitState | StartedState);
604 cpu_signal = os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, acq_rel);
605 }
606
607 while (cpu_signal & ~SIGPdisabled) {
608 if (cpu_signal & SIGPdebug) {
609 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, acquire);
610 ml_interrupt_masked_debug_start(DebuggerXCall, DBG_INTR_TYPE_IPI);
611 DebuggerXCall(cpu_data_ptr->cpu_int_state);
612 ml_interrupt_masked_debug_end();
613 }
614 #if KPERF
615 if (cpu_signal & SIGPkppet) {
616 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, acquire);
617 extern void kperf_signal_handler(void);
618 ml_interrupt_masked_debug_start(kperf_signal_handler, DBG_INTR_TYPE_IPI);
619 kperf_signal_handler();
620 ml_interrupt_masked_debug_end();
621 }
622 #endif /* KPERF */
623 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
624 cpu_handle_xcall(cpu_data_ptr);
625 }
626 if (cpu_signal & SIGPast) {
627 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, acquire);
628 ml_interrupt_masked_debug_start(ast_check, DBG_INTR_TYPE_IPI);
629 ast_check(current_processor());
630 ml_interrupt_masked_debug_end();
631 }
632 if (cpu_signal & SIGPTimerLocal) {
633 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPTimerLocal, acquire);
634 ml_interrupt_masked_debug_start(timer_queue_expire_local, DBG_INTR_TYPE_IPI);
635 timer_queue_expire_local(current_processor());
636 ml_interrupt_masked_debug_end();
637 }
638 if (cpu_signal & SIGPdeferred) {
639 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdeferred, acquire);
640 }
641
642 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire);
643 }
644 }
645
646 void
cpu_exit_wait(int cpu_id)647 cpu_exit_wait(int cpu_id)
648 {
649 #if !APPLEVIRTUALPLATFORM /* AVP doesn't provide an equivalent poll-for-powerdown operation */
650 #if USE_APPLEARMSMP
651 if (!ml_is_quiescing()) {
652 // For runtime disable (non S2R) the CPU will shut down immediately.
653 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
654 assert(cpu && cpu->cpu_IMPL_regs);
655 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
656
657 // Poll the "CPU running state" field until it is 0 (off)
658 // This loop typically finishes in about 600ns. Sometimes it takes as long as 10us.
659 // If it takes longer than 10s, assume something went horribly wrong and panic.
660 uint64_t start = mach_absolute_time(), interval;
661 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
662
663 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
664 __builtin_arm_dsb(DSB_ISH);
665 if (mach_absolute_time() > start + interval) {
666 #if NO_CPU_OVRD
667 // On platforms where CPU_OVRD is unavailable, a core can get stuck
668 // in a loop where it tries to enter WFI but is constantly woken up
669 // by an IRQ or FIQ. This condition persists until the cluster-wide
670 // deep sleep bits are set.
671 //
672 // Making this a fatal condition would be a poor UX, but it's good to
673 // print a warning so we know how often it happens.
674 kprintf("CPU%d failed to shut down\n", cpu_id);
675 #else
676 panic("CPU%d failed to shut down", cpu_id);
677 #endif
678 }
679 }
680 }
681 #endif /* USE_APPLEARMSMP */
682 #endif /* !APPLEVIRTUALPLATFORM */
683
684 if (cpu_id != master_cpu || (support_bootcpu_shutdown && !ml_is_quiescing())) {
685 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
686 cpu_data_t *cpu_data_ptr;
687
688 uint64_t start = mach_absolute_time(), interval;
689 nanoseconds_to_absolutetime(10 * NSEC_PER_SEC, &interval);
690
691 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
692 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
693 if (mach_absolute_time() > start + interval) {
694 panic("CPU %d failed to reach ARM_CPU_ON_SLEEP_PATH: %d", cpu_id, cpu_data_ptr->cpu_sleep_token);
695 }
696 }
697 }
698 }
699
700 void
cpu_machine_init(void)701 cpu_machine_init(void)
702 {
703 cpu_data_t *cpu_data_ptr = getCpuDatap();
704
705 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
706 platform_cache_init();
707 }
708
709 bool started = os_atomic_or_orig(&cpu_data_ptr->cpu_flags, StartedState, relaxed) & StartedState;
710
711 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
712 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
713
714 ml_init_interrupt();
715 }
716
717 processor_t
current_processor(void)718 current_processor(void)
719 {
720 return PERCPU_GET(processor);
721 }
722
723 processor_t
cpu_to_processor(int cpu)724 cpu_to_processor(int cpu)
725 {
726 cpu_data_t *cpu_data = cpu_datap(cpu);
727 if (cpu_data != NULL) {
728 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
729 } else {
730 return NULL;
731 }
732 }
733
734 cpu_data_t *
processor_to_cpu_datap(processor_t processor)735 processor_to_cpu_datap(processor_t processor)
736 {
737 assert(processor->cpu_id <= ml_get_max_cpu_number());
738 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
739
740 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
741 }
742
743 __startup_func
744 static void
cpu_data_startup_init(void)745 cpu_data_startup_init(void)
746 {
747 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
748
749 percpu_base.size = percpu_section_size();
750 if (ml_get_cpu_count() == 1) {
751 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
752 return;
753 }
754
755 /*
756 * The memory needs to be physically contiguous because it contains
757 * cpu_data_t structures sometimes accessed during reset
758 * with the MMU off.
759 *
760 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
761 * normally runs, so we instead steal the memory for the PERCPU subsystem
762 * even earlier.
763 */
764 percpu_base.start = (vm_offset_t)pmap_steal_memory(size, PAGE_SIZE);
765 bzero((void *)percpu_base.start, size);
766
767 percpu_base.start -= percpu_section_start();
768 percpu_base.end = percpu_base.start + size - 1;
769 percpu_base_cur = percpu_base.start;
770 }
771 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
772
773 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)774 cpu_data_alloc(boolean_t is_boot_cpu)
775 {
776 cpu_data_t *cpu_data_ptr = NULL;
777 vm_address_t base;
778
779 if (is_boot_cpu) {
780 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
781 } else {
782 base = os_atomic_add_orig(&percpu_base_cur,
783 percpu_section_size(), relaxed);
784
785 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
786 cpu_stack_alloc(cpu_data_ptr);
787 }
788
789 return cpu_data_ptr;
790 }
791
792 ast_t *
ast_pending(void)793 ast_pending(void)
794 {
795 return &getCpuDatap()->cpu_pending_ast;
796 }
797
798 cpu_type_t
slot_type(int slot_num)799 slot_type(int slot_num)
800 {
801 return cpu_datap(slot_num)->cpu_type;
802 }
803
804 cpu_subtype_t
slot_subtype(int slot_num)805 slot_subtype(int slot_num)
806 {
807 return cpu_datap(slot_num)->cpu_subtype;
808 }
809
810 cpu_threadtype_t
slot_threadtype(int slot_num)811 slot_threadtype(int slot_num)
812 {
813 return cpu_datap(slot_num)->cpu_threadtype;
814 }
815
816 cpu_type_t
cpu_type(void)817 cpu_type(void)
818 {
819 return getCpuDatap()->cpu_type;
820 }
821
822 cpu_subtype_t
cpu_subtype(void)823 cpu_subtype(void)
824 {
825 return getCpuDatap()->cpu_subtype;
826 }
827
828 cpu_threadtype_t
cpu_threadtype(void)829 cpu_threadtype(void)
830 {
831 return getCpuDatap()->cpu_threadtype;
832 }
833
834 int
cpu_number(void)835 cpu_number(void)
836 {
837 return current_thread()->machine.cpu_number;
838 }
839
840 vm_offset_t
current_percpu_base(void)841 current_percpu_base(void)
842 {
843 long base = current_thread()->machine.pcpu_data_base_and_cpu_number;
844
845 return (vm_offset_t)(base >> 16);
846 }
847
848 vm_offset_t
other_percpu_base(int cpu)849 other_percpu_base(int cpu)
850 {
851 return (vm_address_t)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
852 }
853
854 uint64_t
ml_get_wake_timebase(void)855 ml_get_wake_timebase(void)
856 {
857 return wake_abstime;
858 }
859
860 /*
861 * Called while running on a specific CPU to wait for it to handle the
862 * self-IPI that clears SIGPdisabled.
863 */
864 void
ml_wait_for_cpu_signal_to_enable(void)865 ml_wait_for_cpu_signal_to_enable(void)
866 {
867 assert(ml_get_interrupts_enabled());
868 cpu_data_t *cpu_data_ptr = getCpuDatap();
869
870 hw_wait_while_equals32(__DEVOLATILE(uint32_t*, &cpu_data_ptr->cpu_signal), SIGPdisabled);
871 }
872
873 void
assert_ml_cpu_signal_is_enabled(bool enabled)874 assert_ml_cpu_signal_is_enabled(bool enabled)
875 {
876 if (enabled) {
877 assert((getCpuDatap()->cpu_signal & SIGPdisabled) == 0);
878 } else {
879 assert3u(getCpuDatap()->cpu_signal, ==, SIGPdisabled);
880 }
881 }
882
883 bool
ml_cpu_can_exit(__unused int cpu_id)884 ml_cpu_can_exit(__unused int cpu_id)
885 {
886 #if USE_APPLEARMSMP
887 /*
888 * Cyprus and newer chips can disable individual non-boot CPUs. The
889 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
890 * Until the feature is known to be stable, guard it with a boot-arg.
891 */
892
893 bool cpu_supported = true;
894
895 if (cpu_id == master_cpu && !support_bootcpu_shutdown) {
896 cpu_supported = false;
897 }
898
899 if (enable_processor_exit) {
900 return cpu_supported;
901 }
902
903 #if HAS_CLUSTER
904 if (ml_get_topology_info()->cluster_power_down) {
905 return cpu_supported;
906 }
907 #endif /* HAS_CLUSTER */
908
909 #endif /* USE_APPLEARMSMP */
910
911 return false;
912 }
913
914 #ifdef USE_APPLEARMSMP
915
916 void
ml_cpu_begin_state_transition(int cpu_id)917 ml_cpu_begin_state_transition(int cpu_id)
918 {
919 lck_rw_lock_exclusive(&cpu_state_lock);
920 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
921 lck_rw_unlock_exclusive(&cpu_state_lock);
922 }
923
924 void
ml_cpu_end_state_transition(int cpu_id)925 ml_cpu_end_state_transition(int cpu_id)
926 {
927 lck_rw_lock_exclusive(&cpu_state_lock);
928 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
929 lck_rw_unlock_exclusive(&cpu_state_lock);
930 }
931
932 void
ml_cpu_begin_loop(void)933 ml_cpu_begin_loop(void)
934 {
935 lck_rw_lock_shared(&cpu_state_lock);
936 }
937
938 void
ml_cpu_end_loop(void)939 ml_cpu_end_loop(void)
940 {
941 lck_rw_unlock_shared(&cpu_state_lock);
942 }
943
944 void
ml_cpu_power_enable(int cpu_id)945 ml_cpu_power_enable(int cpu_id)
946 {
947 PE_cpu_power_enable(cpu_id);
948 }
949
950 void
ml_cpu_power_disable(int cpu_id)951 ml_cpu_power_disable(int cpu_id)
952 {
953 PE_cpu_power_disable(cpu_id);
954 }
955
956 #else /* USE_APPLEARMSMP */
957
958 void
ml_cpu_begin_state_transition(__unused int cpu_id)959 ml_cpu_begin_state_transition(__unused int cpu_id)
960 {
961 }
962
963 void
ml_cpu_end_state_transition(__unused int cpu_id)964 ml_cpu_end_state_transition(__unused int cpu_id)
965 {
966 }
967
968 void
ml_cpu_begin_loop(void)969 ml_cpu_begin_loop(void)
970 {
971 }
972
973 void
ml_cpu_end_loop(void)974 ml_cpu_end_loop(void)
975 {
976 }
977
978 void
ml_cpu_power_enable(__unused int cpu_id)979 ml_cpu_power_enable(__unused int cpu_id)
980 {
981 }
982
983 void
ml_cpu_power_disable(__unused int cpu_id)984 ml_cpu_power_disable(__unused int cpu_id)
985 {
986 }
987
988 #endif /* USE_APPLEARMSMP */
989