1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <arm/cpu_data.h>
41 #include <arm/cpuid.h>
42 #include <arm/caches_internal.h>
43 #include <arm/cpu_data_internal.h>
44 #include <arm/cpu_internal.h>
45 #include <arm/misc_protos.h>
46 #include <arm/machine_cpu.h>
47 #include <arm/rtclock.h>
48 #include <mach/processor_info.h>
49 #include <machine/atomic.h>
50 #include <machine/config.h>
51 #include <vm/vm_kern.h>
52 #include <vm/vm_map.h>
53 #include <pexpert/arm/protos.h>
54 #include <pexpert/device_tree.h>
55 #include <sys/kdebug.h>
56 #include <arm/machine_routines.h>
57 #include <arm/proc_reg.h>
58 #include <libkern/OSAtomic.h>
59
60 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
61 vm_address_t percpu_base_cur;
62 cpu_data_t PERCPU_DATA(cpu_data);
63 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
64
65 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
66 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
67
68 unsigned int real_ncpus = 1;
69 boolean_t idle_enable = FALSE;
70 uint64_t wake_abstime = 0x0ULL;
71
72 extern uint64_t xcall_ack_timeout_abstime;
73
74 #if defined(HAS_IPI)
75 extern unsigned int gFastIPI;
76 #endif /* defined(HAS_IPI) */
77
78 cpu_data_t *
cpu_datap(int cpu)79 cpu_datap(int cpu)
80 {
81 assert(cpu <= ml_get_max_cpu_number());
82 return CpuDataEntries[cpu].cpu_data_vaddr;
83 }
84
85 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)86 cpu_control(int slot_num,
87 processor_info_t info,
88 unsigned int count)
89 {
90 printf("cpu_control(%d,%p,%d) not implemented\n",
91 slot_num, info, count);
92 return KERN_FAILURE;
93 }
94
95 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)96 cpu_info_count(processor_flavor_t flavor,
97 unsigned int *count)
98 {
99 switch (flavor) {
100 case PROCESSOR_CPU_STAT:
101 *count = PROCESSOR_CPU_STAT_COUNT;
102 return KERN_SUCCESS;
103
104 case PROCESSOR_CPU_STAT64:
105 *count = PROCESSOR_CPU_STAT64_COUNT;
106 return KERN_SUCCESS;
107
108 default:
109 *count = 0;
110 return KERN_FAILURE;
111 }
112 }
113
114 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)115 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
116 unsigned int *count)
117 {
118 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
119
120 switch (flavor) {
121 case PROCESSOR_CPU_STAT:
122 {
123 if (*count < PROCESSOR_CPU_STAT_COUNT) {
124 return KERN_FAILURE;
125 }
126
127 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
128 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
129 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
130 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
131 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
132 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
133 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
134 cpu_stat->vfp_shortv_cnt = 0;
135 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
136 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
137
138 *count = PROCESSOR_CPU_STAT_COUNT;
139
140 return KERN_SUCCESS;
141 }
142
143 case PROCESSOR_CPU_STAT64:
144 {
145 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
146 return KERN_FAILURE;
147 }
148
149 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
150 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
151 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
152 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
153 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
154 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
155 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
156 cpu_stat->vfp_shortv_cnt = 0;
157 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
158 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
159 #if MONOTONIC
160 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
161 #endif /* MONOTONIC */
162
163 *count = PROCESSOR_CPU_STAT64_COUNT;
164
165 return KERN_SUCCESS;
166 }
167
168 default:
169 return KERN_FAILURE;
170 }
171 }
172
173 /*
174 * Routine: cpu_doshutdown
175 * Function:
176 */
177 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)178 cpu_doshutdown(void (*doshutdown)(processor_t),
179 processor_t processor)
180 {
181 doshutdown(processor);
182 }
183
184 /*
185 * Routine: cpu_idle_tickle
186 *
187 */
188 void
cpu_idle_tickle(void)189 cpu_idle_tickle(void)
190 {
191 boolean_t intr;
192 cpu_data_t *cpu_data_ptr;
193 uint64_t new_idle_timeout_ticks = 0x0ULL;
194
195 intr = ml_set_interrupts_enabled(FALSE);
196 cpu_data_ptr = getCpuDatap();
197
198 if (cpu_data_ptr->idle_timer_notify != NULL) {
199 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
200 if (new_idle_timeout_ticks != 0x0ULL) {
201 /* if a new idle timeout was requested set the new idle timer deadline */
202 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
203 } else {
204 /* turn off the idle timer */
205 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
206 }
207 timer_resync_deadlines();
208 }
209 (void) ml_set_interrupts_enabled(intr);
210 }
211
212 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)213 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
214 {
215 broadcastFunc xfunc;
216 void *xparam;
217
218 os_atomic_thread_fence(acquire);
219 /* Come back around if cpu_signal_internal is running on another CPU and has just
220 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
221 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
222 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
223 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
224 xparam = cpu_data_ptr->cpu_xcall_p1;
225 cpu_data_ptr->cpu_xcall_p0 = NULL;
226 cpu_data_ptr->cpu_xcall_p1 = NULL;
227 os_atomic_thread_fence(acq_rel);
228 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
229 xfunc(xparam);
230 INTERRUPT_MASKED_DEBUG_END();
231 }
232 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
233 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
234 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
235 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
236 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
237 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
238 os_atomic_thread_fence(acq_rel);
239 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
240 xfunc(xparam);
241 INTERRUPT_MASKED_DEBUG_END();
242 }
243 }
244
245 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)246 cpu_broadcast_xcall_internal(unsigned int signal,
247 uint32_t *synch,
248 boolean_t self_xcall,
249 broadcastFunc func,
250 void *parm)
251 {
252 boolean_t intr;
253 cpu_data_t *cpu_data_ptr;
254 cpu_data_t *target_cpu_datap;
255 unsigned int failsig;
256 int cpu;
257 int max_cpu = ml_get_max_cpu_number() + 1;
258
259 //yes, param ALSO cannot be NULL
260 assert(func);
261 assert(parm);
262
263 intr = ml_set_interrupts_enabled(FALSE);
264 cpu_data_ptr = getCpuDatap();
265
266 failsig = 0;
267
268 if (synch != NULL) {
269 *synch = max_cpu;
270 assert_wait((event_t)synch, THREAD_UNINT);
271 }
272
273 for (cpu = 0; cpu < max_cpu; cpu++) {
274 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
275
276 if (target_cpu_datap == cpu_data_ptr) {
277 continue;
278 }
279
280 if ((target_cpu_datap == NULL) ||
281 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
282 failsig++;
283 }
284 }
285
286 if (self_xcall) {
287 func(parm);
288 }
289
290 (void) ml_set_interrupts_enabled(intr);
291
292 if (synch != NULL) {
293 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
294 clear_wait(current_thread(), THREAD_AWAKENED);
295 } else {
296 thread_block(THREAD_CONTINUE_NULL);
297 }
298 }
299
300 if (!self_xcall) {
301 return max_cpu - failsig - 1;
302 } else {
303 return max_cpu - failsig;
304 }
305 }
306
307 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)308 cpu_broadcast_xcall(uint32_t *synch,
309 boolean_t self_xcall,
310 broadcastFunc func,
311 void *parm)
312 {
313 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
314 }
315
316 struct cpu_broadcast_xcall_simple_data {
317 broadcastFunc func;
318 void* parm;
319 uint32_t sync;
320 };
321
322 static void
cpu_broadcast_xcall_simple_cbk(void * parm)323 cpu_broadcast_xcall_simple_cbk(void *parm)
324 {
325 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
326
327 data->func(data->parm);
328
329 if (os_atomic_dec(&data->sync, relaxed) == 0) {
330 thread_wakeup((event_t)&data->sync);
331 }
332 }
333
334 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)335 cpu_xcall_simple(boolean_t self_xcall,
336 broadcastFunc func,
337 void *parm,
338 bool immediate)
339 {
340 struct cpu_broadcast_xcall_simple_data data = {};
341
342 data.func = func;
343 data.parm = parm;
344
345 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
346 }
347
348 unsigned int
cpu_broadcast_immediate_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)349 cpu_broadcast_immediate_xcall(uint32_t *synch,
350 boolean_t self_xcall,
351 broadcastFunc func,
352 void *parm)
353 {
354 return cpu_broadcast_xcall_internal(SIGPxcallImm, synch, self_xcall, func, parm);
355 }
356
357 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)358 cpu_broadcast_xcall_simple(boolean_t self_xcall,
359 broadcastFunc func,
360 void *parm)
361 {
362 return cpu_xcall_simple(self_xcall, func, parm, false);
363 }
364
365 unsigned int
cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)366 cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,
367 broadcastFunc func,
368 void *parm)
369 {
370 return cpu_xcall_simple(self_xcall, func, parm, true);
371 }
372
373 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)374 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
375 {
376 cpu_data_t *target_cpu_datap;
377
378 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
379 return KERN_INVALID_ARGUMENT;
380 }
381
382 if (func == NULL || param == NULL) {
383 return KERN_INVALID_ARGUMENT;
384 }
385
386 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
387 if (target_cpu_datap == NULL) {
388 return KERN_INVALID_ARGUMENT;
389 }
390
391 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
392 }
393
394 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)395 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
396 {
397 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
398 }
399
400 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)401 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
402 {
403 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
404 }
405
406 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1,boolean_t defer)407 cpu_signal_internal(cpu_data_t *target_proc,
408 unsigned int signal,
409 void *p0,
410 void *p1,
411 boolean_t defer)
412 {
413 unsigned int Check_SIGPdisabled;
414 int current_signals;
415 Boolean swap_success;
416 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
417 cpu_data_t *current_proc = getCpuDatap();
418
419 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
420 if (defer) {
421 assert(signal == SIGPnop);
422 }
423
424 if (current_proc != target_proc) {
425 Check_SIGPdisabled = SIGPdisabled;
426 } else {
427 Check_SIGPdisabled = 0;
428 }
429
430 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
431 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
432 current_mabs_time = start_mabs_time = mach_absolute_time();
433 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
434 assert(max_mabs_time > current_mabs_time);
435
436 do {
437 current_signals = target_proc->cpu_signal;
438 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
439 ml_set_interrupts_enabled(interruptible);
440 return KERN_FAILURE;
441 }
442 swap_success = OSCompareAndSwap(current_signals & (~signal), current_signals | signal,
443 &target_proc->cpu_signal);
444
445 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
446 ml_set_interrupts_enabled(interruptible);
447 return KERN_ALREADY_WAITING;
448 }
449
450 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
451 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
452 * so break the deadlock by draining pending xcalls. */
453 if (!swap_success && (current_proc->cpu_signal & signal)) {
454 cpu_handle_xcall(current_proc);
455 }
456 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
457
458 /*
459 * If we time out while waiting for the target CPU to respond, it's possible that no
460 * other CPU is available to handle the watchdog interrupt that would eventually trigger
461 * a panic. To prevent this from happening, we just panic here to flag this condition.
462 */
463 if (__improbable(current_mabs_time >= max_mabs_time)) {
464 uint64_t end_time_ns, xcall_ack_timeout_ns;
465 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
466 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
467 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
468 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
469 }
470
471 if (signal == SIGPxcallImm) {
472 target_proc->cpu_imm_xcall_p0 = p0;
473 target_proc->cpu_imm_xcall_p1 = p1;
474 } else {
475 target_proc->cpu_xcall_p0 = p0;
476 target_proc->cpu_xcall_p1 = p1;
477 }
478 } else {
479 do {
480 current_signals = target_proc->cpu_signal;
481 if ((Check_SIGPdisabled != 0) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) {
482 ml_set_interrupts_enabled(interruptible);
483 return KERN_FAILURE;
484 }
485
486 swap_success = OSCompareAndSwap(current_signals, current_signals | signal,
487 &target_proc->cpu_signal);
488 } while (!swap_success);
489 }
490
491 /*
492 * Issue DSB here to guarantee: 1) prior stores to pending signal mask and xcall params
493 * will be visible to other cores when the IPI is dispatched, and 2) subsequent
494 * instructions to signal the other cores will not execute until after the barrier.
495 * DMB would be sufficient to guarantee 1) but not 2).
496 */
497 __builtin_arm_dsb(DSB_ISH);
498
499 if (!(target_proc->cpu_signal & SIGPdisabled)) {
500 if (defer) {
501 #if defined(HAS_IPI)
502 if (gFastIPI) {
503 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
504 } else {
505 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
506 }
507 #else
508 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
509 #endif /* defined(HAS_IPI) */
510 } else {
511 #if defined(HAS_IPI)
512 if (gFastIPI) {
513 ml_cpu_signal(target_proc->cpu_phys_id);
514 } else {
515 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
516 }
517 #else
518 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
519 #endif /* defined(HAS_IPI) */
520 }
521 }
522
523 ml_set_interrupts_enabled(interruptible);
524 return KERN_SUCCESS;
525 }
526
527 kern_return_t
cpu_signal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1)528 cpu_signal(cpu_data_t *target_proc,
529 unsigned int signal,
530 void *p0,
531 void *p1)
532 {
533 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
534 }
535
536 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc)537 cpu_signal_deferred(cpu_data_t *target_proc)
538 {
539 return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE);
540 }
541
542 void
cpu_signal_cancel(cpu_data_t * target_proc)543 cpu_signal_cancel(cpu_data_t *target_proc)
544 {
545 /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */
546 if (!(target_proc->cpu_signal & SIGPdisabled)) {
547 #if defined(HAS_IPI)
548 if (gFastIPI) {
549 ml_cpu_signal_retract(target_proc->cpu_phys_id);
550 } else {
551 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
552 }
553 #else
554 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
555 #endif /* defined(HAS_IPI) */
556 }
557 }
558
559 void
cpu_signal_handler(void)560 cpu_signal_handler(void)
561 {
562 cpu_signal_handler_internal(FALSE);
563 }
564
565 void
cpu_signal_handler_internal(boolean_t disable_signal)566 cpu_signal_handler_internal(boolean_t disable_signal)
567 {
568 cpu_data_t *cpu_data_ptr = getCpuDatap();
569 unsigned int cpu_signal;
570
571 cpu_data_ptr->cpu_stat.ipi_cnt++;
572 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
573 SCHED_STATS_INC(ipi_count);
574
575 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, relaxed);
576
577 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
578 os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
579 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
580 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
581 }
582
583 while (cpu_signal & ~SIGPdisabled) {
584 if (cpu_signal & SIGPdec) {
585 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdec, relaxed);
586 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_IPI);
587 rtclock_intr(FALSE);
588 INTERRUPT_MASKED_DEBUG_END();
589 }
590 #if KPERF
591 if (cpu_signal & SIGPkppet) {
592 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, relaxed);
593 extern void kperf_signal_handler(void);
594 INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI);
595 kperf_signal_handler();
596 INTERRUPT_MASKED_DEBUG_END();
597 }
598 #endif /* KPERF */
599 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
600 cpu_handle_xcall(cpu_data_ptr);
601 }
602 if (cpu_signal & SIGPast) {
603 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, relaxed);
604 INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI);
605 ast_check(current_processor());
606 INTERRUPT_MASKED_DEBUG_END();
607 }
608 if (cpu_signal & SIGPdebug) {
609 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, relaxed);
610 INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI);
611 DebuggerXCall(cpu_data_ptr->cpu_int_state);
612 INTERRUPT_MASKED_DEBUG_END();
613 }
614 #if defined(ARMA7)
615 if (cpu_signal & SIGPLWFlush) {
616 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPLWFlush, relaxed);
617 INTERRUPT_MASKED_DEBUG_START(cache_xcall_handler, DBG_INTR_TYPE_IPI);
618 cache_xcall_handler(LWFlush);
619 INTERRUPT_MASKED_DEBUG_END();
620 }
621 if (cpu_signal & SIGPLWClean) {
622 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPLWClean, relaxed);
623 INTERRUPT_MASKED_DEBUG_START(cache_xcall_handler, DBG_INTR_TYPE_IPI);
624 cache_xcall_handler(LWClean);
625 INTERRUPT_MASKED_DEBUG_END();
626 }
627 #endif
628
629 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, relaxed);
630 }
631 }
632
633 void
cpu_exit_wait(int cpu_id)634 cpu_exit_wait(int cpu_id)
635 {
636 #if USE_APPLEARMSMP
637 if (!ml_is_quiescing()) {
638 // For runtime disable (non S2R) the CPU will shut down immediately.
639 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
640 assert(cpu && cpu->cpu_IMPL_regs);
641 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
642
643 // Poll the "CPU running state" field until it is 0 (off)
644 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
645 __builtin_arm_dsb(DSB_ISH);
646 }
647 return;
648 }
649 #endif /* USE_APPLEARMSMP */
650
651 if (cpu_id != master_cpu) {
652 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
653 cpu_data_t *cpu_data_ptr;
654
655 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
656 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
657 }
658 ;
659 }
660 }
661
662 boolean_t
cpu_can_exit(__unused int cpu)663 cpu_can_exit(__unused int cpu)
664 {
665 return TRUE;
666 }
667
668 void
cpu_machine_init(void)669 cpu_machine_init(void)
670 {
671 static boolean_t started = FALSE;
672 cpu_data_t *cpu_data_ptr;
673
674 cpu_data_ptr = getCpuDatap();
675 started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState);
676 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
677 platform_cache_init();
678 }
679
680 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
681 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
682
683 cpu_data_ptr->cpu_flags |= StartedState;
684 ml_init_interrupt();
685 }
686
687 processor_t
current_processor(void)688 current_processor(void)
689 {
690 return PERCPU_GET(processor);
691 }
692
693 processor_t
cpu_to_processor(int cpu)694 cpu_to_processor(int cpu)
695 {
696 cpu_data_t *cpu_data = cpu_datap(cpu);
697 if (cpu_data != NULL) {
698 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
699 } else {
700 return NULL;
701 }
702 }
703
704 cpu_data_t *
processor_to_cpu_datap(processor_t processor)705 processor_to_cpu_datap(processor_t processor)
706 {
707 assert(processor->cpu_id <= ml_get_max_cpu_number());
708 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
709
710 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
711 }
712
713 __startup_func
714 static void
cpu_data_startup_init(void)715 cpu_data_startup_init(void)
716 {
717 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
718
719 percpu_base.size = percpu_section_size();
720 if (ml_get_cpu_count() == 1) {
721 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
722 return;
723 }
724
725 /*
726 * The memory needs to be physically contiguous because it contains
727 * cpu_data_t structures sometimes accessed during reset
728 * with the MMU off.
729 *
730 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
731 * normally runs, so we instead steal the memory for the PERCPU subsystem
732 * even earlier.
733 */
734 percpu_base.start = (vm_offset_t)pmap_steal_memory(round_page(size));
735 bzero((void *)percpu_base.start, round_page(size));
736
737 percpu_base.start -= percpu_section_start();
738 percpu_base.end = percpu_base.start + size - 1;
739 percpu_base_cur = percpu_base.start;
740 }
741 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
742
743 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)744 cpu_data_alloc(boolean_t is_boot_cpu)
745 {
746 cpu_data_t *cpu_data_ptr = NULL;
747 vm_address_t base;
748
749 if (is_boot_cpu) {
750 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
751 } else {
752 base = os_atomic_add_orig(&percpu_base_cur,
753 percpu_section_size(), relaxed);
754
755 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
756 cpu_stack_alloc(cpu_data_ptr);
757 }
758
759 return cpu_data_ptr;
760 }
761
762 ast_t *
ast_pending(void)763 ast_pending(void)
764 {
765 return &getCpuDatap()->cpu_pending_ast;
766 }
767
768 cpu_type_t
slot_type(int slot_num)769 slot_type(int slot_num)
770 {
771 return cpu_datap(slot_num)->cpu_type;
772 }
773
774 cpu_subtype_t
slot_subtype(int slot_num)775 slot_subtype(int slot_num)
776 {
777 return cpu_datap(slot_num)->cpu_subtype;
778 }
779
780 cpu_threadtype_t
slot_threadtype(int slot_num)781 slot_threadtype(int slot_num)
782 {
783 return cpu_datap(slot_num)->cpu_threadtype;
784 }
785
786 cpu_type_t
cpu_type(void)787 cpu_type(void)
788 {
789 return getCpuDatap()->cpu_type;
790 }
791
792 cpu_subtype_t
cpu_subtype(void)793 cpu_subtype(void)
794 {
795 return getCpuDatap()->cpu_subtype;
796 }
797
798 cpu_threadtype_t
cpu_threadtype(void)799 cpu_threadtype(void)
800 {
801 return getCpuDatap()->cpu_threadtype;
802 }
803
804 int
cpu_number(void)805 cpu_number(void)
806 {
807 return getCpuDatap()->cpu_number;
808 }
809
810 vm_offset_t
current_percpu_base(void)811 current_percpu_base(void)
812 {
813 return current_thread()->machine.pcpu_data_base;
814 }
815
816 vm_offset_t
other_percpu_base(int cpu)817 other_percpu_base(int cpu)
818 {
819 return (vm_offset_t)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
820 }
821
822 uint64_t
ml_get_wake_timebase(void)823 ml_get_wake_timebase(void)
824 {
825 return wake_abstime;
826 }
827
828 bool
ml_cpu_signal_is_enabled(void)829 ml_cpu_signal_is_enabled(void)
830 {
831 return !(getCpuDatap()->cpu_signal & SIGPdisabled);
832 }
833
834 bool
ml_cpu_can_exit(__unused int cpu_id)835 ml_cpu_can_exit(__unused int cpu_id)
836 {
837 /* processor_exit() is always allowed on the S2R path */
838 if (ml_is_quiescing()) {
839 return true;
840 }
841 #if HAS_CLUSTER && USE_APPLEARMSMP
842 /*
843 * Until the feature is known to be stable, guard it with a boot-arg
844 */
845 extern bool enable_processor_exit;
846 if (!enable_processor_exit) {
847 return false;
848 }
849 /*
850 * Cyprus and newer chips can disable individual non-boot CPUs. The
851 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
852 */
853 if (CpuDataEntries[cpu_id].cpu_data_vaddr != &BootCpuData) {
854 return true;
855 }
856 #endif
857 return false;
858 }
859
860 #ifdef USE_APPLEARMSMP
861
862 void
ml_cpu_begin_state_transition(int cpu_id)863 ml_cpu_begin_state_transition(int cpu_id)
864 {
865 lck_rw_lock_exclusive(&cpu_state_lock);
866 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
867 lck_rw_unlock_exclusive(&cpu_state_lock);
868 }
869
870 void
ml_cpu_end_state_transition(int cpu_id)871 ml_cpu_end_state_transition(int cpu_id)
872 {
873 lck_rw_lock_exclusive(&cpu_state_lock);
874 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
875 lck_rw_unlock_exclusive(&cpu_state_lock);
876 }
877
878 void
ml_cpu_begin_loop(void)879 ml_cpu_begin_loop(void)
880 {
881 lck_rw_lock_shared(&cpu_state_lock);
882 }
883
884 void
ml_cpu_end_loop(void)885 ml_cpu_end_loop(void)
886 {
887 lck_rw_unlock_shared(&cpu_state_lock);
888 }
889
890 #else /* USE_APPLEARMSMP */
891
892 void
ml_cpu_begin_state_transition(__unused int cpu_id)893 ml_cpu_begin_state_transition(__unused int cpu_id)
894 {
895 }
896
897 void
ml_cpu_end_state_transition(__unused int cpu_id)898 ml_cpu_end_state_transition(__unused int cpu_id)
899 {
900 }
901
902 void
ml_cpu_begin_loop(void)903 ml_cpu_begin_loop(void)
904 {
905 }
906
907 void
ml_cpu_end_loop(void)908 ml_cpu_end_loop(void)
909 {
910 }
911
912 #endif /* USE_APPLEARMSMP */
913