1 /*
2 * Copyright (c) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm/cpu_common.c
30 *
31 * cpu routines common to all supported arm variants
32 */
33
34 #include <kern/machine.h>
35 #include <kern/cpu_number.h>
36 #include <kern/thread.h>
37 #include <kern/percpu.h>
38 #include <kern/timer_queue.h>
39 #include <kern/locks.h>
40 #include <arm/cpu_data.h>
41 #include <arm/cpuid.h>
42 #include <arm/caches_internal.h>
43 #include <arm/cpu_data_internal.h>
44 #include <arm/cpu_internal.h>
45 #include <arm/misc_protos.h>
46 #include <arm/machine_cpu.h>
47 #include <arm/rtclock.h>
48 #include <mach/processor_info.h>
49 #include <machine/atomic.h>
50 #include <machine/config.h>
51 #include <vm/vm_kern.h>
52 #include <vm/vm_map.h>
53 #include <pexpert/arm/protos.h>
54 #include <pexpert/device_tree.h>
55 #include <sys/kdebug.h>
56 #include <arm/machine_routines.h>
57 #include <arm/proc_reg.h>
58 #include <libkern/OSAtomic.h>
59
60 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
61 vm_address_t percpu_base_cur;
62 cpu_data_t PERCPU_DATA(cpu_data);
63 cpu_data_entry_t CpuDataEntries[MAX_CPUS];
64
65 static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp");
66 static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp);
67
68 unsigned int real_ncpus = 1;
69 boolean_t idle_enable = FALSE;
70 uint64_t wake_abstime = 0x0ULL;
71
72 extern uint64_t xcall_ack_timeout_abstime;
73
74 #if defined(HAS_IPI)
75 extern unsigned int gFastIPI;
76 #endif /* defined(HAS_IPI) */
77
78 cpu_data_t *
cpu_datap(int cpu)79 cpu_datap(int cpu)
80 {
81 assert(cpu <= ml_get_max_cpu_number());
82 return CpuDataEntries[cpu].cpu_data_vaddr;
83 }
84
85 kern_return_t
cpu_control(int slot_num,processor_info_t info,unsigned int count)86 cpu_control(int slot_num,
87 processor_info_t info,
88 unsigned int count)
89 {
90 printf("cpu_control(%d,%p,%d) not implemented\n",
91 slot_num, info, count);
92 return KERN_FAILURE;
93 }
94
95 kern_return_t
cpu_info_count(processor_flavor_t flavor,unsigned int * count)96 cpu_info_count(processor_flavor_t flavor,
97 unsigned int *count)
98 {
99 switch (flavor) {
100 case PROCESSOR_CPU_STAT:
101 *count = PROCESSOR_CPU_STAT_COUNT;
102 return KERN_SUCCESS;
103
104 case PROCESSOR_CPU_STAT64:
105 *count = PROCESSOR_CPU_STAT64_COUNT;
106 return KERN_SUCCESS;
107
108 default:
109 *count = 0;
110 return KERN_FAILURE;
111 }
112 }
113
114 kern_return_t
cpu_info(processor_flavor_t flavor,int slot_num,processor_info_t info,unsigned int * count)115 cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info,
116 unsigned int *count)
117 {
118 cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr;
119
120 switch (flavor) {
121 case PROCESSOR_CPU_STAT:
122 {
123 if (*count < PROCESSOR_CPU_STAT_COUNT) {
124 return KERN_FAILURE;
125 }
126
127 processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info;
128 cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt;
129 cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt;
130 cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt;
131 cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt;
132 cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt;
133 cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt;
134 cpu_stat->vfp_shortv_cnt = 0;
135 cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt;
136 cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt;
137
138 *count = PROCESSOR_CPU_STAT_COUNT;
139
140 return KERN_SUCCESS;
141 }
142
143 case PROCESSOR_CPU_STAT64:
144 {
145 if (*count < PROCESSOR_CPU_STAT64_COUNT) {
146 return KERN_FAILURE;
147 }
148
149 processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info;
150 cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt;
151 cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt;
152 cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt;
153 cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt;
154 cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt;
155 cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt;
156 cpu_stat->vfp_shortv_cnt = 0;
157 cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt;
158 cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt;
159 #if MONOTONIC
160 cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis;
161 #endif /* MONOTONIC */
162
163 *count = PROCESSOR_CPU_STAT64_COUNT;
164
165 return KERN_SUCCESS;
166 }
167
168 default:
169 return KERN_FAILURE;
170 }
171 }
172
173 /*
174 * Routine: cpu_doshutdown
175 * Function:
176 */
177 void
cpu_doshutdown(void (* doshutdown)(processor_t),processor_t processor)178 cpu_doshutdown(void (*doshutdown)(processor_t),
179 processor_t processor)
180 {
181 doshutdown(processor);
182 }
183
184 /*
185 * Routine: cpu_idle_tickle
186 *
187 */
188 void
cpu_idle_tickle(void)189 cpu_idle_tickle(void)
190 {
191 boolean_t intr;
192 cpu_data_t *cpu_data_ptr;
193 uint64_t new_idle_timeout_ticks = 0x0ULL;
194
195 intr = ml_set_interrupts_enabled(FALSE);
196 cpu_data_ptr = getCpuDatap();
197
198 if (cpu_data_ptr->idle_timer_notify != NULL) {
199 cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks);
200 if (new_idle_timeout_ticks != 0x0ULL) {
201 /* if a new idle timeout was requested set the new idle timer deadline */
202 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
203 } else {
204 /* turn off the idle timer */
205 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
206 }
207 timer_resync_deadlines();
208 }
209 (void) ml_set_interrupts_enabled(intr);
210 }
211
212 static void
cpu_handle_xcall(cpu_data_t * cpu_data_ptr)213 cpu_handle_xcall(cpu_data_t *cpu_data_ptr)
214 {
215 broadcastFunc xfunc;
216 void *xparam;
217
218 os_atomic_thread_fence(acquire);
219 /* Come back around if cpu_signal_internal is running on another CPU and has just
220 * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/
221 if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) {
222 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
223 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
224 xparam = cpu_data_ptr->cpu_xcall_p1;
225 cpu_data_ptr->cpu_xcall_p0 = NULL;
226 cpu_data_ptr->cpu_xcall_p1 = NULL;
227 os_atomic_thread_fence(acq_rel);
228 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed);
229 xfunc(xparam);
230 INTERRUPT_MASKED_DEBUG_END();
231 }
232 if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) {
233 xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr);
234 INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI);
235 xparam = cpu_data_ptr->cpu_imm_xcall_p1;
236 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
237 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
238 os_atomic_thread_fence(acq_rel);
239 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed);
240 xfunc(xparam);
241 INTERRUPT_MASKED_DEBUG_END();
242 }
243 }
244
245 static unsigned int
cpu_broadcast_xcall_internal(unsigned int signal,uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)246 cpu_broadcast_xcall_internal(unsigned int signal,
247 uint32_t *synch,
248 boolean_t self_xcall,
249 broadcastFunc func,
250 void *parm)
251 {
252 boolean_t intr;
253 cpu_data_t *cpu_data_ptr;
254 cpu_data_t *target_cpu_datap;
255 unsigned int failsig;
256 int cpu;
257 int max_cpu = ml_get_max_cpu_number() + 1;
258
259 //yes, param ALSO cannot be NULL
260 assert(func);
261 assert(parm);
262
263 intr = ml_set_interrupts_enabled(FALSE);
264 cpu_data_ptr = getCpuDatap();
265
266 failsig = 0;
267
268 if (synch != NULL) {
269 *synch = max_cpu;
270 assert_wait((event_t)synch, THREAD_UNINT);
271 }
272
273 for (cpu = 0; cpu < max_cpu; cpu++) {
274 target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
275
276 if (target_cpu_datap == cpu_data_ptr) {
277 continue;
278 }
279
280 if ((target_cpu_datap == NULL) ||
281 KERN_SUCCESS != cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), parm)) {
282 failsig++;
283 }
284 }
285
286 if (self_xcall) {
287 func(parm);
288 }
289
290 (void) ml_set_interrupts_enabled(intr);
291
292 if (synch != NULL) {
293 if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) {
294 clear_wait(current_thread(), THREAD_AWAKENED);
295 } else {
296 thread_block(THREAD_CONTINUE_NULL);
297 }
298 }
299
300 if (!self_xcall) {
301 return max_cpu - failsig - 1;
302 } else {
303 return max_cpu - failsig;
304 }
305 }
306
307 unsigned int
cpu_broadcast_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)308 cpu_broadcast_xcall(uint32_t *synch,
309 boolean_t self_xcall,
310 broadcastFunc func,
311 void *parm)
312 {
313 return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm);
314 }
315
316 struct cpu_broadcast_xcall_simple_data {
317 broadcastFunc func;
318 void* parm;
319 uint32_t sync;
320 };
321
322 static void
cpu_broadcast_xcall_simple_cbk(void * parm)323 cpu_broadcast_xcall_simple_cbk(void *parm)
324 {
325 struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm;
326
327 data->func(data->parm);
328
329 if (os_atomic_dec(&data->sync, relaxed) == 0) {
330 thread_wakeup((event_t)&data->sync);
331 }
332 }
333
334 static unsigned int
cpu_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm,bool immediate)335 cpu_xcall_simple(boolean_t self_xcall,
336 broadcastFunc func,
337 void *parm,
338 bool immediate)
339 {
340 struct cpu_broadcast_xcall_simple_data data = {};
341
342 data.func = func;
343 data.parm = parm;
344
345 return cpu_broadcast_xcall_internal(immediate ? SIGPxcallImm : SIGPxcall, &data.sync, self_xcall, cpu_broadcast_xcall_simple_cbk, &data);
346 }
347
348 unsigned int
cpu_broadcast_immediate_xcall(uint32_t * synch,boolean_t self_xcall,broadcastFunc func,void * parm)349 cpu_broadcast_immediate_xcall(uint32_t *synch,
350 boolean_t self_xcall,
351 broadcastFunc func,
352 void *parm)
353 {
354 return cpu_broadcast_xcall_internal(SIGPxcallImm, synch, self_xcall, func, parm);
355 }
356
357 unsigned int
cpu_broadcast_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)358 cpu_broadcast_xcall_simple(boolean_t self_xcall,
359 broadcastFunc func,
360 void *parm)
361 {
362 return cpu_xcall_simple(self_xcall, func, parm, false);
363 }
364
365 unsigned int
cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,broadcastFunc func,void * parm)366 cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall,
367 broadcastFunc func,
368 void *parm)
369 {
370 return cpu_xcall_simple(self_xcall, func, parm, true);
371 }
372
373 static kern_return_t
cpu_xcall_internal(unsigned int signal,int cpu_number,broadcastFunc func,void * param)374 cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param)
375 {
376 cpu_data_t *target_cpu_datap;
377
378 if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) {
379 panic("cpu_xcall_internal: invalid cpu_number %d", cpu_number);
380 }
381
382 if (func == NULL || param == NULL) {
383 // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready
384 panic("cpu_xcall_internal: cannot have null func/param: %p %p", func, param);
385 }
386
387 target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr;
388 if (target_cpu_datap == NULL) {
389 panic("cpu_xcall_internal: cpu %d not initialized", cpu_number);
390 }
391
392 return cpu_signal(target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), param);
393 }
394
395 kern_return_t
cpu_xcall(int cpu_number,broadcastFunc func,void * param)396 cpu_xcall(int cpu_number, broadcastFunc func, void *param)
397 {
398 return cpu_xcall_internal(SIGPxcall, cpu_number, func, param);
399 }
400
401 kern_return_t
cpu_immediate_xcall(int cpu_number,broadcastFunc func,void * param)402 cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param)
403 {
404 return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param);
405 }
406
407 static kern_return_t
cpu_signal_internal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1,boolean_t defer)408 cpu_signal_internal(cpu_data_t *target_proc,
409 unsigned int signal,
410 void *p0,
411 void *p1,
412 boolean_t defer)
413 {
414 unsigned int Check_SIGPdisabled;
415 int current_signals;
416 Boolean swap_success;
417 boolean_t interruptible = ml_set_interrupts_enabled(FALSE);
418 cpu_data_t *current_proc = getCpuDatap();
419
420 /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */
421 if (defer) {
422 assert(signal == SIGPnop);
423 }
424
425 if (current_proc != target_proc) {
426 Check_SIGPdisabled = SIGPdisabled;
427 } else {
428 Check_SIGPdisabled = 0;
429 }
430
431 if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) {
432 uint64_t start_mabs_time, max_mabs_time, current_mabs_time;
433 current_mabs_time = start_mabs_time = mach_absolute_time();
434 max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time;
435 assert(max_mabs_time > current_mabs_time);
436
437 do {
438 current_signals = target_proc->cpu_signal;
439 if ((current_signals & SIGPdisabled) == SIGPdisabled) {
440 ml_set_interrupts_enabled(interruptible);
441 return KERN_FAILURE;
442 }
443 swap_success = OSCompareAndSwap(current_signals & (~signal), current_signals | signal,
444 &target_proc->cpu_signal);
445
446 if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) {
447 ml_set_interrupts_enabled(interruptible);
448 return KERN_ALREADY_WAITING;
449 }
450
451 /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn
452 * be trying to xcall us. Since we have interrupts disabled that can deadlock,
453 * so break the deadlock by draining pending xcalls. */
454 if (!swap_success && (current_proc->cpu_signal & signal)) {
455 cpu_handle_xcall(current_proc);
456 }
457 } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time));
458
459 /*
460 * If we time out while waiting for the target CPU to respond, it's possible that no
461 * other CPU is available to handle the watchdog interrupt that would eventually trigger
462 * a panic. To prevent this from happening, we just panic here to flag this condition.
463 */
464 if (__improbable(current_mabs_time >= max_mabs_time)) {
465 uint64_t end_time_ns, xcall_ack_timeout_ns;
466 absolutetime_to_nanoseconds(current_mabs_time - start_mabs_time, &end_time_ns);
467 absolutetime_to_nanoseconds(xcall_ack_timeout_abstime, &xcall_ack_timeout_ns);
468 panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)",
469 target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns);
470 }
471
472 if (signal == SIGPxcallImm) {
473 target_proc->cpu_imm_xcall_p0 = p0;
474 target_proc->cpu_imm_xcall_p1 = p1;
475 } else {
476 target_proc->cpu_xcall_p0 = p0;
477 target_proc->cpu_xcall_p1 = p1;
478 }
479 } else {
480 do {
481 current_signals = target_proc->cpu_signal;
482 if ((Check_SIGPdisabled != 0) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) {
483 ml_set_interrupts_enabled(interruptible);
484 return KERN_FAILURE;
485 }
486
487 swap_success = OSCompareAndSwap(current_signals, current_signals | signal,
488 &target_proc->cpu_signal);
489 } while (!swap_success);
490 }
491
492 /*
493 * Issue DSB here to guarantee: 1) prior stores to pending signal mask and xcall params
494 * will be visible to other cores when the IPI is dispatched, and 2) subsequent
495 * instructions to signal the other cores will not execute until after the barrier.
496 * DMB would be sufficient to guarantee 1) but not 2).
497 */
498 __builtin_arm_dsb(DSB_ISH);
499
500 if (!(target_proc->cpu_signal & SIGPdisabled)) {
501 if (defer) {
502 #if defined(HAS_IPI)
503 if (gFastIPI) {
504 ml_cpu_signal_deferred(target_proc->cpu_phys_id);
505 } else {
506 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
507 }
508 #else
509 PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id);
510 #endif /* defined(HAS_IPI) */
511 } else {
512 #if defined(HAS_IPI)
513 if (gFastIPI) {
514 ml_cpu_signal(target_proc->cpu_phys_id);
515 } else {
516 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
517 }
518 #else
519 PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id);
520 #endif /* defined(HAS_IPI) */
521 }
522 }
523
524 ml_set_interrupts_enabled(interruptible);
525 return KERN_SUCCESS;
526 }
527
528 kern_return_t
cpu_signal(cpu_data_t * target_proc,unsigned int signal,void * p0,void * p1)529 cpu_signal(cpu_data_t *target_proc,
530 unsigned int signal,
531 void *p0,
532 void *p1)
533 {
534 return cpu_signal_internal(target_proc, signal, p0, p1, FALSE);
535 }
536
537 kern_return_t
cpu_signal_deferred(cpu_data_t * target_proc)538 cpu_signal_deferred(cpu_data_t *target_proc)
539 {
540 return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE);
541 }
542
543 void
cpu_signal_cancel(cpu_data_t * target_proc)544 cpu_signal_cancel(cpu_data_t *target_proc)
545 {
546 /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */
547 if (!(target_proc->cpu_signal & SIGPdisabled)) {
548 #if defined(HAS_IPI)
549 if (gFastIPI) {
550 ml_cpu_signal_retract(target_proc->cpu_phys_id);
551 } else {
552 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
553 }
554 #else
555 PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id);
556 #endif /* defined(HAS_IPI) */
557 }
558 }
559
560 void
cpu_signal_handler(void)561 cpu_signal_handler(void)
562 {
563 cpu_signal_handler_internal(FALSE);
564 }
565
566 void
cpu_signal_handler_internal(boolean_t disable_signal)567 cpu_signal_handler_internal(boolean_t disable_signal)
568 {
569 cpu_data_t *cpu_data_ptr = getCpuDatap();
570 unsigned int cpu_signal;
571
572 cpu_data_ptr->cpu_stat.ipi_cnt++;
573 cpu_data_ptr->cpu_stat.ipi_cnt_wake++;
574 SCHED_STATS_INC(ipi_count);
575
576 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, relaxed);
577
578 if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) {
579 os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
580 } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) {
581 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed);
582 }
583
584 while (cpu_signal & ~SIGPdisabled) {
585 if (cpu_signal & SIGPdec) {
586 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdec, relaxed);
587 INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_IPI);
588 rtclock_intr(FALSE);
589 INTERRUPT_MASKED_DEBUG_END();
590 }
591 #if KPERF
592 if (cpu_signal & SIGPkppet) {
593 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, relaxed);
594 extern void kperf_signal_handler(void);
595 INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI);
596 kperf_signal_handler();
597 INTERRUPT_MASKED_DEBUG_END();
598 }
599 #endif /* KPERF */
600 if (cpu_signal & (SIGPxcall | SIGPxcallImm)) {
601 cpu_handle_xcall(cpu_data_ptr);
602 }
603 if (cpu_signal & SIGPast) {
604 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, relaxed);
605 INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI);
606 ast_check(current_processor());
607 INTERRUPT_MASKED_DEBUG_END();
608 }
609 if (cpu_signal & SIGPdebug) {
610 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, relaxed);
611 INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI);
612 DebuggerXCall(cpu_data_ptr->cpu_int_state);
613 INTERRUPT_MASKED_DEBUG_END();
614 }
615 #if defined(ARMA7)
616 if (cpu_signal & SIGPLWFlush) {
617 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPLWFlush, relaxed);
618 INTERRUPT_MASKED_DEBUG_START(cache_xcall_handler, DBG_INTR_TYPE_IPI);
619 cache_xcall_handler(LWFlush);
620 INTERRUPT_MASKED_DEBUG_END();
621 }
622 if (cpu_signal & SIGPLWClean) {
623 os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPLWClean, relaxed);
624 INTERRUPT_MASKED_DEBUG_START(cache_xcall_handler, DBG_INTR_TYPE_IPI);
625 cache_xcall_handler(LWClean);
626 INTERRUPT_MASKED_DEBUG_END();
627 }
628 #endif
629
630 cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, relaxed);
631 }
632 }
633
634 void
cpu_exit_wait(int cpu_id)635 cpu_exit_wait(int cpu_id)
636 {
637 #if USE_APPLEARMSMP
638 if (!ml_is_quiescing()) {
639 // For runtime disable (non S2R) the CPU will shut down immediately.
640 ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id];
641 assert(cpu && cpu->cpu_IMPL_regs);
642 volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET);
643
644 // Poll the "CPU running state" field until it is 0 (off)
645 while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) {
646 __builtin_arm_dsb(DSB_ISH);
647 }
648 return;
649 }
650 #endif /* USE_APPLEARMSMP */
651
652 if (cpu_id != master_cpu) {
653 // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH.
654 cpu_data_t *cpu_data_ptr;
655
656 cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr;
657 while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {
658 }
659 ;
660 }
661 }
662
663 boolean_t
cpu_can_exit(__unused int cpu)664 cpu_can_exit(__unused int cpu)
665 {
666 return TRUE;
667 }
668
669 void
cpu_machine_init(void)670 cpu_machine_init(void)
671 {
672 static boolean_t started = FALSE;
673 cpu_data_t *cpu_data_ptr;
674
675 cpu_data_ptr = getCpuDatap();
676 started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState);
677 if (cpu_data_ptr->cpu_cache_dispatch != NULL) {
678 platform_cache_init();
679 }
680
681 /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */
682 PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started);
683
684 cpu_data_ptr->cpu_flags |= StartedState;
685 ml_init_interrupt();
686 }
687
688 processor_t
current_processor(void)689 current_processor(void)
690 {
691 return PERCPU_GET(processor);
692 }
693
694 processor_t
cpu_to_processor(int cpu)695 cpu_to_processor(int cpu)
696 {
697 cpu_data_t *cpu_data = cpu_datap(cpu);
698 if (cpu_data != NULL) {
699 return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data);
700 } else {
701 return NULL;
702 }
703 }
704
705 cpu_data_t *
processor_to_cpu_datap(processor_t processor)706 processor_to_cpu_datap(processor_t processor)
707 {
708 assert(processor->cpu_id <= ml_get_max_cpu_number());
709 assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL);
710
711 return PERCPU_GET_RELATIVE(cpu_data, processor, processor);
712 }
713
714 __startup_func
715 static void
cpu_data_startup_init(void)716 cpu_data_startup_init(void)
717 {
718 vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1);
719
720 percpu_base.size = percpu_section_size();
721 if (ml_get_cpu_count() == 1) {
722 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
723 return;
724 }
725
726 /*
727 * The memory needs to be physically contiguous because it contains
728 * cpu_data_t structures sometimes accessed during reset
729 * with the MMU off.
730 *
731 * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU
732 * normally runs, so we instead steal the memory for the PERCPU subsystem
733 * even earlier.
734 */
735 percpu_base.start = (vm_offset_t)pmap_steal_memory(round_page(size));
736 bzero((void *)percpu_base.start, round_page(size));
737
738 percpu_base.start -= percpu_section_start();
739 percpu_base.end = percpu_base.start + size - 1;
740 percpu_base_cur = percpu_base.start;
741 }
742 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init);
743
744 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)745 cpu_data_alloc(boolean_t is_boot_cpu)
746 {
747 cpu_data_t *cpu_data_ptr = NULL;
748 vm_address_t base;
749
750 if (is_boot_cpu) {
751 cpu_data_ptr = PERCPU_GET_MASTER(cpu_data);
752 } else {
753 base = os_atomic_add_orig(&percpu_base_cur,
754 percpu_section_size(), relaxed);
755
756 cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data);
757 cpu_stack_alloc(cpu_data_ptr);
758 }
759
760 return cpu_data_ptr;
761 }
762
763 ast_t *
ast_pending(void)764 ast_pending(void)
765 {
766 return &getCpuDatap()->cpu_pending_ast;
767 }
768
769 cpu_type_t
slot_type(int slot_num)770 slot_type(int slot_num)
771 {
772 return cpu_datap(slot_num)->cpu_type;
773 }
774
775 cpu_subtype_t
slot_subtype(int slot_num)776 slot_subtype(int slot_num)
777 {
778 return cpu_datap(slot_num)->cpu_subtype;
779 }
780
781 cpu_threadtype_t
slot_threadtype(int slot_num)782 slot_threadtype(int slot_num)
783 {
784 return cpu_datap(slot_num)->cpu_threadtype;
785 }
786
787 cpu_type_t
cpu_type(void)788 cpu_type(void)
789 {
790 return getCpuDatap()->cpu_type;
791 }
792
793 cpu_subtype_t
cpu_subtype(void)794 cpu_subtype(void)
795 {
796 return getCpuDatap()->cpu_subtype;
797 }
798
799 cpu_threadtype_t
cpu_threadtype(void)800 cpu_threadtype(void)
801 {
802 return getCpuDatap()->cpu_threadtype;
803 }
804
805 int
cpu_number(void)806 cpu_number(void)
807 {
808 return getCpuDatap()->cpu_number;
809 }
810
811 vm_offset_t
current_percpu_base(void)812 current_percpu_base(void)
813 {
814 return current_thread()->machine.pcpu_data_base;
815 }
816
817 vm_offset_t
other_percpu_base(int cpu)818 other_percpu_base(int cpu)
819 {
820 return (vm_offset_t)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data);
821 }
822
823 uint64_t
ml_get_wake_timebase(void)824 ml_get_wake_timebase(void)
825 {
826 return wake_abstime;
827 }
828
829 bool
ml_cpu_signal_is_enabled(void)830 ml_cpu_signal_is_enabled(void)
831 {
832 return !(getCpuDatap()->cpu_signal & SIGPdisabled);
833 }
834
835 bool
ml_cpu_can_exit(__unused int cpu_id)836 ml_cpu_can_exit(__unused int cpu_id)
837 {
838 /* processor_exit() is always allowed on the S2R path */
839 if (ml_is_quiescing()) {
840 return true;
841 }
842 #if HAS_CLUSTER && USE_APPLEARMSMP
843 /*
844 * Until the feature is known to be stable, guard it with a boot-arg
845 */
846 extern bool enable_processor_exit;
847 if (!enable_processor_exit) {
848 return false;
849 }
850 /*
851 * Cyprus and newer chips can disable individual non-boot CPUs. The
852 * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips.
853 */
854 if (CpuDataEntries[cpu_id].cpu_data_vaddr != &BootCpuData) {
855 return true;
856 }
857 #endif
858 return false;
859 }
860
861 #ifdef USE_APPLEARMSMP
862
863 void
ml_cpu_begin_state_transition(int cpu_id)864 ml_cpu_begin_state_transition(int cpu_id)
865 {
866 lck_rw_lock_exclusive(&cpu_state_lock);
867 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true;
868 lck_rw_unlock_exclusive(&cpu_state_lock);
869 }
870
871 void
ml_cpu_end_state_transition(int cpu_id)872 ml_cpu_end_state_transition(int cpu_id)
873 {
874 lck_rw_lock_exclusive(&cpu_state_lock);
875 CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false;
876 lck_rw_unlock_exclusive(&cpu_state_lock);
877 }
878
879 void
ml_cpu_begin_loop(void)880 ml_cpu_begin_loop(void)
881 {
882 lck_rw_lock_shared(&cpu_state_lock);
883 }
884
885 void
ml_cpu_end_loop(void)886 ml_cpu_end_loop(void)
887 {
888 lck_rw_unlock_shared(&cpu_state_lock);
889 }
890
891 #else /* USE_APPLEARMSMP */
892
893 void
ml_cpu_begin_state_transition(__unused int cpu_id)894 ml_cpu_begin_state_transition(__unused int cpu_id)
895 {
896 }
897
898 void
ml_cpu_end_state_transition(__unused int cpu_id)899 ml_cpu_end_state_transition(__unused int cpu_id)
900 {
901 }
902
903 void
ml_cpu_begin_loop(void)904 ml_cpu_begin_loop(void)
905 {
906 }
907
908 void
ml_cpu_end_loop(void)909 ml_cpu_end_loop(void)
910 {
911 }
912
913 #endif /* USE_APPLEARMSMP */
914