1 /*
2 * Copyright (c) 2011-2022 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <kern/cambria_layout.h>
35 #include <vm/vm_map.h>
36 #include <kperf/buffer.h>
37 #include <kperf/context.h>
38 #include <kperf/callstack.h>
39 #include <kperf/ast.h>
40 #include <sys/errno.h>
41
42 #if defined(__arm64__)
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #endif
46
47 static void
callstack_fixup_user(struct kp_ucallstack * cs,thread_t thread)48 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
49 {
50 uint64_t fixup_val = 0;
51 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
52
53 #if defined(__x86_64__)
54 user_addr_t sp_user;
55 bool user_64;
56 x86_saved_state_t *state;
57
58 state = get_user_regs(thread);
59 if (!state) {
60 goto out;
61 }
62
63 user_64 = is_saved_state64(state);
64 if (user_64) {
65 sp_user = saved_state64(state)->isf.rsp;
66 } else {
67 sp_user = saved_state32(state)->uesp;
68 }
69
70 if (thread == current_thread()) {
71 (void)copyin(sp_user, (char *)&fixup_val,
72 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
73 } else {
74 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
75 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
76 }
77
78 #elif defined(__arm64__)
79
80 struct arm_saved_state *state = get_user_regs(thread);
81 if (!state) {
82 goto out;
83 }
84
85 /* encode thumb mode into low bit of PC */
86 if (is_saved_state32(state) && (get_saved_state_cpsr(state) & PSR_TF)) {
87 cs->kpuc_frames[0] |= 1ULL;
88 }
89
90
91 fixup_val = get_saved_state_lr(state);
92
93 #else
94 #error "callstack_fixup_user: unsupported architecture"
95 #endif
96
97 out:
98 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
99 }
100
101 #if defined(__x86_64__)
102
103 __attribute__((used))
104 static kern_return_t
interrupted_kernel_sp_value(uintptr_t * sp_val)105 interrupted_kernel_sp_value(uintptr_t *sp_val)
106 {
107 x86_saved_state_t *state;
108 uintptr_t sp;
109 bool state_64;
110 uint64_t cs;
111 uintptr_t top, bottom;
112
113 state = current_cpu_datap()->cpu_int_state;
114 if (!state) {
115 return KERN_FAILURE;
116 }
117
118 state_64 = is_saved_state64(state);
119
120 if (state_64) {
121 cs = saved_state64(state)->isf.cs;
122 } else {
123 cs = saved_state32(state)->cs;
124 }
125 /* return early if interrupted a thread in user space */
126 if ((cs & SEL_PL) == SEL_PL_U) {
127 return KERN_FAILURE;
128 }
129
130 if (state_64) {
131 sp = saved_state64(state)->isf.rsp;
132 } else {
133 sp = saved_state32(state)->uesp;
134 }
135
136 /* make sure the stack pointer is pointing somewhere in this stack */
137 bottom = current_thread()->kernel_stack;
138 top = bottom + kernel_stack_size;
139 if (sp >= bottom && sp < top) {
140 return KERN_FAILURE;
141 }
142
143 *sp_val = *(uintptr_t *)sp;
144 return KERN_SUCCESS;
145 }
146
147 #elif defined(__arm64__)
148
149 __attribute__((used))
150 static kern_return_t
interrupted_kernel_lr(uintptr_t * lr)151 interrupted_kernel_lr(uintptr_t *lr)
152 {
153 struct arm_saved_state *state;
154
155 state = getCpuDatap()->cpu_int_state;
156
157 /* return early if interrupted a thread in user space */
158 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
159 return KERN_FAILURE;
160 }
161
162 *lr = get_saved_state_lr(state);
163 return KERN_SUCCESS;
164 }
165 #else /* defined(__arm64__) */
166 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
167 #endif /* !defined(__arm64__) */
168
169
170 static void
callstack_fixup_interrupted(struct kp_kcallstack * cs)171 callstack_fixup_interrupted(struct kp_kcallstack *cs)
172 {
173 uintptr_t fixup_val = 0;
174 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
175
176 /*
177 * Only provide arbitrary data on development or debug kernels.
178 */
179 #if DEVELOPMENT || DEBUG
180 #if defined(__x86_64__)
181 (void)interrupted_kernel_sp_value(&fixup_val);
182 #elif defined(__arm64__)
183 (void)interrupted_kernel_lr(&fixup_val);
184 #endif /* defined(__x86_64__) */
185 #endif /* DEVELOPMENT || DEBUG */
186
187 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
188 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
189 }
190
191 void
kperf_continuation_sample(struct kp_kcallstack * cs,struct kperf_context * context)192 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
193 {
194 thread_t thread;
195
196 assert(cs != NULL);
197 assert(context != NULL);
198
199 thread = context->cur_thread;
200 assert(thread != NULL);
201 assert(thread->continuation != NULL);
202
203 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
204 #ifdef __LP64__
205 cs->kpkc_flags |= CALLSTACK_64BIT;
206 #endif
207
208 cs->kpkc_nframes = 1;
209 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
210 }
211
212 void
kperf_backtrace_sample(struct kp_kcallstack * cs,struct kperf_context * context)213 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
214 {
215 assert(cs != NULL);
216 assert(context != NULL);
217 assert(context->cur_thread == current_thread());
218
219 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
220 #ifdef __LP64__
221 cs->kpkc_flags |= CALLSTACK_64BIT;
222 #endif
223
224 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
225
226 backtrace_info_t btinfo = BTI_NONE;
227 struct backtrace_control ctl = {
228 .btc_frame_addr = (uintptr_t)context->starting_fp,
229 };
230 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
231 &ctl, &btinfo);
232 if (cs->kpkc_nframes > 0) {
233 cs->kpkc_flags |= CALLSTACK_VALID;
234 /*
235 * Fake the value pointed to by the stack pointer or the link
236 * register for symbolicators.
237 */
238 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
239 cs->kpkc_nframes += 1;
240 }
241 if ((btinfo & BTI_TRUNCATED)) {
242 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
243 }
244
245 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
246 }
247
248 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
249 uint64_t *callStack, mach_msg_type_number_t *count,
250 boolean_t user_only);
251
252 void
kperf_kcallstack_sample(struct kp_kcallstack * cs,struct kperf_context * context)253 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
254 {
255 thread_t thread;
256
257 assert(cs != NULL);
258 assert(context != NULL);
259 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
260
261 thread = context->cur_thread;
262 assert(thread != NULL);
263
264 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
265 cs->kpkc_nframes);
266
267 cs->kpkc_flags = CALLSTACK_KERNEL;
268 #ifdef __LP64__
269 cs->kpkc_flags |= CALLSTACK_64BIT;
270 #endif
271
272 if (ml_at_interrupt_context()) {
273 assert(thread == current_thread());
274 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
275 backtrace_info_t btinfo = BTI_NONE;
276 struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, };
277 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
278 &ctl, &btinfo);
279 if (cs->kpkc_nframes != 0) {
280 callstack_fixup_interrupted(cs);
281 }
282 if ((btinfo & BTI_TRUNCATED)) {
283 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
284 }
285 } else {
286 /*
287 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
288 * other threads.
289 */
290 kern_return_t kr;
291 kr = chudxnu_thread_get_callstack64_kperf(thread,
292 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
293 if (kr == KERN_SUCCESS) {
294 cs->kpkc_flags |= CALLSTACK_VALID;
295 } else if (kr == KERN_RESOURCE_SHORTAGE) {
296 cs->kpkc_flags |= CALLSTACK_VALID;
297 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
298 } else {
299 cs->kpkc_nframes = 0;
300 }
301 }
302
303 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
304 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
305 }
306
307 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
308 cs->kpkc_flags, cs->kpkc_nframes);
309 }
310
311 void
kperf_ucallstack_sample(struct kp_ucallstack * cs,struct kperf_context * context)312 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
313 {
314 assert(ml_get_interrupts_enabled() == TRUE);
315
316 thread_t thread = context->cur_thread;
317 assert(thread != NULL);
318
319 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
320 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
321
322 struct backtrace_user_info btinfo = BTUINFO_INIT;
323 /*
324 * Leave space for the fixup information.
325 */
326 unsigned int maxnframes = cs->kpuc_nframes - 1;
327 struct backtrace_control ctl = { .btc_user_thread = thread, };
328 unsigned int nframes = backtrace_user(cs->kpuc_frames, maxnframes, &ctl,
329 &btinfo);
330 cs->kpuc_nframes = MIN(maxnframes, nframes);
331
332 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS |
333 ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) |
334 ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0);
335
336 /*
337 * Ignore EFAULT to get as much of the stack as possible.
338 */
339 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
340 callstack_fixup_user(cs, thread);
341 cs->kpuc_flags |= CALLSTACK_VALID;
342
343 if (cs->kpuc_nframes < maxnframes &&
344 btinfo.btui_async_frame_addr != 0) {
345 cs->kpuc_async_index = btinfo.btui_async_start_index;
346 ctl.btc_frame_addr = btinfo.btui_async_frame_addr;
347 ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET;
348 maxnframes -= cs->kpuc_nframes;
349 btinfo = BTUINFO_INIT;
350 unsigned int nasync_frames = backtrace_user(
351 &cs->kpuc_frames[cs->kpuc_nframes], maxnframes, &ctl, &btinfo);
352 if (btinfo.btui_info & BTI_TRUNCATED) {
353 cs->kpuc_flags |= CALLSTACK_TRUNCATED;
354 }
355 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
356 cs->kpuc_flags |= CALLSTACK_HAS_ASYNC;
357 cs->kpuc_async_nframes = nasync_frames;
358 }
359 }
360 } else {
361 cs->kpuc_nframes = 0;
362 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error);
363 }
364
365 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
366 cs->kpuc_flags, cs->kpuc_nframes);
367 }
368
369 static inline uintptr_t
scrub_word(uintptr_t * bt,int n_frames,int frame,bool kern)370 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
371 {
372 if (frame < n_frames) {
373 if (kern) {
374 return VM_KERNEL_UNSLIDE(bt[frame]);
375 } else {
376 return bt[frame];
377 }
378 } else {
379 return 0;
380 }
381 }
382
383 static inline uintptr_t
scrub_frame(uint64_t * bt,int n_frames,int frame)384 scrub_frame(uint64_t *bt, int n_frames, int frame)
385 {
386 if (frame < n_frames) {
387 return (uintptr_t)(bt[frame]);
388 } else {
389 return 0;
390 }
391 }
392
393 static void
callstack_log(uint32_t hdrid,uint32_t dataid,void * vframes,unsigned int nframes,unsigned int flags,unsigned int async_index,unsigned int async_nframes)394 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
395 unsigned int nframes, unsigned int flags, unsigned int async_index,
396 unsigned int async_nframes)
397 {
398 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
399 BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes);
400
401 unsigned int nevts = nframes / 4;
402 unsigned int ovf = nframes % 4;
403 if (ovf != 0) {
404 nevts++;
405 }
406
407 bool kern = flags & CALLSTACK_KERNEL;
408
409 if (flags & CALLSTACK_KERNEL_WORDS) {
410 uintptr_t *frames = vframes;
411 for (unsigned int i = 0; i < nevts; i++) {
412 unsigned int j = i * 4;
413 BUF_DATA(dataid,
414 scrub_word(frames, nframes, j + 0, kern),
415 scrub_word(frames, nframes, j + 1, kern),
416 scrub_word(frames, nframes, j + 2, kern),
417 scrub_word(frames, nframes, j + 3, kern));
418 }
419 } else {
420 for (unsigned int i = 0; i < nevts; i++) {
421 uint64_t *frames = vframes;
422 unsigned int j = i * 4;
423 BUF_DATA(dataid,
424 scrub_frame(frames, nframes, j + 0),
425 scrub_frame(frames, nframes, j + 1),
426 scrub_frame(frames, nframes, j + 2),
427 scrub_frame(frames, nframes, j + 3));
428 }
429 }
430
431 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
432 }
433
434 void
kperf_kcallstack_log(struct kp_kcallstack * cs)435 kperf_kcallstack_log(struct kp_kcallstack *cs)
436 {
437 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
438 cs->kpkc_nframes, cs->kpkc_flags, 0, 0);
439 }
440
441 void
kperf_ucallstack_log(struct kp_ucallstack * cs)442 kperf_ucallstack_log(struct kp_ucallstack *cs)
443 {
444 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
445 cs->kpuc_nframes + cs->kpuc_async_nframes, cs->kpuc_flags,
446 cs->kpuc_async_index, cs->kpuc_async_nframes);
447 }
448
449 #if CONFIG_EXCLAVES
450 void
kperf_excallstack_log(const stackshot_ipcstackentry_s * ipcstack)451 kperf_excallstack_log(const stackshot_ipcstackentry_s *ipcstack)
452 {
453 __block unsigned int nframes = 0;
454 __block unsigned int flags = CALLSTACK_VALID;
455 uint64_t frames[MAX_EXCALLSTACK_FRAMES] = {};
456 uint64_t *frames_block = frames;
457
458 BUF_DATA(PERF_CS_EXSTACK, ipcstack->asid);
459
460 if (ipcstack->stacktrace.has_value) {
461 address__v_visit(&ipcstack->stacktrace.value, ^(size_t i, const stackshot_address_s item) {
462 if (i >= MAX_EXCALLSTACK_FRAMES) {
463 flags |= CALLSTACK_TRUNCATED;
464 return;
465 }
466 frames_block[i] = item;
467 nframes += 1;
468 });
469 callstack_log(PERF_CS_EXHDR, PERF_CS_EXDATA, frames, nframes, flags, 0, 0);
470 }
471 }
472
473 bool
kperf_exclave_callstack_pend(struct kperf_context * context,unsigned int actionid)474 kperf_exclave_callstack_pend(struct kperf_context *context, unsigned int actionid)
475 {
476 if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC)
477 && (os_atomic_load(&context->cur_thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) == 0) {
478 os_atomic_or(&context->cur_thread->th_exclaves_inspection_state, TH_EXCLAVES_INSPECTION_KPERF, relaxed);
479 context->cur_thread->kperf_exclaves_ast |= T_KPERF_SET_ACTIONID(actionid);
480 return true;
481 }
482 return false;
483 }
484 #endif /* CONFIG_EXCLAVES */
485
486 int
kperf_ucallstack_pend(struct kperf_context * context,uint32_t depth,unsigned int actionid)487 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
488 unsigned int actionid)
489 {
490 if (depth < 2) {
491 panic("HUH");
492 }
493 kperf_ast_set_callstack_depth(context->cur_thread, depth);
494 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
495 actionid);
496 }
497
498 static kern_return_t
chudxnu_kern_read(void * dstaddr,vm_offset_t srcaddr,vm_size_t size)499 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
500 {
501 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
502 KERN_SUCCESS : KERN_FAILURE;
503 }
504
505 static kern_return_t
chudxnu_task_read(task_t task,void * kernaddr,uint64_t usraddr,vm_size_t size)506 chudxnu_task_read(
507 task_t task,
508 void *kernaddr,
509 uint64_t usraddr,
510 vm_size_t size)
511 {
512 //ppc version ported to arm
513 kern_return_t ret = KERN_SUCCESS;
514
515 if (ml_at_interrupt_context()) {
516 return KERN_FAILURE; // can't look at tasks on interrupt stack
517 }
518
519 if (current_task() == task) {
520 if (copyin(usraddr, kernaddr, size)) {
521 ret = KERN_FAILURE;
522 }
523 } else {
524 vm_map_t map = get_task_map(task);
525 ret = vm_map_read_user(map, usraddr, kernaddr, size);
526 }
527
528 return ret;
529 }
530
531 static inline uint64_t
chudxnu_vm_unslide(uint64_t ptr,int kaddr)532 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
533 {
534 if (!kaddr) {
535 return ptr;
536 }
537
538 return VM_KERNEL_UNSLIDE(ptr);
539 }
540
541 #if __arm64__
542
543 #if defined(HAS_APPLE_PAC)
544 #include <ptrauth.h>
545 #endif
546
547 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
548 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
549 // after sampling has finished.
550 //
551 // For an N-entry callstack:
552 //
553 // [0] current pc
554 // [1..N-3] stack frames (including current one)
555 // [N-2] current LR (return value if we're in a leaf function)
556 // [N-1] current r0 (in case we've saved LR in r0) (optional)
557 //
558 //
559 #define CS_FLAG_EXTRASP 1 // capture extra sp register
560
561 static kern_return_t
chudxnu_thread_get_callstack64_internal(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only,int flags)562 chudxnu_thread_get_callstack64_internal(
563 thread_t thread,
564 uint64_t *callStack,
565 mach_msg_type_number_t *count,
566 boolean_t user_only,
567 int flags)
568 {
569 kern_return_t kr = KERN_SUCCESS;
570 task_t task;
571 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
572 uint64_t prevPC = 0ULL;
573 uint64_t kernStackMin = thread->kernel_stack;
574 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
575 uint64_t *buffer = callStack;
576 int bufferIndex = 0;
577 int bufferMaxIndex = 0;
578 boolean_t kernel = FALSE;
579 struct arm_saved_state *sstate = NULL;
580 uint64_t pc = 0ULL;
581
582 task = get_threadtask(thread);
583 bufferMaxIndex = *count;
584 //get thread state
585 if (user_only) {
586 sstate = find_user_regs(thread);
587 } else {
588 sstate = find_kern_regs(thread);
589 }
590
591 if (!sstate) {
592 *count = 0;
593 return KERN_FAILURE;
594 }
595
596 if (is_saved_state64(sstate)) {
597 struct arm_saved_state64 *state = NULL;
598 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
599 uint64_t frame[2];
600
601 state = saved_state64(sstate);
602
603 /* make sure it is safe to dereference before you do it */
604 kernel = PSR64_IS_KERNEL(state->cpsr);
605
606 /* can't take a kernel callstack if we've got a user frame */
607 if (!user_only && !kernel) {
608 return KERN_FAILURE;
609 }
610
611 /*
612 * Reserve space for saving LR (and sometimes SP) at the end of the
613 * backtrace.
614 */
615 if (flags & CS_FLAG_EXTRASP) {
616 bufferMaxIndex -= 2;
617 } else {
618 bufferMaxIndex -= 1;
619 }
620
621 if (bufferMaxIndex < 2) {
622 *count = 0;
623 return KERN_RESOURCE_SHORTAGE;
624 }
625
626 currPC = state->pc;
627 currLR = state->lr;
628 currSP = state->sp;
629
630 fp = (uint64_t *)state->fp; /* frame pointer */
631 #if defined(HAS_APPLE_PAC)
632 /* frame pointers on stack will be signed by arm64e ABI */
633 fp = ptrauth_strip(fp, ptrauth_key_frame_pointer);
634 #endif
635 topfp = fp;
636
637 bufferIndex = 0; // start with a stack of size zero
638 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
639
640 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
641
642 // Now, fill buffer with stack backtraces.
643 while (bufferIndex < bufferMaxIndex) {
644 pc = 0ULL;
645 /*
646 * Below the frame pointer, the following values are saved:
647 * -> FP
648 */
649
650 /*
651 * Note that we read the pc even for the first stack frame
652 * (which, in theory, is always empty because the callee fills
653 * it in just before it lowers the stack. However, if we
654 * catch the program in between filling in the return address
655 * and lowering the stack, we want to still have a valid
656 * backtrace. FixupStack correctly disregards this value if
657 * necessary.
658 */
659
660 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
661 /* frame pointer is invalid - stop backtracing */
662 pc = 0ULL;
663 break;
664 }
665
666 if (kernel) {
667 if (((uint64_t)fp > kernStackMax) ||
668 ((uint64_t)fp < kernStackMin)) {
669 kr = KERN_FAILURE;
670 } else {
671 kr = chudxnu_kern_read(&frame,
672 (vm_offset_t)fp,
673 (vm_size_t)sizeof(frame));
674 if (kr == KERN_SUCCESS) {
675 #if defined(HAS_APPLE_PAC)
676 /* return addresses on stack will be signed by arm64e ABI */
677 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
678 #else
679 pc = frame[1];
680 #endif
681 nextFramePointer = (uint64_t *)frame[0];
682 #if defined(HAS_APPLE_PAC)
683 /* frame pointers on stack will be signed by arm64e ABI */
684 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
685 #endif
686 } else {
687 pc = 0ULL;
688 nextFramePointer = 0ULL;
689 kr = KERN_FAILURE;
690 }
691 }
692 } else {
693 kr = chudxnu_task_read(task,
694 &frame,
695 (vm_offset_t)fp,
696 (vm_size_t)sizeof(frame));
697 if (kr == KERN_SUCCESS) {
698 #if defined(HAS_APPLE_PAC)
699 /* return addresses on stack will be signed by arm64e ABI */
700 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
701 #else
702 pc = frame[1];
703 #endif
704 nextFramePointer = (uint64_t *)(frame[0]);
705 #if defined(HAS_APPLE_PAC)
706 /* frame pointers on stack will be signed by arm64e ABI */
707 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
708 #endif
709 } else {
710 pc = 0ULL;
711 nextFramePointer = 0ULL;
712 kr = KERN_FAILURE;
713 }
714 }
715
716 if (kr != KERN_SUCCESS) {
717 pc = 0ULL;
718 break;
719 }
720
721 if (nextFramePointer) {
722 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
723 prevPC = pc;
724 }
725
726 if (nextFramePointer < fp) {
727 break;
728 } else {
729 fp = nextFramePointer;
730 }
731 }
732
733 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
734
735 if (bufferIndex >= bufferMaxIndex) {
736 bufferIndex = bufferMaxIndex;
737 kr = KERN_RESOURCE_SHORTAGE;
738 } else {
739 kr = KERN_SUCCESS;
740 }
741
742 // Save link register and SP at bottom of stack (used for later fixup).
743 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
744 if (flags & CS_FLAG_EXTRASP) {
745 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
746 }
747 } else {
748 struct arm_saved_state32 *state = NULL;
749 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
750
751 /* 64-bit kernel stacks, 32-bit user stacks */
752 uint64_t frame[2];
753 uint32_t frame32[2];
754
755 state = saved_state32(sstate);
756
757 /* make sure it is safe to dereference before you do it */
758 kernel = PSR_IS_KERNEL(state->cpsr);
759
760 /* can't take a kernel callstack if we've got a user frame */
761 if (!user_only && !kernel) {
762 return KERN_FAILURE;
763 }
764
765 /*
766 * Reserve space for saving LR (and sometimes SP) at the end of the
767 * backtrace.
768 */
769 if (flags & CS_FLAG_EXTRASP) {
770 bufferMaxIndex -= 2;
771 } else {
772 bufferMaxIndex -= 1;
773 }
774
775 if (bufferMaxIndex < 2) {
776 *count = 0;
777 return KERN_RESOURCE_SHORTAGE;
778 }
779
780 currPC = (uint64_t)state->pc; /* r15 */
781 if (state->cpsr & PSR_TF) {
782 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
783 }
784 currLR = (uint64_t)state->lr; /* r14 */
785 currSP = (uint64_t)state->sp; /* r13 */
786
787 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
788 topfp = fp;
789
790 bufferIndex = 0; // start with a stack of size zero
791 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
792
793 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
794
795 // Now, fill buffer with stack backtraces.
796 while (bufferIndex < bufferMaxIndex) {
797 pc = 0ULL;
798 /*
799 * Below the frame pointer, the following values are saved:
800 * -> FP
801 */
802
803 /*
804 * Note that we read the pc even for the first stack frame
805 * (which, in theory, is always empty because the callee fills
806 * it in just before it lowers the stack. However, if we
807 * catch the program in between filling in the return address
808 * and lowering the stack, we want to still have a valid
809 * backtrace. FixupStack correctly disregards this value if
810 * necessary.
811 */
812
813 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
814 /* frame pointer is invalid - stop backtracing */
815 pc = 0ULL;
816 break;
817 }
818
819 if (kernel) {
820 if (((uint32_t)fp > kernStackMax) ||
821 ((uint32_t)fp < kernStackMin)) {
822 kr = KERN_FAILURE;
823 } else {
824 kr = chudxnu_kern_read(&frame,
825 (vm_offset_t)fp,
826 (vm_size_t)sizeof(frame));
827 if (kr == KERN_SUCCESS) {
828 pc = (uint64_t)frame[1];
829 nextFramePointer = (uint32_t *) (frame[0]);
830 } else {
831 pc = 0ULL;
832 nextFramePointer = 0ULL;
833 kr = KERN_FAILURE;
834 }
835 }
836 } else {
837 kr = chudxnu_task_read(task,
838 &frame32,
839 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
840 sizeof(frame32));
841 if (kr == KERN_SUCCESS) {
842 pc = (uint64_t)frame32[1];
843 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
844 } else {
845 pc = 0ULL;
846 nextFramePointer = 0ULL;
847 kr = KERN_FAILURE;
848 }
849 }
850
851 if (kr != KERN_SUCCESS) {
852 pc = 0ULL;
853 break;
854 }
855
856 if (nextFramePointer) {
857 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
858 prevPC = pc;
859 }
860
861 if (nextFramePointer < fp) {
862 break;
863 } else {
864 fp = nextFramePointer;
865 }
866 }
867
868 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
869
870 /* clamp callstack size to max */
871 if (bufferIndex >= bufferMaxIndex) {
872 bufferIndex = bufferMaxIndex;
873 kr = KERN_RESOURCE_SHORTAGE;
874 } else {
875 /* ignore all other failures */
876 kr = KERN_SUCCESS;
877 }
878
879 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
880 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
881 if (flags & CS_FLAG_EXTRASP) {
882 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
883 }
884 }
885
886 *count = bufferIndex;
887 return kr;
888 }
889
890 kern_return_t
chudxnu_thread_get_callstack64_kperf(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only)891 chudxnu_thread_get_callstack64_kperf(
892 thread_t thread,
893 uint64_t *callStack,
894 mach_msg_type_number_t *count,
895 boolean_t user_only)
896 {
897 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
898 }
899 #elif __x86_64__
900
901 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
902 // don't try to read in the hole
903 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
904 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
905 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
906
907 typedef struct _cframe64_t {
908 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
909 uint64_t caller;
910 uint64_t args[0];
911 }cframe64_t;
912
913
914 typedef struct _cframe_t {
915 uint32_t prev; // this is really a user32-space pointer to the previous frame
916 uint32_t caller;
917 uint32_t args[0];
918 } cframe_t;
919
920 extern void * find_user_regs(thread_t);
921 extern x86_saved_state32_t *find_kern_regs(thread_t);
922
923 static kern_return_t
do_kernel_backtrace(thread_t thread,struct x86_kernel_state * regs,uint64_t * frames,mach_msg_type_number_t * start_idx,mach_msg_type_number_t max_idx)924 do_kernel_backtrace(
925 thread_t thread,
926 struct x86_kernel_state *regs,
927 uint64_t *frames,
928 mach_msg_type_number_t *start_idx,
929 mach_msg_type_number_t max_idx)
930 {
931 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
932 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
933 mach_msg_type_number_t ct = *start_idx;
934 kern_return_t kr = KERN_FAILURE;
935
936 #if __LP64__
937 uint64_t currPC = 0ULL;
938 uint64_t currFP = 0ULL;
939 uint64_t prevPC = 0ULL;
940 uint64_t prevFP = 0ULL;
941 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
942 return KERN_FAILURE;
943 }
944 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
945 return KERN_FAILURE;
946 }
947 #else
948 uint32_t currPC = 0U;
949 uint32_t currFP = 0U;
950 uint32_t prevPC = 0U;
951 uint32_t prevFP = 0U;
952 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
953 return KERN_FAILURE;
954 }
955 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
956 return KERN_FAILURE;
957 }
958 #endif
959
960 if (*start_idx >= max_idx) {
961 return KERN_RESOURCE_SHORTAGE; // no frames traced
962 }
963 if (!currPC) {
964 return KERN_FAILURE;
965 }
966
967 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
968
969 // build a backtrace of this kernel state
970 #if __LP64__
971 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
972 // this is the address where caller lives in the user thread
973 uint64_t caller = currFP + sizeof(uint64_t);
974 #else
975 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
976 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
977 #endif
978
979 if (!currFP || !currPC) {
980 currPC = 0;
981 break;
982 }
983
984 if (ct >= max_idx) {
985 *start_idx = ct;
986 return KERN_RESOURCE_SHORTAGE;
987 }
988
989 /* read our caller */
990 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
991
992 if (kr != KERN_SUCCESS || !currPC) {
993 currPC = 0UL;
994 break;
995 }
996
997 /*
998 * retrive contents of the frame pointer and advance to the next stack
999 * frame if it's valid
1000 */
1001 prevFP = 0;
1002 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1003
1004 #if __LP64__
1005 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1006 #else
1007 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1008 #endif
1009 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1010 prevPC = currPC;
1011 }
1012 if (prevFP <= currFP) {
1013 break;
1014 } else {
1015 currFP = prevFP;
1016 }
1017 }
1018
1019 *start_idx = ct;
1020 return KERN_SUCCESS;
1021 }
1022
1023
1024
1025 static kern_return_t
1026 do_backtrace32(
1027 task_t task,
1028 thread_t thread,
1029 x86_saved_state32_t *regs,
1030 uint64_t *frames,
1031 mach_msg_type_number_t *start_idx,
1032 mach_msg_type_number_t max_idx,
1033 boolean_t supervisor)
1034 {
1035 uint32_t tmpWord = 0UL;
1036 uint64_t currPC = (uint64_t) regs->eip;
1037 uint64_t currFP = (uint64_t) regs->ebp;
1038 uint64_t prevPC = 0ULL;
1039 uint64_t prevFP = 0ULL;
1040 uint64_t kernStackMin = thread->kernel_stack;
1041 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1042 mach_msg_type_number_t ct = *start_idx;
1043 kern_return_t kr = KERN_FAILURE;
1044
1045 if (ct >= max_idx) {
1046 return KERN_RESOURCE_SHORTAGE; // no frames traced
1047 }
1048 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1049
1050 // build a backtrace of this 32 bit state.
1051 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1052 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1053
1054 if (!currFP) {
1055 currPC = 0;
1056 break;
1057 }
1058
1059 if (ct >= max_idx) {
1060 *start_idx = ct;
1061 return KERN_RESOURCE_SHORTAGE;
1062 }
1063
1064 /* read our caller */
1065 if (supervisor) {
1066 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1067 } else {
1068 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1069 }
1070
1071 if (kr != KERN_SUCCESS) {
1072 currPC = 0ULL;
1073 break;
1074 }
1075
1076 currPC = (uint64_t) tmpWord; // promote 32 bit address
1077
1078 /*
1079 * retrive contents of the frame pointer and advance to the next stack
1080 * frame if it's valid
1081 */
1082 prevFP = 0;
1083 if (supervisor) {
1084 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1085 } else {
1086 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1087 }
1088 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1089
1090 if (prevFP) {
1091 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1092 prevPC = currPC;
1093 }
1094 if (prevFP < currFP) {
1095 break;
1096 } else {
1097 currFP = prevFP;
1098 }
1099 }
1100
1101 *start_idx = ct;
1102 return KERN_SUCCESS;
1103 }
1104
1105 static kern_return_t
1106 do_backtrace64(
1107 task_t task,
1108 thread_t thread,
1109 x86_saved_state64_t *regs,
1110 uint64_t *frames,
1111 mach_msg_type_number_t *start_idx,
1112 mach_msg_type_number_t max_idx,
1113 boolean_t supervisor)
1114 {
1115 uint64_t currPC = regs->isf.rip;
1116 uint64_t currFP = regs->rbp;
1117 uint64_t prevPC = 0ULL;
1118 uint64_t prevFP = 0ULL;
1119 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1120 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1121 mach_msg_type_number_t ct = *start_idx;
1122 kern_return_t kr = KERN_FAILURE;
1123
1124 if (*start_idx >= max_idx) {
1125 return KERN_RESOURCE_SHORTAGE; // no frames traced
1126 }
1127 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1128
1129 // build a backtrace of this 32 bit state.
1130 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1131 // this is the address where caller lives in the user thread
1132 uint64_t caller = currFP + sizeof(uint64_t);
1133
1134 if (!currFP) {
1135 currPC = 0;
1136 break;
1137 }
1138
1139 if (ct >= max_idx) {
1140 *start_idx = ct;
1141 return KERN_RESOURCE_SHORTAGE;
1142 }
1143
1144 /* read our caller */
1145 if (supervisor) {
1146 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1147 } else {
1148 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1149 }
1150
1151 if (kr != KERN_SUCCESS) {
1152 currPC = 0ULL;
1153 break;
1154 }
1155
1156 /*
1157 * retrive contents of the frame pointer and advance to the next stack
1158 * frame if it's valid
1159 */
1160 prevFP = 0;
1161 if (supervisor) {
1162 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1163 } else {
1164 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1165 }
1166
1167 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1168 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1169 prevPC = currPC;
1170 }
1171 if (prevFP < currFP) {
1172 break;
1173 } else {
1174 currFP = prevFP;
1175 }
1176 }
1177
1178 *start_idx = ct;
1179 return KERN_SUCCESS;
1180 }
1181
1182 static kern_return_t
1183 chudxnu_thread_get_callstack64_internal(
1184 thread_t thread,
1185 uint64_t *callstack,
1186 mach_msg_type_number_t *count,
1187 boolean_t user_only,
1188 boolean_t kern_only)
1189 {
1190 kern_return_t kr = KERN_FAILURE;
1191 task_t task = get_threadtask(thread);
1192 uint64_t currPC = 0ULL;
1193 boolean_t supervisor = FALSE;
1194 mach_msg_type_number_t bufferIndex = 0;
1195 mach_msg_type_number_t bufferMaxIndex = *count;
1196 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1197 x86_saved_state64_t *regs64 = NULL;
1198 x86_saved_state32_t *regs32 = NULL;
1199 x86_saved_state32_t *u_regs32 = NULL;
1200 x86_saved_state64_t *u_regs64 = NULL;
1201 struct x86_kernel_state *kregs = NULL;
1202
1203 if (ml_at_interrupt_context()) {
1204 if (user_only) {
1205 /* can't backtrace user state on interrupt stack. */
1206 return KERN_FAILURE;
1207 }
1208
1209 /* backtracing at interrupt context? */
1210 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1211 /*
1212 * Locate the registers for the interrupted thread, assuming it is
1213 * current_thread().
1214 */
1215 tagged_regs = current_cpu_datap()->cpu_int_state;
1216
1217 if (is_saved_state64(tagged_regs)) {
1218 /* 64 bit registers */
1219 regs64 = saved_state64(tagged_regs);
1220 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1221 } else {
1222 /* 32 bit registers */
1223 regs32 = saved_state32(tagged_regs);
1224 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1225 }
1226 }
1227 }
1228
1229 if (!ml_at_interrupt_context() && kernel_task == task) {
1230 if (!thread->kernel_stack) {
1231 return KERN_FAILURE;
1232 }
1233
1234 // Kernel thread not at interrupt context
1235 kregs = (struct x86_kernel_state *)NULL;
1236
1237 // nofault read of the thread->kernel_stack pointer
1238 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1239 return KERN_FAILURE;
1240 }
1241
1242 // Adjust to find the saved kernel state
1243 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1244
1245 supervisor = TRUE;
1246 } else if (!tagged_regs) {
1247 /*
1248 * not at interrupt context, or tracing a different thread than
1249 * current_thread() at interrupt context
1250 */
1251 tagged_regs = USER_STATE(thread);
1252 if (is_saved_state64(tagged_regs)) {
1253 /* 64 bit registers */
1254 regs64 = saved_state64(tagged_regs);
1255 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1256 } else {
1257 /* 32 bit registers */
1258 regs32 = saved_state32(tagged_regs);
1259 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1260 }
1261 }
1262
1263 *count = 0;
1264
1265 if (supervisor) {
1266 // the caller only wants a user callstack.
1267 if (user_only) {
1268 // bail - we've only got kernel state
1269 return KERN_FAILURE;
1270 }
1271 } else {
1272 // regs32(64) is not in supervisor mode.
1273 u_regs32 = regs32;
1274 u_regs64 = regs64;
1275 regs32 = NULL;
1276 regs64 = NULL;
1277 }
1278
1279 if (user_only) {
1280 /* we only want to backtrace the user mode */
1281 if (!(u_regs32 || u_regs64)) {
1282 /* no user state to look at */
1283 return KERN_FAILURE;
1284 }
1285 }
1286
1287 /*
1288 * Order of preference for top of stack:
1289 * 64 bit kernel state (not likely)
1290 * 32 bit kernel state
1291 * 64 bit user land state
1292 * 32 bit user land state
1293 */
1294
1295 if (kregs) {
1296 /*
1297 * nofault read of the registers from the kernel stack (as they can
1298 * disappear on the fly).
1299 */
1300
1301 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1302 return KERN_FAILURE;
1303 }
1304 } else if (regs64) {
1305 currPC = regs64->isf.rip;
1306 } else if (regs32) {
1307 currPC = (uint64_t) regs32->eip;
1308 } else if (u_regs64) {
1309 currPC = u_regs64->isf.rip;
1310 } else if (u_regs32) {
1311 currPC = (uint64_t) u_regs32->eip;
1312 }
1313
1314 if (!currPC) {
1315 /* no top of the stack, bail out */
1316 return KERN_FAILURE;
1317 }
1318
1319 bufferIndex = 0;
1320
1321 if (bufferMaxIndex < 1) {
1322 *count = 0;
1323 return KERN_RESOURCE_SHORTAGE;
1324 }
1325
1326 /* backtrace kernel */
1327 if (kregs) {
1328 addr64_t address = 0ULL;
1329 size_t size = 0UL;
1330
1331 // do the backtrace
1332 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1333
1334 // and do a nofault read of (r|e)sp
1335 uint64_t rsp = 0ULL;
1336 size = sizeof(uint64_t);
1337
1338 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1339 address = 0ULL;
1340 }
1341
1342 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1343 callstack[bufferIndex++] = (uint64_t)rsp;
1344 }
1345 } else if (regs64) {
1346 uint64_t rsp = 0ULL;
1347
1348 // backtrace the 64bit side.
1349 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1350 bufferMaxIndex - 1, TRUE);
1351
1352 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1353 bufferIndex < bufferMaxIndex) {
1354 callstack[bufferIndex++] = rsp;
1355 }
1356 } else if (regs32) {
1357 uint32_t esp = 0UL;
1358
1359 // backtrace the 32bit side.
1360 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1361 bufferMaxIndex - 1, TRUE);
1362
1363 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1364 bufferIndex < bufferMaxIndex) {
1365 callstack[bufferIndex++] = (uint64_t) esp;
1366 }
1367 } else if (u_regs64 && !kern_only) {
1368 /* backtrace user land */
1369 uint64_t rsp = 0ULL;
1370
1371 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1372 bufferMaxIndex - 1, FALSE);
1373
1374 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1375 bufferIndex < bufferMaxIndex) {
1376 callstack[bufferIndex++] = rsp;
1377 }
1378 } else if (u_regs32 && !kern_only) {
1379 uint32_t esp = 0UL;
1380
1381 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1382 bufferMaxIndex - 1, FALSE);
1383
1384 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1385 bufferIndex < bufferMaxIndex) {
1386 callstack[bufferIndex++] = (uint64_t) esp;
1387 }
1388 }
1389
1390 *count = bufferIndex;
1391 return kr;
1392 }
1393
1394 __private_extern__
1395 kern_return_t
1396 chudxnu_thread_get_callstack64_kperf(
1397 thread_t thread,
1398 uint64_t *callstack,
1399 mach_msg_type_number_t *count,
1400 boolean_t is_user)
1401 {
1402 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1403 }
1404 #else /* !__arm64__ && !__x86_64__ */
1405 #error kperf: unsupported architecture
1406 #endif /* !__arm64__ && !__x86_64__ */
1407