1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <kern/cambria_layout.h>
35 #include <vm/vm_map.h>
36 #include <kperf/buffer.h>
37 #include <kperf/context.h>
38 #include <kperf/callstack.h>
39 #include <kperf/ast.h>
40 #include <sys/errno.h>
41
42 #if defined(__arm__) || defined(__arm64__)
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #endif
46
47 static void
callstack_fixup_user(struct kp_ucallstack * cs,thread_t thread)48 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
49 {
50 uint64_t fixup_val = 0;
51 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
52
53 #if defined(__x86_64__)
54 user_addr_t sp_user;
55 bool user_64;
56 x86_saved_state_t *state;
57
58 state = get_user_regs(thread);
59 if (!state) {
60 goto out;
61 }
62
63 user_64 = is_saved_state64(state);
64 if (user_64) {
65 sp_user = saved_state64(state)->isf.rsp;
66 } else {
67 sp_user = saved_state32(state)->uesp;
68 }
69
70 if (thread == current_thread()) {
71 (void)copyin(sp_user, (char *)&fixup_val,
72 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
73 } else {
74 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
75 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
76 }
77
78 #elif defined(__arm64__) || defined(__arm__)
79
80 struct arm_saved_state *state = get_user_regs(thread);
81 if (!state) {
82 goto out;
83 }
84
85 /* encode thumb mode into low bit of PC */
86 if (get_saved_state_cpsr(state) & PSR_TF) {
87 cs->kpuc_frames[0] |= 1ULL;
88 }
89
90
91 fixup_val = get_saved_state_lr(state);
92
93 #else
94 #error "callstack_fixup_user: unsupported architecture"
95 #endif
96
97 out:
98 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
99 }
100
101 #if defined(__x86_64__)
102
103 __attribute__((used))
104 static kern_return_t
interrupted_kernel_sp_value(uintptr_t * sp_val)105 interrupted_kernel_sp_value(uintptr_t *sp_val)
106 {
107 x86_saved_state_t *state;
108 uintptr_t sp;
109 bool state_64;
110 uint64_t cs;
111 uintptr_t top, bottom;
112
113 state = current_cpu_datap()->cpu_int_state;
114 if (!state) {
115 return KERN_FAILURE;
116 }
117
118 state_64 = is_saved_state64(state);
119
120 if (state_64) {
121 cs = saved_state64(state)->isf.cs;
122 } else {
123 cs = saved_state32(state)->cs;
124 }
125 /* return early if interrupted a thread in user space */
126 if ((cs & SEL_PL) == SEL_PL_U) {
127 return KERN_FAILURE;
128 }
129
130 if (state_64) {
131 sp = saved_state64(state)->isf.rsp;
132 } else {
133 sp = saved_state32(state)->uesp;
134 }
135
136 /* make sure the stack pointer is pointing somewhere in this stack */
137 bottom = current_thread()->kernel_stack;
138 top = bottom + kernel_stack_size;
139 if (sp >= bottom && sp < top) {
140 return KERN_FAILURE;
141 }
142
143 *sp_val = *(uintptr_t *)sp;
144 return KERN_SUCCESS;
145 }
146
147 #elif defined(__arm64__)
148
149 __attribute__((used))
150 static kern_return_t
interrupted_kernel_lr(uintptr_t * lr)151 interrupted_kernel_lr(uintptr_t *lr)
152 {
153 struct arm_saved_state *state;
154
155 state = getCpuDatap()->cpu_int_state;
156
157 /* return early if interrupted a thread in user space */
158 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
159 return KERN_FAILURE;
160 }
161
162 *lr = get_saved_state_lr(state);
163 return KERN_SUCCESS;
164 }
165
166 #elif defined(__arm__)
167
168 __attribute__((used))
169 static kern_return_t
interrupted_kernel_lr(uintptr_t * lr)170 interrupted_kernel_lr(uintptr_t *lr)
171 {
172 struct arm_saved_state *state;
173
174 state = getCpuDatap()->cpu_int_state;
175
176 /* return early if interrupted a thread in user space */
177 if (PSR_IS_USER(get_saved_state_cpsr(state))) {
178 return KERN_FAILURE;
179 }
180
181 *lr = get_saved_state_lr(state);
182 return KERN_SUCCESS;
183 }
184
185 #else /* defined(__arm__) */
186 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
187 #endif /* !defined(__arm__) */
188
189
190 static void
callstack_fixup_interrupted(struct kp_kcallstack * cs)191 callstack_fixup_interrupted(struct kp_kcallstack *cs)
192 {
193 uintptr_t fixup_val = 0;
194 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
195
196 /*
197 * Only provide arbitrary data on development or debug kernels.
198 */
199 #if DEVELOPMENT || DEBUG
200 #if defined(__x86_64__)
201 (void)interrupted_kernel_sp_value(&fixup_val);
202 #elif defined(__arm64__) || defined(__arm__)
203 (void)interrupted_kernel_lr(&fixup_val);
204 #endif /* defined(__x86_64__) */
205 #endif /* DEVELOPMENT || DEBUG */
206
207 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
208 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
209 }
210
211 void
kperf_continuation_sample(struct kp_kcallstack * cs,struct kperf_context * context)212 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
213 {
214 thread_t thread;
215
216 assert(cs != NULL);
217 assert(context != NULL);
218
219 thread = context->cur_thread;
220 assert(thread != NULL);
221 assert(thread->continuation != NULL);
222
223 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
224 #ifdef __LP64__
225 cs->kpkc_flags |= CALLSTACK_64BIT;
226 #endif
227
228 cs->kpkc_nframes = 1;
229 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
230 }
231
232 void
kperf_backtrace_sample(struct kp_kcallstack * cs,struct kperf_context * context)233 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
234 {
235 assert(cs != NULL);
236 assert(context != NULL);
237 assert(context->cur_thread == current_thread());
238
239 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
240 #ifdef __LP64__
241 cs->kpkc_flags |= CALLSTACK_64BIT;
242 #endif
243
244 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
245
246 backtrace_info_t btinfo = BTI_NONE;
247 struct backtrace_control ctl = {
248 .btc_frame_addr = (uintptr_t)context->starting_fp,
249 };
250 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
251 &ctl, &btinfo);
252 if (cs->kpkc_nframes > 0) {
253 cs->kpkc_flags |= CALLSTACK_VALID;
254 /*
255 * Fake the value pointed to by the stack pointer or the link
256 * register for symbolicators.
257 */
258 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
259 cs->kpkc_nframes += 1;
260 }
261 if ((btinfo & BTI_TRUNCATED)) {
262 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
263 }
264
265 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
266 }
267
268 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
269 uint64_t *callStack, mach_msg_type_number_t *count,
270 boolean_t user_only);
271
272 void
kperf_kcallstack_sample(struct kp_kcallstack * cs,struct kperf_context * context)273 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
274 {
275 thread_t thread;
276
277 assert(cs != NULL);
278 assert(context != NULL);
279 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
280
281 thread = context->cur_thread;
282 assert(thread != NULL);
283
284 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
285 cs->kpkc_nframes);
286
287 cs->kpkc_flags = CALLSTACK_KERNEL;
288 #ifdef __LP64__
289 cs->kpkc_flags |= CALLSTACK_64BIT;
290 #endif
291
292 if (ml_at_interrupt_context()) {
293 assert(thread == current_thread());
294 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
295 backtrace_info_t btinfo = BTI_NONE;
296 struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, };
297 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
298 &ctl, &btinfo);
299 if (cs->kpkc_nframes != 0) {
300 callstack_fixup_interrupted(cs);
301 }
302 if ((btinfo & BTI_TRUNCATED)) {
303 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
304 }
305 } else {
306 /*
307 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
308 * other threads.
309 */
310 kern_return_t kr;
311 kr = chudxnu_thread_get_callstack64_kperf(thread,
312 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
313 if (kr == KERN_SUCCESS) {
314 cs->kpkc_flags |= CALLSTACK_VALID;
315 } else if (kr == KERN_RESOURCE_SHORTAGE) {
316 cs->kpkc_flags |= CALLSTACK_VALID;
317 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
318 } else {
319 cs->kpkc_nframes = 0;
320 }
321 }
322
323 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
324 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
325 }
326
327 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
328 cs->kpkc_flags, cs->kpkc_nframes);
329 }
330
331 void
kperf_ucallstack_sample(struct kp_ucallstack * cs,struct kperf_context * context)332 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
333 {
334 assert(ml_get_interrupts_enabled() == TRUE);
335
336 thread_t thread = context->cur_thread;
337 assert(thread != NULL);
338
339 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
340 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
341
342 struct backtrace_user_info btinfo = BTUINFO_INIT;
343 /*
344 * Leave space for the fixup information.
345 */
346 unsigned int maxnframes = cs->kpuc_nframes - 1;
347 struct backtrace_control ctl = { .btc_user_thread = thread, };
348 unsigned int nframes = backtrace_user(cs->kpuc_frames, maxnframes, &ctl,
349 &btinfo);
350 cs->kpuc_nframes = MIN(maxnframes, nframes);
351
352 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS |
353 ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) |
354 ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0);
355
356 /*
357 * Ignore EFAULT to get as much of the stack as possible.
358 */
359 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
360 callstack_fixup_user(cs, thread);
361 cs->kpuc_flags |= CALLSTACK_VALID;
362
363 if (cs->kpuc_nframes < maxnframes &&
364 btinfo.btui_async_frame_addr != 0) {
365 cs->kpuc_async_index = btinfo.btui_async_start_index;
366 ctl.btc_frame_addr = btinfo.btui_async_frame_addr;
367 ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET;
368 maxnframes -= cs->kpuc_nframes;
369 btinfo = BTUINFO_INIT;
370 unsigned int nasync_frames = backtrace_user(
371 &cs->kpuc_frames[cs->kpuc_nframes], maxnframes, &ctl, &btinfo);
372 if (btinfo.btui_info & BTI_TRUNCATED) {
373 cs->kpuc_flags |= CALLSTACK_TRUNCATED;
374 }
375 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
376 cs->kpuc_flags |= CALLSTACK_HAS_ASYNC;
377 cs->kpuc_async_nframes = nasync_frames;
378 }
379 }
380 } else {
381 cs->kpuc_nframes = 0;
382 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error);
383 }
384
385 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
386 cs->kpuc_flags, cs->kpuc_nframes);
387 }
388
389 static inline uintptr_t
scrub_word(uintptr_t * bt,int n_frames,int frame,bool kern)390 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
391 {
392 if (frame < n_frames) {
393 if (kern) {
394 return VM_KERNEL_UNSLIDE(bt[frame]);
395 } else {
396 return bt[frame];
397 }
398 } else {
399 return 0;
400 }
401 }
402
403 static inline uintptr_t
scrub_frame(uint64_t * bt,int n_frames,int frame)404 scrub_frame(uint64_t *bt, int n_frames, int frame)
405 {
406 if (frame < n_frames) {
407 return (uintptr_t)(bt[frame]);
408 } else {
409 return 0;
410 }
411 }
412
413 static void
callstack_log(uint32_t hdrid,uint32_t dataid,void * vframes,unsigned int nframes,unsigned int flags,unsigned int async_index,unsigned int async_nframes)414 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
415 unsigned int nframes, unsigned int flags, unsigned int async_index,
416 unsigned int async_nframes)
417 {
418 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
419 BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes);
420
421 unsigned int nevts = nframes / 4;
422 unsigned int ovf = nframes % 4;
423 if (ovf != 0) {
424 nevts++;
425 }
426
427 bool kern = flags & CALLSTACK_KERNEL;
428
429 if (flags & CALLSTACK_KERNEL_WORDS) {
430 uintptr_t *frames = vframes;
431 for (unsigned int i = 0; i < nevts; i++) {
432 unsigned int j = i * 4;
433 BUF_DATA(dataid,
434 scrub_word(frames, nframes, j + 0, kern),
435 scrub_word(frames, nframes, j + 1, kern),
436 scrub_word(frames, nframes, j + 2, kern),
437 scrub_word(frames, nframes, j + 3, kern));
438 }
439 } else {
440 for (unsigned int i = 0; i < nevts; i++) {
441 uint64_t *frames = vframes;
442 unsigned int j = i * 4;
443 BUF_DATA(dataid,
444 scrub_frame(frames, nframes, j + 0),
445 scrub_frame(frames, nframes, j + 1),
446 scrub_frame(frames, nframes, j + 2),
447 scrub_frame(frames, nframes, j + 3));
448 }
449 }
450
451 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
452 }
453
454 void
kperf_kcallstack_log(struct kp_kcallstack * cs)455 kperf_kcallstack_log(struct kp_kcallstack *cs)
456 {
457 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
458 cs->kpkc_nframes, cs->kpkc_flags, 0, 0);
459 }
460
461 void
kperf_ucallstack_log(struct kp_ucallstack * cs)462 kperf_ucallstack_log(struct kp_ucallstack *cs)
463 {
464 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
465 cs->kpuc_nframes + cs->kpuc_async_nframes, cs->kpuc_flags,
466 cs->kpuc_async_index, cs->kpuc_async_nframes);
467 }
468
469 int
kperf_ucallstack_pend(struct kperf_context * context,uint32_t depth,unsigned int actionid)470 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
471 unsigned int actionid)
472 {
473 if (depth < 2) {
474 panic("HUH");
475 }
476 kperf_ast_set_callstack_depth(context->cur_thread, depth);
477 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
478 actionid);
479 }
480
481 static kern_return_t
chudxnu_kern_read(void * dstaddr,vm_offset_t srcaddr,vm_size_t size)482 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
483 {
484 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
485 KERN_SUCCESS : KERN_FAILURE;
486 }
487
488 static kern_return_t
chudxnu_task_read(task_t task,void * kernaddr,uint64_t usraddr,vm_size_t size)489 chudxnu_task_read(
490 task_t task,
491 void *kernaddr,
492 uint64_t usraddr,
493 vm_size_t size)
494 {
495 //ppc version ported to arm
496 kern_return_t ret = KERN_SUCCESS;
497
498 if (ml_at_interrupt_context()) {
499 return KERN_FAILURE; // can't look at tasks on interrupt stack
500 }
501
502 if (current_task() == task) {
503 if (copyin(usraddr, kernaddr, size)) {
504 ret = KERN_FAILURE;
505 }
506 } else {
507 vm_map_t map = get_task_map(task);
508 ret = vm_map_read_user(map, usraddr, kernaddr, size);
509 }
510
511 return ret;
512 }
513
514 static inline uint64_t
chudxnu_vm_unslide(uint64_t ptr,int kaddr)515 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
516 {
517 if (!kaddr) {
518 return ptr;
519 }
520
521 return VM_KERNEL_UNSLIDE(ptr);
522 }
523
524 #if __arm__
525 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
526 #define CS_FLAG_EXTRASP 1 // capture extra sp register
527 static kern_return_t
chudxnu_thread_get_callstack64_internal(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only,int flags)528 chudxnu_thread_get_callstack64_internal(
529 thread_t thread,
530 uint64_t *callStack,
531 mach_msg_type_number_t *count,
532 boolean_t user_only,
533 int flags)
534 {
535 kern_return_t kr;
536 task_t task;
537 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
538 uint64_t prevPC = 0ULL;
539 uint32_t kernStackMin = thread->kernel_stack;
540 uint32_t kernStackMax = kernStackMin + kernel_stack_size;
541 uint64_t *buffer = callStack;
542 uint32_t frame[2];
543 int bufferIndex = 0;
544 int bufferMaxIndex = 0;
545 boolean_t supervisor = FALSE;
546 struct arm_saved_state *state = NULL;
547 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
548 uint64_t pc = 0ULL;
549
550 task = get_threadtask(thread);
551
552 bufferMaxIndex = *count;
553 //get thread state
554 if (user_only) {
555 state = find_user_regs(thread);
556 } else {
557 state = find_kern_regs(thread);
558 }
559
560 if (!state) {
561 *count = 0;
562 return KERN_FAILURE;
563 }
564
565 /* make sure it is safe to dereference before you do it */
566 supervisor = ARM_SUPERVISOR_MODE(state->cpsr);
567
568 /* can't take a kernel callstack if we've got a user frame */
569 if (!user_only && !supervisor) {
570 return KERN_FAILURE;
571 }
572
573 /*
574 * Reserve space for saving LR (and sometimes SP) at the end of the
575 * backtrace.
576 */
577 if (flags & CS_FLAG_EXTRASP) {
578 bufferMaxIndex -= 2;
579 } else {
580 bufferMaxIndex -= 1;
581 }
582
583 if (bufferMaxIndex < 2) {
584 *count = 0;
585 return KERN_RESOURCE_SHORTAGE;
586 }
587
588 currPC = (uint64_t)state->pc; /* r15 */
589 if (state->cpsr & PSR_TF) {
590 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
591 }
592 currLR = (uint64_t)state->lr; /* r14 */
593 currSP = (uint64_t)state->sp; /* r13 */
594
595 fp = (uint32_t *)state->r[7]; /* frame pointer */
596 topfp = fp;
597
598 bufferIndex = 0; // start with a stack of size zero
599 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, supervisor); // save PC in position 0.
600
601 // Now, fill buffer with stack backtraces.
602 while (bufferIndex < bufferMaxIndex) {
603 pc = 0ULL;
604 /*
605 * Below the frame pointer, the following values are saved:
606 * -> FP
607 */
608
609 /*
610 * Note that we read the pc even for the first stack frame
611 * (which, in theory, is always empty because the callee fills
612 * it in just before it lowers the stack. However, if we
613 * catch the program in between filling in the return address
614 * and lowering the stack, we want to still have a valid
615 * backtrace. FixupStack correctly disregards this value if
616 * necessary.
617 */
618
619 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
620 /* frame pointer is invalid - stop backtracing */
621 pc = 0ULL;
622 break;
623 }
624
625 if (supervisor) {
626 if (((uint32_t)fp > kernStackMax) ||
627 ((uint32_t)fp < kernStackMin)) {
628 kr = KERN_FAILURE;
629 } else {
630 kr = chudxnu_kern_read(&frame,
631 (vm_offset_t)fp,
632 (vm_size_t)sizeof(frame));
633 if (kr == KERN_SUCCESS) {
634 pc = (uint64_t)frame[1];
635 nextFramePointer = (uint32_t *) (frame[0]);
636 } else {
637 pc = 0ULL;
638 nextFramePointer = 0ULL;
639 kr = KERN_FAILURE;
640 }
641 }
642 } else {
643 kr = chudxnu_task_read(task,
644 &frame,
645 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
646 sizeof(frame));
647 if (kr == KERN_SUCCESS) {
648 pc = (uint64_t) frame[1];
649 nextFramePointer = (uint32_t *) (frame[0]);
650 } else {
651 pc = 0ULL;
652 nextFramePointer = 0ULL;
653 kr = KERN_FAILURE;
654 }
655 }
656
657 if (kr != KERN_SUCCESS) {
658 pc = 0ULL;
659 break;
660 }
661
662 if (nextFramePointer) {
663 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, supervisor);
664 prevPC = pc;
665 }
666
667 if (nextFramePointer < fp) {
668 break;
669 } else {
670 fp = nextFramePointer;
671 }
672 }
673
674 if (bufferIndex >= bufferMaxIndex) {
675 bufferIndex = bufferMaxIndex;
676 kr = KERN_RESOURCE_SHORTAGE;
677 } else {
678 kr = KERN_SUCCESS;
679 }
680
681 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
682 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, supervisor);
683 if (flags & CS_FLAG_EXTRASP) {
684 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, supervisor);
685 }
686
687 *count = bufferIndex;
688 return kr;
689 }
690
691 kern_return_t
chudxnu_thread_get_callstack64_kperf(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only)692 chudxnu_thread_get_callstack64_kperf(
693 thread_t thread,
694 uint64_t *callStack,
695 mach_msg_type_number_t *count,
696 boolean_t user_only)
697 {
698 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
699 }
700 #elif __arm64__
701
702 #if defined(HAS_APPLE_PAC)
703 #include <ptrauth.h>
704 #endif
705
706 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
707 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
708 // after sampling has finished.
709 //
710 // For an N-entry callstack:
711 //
712 // [0] current pc
713 // [1..N-3] stack frames (including current one)
714 // [N-2] current LR (return value if we're in a leaf function)
715 // [N-1] current r0 (in case we've saved LR in r0) (optional)
716 //
717 //
718 #define ARM_SUPERVISOR_MODE(cpsr) ((((cpsr) & PSR_MODE_MASK) != PSR_USER_MODE) ? TRUE : FALSE)
719
720 #define CS_FLAG_EXTRASP 1 // capture extra sp register
721
722 static kern_return_t
chudxnu_thread_get_callstack64_internal(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only,int flags)723 chudxnu_thread_get_callstack64_internal(
724 thread_t thread,
725 uint64_t *callStack,
726 mach_msg_type_number_t *count,
727 boolean_t user_only,
728 int flags)
729 {
730 kern_return_t kr = KERN_SUCCESS;
731 task_t task;
732 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
733 uint64_t prevPC = 0ULL;
734 uint64_t kernStackMin = thread->kernel_stack;
735 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
736 uint64_t *buffer = callStack;
737 int bufferIndex = 0;
738 int bufferMaxIndex = 0;
739 boolean_t kernel = FALSE;
740 struct arm_saved_state *sstate = NULL;
741 uint64_t pc = 0ULL;
742
743 task = get_threadtask(thread);
744 bufferMaxIndex = *count;
745 //get thread state
746 if (user_only) {
747 sstate = find_user_regs(thread);
748 } else {
749 sstate = find_kern_regs(thread);
750 }
751
752 if (!sstate) {
753 *count = 0;
754 return KERN_FAILURE;
755 }
756
757 if (is_saved_state64(sstate)) {
758 struct arm_saved_state64 *state = NULL;
759 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
760 uint64_t frame[2];
761
762 state = saved_state64(sstate);
763
764 /* make sure it is safe to dereference before you do it */
765 kernel = PSR64_IS_KERNEL(state->cpsr);
766
767 /* can't take a kernel callstack if we've got a user frame */
768 if (!user_only && !kernel) {
769 return KERN_FAILURE;
770 }
771
772 /*
773 * Reserve space for saving LR (and sometimes SP) at the end of the
774 * backtrace.
775 */
776 if (flags & CS_FLAG_EXTRASP) {
777 bufferMaxIndex -= 2;
778 } else {
779 bufferMaxIndex -= 1;
780 }
781
782 if (bufferMaxIndex < 2) {
783 *count = 0;
784 return KERN_RESOURCE_SHORTAGE;
785 }
786
787 currPC = state->pc;
788 currLR = state->lr;
789 currSP = state->sp;
790
791 fp = (uint64_t *)state->fp; /* frame pointer */
792 topfp = fp;
793
794 bufferIndex = 0; // start with a stack of size zero
795 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
796
797 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
798
799 // Now, fill buffer with stack backtraces.
800 while (bufferIndex < bufferMaxIndex) {
801 pc = 0ULL;
802 /*
803 * Below the frame pointer, the following values are saved:
804 * -> FP
805 */
806
807 /*
808 * Note that we read the pc even for the first stack frame
809 * (which, in theory, is always empty because the callee fills
810 * it in just before it lowers the stack. However, if we
811 * catch the program in between filling in the return address
812 * and lowering the stack, we want to still have a valid
813 * backtrace. FixupStack correctly disregards this value if
814 * necessary.
815 */
816
817 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
818 /* frame pointer is invalid - stop backtracing */
819 pc = 0ULL;
820 break;
821 }
822
823 if (kernel) {
824 if (((uint64_t)fp > kernStackMax) ||
825 ((uint64_t)fp < kernStackMin)) {
826 kr = KERN_FAILURE;
827 } else {
828 kr = chudxnu_kern_read(&frame,
829 (vm_offset_t)fp,
830 (vm_size_t)sizeof(frame));
831 if (kr == KERN_SUCCESS) {
832 #if defined(HAS_APPLE_PAC)
833 /* return addresses on stack will be signed by arm64e ABI */
834 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
835 #else
836 pc = frame[1];
837 #endif
838 nextFramePointer = (uint64_t *)frame[0];
839 } else {
840 pc = 0ULL;
841 nextFramePointer = 0ULL;
842 kr = KERN_FAILURE;
843 }
844 }
845 } else {
846 kr = chudxnu_task_read(task,
847 &frame,
848 (vm_offset_t)fp,
849 (vm_size_t)sizeof(frame));
850 if (kr == KERN_SUCCESS) {
851 #if defined(HAS_APPLE_PAC)
852 /* return addresses on stack will be signed by arm64e ABI */
853 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
854 #else
855 pc = frame[1];
856 #endif
857 nextFramePointer = (uint64_t *)(frame[0]);
858 } else {
859 pc = 0ULL;
860 nextFramePointer = 0ULL;
861 kr = KERN_FAILURE;
862 }
863 }
864
865 if (kr != KERN_SUCCESS) {
866 pc = 0ULL;
867 break;
868 }
869
870 if (nextFramePointer) {
871 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
872 prevPC = pc;
873 }
874
875 if (nextFramePointer < fp) {
876 break;
877 } else {
878 fp = nextFramePointer;
879 }
880 }
881
882 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
883
884 if (bufferIndex >= bufferMaxIndex) {
885 bufferIndex = bufferMaxIndex;
886 kr = KERN_RESOURCE_SHORTAGE;
887 } else {
888 kr = KERN_SUCCESS;
889 }
890
891 // Save link register and SP at bottom of stack (used for later fixup).
892 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
893 if (flags & CS_FLAG_EXTRASP) {
894 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
895 }
896 } else {
897 struct arm_saved_state32 *state = NULL;
898 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
899
900 /* 64-bit kernel stacks, 32-bit user stacks */
901 uint64_t frame[2];
902 uint32_t frame32[2];
903
904 state = saved_state32(sstate);
905
906 /* make sure it is safe to dereference before you do it */
907 kernel = ARM_SUPERVISOR_MODE(state->cpsr);
908
909 /* can't take a kernel callstack if we've got a user frame */
910 if (!user_only && !kernel) {
911 return KERN_FAILURE;
912 }
913
914 /*
915 * Reserve space for saving LR (and sometimes SP) at the end of the
916 * backtrace.
917 */
918 if (flags & CS_FLAG_EXTRASP) {
919 bufferMaxIndex -= 2;
920 } else {
921 bufferMaxIndex -= 1;
922 }
923
924 if (bufferMaxIndex < 2) {
925 *count = 0;
926 return KERN_RESOURCE_SHORTAGE;
927 }
928
929 currPC = (uint64_t)state->pc; /* r15 */
930 if (state->cpsr & PSR_TF) {
931 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
932 }
933 currLR = (uint64_t)state->lr; /* r14 */
934 currSP = (uint64_t)state->sp; /* r13 */
935
936 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
937 topfp = fp;
938
939 bufferIndex = 0; // start with a stack of size zero
940 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
941
942 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
943
944 // Now, fill buffer with stack backtraces.
945 while (bufferIndex < bufferMaxIndex) {
946 pc = 0ULL;
947 /*
948 * Below the frame pointer, the following values are saved:
949 * -> FP
950 */
951
952 /*
953 * Note that we read the pc even for the first stack frame
954 * (which, in theory, is always empty because the callee fills
955 * it in just before it lowers the stack. However, if we
956 * catch the program in between filling in the return address
957 * and lowering the stack, we want to still have a valid
958 * backtrace. FixupStack correctly disregards this value if
959 * necessary.
960 */
961
962 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
963 /* frame pointer is invalid - stop backtracing */
964 pc = 0ULL;
965 break;
966 }
967
968 if (kernel) {
969 if (((uint32_t)fp > kernStackMax) ||
970 ((uint32_t)fp < kernStackMin)) {
971 kr = KERN_FAILURE;
972 } else {
973 kr = chudxnu_kern_read(&frame,
974 (vm_offset_t)fp,
975 (vm_size_t)sizeof(frame));
976 if (kr == KERN_SUCCESS) {
977 pc = (uint64_t)frame[1];
978 nextFramePointer = (uint32_t *) (frame[0]);
979 } else {
980 pc = 0ULL;
981 nextFramePointer = 0ULL;
982 kr = KERN_FAILURE;
983 }
984 }
985 } else {
986 kr = chudxnu_task_read(task,
987 &frame32,
988 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
989 sizeof(frame32));
990 if (kr == KERN_SUCCESS) {
991 pc = (uint64_t)frame32[1];
992 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
993 } else {
994 pc = 0ULL;
995 nextFramePointer = 0ULL;
996 kr = KERN_FAILURE;
997 }
998 }
999
1000 if (kr != KERN_SUCCESS) {
1001 pc = 0ULL;
1002 break;
1003 }
1004
1005 if (nextFramePointer) {
1006 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
1007 prevPC = pc;
1008 }
1009
1010 if (nextFramePointer < fp) {
1011 break;
1012 } else {
1013 fp = nextFramePointer;
1014 }
1015 }
1016
1017 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
1018
1019 /* clamp callstack size to max */
1020 if (bufferIndex >= bufferMaxIndex) {
1021 bufferIndex = bufferMaxIndex;
1022 kr = KERN_RESOURCE_SHORTAGE;
1023 } else {
1024 /* ignore all other failures */
1025 kr = KERN_SUCCESS;
1026 }
1027
1028 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
1029 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
1030 if (flags & CS_FLAG_EXTRASP) {
1031 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
1032 }
1033 }
1034
1035 *count = bufferIndex;
1036 return kr;
1037 }
1038
1039 kern_return_t
chudxnu_thread_get_callstack64_kperf(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only)1040 chudxnu_thread_get_callstack64_kperf(
1041 thread_t thread,
1042 uint64_t *callStack,
1043 mach_msg_type_number_t *count,
1044 boolean_t user_only)
1045 {
1046 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
1047 }
1048 #elif __x86_64__
1049
1050 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
1051 // don't try to read in the hole
1052 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
1053 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
1054 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
1055
1056 typedef struct _cframe64_t {
1057 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
1058 uint64_t caller;
1059 uint64_t args[0];
1060 }cframe64_t;
1061
1062
1063 typedef struct _cframe_t {
1064 uint32_t prev; // this is really a user32-space pointer to the previous frame
1065 uint32_t caller;
1066 uint32_t args[0];
1067 } cframe_t;
1068
1069 extern void * find_user_regs(thread_t);
1070 extern x86_saved_state32_t *find_kern_regs(thread_t);
1071
1072 static kern_return_t
do_kernel_backtrace(thread_t thread,struct x86_kernel_state * regs,uint64_t * frames,mach_msg_type_number_t * start_idx,mach_msg_type_number_t max_idx)1073 do_kernel_backtrace(
1074 thread_t thread,
1075 struct x86_kernel_state *regs,
1076 uint64_t *frames,
1077 mach_msg_type_number_t *start_idx,
1078 mach_msg_type_number_t max_idx)
1079 {
1080 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1081 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1082 mach_msg_type_number_t ct = *start_idx;
1083 kern_return_t kr = KERN_FAILURE;
1084
1085 #if __LP64__
1086 uint64_t currPC = 0ULL;
1087 uint64_t currFP = 0ULL;
1088 uint64_t prevPC = 0ULL;
1089 uint64_t prevFP = 0ULL;
1090 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
1091 return KERN_FAILURE;
1092 }
1093 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
1094 return KERN_FAILURE;
1095 }
1096 #else
1097 uint32_t currPC = 0U;
1098 uint32_t currFP = 0U;
1099 uint32_t prevPC = 0U;
1100 uint32_t prevFP = 0U;
1101 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
1102 return KERN_FAILURE;
1103 }
1104 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
1105 return KERN_FAILURE;
1106 }
1107 #endif
1108
1109 if (*start_idx >= max_idx) {
1110 return KERN_RESOURCE_SHORTAGE; // no frames traced
1111 }
1112 if (!currPC) {
1113 return KERN_FAILURE;
1114 }
1115
1116 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1117
1118 // build a backtrace of this kernel state
1119 #if __LP64__
1120 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
1121 // this is the address where caller lives in the user thread
1122 uint64_t caller = currFP + sizeof(uint64_t);
1123 #else
1124 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
1125 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
1126 #endif
1127
1128 if (!currFP || !currPC) {
1129 currPC = 0;
1130 break;
1131 }
1132
1133 if (ct >= max_idx) {
1134 *start_idx = ct;
1135 return KERN_RESOURCE_SHORTAGE;
1136 }
1137
1138 /* read our caller */
1139 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1140
1141 if (kr != KERN_SUCCESS || !currPC) {
1142 currPC = 0UL;
1143 break;
1144 }
1145
1146 /*
1147 * retrive contents of the frame pointer and advance to the next stack
1148 * frame if it's valid
1149 */
1150 prevFP = 0;
1151 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1152
1153 #if __LP64__
1154 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1155 #else
1156 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1157 #endif
1158 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1159 prevPC = currPC;
1160 }
1161 if (prevFP <= currFP) {
1162 break;
1163 } else {
1164 currFP = prevFP;
1165 }
1166 }
1167
1168 *start_idx = ct;
1169 return KERN_SUCCESS;
1170 }
1171
1172
1173
1174 static kern_return_t
1175 do_backtrace32(
1176 task_t task,
1177 thread_t thread,
1178 x86_saved_state32_t *regs,
1179 uint64_t *frames,
1180 mach_msg_type_number_t *start_idx,
1181 mach_msg_type_number_t max_idx,
1182 boolean_t supervisor)
1183 {
1184 uint32_t tmpWord = 0UL;
1185 uint64_t currPC = (uint64_t) regs->eip;
1186 uint64_t currFP = (uint64_t) regs->ebp;
1187 uint64_t prevPC = 0ULL;
1188 uint64_t prevFP = 0ULL;
1189 uint64_t kernStackMin = thread->kernel_stack;
1190 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1191 mach_msg_type_number_t ct = *start_idx;
1192 kern_return_t kr = KERN_FAILURE;
1193
1194 if (ct >= max_idx) {
1195 return KERN_RESOURCE_SHORTAGE; // no frames traced
1196 }
1197 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1198
1199 // build a backtrace of this 32 bit state.
1200 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1201 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1202
1203 if (!currFP) {
1204 currPC = 0;
1205 break;
1206 }
1207
1208 if (ct >= max_idx) {
1209 *start_idx = ct;
1210 return KERN_RESOURCE_SHORTAGE;
1211 }
1212
1213 /* read our caller */
1214 if (supervisor) {
1215 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1216 } else {
1217 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1218 }
1219
1220 if (kr != KERN_SUCCESS) {
1221 currPC = 0ULL;
1222 break;
1223 }
1224
1225 currPC = (uint64_t) tmpWord; // promote 32 bit address
1226
1227 /*
1228 * retrive contents of the frame pointer and advance to the next stack
1229 * frame if it's valid
1230 */
1231 prevFP = 0;
1232 if (supervisor) {
1233 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1234 } else {
1235 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1236 }
1237 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1238
1239 if (prevFP) {
1240 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1241 prevPC = currPC;
1242 }
1243 if (prevFP < currFP) {
1244 break;
1245 } else {
1246 currFP = prevFP;
1247 }
1248 }
1249
1250 *start_idx = ct;
1251 return KERN_SUCCESS;
1252 }
1253
1254 static kern_return_t
1255 do_backtrace64(
1256 task_t task,
1257 thread_t thread,
1258 x86_saved_state64_t *regs,
1259 uint64_t *frames,
1260 mach_msg_type_number_t *start_idx,
1261 mach_msg_type_number_t max_idx,
1262 boolean_t supervisor)
1263 {
1264 uint64_t currPC = regs->isf.rip;
1265 uint64_t currFP = regs->rbp;
1266 uint64_t prevPC = 0ULL;
1267 uint64_t prevFP = 0ULL;
1268 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1269 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1270 mach_msg_type_number_t ct = *start_idx;
1271 kern_return_t kr = KERN_FAILURE;
1272
1273 if (*start_idx >= max_idx) {
1274 return KERN_RESOURCE_SHORTAGE; // no frames traced
1275 }
1276 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1277
1278 // build a backtrace of this 32 bit state.
1279 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1280 // this is the address where caller lives in the user thread
1281 uint64_t caller = currFP + sizeof(uint64_t);
1282
1283 if (!currFP) {
1284 currPC = 0;
1285 break;
1286 }
1287
1288 if (ct >= max_idx) {
1289 *start_idx = ct;
1290 return KERN_RESOURCE_SHORTAGE;
1291 }
1292
1293 /* read our caller */
1294 if (supervisor) {
1295 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1296 } else {
1297 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1298 }
1299
1300 if (kr != KERN_SUCCESS) {
1301 currPC = 0ULL;
1302 break;
1303 }
1304
1305 /*
1306 * retrive contents of the frame pointer and advance to the next stack
1307 * frame if it's valid
1308 */
1309 prevFP = 0;
1310 if (supervisor) {
1311 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1312 } else {
1313 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1314 }
1315
1316 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1317 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1318 prevPC = currPC;
1319 }
1320 if (prevFP < currFP) {
1321 break;
1322 } else {
1323 currFP = prevFP;
1324 }
1325 }
1326
1327 *start_idx = ct;
1328 return KERN_SUCCESS;
1329 }
1330
1331 static kern_return_t
1332 chudxnu_thread_get_callstack64_internal(
1333 thread_t thread,
1334 uint64_t *callstack,
1335 mach_msg_type_number_t *count,
1336 boolean_t user_only,
1337 boolean_t kern_only)
1338 {
1339 kern_return_t kr = KERN_FAILURE;
1340 task_t task = get_threadtask(thread);
1341 uint64_t currPC = 0ULL;
1342 boolean_t supervisor = FALSE;
1343 mach_msg_type_number_t bufferIndex = 0;
1344 mach_msg_type_number_t bufferMaxIndex = *count;
1345 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1346 x86_saved_state64_t *regs64 = NULL;
1347 x86_saved_state32_t *regs32 = NULL;
1348 x86_saved_state32_t *u_regs32 = NULL;
1349 x86_saved_state64_t *u_regs64 = NULL;
1350 struct x86_kernel_state *kregs = NULL;
1351
1352 if (ml_at_interrupt_context()) {
1353 if (user_only) {
1354 /* can't backtrace user state on interrupt stack. */
1355 return KERN_FAILURE;
1356 }
1357
1358 /* backtracing at interrupt context? */
1359 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1360 /*
1361 * Locate the registers for the interrupted thread, assuming it is
1362 * current_thread().
1363 */
1364 tagged_regs = current_cpu_datap()->cpu_int_state;
1365
1366 if (is_saved_state64(tagged_regs)) {
1367 /* 64 bit registers */
1368 regs64 = saved_state64(tagged_regs);
1369 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1370 } else {
1371 /* 32 bit registers */
1372 regs32 = saved_state32(tagged_regs);
1373 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1374 }
1375 }
1376 }
1377
1378 if (!ml_at_interrupt_context() && kernel_task == task) {
1379 if (!thread->kernel_stack) {
1380 return KERN_FAILURE;
1381 }
1382
1383 // Kernel thread not at interrupt context
1384 kregs = (struct x86_kernel_state *)NULL;
1385
1386 // nofault read of the thread->kernel_stack pointer
1387 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1388 return KERN_FAILURE;
1389 }
1390
1391 // Adjust to find the saved kernel state
1392 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1393
1394 supervisor = TRUE;
1395 } else if (!tagged_regs) {
1396 /*
1397 * not at interrupt context, or tracing a different thread than
1398 * current_thread() at interrupt context
1399 */
1400 tagged_regs = USER_STATE(thread);
1401 if (is_saved_state64(tagged_regs)) {
1402 /* 64 bit registers */
1403 regs64 = saved_state64(tagged_regs);
1404 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1405 } else {
1406 /* 32 bit registers */
1407 regs32 = saved_state32(tagged_regs);
1408 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1409 }
1410 }
1411
1412 *count = 0;
1413
1414 if (supervisor) {
1415 // the caller only wants a user callstack.
1416 if (user_only) {
1417 // bail - we've only got kernel state
1418 return KERN_FAILURE;
1419 }
1420 } else {
1421 // regs32(64) is not in supervisor mode.
1422 u_regs32 = regs32;
1423 u_regs64 = regs64;
1424 regs32 = NULL;
1425 regs64 = NULL;
1426 }
1427
1428 if (user_only) {
1429 /* we only want to backtrace the user mode */
1430 if (!(u_regs32 || u_regs64)) {
1431 /* no user state to look at */
1432 return KERN_FAILURE;
1433 }
1434 }
1435
1436 /*
1437 * Order of preference for top of stack:
1438 * 64 bit kernel state (not likely)
1439 * 32 bit kernel state
1440 * 64 bit user land state
1441 * 32 bit user land state
1442 */
1443
1444 if (kregs) {
1445 /*
1446 * nofault read of the registers from the kernel stack (as they can
1447 * disappear on the fly).
1448 */
1449
1450 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1451 return KERN_FAILURE;
1452 }
1453 } else if (regs64) {
1454 currPC = regs64->isf.rip;
1455 } else if (regs32) {
1456 currPC = (uint64_t) regs32->eip;
1457 } else if (u_regs64) {
1458 currPC = u_regs64->isf.rip;
1459 } else if (u_regs32) {
1460 currPC = (uint64_t) u_regs32->eip;
1461 }
1462
1463 if (!currPC) {
1464 /* no top of the stack, bail out */
1465 return KERN_FAILURE;
1466 }
1467
1468 bufferIndex = 0;
1469
1470 if (bufferMaxIndex < 1) {
1471 *count = 0;
1472 return KERN_RESOURCE_SHORTAGE;
1473 }
1474
1475 /* backtrace kernel */
1476 if (kregs) {
1477 addr64_t address = 0ULL;
1478 size_t size = 0UL;
1479
1480 // do the backtrace
1481 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1482
1483 // and do a nofault read of (r|e)sp
1484 uint64_t rsp = 0ULL;
1485 size = sizeof(uint64_t);
1486
1487 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1488 address = 0ULL;
1489 }
1490
1491 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1492 callstack[bufferIndex++] = (uint64_t)rsp;
1493 }
1494 } else if (regs64) {
1495 uint64_t rsp = 0ULL;
1496
1497 // backtrace the 64bit side.
1498 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1499 bufferMaxIndex - 1, TRUE);
1500
1501 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1502 bufferIndex < bufferMaxIndex) {
1503 callstack[bufferIndex++] = rsp;
1504 }
1505 } else if (regs32) {
1506 uint32_t esp = 0UL;
1507
1508 // backtrace the 32bit side.
1509 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1510 bufferMaxIndex - 1, TRUE);
1511
1512 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1513 bufferIndex < bufferMaxIndex) {
1514 callstack[bufferIndex++] = (uint64_t) esp;
1515 }
1516 } else if (u_regs64 && !kern_only) {
1517 /* backtrace user land */
1518 uint64_t rsp = 0ULL;
1519
1520 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1521 bufferMaxIndex - 1, FALSE);
1522
1523 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1524 bufferIndex < bufferMaxIndex) {
1525 callstack[bufferIndex++] = rsp;
1526 }
1527 } else if (u_regs32 && !kern_only) {
1528 uint32_t esp = 0UL;
1529
1530 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1531 bufferMaxIndex - 1, FALSE);
1532
1533 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1534 bufferIndex < bufferMaxIndex) {
1535 callstack[bufferIndex++] = (uint64_t) esp;
1536 }
1537 }
1538
1539 *count = bufferIndex;
1540 return kr;
1541 }
1542
1543 __private_extern__
1544 kern_return_t
1545 chudxnu_thread_get_callstack64_kperf(
1546 thread_t thread,
1547 uint64_t *callstack,
1548 mach_msg_type_number_t *count,
1549 boolean_t is_user)
1550 {
1551 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1552 }
1553 #else /* !__arm__ && !__arm64__ && !__x86_64__ */
1554 #error kperf: unsupported architecture
1555 #endif /* !__arm__ && !__arm64__ && !__x86_64__ */
1556