1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Called from a trigger. Actually takes the data from the different
31 * modules and puts them in a buffer
32 */
33
34 #include <mach/mach_types.h>
35 #include <machine/machine_routines.h>
36 #include <kern/kalloc.h>
37 #include <kern/debug.h> /* panic */
38 #include <kern/thread.h>
39 #include <sys/errno.h>
40 #include <sys/vm.h>
41 #include <vm/vm_object_xnu.h>
42 #include <vm/vm_page.h>
43 #include <vm/vm_pageout_xnu.h>
44
45 #ifdef CONFIG_EXCLAVES
46 #include <kern/exclaves.tightbeam.h>
47 #endif /* CONFIG_EXCLAVES */
48
49 #include <kperf/action.h>
50 #include <kperf/ast.h>
51 #include <kperf/buffer.h>
52 #include <kperf/callstack.h>
53 #include <kperf/context.h>
54 #include <kperf/kdebug_trigger.h>
55 #include <kperf/kperf.h>
56 #include <kperf/kperf_kpc.h>
57 #include <kperf/kptimer.h>
58 #include <kperf/pet.h>
59 #include <kperf/sample.h>
60 #include <kperf/thread_samplers.h>
61
62 #define ACTION_MAX (32)
63
64 /* the list of different actions to take */
65 struct action {
66 uint32_t sample;
67 uint32_t ucallstack_depth;
68 uint32_t kcallstack_depth;
69 uint32_t userdata;
70 int pid_filter;
71 };
72
73 /* the list of actions */
74 static unsigned int actionc = 0;
75 static struct action *actionv = NULL;
76
77 /* should emit tracepoint on context switch */
78 int kperf_kdebug_cswitch = 0;
79
80 int kperf_max_actions = ACTION_MAX;
81 bool
kperf_action_has_non_system(unsigned int actionid)82 kperf_action_has_non_system(unsigned int actionid)
83 {
84 if (actionid > actionc) {
85 return false;
86 }
87
88 if (actionv[actionid - 1].sample & ~SAMPLER_SYS_MEM) {
89 return true;
90 } else {
91 return false;
92 }
93 }
94
95 bool
kperf_action_has_task(unsigned int actionid)96 kperf_action_has_task(unsigned int actionid)
97 {
98 if (actionid > actionc) {
99 return false;
100 }
101
102 return actionv[actionid - 1].sample & SAMPLER_TASK_MASK;
103 }
104
105 bool
kperf_action_has_thread(unsigned int actionid)106 kperf_action_has_thread(unsigned int actionid)
107 {
108 if (actionid > actionc) {
109 return false;
110 }
111
112 return actionv[actionid - 1].sample & SAMPLER_THREAD_MASK;
113 }
114
115 static void
kperf_system_memory_log(void)116 kperf_system_memory_log(void)
117 {
118 extern unsigned int memorystatus_level;
119
120 BUF_DATA(PERF_MI_SYS_DATA, (uintptr_t)vm_page_free_count,
121 (uintptr_t)vm_page_wire_count, (uintptr_t)vm_page_external_count,
122 (uintptr_t)(vm_page_active_count + vm_page_inactive_count +
123 vm_page_speculative_count));
124 BUF_DATA(PERF_MI_SYS_DATA_2, (uintptr_t)vm_page_anonymous_count,
125 (uintptr_t)vm_page_internal_count,
126 (uintptr_t)vm_pageout_vminfo.vm_pageout_compressions,
127 (uintptr_t)VM_PAGE_COMPRESSOR_COUNT);
128 BUF_DATA(PERF_MI_SYS_DATA_3,
129 #if CONFIG_SECLUDED_MEMORY
130 (uintptr_t)vm_page_secluded_count,
131 #else // CONFIG_SECLUDED_MEMORY
132 0,
133 #endif // !CONFIG_SECLUDED_MEMORY
134 (uintptr_t)vm_page_purgeable_count,
135 memorystatus_level);
136 }
137
138 static void
kperf_sample_user_internal(struct kperf_usample * sbuf,struct kperf_context * context,unsigned int actionid,unsigned int sample_what)139 kperf_sample_user_internal(struct kperf_usample *sbuf,
140 struct kperf_context *context, unsigned int actionid,
141 unsigned int sample_what)
142 {
143 if (sample_what & SAMPLER_USTACK) {
144 kperf_ucallstack_sample(&sbuf->ucallstack, context);
145 }
146 if (sample_what & SAMPLER_TH_INFO) {
147 kperf_thread_info_sample(&sbuf->th_info, context);
148 }
149
150 boolean_t intren = ml_set_interrupts_enabled(FALSE);
151
152 /*
153 * No userdata or sample_flags for this one.
154 */
155 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what, actionid);
156
157 if (sample_what & SAMPLER_USTACK) {
158 kperf_ucallstack_log(&sbuf->ucallstack);
159 }
160 if (sample_what & SAMPLER_TH_DISPATCH) {
161 kperf_thread_dispatch_log(&sbuf->usample_min->th_dispatch);
162 }
163 if (sample_what & SAMPLER_TH_INFO) {
164 kperf_thread_info_log(&sbuf->th_info);
165 }
166
167 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what);
168
169 ml_set_interrupts_enabled(intren);
170 }
171
172 static unsigned int
kperf_prepare_sample_what(unsigned int sample_what,unsigned int sample_flags)173 kperf_prepare_sample_what(unsigned int sample_what, unsigned int sample_flags)
174 {
175 /* callstacks should be explicitly ignored */
176 if (sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) {
177 sample_what &= ~(SAMPLER_KSTACK | SAMPLER_USTACK | SAMPLER_EXSTACK);
178 }
179 if (sample_flags & SAMPLE_FLAG_ONLY_SYSTEM) {
180 sample_what &= SAMPLER_SYS_MEM;
181 }
182 assert((sample_flags & (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY))
183 != (SAMPLE_FLAG_THREAD_ONLY | SAMPLE_FLAG_TASK_ONLY));
184 if (sample_flags & SAMPLE_FLAG_THREAD_ONLY) {
185 sample_what &= SAMPLER_THREAD_MASK;
186 }
187 if (sample_flags & SAMPLE_FLAG_TASK_ONLY) {
188 sample_what &= SAMPLER_TASK_MASK;
189 }
190
191 return sample_what;
192 }
193
194 void
kperf_sample_user(struct kperf_usample * sbuf,struct kperf_context * context,unsigned int actionid,unsigned int sample_flags)195 kperf_sample_user(struct kperf_usample *sbuf, struct kperf_context *context,
196 unsigned int actionid, unsigned int sample_flags)
197 {
198 if (actionid == 0 || actionid > actionc) {
199 return;
200 }
201
202 unsigned int sample_what = kperf_prepare_sample_what(
203 actionv[actionid - 1].sample, sample_flags);
204 if (sample_what == 0) {
205 return;
206 }
207
208 unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth;
209 sbuf->ucallstack.kpuc_nframes = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES;
210
211 kperf_sample_user_internal(sbuf, context, actionid, sample_what);
212 }
213
214 static kern_return_t
kperf_sample_internal(struct kperf_sample * sbuf,struct kperf_context * context,unsigned sample_what,unsigned sample_flags,unsigned actionid,unsigned ucallstack_depth)215 kperf_sample_internal(struct kperf_sample *sbuf,
216 struct kperf_context *context,
217 unsigned sample_what, unsigned sample_flags,
218 unsigned actionid, unsigned ucallstack_depth)
219 {
220 int pended_ucallstack = 0;
221 int pended_th_dispatch = 0;
222 uint32_t userdata = actionid;
223 bool task_only = (sample_flags & SAMPLE_FLAG_TASK_ONLY) != 0;
224 bool pended_exclave_callstack = false;
225 uint64_t sample_meta_flags = 0;
226
227 sample_what = kperf_prepare_sample_what(sample_what, sample_flags);
228 if (sample_what == 0) {
229 return SAMPLE_CONTINUE;
230 }
231
232 if (!task_only) {
233 context->cur_thread->kperf_pet_gen =
234 os_atomic_load(&kppet_gencount, relaxed);
235 }
236 bool is_kernel = (context->cur_pid == 0);
237
238 if (actionid && actionid <= actionc) {
239 sbuf->kcallstack.kpkc_nframes =
240 actionv[actionid - 1].kcallstack_depth;
241 } else {
242 sbuf->kcallstack.kpkc_nframes = MAX_KCALLSTACK_FRAMES;
243 }
244
245 ucallstack_depth = ucallstack_depth ?: MAX_UCALLSTACK_FRAMES;
246 sbuf->kcallstack.kpkc_flags = 0;
247 sbuf->usample.ucallstack.kpuc_flags = 0;
248
249 if (sample_what & SAMPLER_TH_INFO) {
250 kperf_thread_info_sample(&sbuf->th_info, context);
251
252 if (!(sample_flags & SAMPLE_FLAG_IDLE_THREADS)) {
253 if (sbuf->th_info.kpthi_runmode & 0x40) {
254 sample_meta_flags |= SAMPLE_META_THREAD_WAS_IDLE;
255 goto log_sample;
256 }
257 }
258 }
259
260 if (sample_what & SAMPLER_TH_SNAPSHOT) {
261 kperf_thread_snapshot_sample(&(sbuf->th_snapshot), context);
262 }
263 if (sample_what & SAMPLER_TH_SCHEDULING) {
264 kperf_thread_scheduling_sample(&(sbuf->th_scheduling), context);
265 }
266 if (sample_what & SAMPLER_KSTACK) {
267 if (sample_flags & SAMPLE_FLAG_CONTINUATION) {
268 kperf_continuation_sample(&(sbuf->kcallstack), context);
269 } else if (sample_flags & SAMPLE_FLAG_NON_INTERRUPT) {
270 /* outside of interrupt context, backtrace the current thread */
271 kperf_backtrace_sample(&(sbuf->kcallstack), context);
272 } else {
273 kperf_kcallstack_sample(&(sbuf->kcallstack), context);
274 }
275 }
276 if (sample_what & SAMPLER_TK_SNAPSHOT) {
277 kperf_task_snapshot_sample(context->cur_task, &(sbuf->tk_snapshot));
278 }
279
280 if (!is_kernel) {
281 if (sample_what & SAMPLER_MEMINFO) {
282 kperf_meminfo_sample(context->cur_task, &(sbuf->meminfo));
283 }
284
285 if (sample_flags & SAMPLE_FLAG_PEND_USER) {
286 if (sample_what & SAMPLER_USTACK) {
287 pended_ucallstack = kperf_ucallstack_pend(context,
288 ucallstack_depth, actionid);
289 }
290
291 if (sample_what & SAMPLER_TH_DISPATCH) {
292 pended_th_dispatch =
293 kperf_thread_dispatch_pend(context, actionid);
294 }
295 }
296 }
297
298 #if CONFIG_EXCLAVES
299 if (sample_what & SAMPLER_EXSTACK) {
300 pended_exclave_callstack = kperf_exclave_callstack_pend(context, actionid);
301 }
302 #endif /* CONFIG_EXCLAVES */
303
304 #if CONFIG_CPU_COUNTERS
305 if (sample_what & SAMPLER_PMC_THREAD) {
306 kperf_kpc_thread_sample(&(sbuf->kpcdata), sample_what);
307 } else if (sample_what & SAMPLER_PMC_CPU) {
308 kperf_kpc_cpu_sample(&(sbuf->kpcdata), sample_what);
309 }
310 #endif /* CONFIG_CPU_COUNTERS */
311
312 log_sample:
313 /* lookup the user tag, if any */
314 if (actionid && (actionid <= actionc)) {
315 userdata = actionv[actionid - 1].userdata;
316 }
317
318 /* avoid logging if this sample only pended samples */
319 if (sample_flags & SAMPLE_FLAG_PEND_USER &&
320 !(sample_what & ~(SAMPLER_USTACK | SAMPLER_TH_DISPATCH))) {
321 return SAMPLE_CONTINUE;
322 }
323
324 /* stash the data into the buffer
325 * interrupts off to ensure we don't get split
326 */
327 boolean_t enabled = ml_set_interrupts_enabled(FALSE);
328
329 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, sample_what,
330 actionid, userdata, sample_flags);
331
332 if (sample_flags & SAMPLE_FLAG_SYSTEM) {
333 if (sample_what & SAMPLER_SYS_MEM) {
334 kperf_system_memory_log();
335 }
336 }
337 if (sample_meta_flags & SAMPLE_META_THREAD_WAS_IDLE) {
338 goto log_sample_end;
339 }
340
341 if (sample_what & SAMPLER_TH_INFO) {
342 kperf_thread_info_log(&sbuf->th_info);
343 }
344 if (sample_what & SAMPLER_TH_SCHEDULING) {
345 kperf_thread_scheduling_log(&(sbuf->th_scheduling));
346 }
347 if (sample_what & SAMPLER_TH_SNAPSHOT) {
348 kperf_thread_snapshot_log(&(sbuf->th_snapshot));
349 }
350 if (sample_what & SAMPLER_KSTACK) {
351 kperf_kcallstack_log(&sbuf->kcallstack);
352 }
353 if (sample_what & SAMPLER_TH_INSCYC) {
354 kperf_thread_inscyc_log(context);
355 }
356 if (sample_what & SAMPLER_TK_SNAPSHOT) {
357 kperf_task_snapshot_log(&(sbuf->tk_snapshot));
358 }
359 if (sample_what & SAMPLER_TK_INFO) {
360 kperf_task_info_log(context);
361 }
362
363 /* dump user stuff */
364 if (!is_kernel) {
365 /* dump meminfo */
366 if (sample_what & SAMPLER_MEMINFO) {
367 kperf_meminfo_log(&(sbuf->meminfo));
368 }
369
370 if (sample_flags & SAMPLE_FLAG_PEND_USER) {
371 if (pended_ucallstack) {
372 BUF_INFO(PERF_CS_UPEND);
373 sample_meta_flags |= SAMPLE_META_UPEND;
374 }
375
376 if (pended_th_dispatch) {
377 BUF_INFO(PERF_TI_DISPPEND);
378 }
379 }
380 }
381
382 if (pended_exclave_callstack) {
383 sample_meta_flags |= SAMPLE_META_EXPEND;
384 }
385
386 #if CONFIG_CPU_COUNTERS
387 if (sample_what & SAMPLER_PMC_CONFIG) {
388 kperf_kpc_config_log(&(sbuf->kpcdata));
389 }
390 if (sample_what & SAMPLER_PMC_THREAD) {
391 kperf_kpc_thread_log(&(sbuf->kpcdata));
392 } else if (sample_what & SAMPLER_PMC_CPU) {
393 kperf_kpc_cpu_log(&(sbuf->kpcdata));
394 }
395 #endif /* CONFIG_CPU_COUNTERS */
396
397 log_sample_end:
398 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, sample_what, sample_meta_flags);
399
400 /* intrs back on */
401 ml_set_interrupts_enabled(enabled);
402
403 return SAMPLE_CONTINUE;
404 }
405
406 /* Translate actionid into sample bits and take a sample */
407 kern_return_t
kperf_sample(struct kperf_sample * sbuf,struct kperf_context * context,unsigned actionid,unsigned sample_flags)408 kperf_sample(struct kperf_sample *sbuf,
409 struct kperf_context *context,
410 unsigned actionid, unsigned sample_flags)
411 {
412 /* work out what to sample, if anything */
413 if ((actionid > actionc) || (actionid == 0)) {
414 return SAMPLE_SHUTDOWN;
415 }
416
417 /* check the pid filter against the context's current pid.
418 * filter pid == -1 means any pid
419 */
420 int pid_filter = actionv[actionid - 1].pid_filter;
421 if ((pid_filter != -1) && (pid_filter != context->cur_pid)) {
422 return SAMPLE_CONTINUE;
423 }
424
425 /* the samplers to run */
426 unsigned int sample_what = actionv[actionid - 1].sample;
427 unsigned int ucallstack_depth = actionv[actionid - 1].ucallstack_depth;
428
429 /* do the actual sample operation */
430 return kperf_sample_internal(sbuf, context, sample_what,
431 sample_flags, actionid, ucallstack_depth);
432 }
433
434 void
kperf_kdebug_handler(uint32_t debugid,uintptr_t * starting_fp)435 kperf_kdebug_handler(uint32_t debugid, uintptr_t *starting_fp)
436 {
437 uint32_t sample_flags = SAMPLE_FLAG_NON_INTERRUPT | SAMPLE_FLAG_PEND_USER;
438 struct kperf_sample *sample = NULL;
439 kern_return_t kr = KERN_SUCCESS;
440 int s;
441
442 if (!kperf_kdebug_should_trigger(debugid)) {
443 return;
444 }
445
446 BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_START, debugid);
447
448 thread_t thread = current_thread();
449 task_t task = get_threadtask(thread);
450 struct kperf_context ctx = {
451 .cur_thread = thread,
452 .cur_task = task,
453 .cur_pid = task_pid(task),
454 .trigger_type = TRIGGER_TYPE_KDEBUG,
455 .trigger_id = 0,
456 .starting_fp = starting_fp,
457 };
458
459 s = ml_set_interrupts_enabled(0);
460
461 sample = kperf_intr_sample_buffer();
462
463 kr = kperf_sample(sample, &ctx, kperf_kdebug_get_action(), sample_flags);
464
465 ml_set_interrupts_enabled(s);
466 BUF_VERB(PERF_KDBG_HNDLR | DBG_FUNC_END, kr);
467 }
468
469 /*
470 * Sample using a minimum of stack space during this phase.
471 */
472 static void
kperf_ast_sample_min_stack_phase(struct kperf_usample_min * sbuf_min,struct kperf_context * context,unsigned int sample_what)473 kperf_ast_sample_min_stack_phase(struct kperf_usample_min *sbuf_min,
474 struct kperf_context *context, unsigned int sample_what)
475 {
476 if (sample_what & SAMPLER_TH_DISPATCH) {
477 kperf_thread_dispatch_sample(&sbuf_min->th_dispatch, context);
478 }
479 }
480
481 /*
482 * This function should not be inlined with its caller, which would pollute
483 * the stack usage of the minimum stack phase, above.
484 */
485 __attribute__((noinline))
486 static void
kperf_ast_sample_max_stack_phase(struct kperf_usample_min * sbuf_min,struct kperf_context * context,uint32_t actionid,unsigned int sample_what,unsigned int nframes)487 kperf_ast_sample_max_stack_phase(struct kperf_usample_min *sbuf_min,
488 struct kperf_context *context, uint32_t actionid, unsigned int sample_what,
489 unsigned int nframes)
490 {
491 struct kperf_usample sbuf = { .usample_min = sbuf_min };
492 sbuf.ucallstack.kpuc_nframes = nframes;
493
494 kperf_sample_user_internal(&sbuf, context, actionid, sample_what);
495 }
496
497 /*
498 * This function allocates >2.3KB of the stack. Prevent the compiler from
499 * inlining this function into ast_taken and ensure the stack memory is only
500 * allocated for the kperf AST.
501 */
502 __attribute__((noinline))
503 void
kperf_thread_ast_handler(thread_t thread)504 kperf_thread_ast_handler(thread_t thread)
505 {
506 uint32_t ast = thread->kperf_ast;
507
508 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_START, thread, ast);
509
510 task_t task = get_threadtask(thread);
511
512 if (task_did_exec(task) || task_is_exec_copy(task)) {
513 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END, SAMPLE_CONTINUE);
514 return;
515 }
516
517 struct kperf_context ctx = {
518 .cur_thread = thread,
519 .cur_task = task,
520 .cur_pid = task_pid(task),
521 };
522
523 unsigned int sample_what = 0;
524 if (ast & T_KPERF_AST_DISPATCH) {
525 sample_what |= SAMPLER_TH_DISPATCH;
526 }
527 if (ast & T_KPERF_AST_CALLSTACK) {
528 /* TH_INFO for backwards compatibility */
529 sample_what |= SAMPLER_USTACK | SAMPLER_TH_INFO;
530 }
531
532 unsigned int actionid = T_KPERF_GET_ACTIONID(ast);
533
534 struct kperf_usample_min sbuf_min = { 0 };
535 kperf_ast_sample_min_stack_phase(&sbuf_min, &ctx, sample_what);
536 kperf_ast_sample_max_stack_phase(&sbuf_min, &ctx, actionid, sample_what,
537 T_KPERF_GET_CALLSTACK_DEPTH(ast) ?: MAX_UCALLSTACK_FRAMES);
538
539 BUF_INFO(PERF_AST_HNDLR | DBG_FUNC_END);
540 }
541
542
543 #if CONFIG_EXCLAVES
544 /* Called from Exclave inspection thread after collecting a sample */
545 __attribute__((noinline))
546 void kperf_thread_exclaves_ast_handler(thread_t thread, const stackshot_stackshotentry_s * _Nonnull entry);
547
548 __attribute__((noinline))
549 void
kperf_thread_exclaves_ast_handler(thread_t thread,const stackshot_stackshotentry_s * _Nonnull entry)550 kperf_thread_exclaves_ast_handler(thread_t thread, const stackshot_stackshotentry_s * _Nonnull entry)
551 {
552 assert3u(entry->scid, ==, thread->th_exclaves_ipc_ctx.scid);
553 uint32_t ast = thread->kperf_exclaves_ast;
554
555 BUF_INFO(PERF_AST_EXCLAVES | DBG_FUNC_START, thread, ast);
556 unsigned int actionid = T_KPERF_GET_ACTIONID(ast);
557
558 boolean_t intren = ml_set_interrupts_enabled(false);
559
560 __block size_t ipcstack_count = 0;
561
562 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_START, SAMPLER_EXSTACK, actionid);
563 if (entry->ipcstack.has_value) {
564 stackshottypes_ipcstackentry__v_visit(&entry->ipcstack.value, ^(size_t __unused i, const stackshottypes_ipcstackentry_s * _Nonnull __unused ipcstack) {
565 ipcstack_count += 1;
566 });
567
568 BUF_DATA(PERF_CS_EXSTACKHDR, ipcstack_count, thread->thread_id, entry->scid);
569
570 stackshottypes_ipcstackentry__v_visit(&entry->ipcstack.value, ^(size_t __unused j, const stackshottypes_ipcstackentry_s * _Nonnull ipcstack) {
571 kperf_excallstack_log(ipcstack);
572 });
573 }
574 BUF_DATA(PERF_GEN_EVENT | DBG_FUNC_END, SAMPLER_EXSTACK);
575
576 ml_set_interrupts_enabled(intren);
577
578 BUF_INFO(PERF_AST_EXCLAVES | DBG_FUNC_END);
579 }
580 #endif /* CONFIG_EXCLAVES */
581
582 int
kperf_ast_pend(thread_t thread,uint32_t set_flags,unsigned int set_actionid)583 kperf_ast_pend(thread_t thread, uint32_t set_flags, unsigned int set_actionid)
584 {
585 if (thread != current_thread()) {
586 panic("kperf: pending AST to non-current thread");
587 }
588
589 uint32_t ast = thread->kperf_ast;
590 unsigned int actionid = T_KPERF_GET_ACTIONID(ast);
591 uint32_t flags = ast & T_KPERF_AST_ALL;
592
593 if ((flags | set_flags) != flags || actionid != set_actionid) {
594 ast &= ~T_KPERF_SET_ACTIONID(actionid);
595 ast |= T_KPERF_SET_ACTIONID(set_actionid);
596 ast |= set_flags;
597
598 thread->kperf_ast = ast;
599
600 /* set the actual AST */
601 act_set_kperf(thread);
602 return 1;
603 }
604
605 return 0;
606 }
607
608 void
kperf_ast_set_callstack_depth(thread_t thread,uint32_t depth)609 kperf_ast_set_callstack_depth(thread_t thread, uint32_t depth)
610 {
611 uint32_t ast = thread->kperf_ast;
612 uint32_t existing_depth = T_KPERF_GET_CALLSTACK_DEPTH(ast);
613 if (existing_depth < depth) {
614 ast &= ~T_KPERF_SET_CALLSTACK_DEPTH(existing_depth);
615 ast |= T_KPERF_SET_CALLSTACK_DEPTH(depth);
616 thread->kperf_ast = ast;
617 }
618 }
619
620 int
kperf_kdbg_cswitch_get(void)621 kperf_kdbg_cswitch_get(void)
622 {
623 return kperf_kdebug_cswitch;
624 }
625
626 int
kperf_kdbg_cswitch_set(int newval)627 kperf_kdbg_cswitch_set(int newval)
628 {
629 kperf_kdebug_cswitch = newval;
630 kperf_on_cpu_update();
631
632 return 0;
633 }
634
635 /*
636 * Action configuration
637 */
638 unsigned int
kperf_action_get_count(void)639 kperf_action_get_count(void)
640 {
641 return actionc;
642 }
643
644 int
kperf_action_set_samplers(unsigned actionid,uint32_t samplers)645 kperf_action_set_samplers(unsigned actionid, uint32_t samplers)
646 {
647 if ((actionid > actionc) || (actionid == 0)) {
648 return EINVAL;
649 }
650
651 /* disallow both CPU and thread counters to be sampled in the same
652 * action */
653 if ((samplers & SAMPLER_PMC_THREAD) && (samplers & SAMPLER_PMC_CPU)) {
654 return EINVAL;
655 }
656
657 actionv[actionid - 1].sample = samplers;
658
659 return 0;
660 }
661
662 int
kperf_action_get_samplers(unsigned actionid,uint32_t * samplers_out)663 kperf_action_get_samplers(unsigned actionid, uint32_t *samplers_out)
664 {
665 if ((actionid > actionc)) {
666 return EINVAL;
667 }
668
669 if (actionid == 0) {
670 *samplers_out = 0; /* "NULL" action */
671 } else {
672 *samplers_out = actionv[actionid - 1].sample;
673 }
674
675 return 0;
676 }
677
678 int
kperf_action_set_userdata(unsigned actionid,uint32_t userdata)679 kperf_action_set_userdata(unsigned actionid, uint32_t userdata)
680 {
681 if ((actionid > actionc) || (actionid == 0)) {
682 return EINVAL;
683 }
684
685 actionv[actionid - 1].userdata = userdata;
686
687 return 0;
688 }
689
690 int
kperf_action_get_userdata(unsigned actionid,uint32_t * userdata_out)691 kperf_action_get_userdata(unsigned actionid, uint32_t *userdata_out)
692 {
693 if ((actionid > actionc)) {
694 return EINVAL;
695 }
696
697 if (actionid == 0) {
698 *userdata_out = 0; /* "NULL" action */
699 } else {
700 *userdata_out = actionv[actionid - 1].userdata;
701 }
702
703 return 0;
704 }
705
706 int
kperf_action_set_filter(unsigned actionid,int pid)707 kperf_action_set_filter(unsigned actionid, int pid)
708 {
709 if ((actionid > actionc) || (actionid == 0)) {
710 return EINVAL;
711 }
712
713 actionv[actionid - 1].pid_filter = pid;
714
715 return 0;
716 }
717
718 int
kperf_action_get_filter(unsigned actionid,int * pid_out)719 kperf_action_get_filter(unsigned actionid, int *pid_out)
720 {
721 if ((actionid > actionc)) {
722 return EINVAL;
723 }
724
725 if (actionid == 0) {
726 *pid_out = -1; /* "NULL" action */
727 } else {
728 *pid_out = actionv[actionid - 1].pid_filter;
729 }
730
731 return 0;
732 }
733
734 void
kperf_action_reset(void)735 kperf_action_reset(void)
736 {
737 for (unsigned int i = 0; i < actionc; i++) {
738 kperf_action_set_samplers(i + 1, 0);
739 kperf_action_set_userdata(i + 1, 0);
740 kperf_action_set_filter(i + 1, -1);
741 kperf_action_set_ucallstack_depth(i + 1, MAX_UCALLSTACK_FRAMES);
742 kperf_action_set_kcallstack_depth(i + 1, MAX_KCALLSTACK_FRAMES);
743 }
744 }
745
746 int
kperf_action_set_count(unsigned count)747 kperf_action_set_count(unsigned count)
748 {
749 struct action *new_actionv = NULL, *old_actionv = NULL;
750 unsigned old_count;
751
752 /* easy no-op */
753 if (count == actionc) {
754 return 0;
755 }
756
757 /* TODO: allow shrinking? */
758 if (count < actionc) {
759 return EINVAL;
760 }
761
762 /* cap it for good measure */
763 if (count > ACTION_MAX) {
764 return EINVAL;
765 }
766
767 /* creating the action arror for the first time. create a few
768 * more things, too.
769 */
770 if (actionc == 0) {
771 kperf_setup();
772 }
773
774 /* create a new array */
775 new_actionv = kalloc_data_tag(count * sizeof(*new_actionv),
776 Z_WAITOK, VM_KERN_MEMORY_DIAG);
777 if (new_actionv == NULL) {
778 return ENOMEM;
779 }
780
781 old_actionv = actionv;
782 old_count = actionc;
783
784 if (old_actionv != NULL) {
785 memcpy(new_actionv, actionv, actionc * sizeof(*actionv));
786 }
787
788 memset(&(new_actionv[actionc]), 0, (count - old_count) * sizeof(*actionv));
789
790 for (unsigned int i = old_count; i < count; i++) {
791 new_actionv[i].pid_filter = -1;
792 new_actionv[i].ucallstack_depth = MAX_UCALLSTACK_FRAMES;
793 new_actionv[i].kcallstack_depth = MAX_KCALLSTACK_FRAMES;
794 }
795
796 actionv = new_actionv;
797 actionc = count;
798
799 kfree_data(old_actionv, old_count * sizeof(*actionv));
800
801 return 0;
802 }
803
804 int
kperf_action_set_ucallstack_depth(unsigned action_id,uint32_t depth)805 kperf_action_set_ucallstack_depth(unsigned action_id, uint32_t depth)
806 {
807 if ((action_id > actionc) || (action_id == 0)) {
808 return EINVAL;
809 }
810
811 if (depth > MAX_UCALLSTACK_FRAMES) {
812 return EINVAL;
813 }
814 if (depth < 2) {
815 return EINVAL;
816 }
817
818 actionv[action_id - 1].ucallstack_depth = depth;
819
820 return 0;
821 }
822
823 int
kperf_action_set_kcallstack_depth(unsigned action_id,uint32_t depth)824 kperf_action_set_kcallstack_depth(unsigned action_id, uint32_t depth)
825 {
826 if ((action_id > actionc) || (action_id == 0)) {
827 return EINVAL;
828 }
829
830 if (depth > MAX_KCALLSTACK_FRAMES) {
831 return EINVAL;
832 }
833 if (depth < 1) {
834 return EINVAL;
835 }
836
837 actionv[action_id - 1].kcallstack_depth = depth;
838
839 return 0;
840 }
841
842 int
kperf_action_get_ucallstack_depth(unsigned action_id,uint32_t * depth_out)843 kperf_action_get_ucallstack_depth(unsigned action_id, uint32_t * depth_out)
844 {
845 if ((action_id > actionc)) {
846 return EINVAL;
847 }
848
849 assert(depth_out);
850
851 if (action_id == 0) {
852 *depth_out = MAX_UCALLSTACK_FRAMES;
853 } else {
854 *depth_out = actionv[action_id - 1].ucallstack_depth;
855 }
856
857 return 0;
858 }
859
860 int
kperf_action_get_kcallstack_depth(unsigned action_id,uint32_t * depth_out)861 kperf_action_get_kcallstack_depth(unsigned action_id, uint32_t * depth_out)
862 {
863 if ((action_id > actionc)) {
864 return EINVAL;
865 }
866
867 assert(depth_out);
868
869 if (action_id == 0) {
870 *depth_out = MAX_KCALLSTACK_FRAMES;
871 } else {
872 *depth_out = actionv[action_id - 1].kcallstack_depth;
873 }
874
875 return 0;
876 }
877