1 /*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Sample thread data */
30
31 #include <kern/debug.h> /* panic */
32 #include <kern/thread.h> /* thread_* */
33 #include <kern/policy_internal.h> /* TASK_POLICY_* */
34 #include <mach/mach_types.h>
35 #include <sys/errno.h>
36
37 #include <kperf/kperf.h>
38 #include <kperf/buffer.h>
39 #include <kperf/context.h>
40 #include <kperf/thread_samplers.h>
41 #include <kperf/ast.h>
42
43 #if MONOTONIC
44 #include <kern/monotonic.h>
45 #include <machine/monotonic.h>
46 #endif /* MONOTONIC */
47
48 extern boolean_t stackshot_thread_is_idle_worker_unsafe(thread_t thread);
49
50 /*
51 * XXX Deprecated, use thread scheduling sampler instead.
52 *
53 * Taken from AppleProfileGetRunModeOfThread and CHUD. Still here for
54 * backwards compatibility.
55 */
56
57 #define KPERF_TI_RUNNING (1U << 0)
58 #define KPERF_TI_RUNNABLE (1U << 1)
59 #define KPERF_TI_WAIT (1U << 2)
60 #define KPERF_TI_UNINT (1U << 3)
61 #define KPERF_TI_SUSP (1U << 4)
62 #define KPERF_TI_TERMINATE (1U << 5)
63 #define KPERF_TI_IDLE (1U << 6)
64
65 static uint32_t
kperf_thread_info_runmode_legacy(thread_t thread)66 kperf_thread_info_runmode_legacy(thread_t thread)
67 {
68 uint32_t kperf_state = 0;
69 int sched_state = thread->state;
70 processor_t last_processor = thread->last_processor;
71
72 if ((last_processor != PROCESSOR_NULL) && (thread == last_processor->active_thread)) {
73 kperf_state |= KPERF_TI_RUNNING;
74 }
75 if (sched_state & TH_RUN) {
76 kperf_state |= KPERF_TI_RUNNABLE;
77 }
78 if (sched_state & TH_WAIT) {
79 kperf_state |= KPERF_TI_WAIT;
80 }
81 if (sched_state & TH_UNINT) {
82 kperf_state |= KPERF_TI_UNINT;
83 }
84 if (sched_state & TH_SUSP) {
85 kperf_state |= KPERF_TI_SUSP;
86 }
87 if (sched_state & TH_TERMINATE) {
88 kperf_state |= KPERF_TI_TERMINATE;
89 }
90 if (sched_state & TH_IDLE) {
91 kperf_state |= KPERF_TI_IDLE;
92 }
93
94 #if defined(XNU_TARGET_OS_OSX)
95 /* on desktop, if state is blank, leave not idle set */
96 if (kperf_state == 0) {
97 return TH_IDLE << 16;
98 }
99 #endif /* defined(XNU_TARGET_OS_OSX) */
100
101 /* high two bytes are inverted mask, low two bytes are normal */
102 return ((~kperf_state & 0xffff) << 16) | (kperf_state & 0xffff);
103 }
104
105 void
kperf_thread_info_sample(struct kperf_thread_info * ti,struct kperf_context * context)106 kperf_thread_info_sample(struct kperf_thread_info *ti, struct kperf_context *context)
107 {
108 thread_t cur_thread = context->cur_thread;
109
110 BUF_INFO(PERF_TI_SAMPLE, (uintptr_t)thread_tid(cur_thread));
111
112 ti->kpthi_pid = context->cur_pid;
113 ti->kpthi_tid = thread_tid(cur_thread);
114 ti->kpthi_dq_addr = thread_dispatchqaddr(cur_thread);
115 ti->kpthi_runmode = kperf_thread_info_runmode_legacy(cur_thread);
116
117 BUF_VERB(PERF_TI_SAMPLE | DBG_FUNC_END);
118 }
119
120 void
kperf_thread_info_log(struct kperf_thread_info * ti)121 kperf_thread_info_log(struct kperf_thread_info *ti)
122 {
123 BUF_DATA(PERF_TI_DATA, ti->kpthi_pid, ti->kpthi_tid /* K64-only */,
124 ti->kpthi_dq_addr, ti->kpthi_runmode);
125 }
126
127 /*
128 * Scheduling information reports inputs and outputs of the scheduler state for
129 * a thread.
130 */
131
132 void
kperf_thread_scheduling_sample(struct kperf_thread_scheduling * thsc,struct kperf_context * context)133 kperf_thread_scheduling_sample(struct kperf_thread_scheduling *thsc,
134 struct kperf_context *context)
135 {
136 assert(thsc != NULL);
137 assert(context != NULL);
138
139 thread_t thread = context->cur_thread;
140
141 BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
142
143 struct recount_times_mach times = { 0 };
144 if (thread == current_thread()) {
145 boolean_t interrupt_state = ml_set_interrupts_enabled(FALSE);
146 times = recount_current_thread_times();
147 ml_set_interrupts_enabled(interrupt_state);
148 } else {
149 times = recount_thread_times(thread);
150 }
151 thsc->kpthsc_user_time = times.rtm_user;
152 thsc->kpthsc_system_time = times.rtm_system;
153
154 thsc->kpthsc_runnable_time = timer_grab(&thread->runnable_timer);
155 thsc->kpthsc_state = thread->state;
156 thsc->kpthsc_base_priority = thread->base_pri;
157 thsc->kpthsc_sched_priority = thread->sched_pri;
158 thsc->kpthsc_effective_qos = thread->effective_policy.thep_qos;
159 thsc->kpthsc_requested_qos = thread->requested_policy.thrp_qos;
160 thsc->kpthsc_requested_qos_override = MAX(thread->requested_policy.thrp_qos_override,
161 thread->requested_policy.thrp_qos_workq_override);
162 thsc->kpthsc_requested_qos_promote = thread->requested_policy.thrp_qos_promote;
163 thsc->kpthsc_requested_qos_kevent_override = MAX(
164 thread->requested_policy.thrp_qos_kevent_override,
165 thread->requested_policy.thrp_qos_wlsvc_override);
166 thsc->kpthsc_requested_qos_sync_ipc_override = THREAD_QOS_UNSPECIFIED;
167 thsc->kpthsc_effective_latency_qos = thread->effective_policy.thep_latency_qos;
168
169 BUF_INFO(PERF_TI_SCHEDSAMPLE | DBG_FUNC_END);
170 }
171
172
173 void
kperf_thread_scheduling_log(struct kperf_thread_scheduling * thsc)174 kperf_thread_scheduling_log(struct kperf_thread_scheduling *thsc)
175 {
176 assert(thsc != NULL);
177 #if defined(__LP64__)
178 BUF_DATA(PERF_TI_SCHEDDATA_2, thsc->kpthsc_user_time,
179 thsc->kpthsc_system_time,
180 (((uint64_t)thsc->kpthsc_base_priority) << 48)
181 | ((uint64_t)thsc->kpthsc_sched_priority << 32)
182 | ((uint64_t)(thsc->kpthsc_state & 0xff) << 24)
183 | (thsc->kpthsc_effective_qos << 6)
184 | (thsc->kpthsc_requested_qos << 3)
185 | thsc->kpthsc_requested_qos_override,
186 ((uint64_t)thsc->kpthsc_effective_latency_qos << 61)
187 | ((uint64_t)thsc->kpthsc_requested_qos_promote << 58)
188 | ((uint64_t)thsc->kpthsc_requested_qos_kevent_override << 55)
189 );
190 BUF_DATA(PERF_TI_SCHEDDATA_3, thsc->kpthsc_runnable_time);
191 #else
192 BUF_DATA(PERF_TI_SCHEDDATA1_32, UPPER_32(thsc->kpthsc_user_time),
193 LOWER_32(thsc->kpthsc_user_time),
194 UPPER_32(thsc->kpthsc_system_time),
195 LOWER_32(thsc->kpthsc_system_time)
196 );
197 BUF_DATA(PERF_TI_SCHEDDATA2_32_2, (((uint32_t)thsc->kpthsc_base_priority) << 16)
198 | thsc->kpthsc_sched_priority,
199 ((thsc->kpthsc_state & 0xff) << 24)
200 | (thsc->kpthsc_effective_qos << 6)
201 | (thsc->kpthsc_requested_qos << 3)
202 | thsc->kpthsc_requested_qos_override,
203 ((uint32_t)thsc->kpthsc_effective_latency_qos << 29)
204 | ((uint32_t)thsc->kpthsc_requested_qos_promote << 26)
205 | ((uint32_t)thsc->kpthsc_requested_qos_kevent_override << 23)
206 );
207 BUF_DATA(PERF_TI_SCHEDDATA3_32, UPPER_32(thsc->kpthsc_runnable_time),
208 LOWER_32(thsc->kpthsc_runnable_time));
209 #endif /* defined(__LP64__) */
210 }
211
212 /*
213 * Snapshot information maintains parity with stackshot information for other,
214 * miscellaneous information about threads.
215 */
216
217 #define KPERF_THREAD_SNAPSHOT_DARWIN_BG (1U << 0);
218 #define KPERF_THREAD_SNAPSHOT_PASSIVE_IO (1U << 1);
219 #define KPERF_THREAD_SNAPSHOT_GFI (1U << 2);
220 #define KPERF_THREAD_SNAPSHOT_IDLE_WQ (1U << 3);
221 /* max is 1U << 7 */
222
223 void
kperf_thread_snapshot_sample(struct kperf_thread_snapshot * thsn,struct kperf_context * context)224 kperf_thread_snapshot_sample(struct kperf_thread_snapshot *thsn,
225 struct kperf_context *context)
226 {
227 assert(thsn != NULL);
228 assert(context != NULL);
229
230 thread_t thread = context->cur_thread;
231
232 BUF_INFO(PERF_TI_SNAPSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread));
233
234 thsn->kpthsn_last_made_runnable_time = thread->last_made_runnable_time;
235
236 thsn->kpthsn_flags = 0;
237 if (thread->effective_policy.thep_darwinbg) {
238 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_DARWIN_BG;
239 }
240 if (proc_get_effective_thread_policy(thread, TASK_POLICY_PASSIVE_IO)) {
241 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_PASSIVE_IO;
242 }
243 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
244 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_GFI
245 }
246 if (stackshot_thread_is_idle_worker_unsafe(thread)) {
247 thsn->kpthsn_flags |= KPERF_THREAD_SNAPSHOT_IDLE_WQ;
248 }
249
250 thsn->kpthsn_suspend_count = thread->suspend_count;
251 /*
252 * Only have room for 8-bits in the trace event, so truncate here.
253 */
254 thsn->kpthsn_io_tier = (uint8_t)proc_get_effective_thread_policy(thread, TASK_POLICY_IO);
255
256 BUF_VERB(PERF_TI_SNAPSAMPLE | DBG_FUNC_END);
257 }
258
259 void
kperf_thread_snapshot_log(struct kperf_thread_snapshot * thsn)260 kperf_thread_snapshot_log(struct kperf_thread_snapshot *thsn)
261 {
262 assert(thsn != NULL);
263 #if defined(__LP64__)
264 BUF_DATA(PERF_TI_SNAPDATA, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
265 | (thsn->kpthsn_io_tier << 24),
266 thsn->kpthsn_last_made_runnable_time);
267 #else
268 BUF_DATA(PERF_TI_SNAPDATA_32, thsn->kpthsn_flags | ((uint32_t)(thsn->kpthsn_suspend_count) << 8)
269 | (thsn->kpthsn_io_tier << 24),
270 UPPER_32(thsn->kpthsn_last_made_runnable_time),
271 LOWER_32(thsn->kpthsn_last_made_runnable_time));
272 #endif /* defined(__LP64__) */
273 }
274
275 /*
276 * Dispatch information only contains the dispatch queue serial number from
277 * libdispatch.
278 *
279 * It's a separate sampler because queue data must be copied in from user space.
280 */
281
282 void
kperf_thread_dispatch_sample(struct kperf_thread_dispatch * thdi,struct kperf_context * context)283 kperf_thread_dispatch_sample(struct kperf_thread_dispatch *thdi,
284 struct kperf_context *context)
285 {
286 assert(thdi != NULL);
287 assert(context != NULL);
288
289 thread_t thread = context->cur_thread;
290
291 BUF_INFO(PERF_TI_DISPSAMPLE | DBG_FUNC_START,
292 (uintptr_t)thread_tid(thread));
293
294 task_t task = get_threadtask(thread);
295 size_t user_addr_size = task_has_64Bit_addr(task) ? 8 : 4;
296 thdi->kpthdi_dq_serialno = 0;
297 thdi->kpthdi_dq_label[0] = '\0';
298 int error = 0;
299
300 /*
301 * The dispatch queue address points to a struct that contains
302 * information about the dispatch queue. Use task-level offsets to
303 * find the serial number and label of the dispatch queue.
304 */
305 assert(task != kernel_task);
306 uint64_t user_dq_key_addr = thread_dispatchqaddr(thread);
307 if (user_dq_key_addr == 0) {
308 error = ENOENT;
309 goto out;
310 }
311
312 uint64_t user_dq_addr = 0;
313 if ((error = copyin((user_addr_t)user_dq_key_addr, &user_dq_addr,
314 user_addr_size)) != 0) {
315 goto out;
316 }
317
318 if (user_dq_addr == 0) {
319 error = EINVAL;
320 goto out;
321 }
322
323 uint64_t serialno_offset = get_task_dispatchqueue_serialno_offset(task);
324 uint64_t user_dq_serialno_addr = 0;
325 if (os_add_overflow(user_dq_addr, serialno_offset,
326 &user_dq_serialno_addr)) {
327 error = EOVERFLOW;
328 goto out;
329 }
330
331 if ((error = copyin((user_addr_t)user_dq_serialno_addr,
332 &(thdi->kpthdi_dq_serialno), user_addr_size)) != 0) {
333 goto out;
334 }
335
336 uint64_t lbl_offset = get_task_dispatchqueue_label_offset(task);
337 if (lbl_offset == 0) {
338 error = ENOBUFS;
339 goto out;
340 }
341
342 uint64_t user_dqlbl_ptr_addr = 0;
343 if (os_add_overflow(user_dq_addr, lbl_offset, &user_dqlbl_ptr_addr)) {
344 error = EOVERFLOW;
345 goto out;
346 }
347
348 uint64_t user_dqlbl_addr = 0;
349 /*
350 * The label isn't embedded in the struct -- it just holds a
351 * pointer to the label string, NUL-terminated.
352 */
353 if ((error = copyin((user_addr_t)user_dqlbl_ptr_addr, &user_dqlbl_addr,
354 user_addr_size)) != 0) {
355 goto out;
356 }
357
358 vm_size_t copied = 0;
359 if ((error = copyinstr((user_addr_t)user_dqlbl_addr,
360 thdi->kpthdi_dq_label, sizeof(thdi->kpthdi_dq_label),
361 &copied)) != 0) {
362 goto out;
363 }
364 thdi->kpthdi_dq_label[sizeof(thdi->kpthdi_dq_label) - 1] = '\0';
365
366 out:
367 BUF_VERB(PERF_TI_DISPSAMPLE | DBG_FUNC_END, error);
368 }
369
370 int
kperf_thread_dispatch_pend(struct kperf_context * context,unsigned int actionid)371 kperf_thread_dispatch_pend(struct kperf_context *context,
372 unsigned int actionid)
373 {
374 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_DISPATCH,
375 actionid);
376 }
377
378 void
kperf_thread_dispatch_log(struct kperf_thread_dispatch * thdi)379 kperf_thread_dispatch_log(struct kperf_thread_dispatch *thdi)
380 {
381 assert(thdi != NULL);
382 #if defined(__LP64__)
383 BUF_DATA(PERF_TI_DISPDATA, thdi->kpthdi_dq_serialno);
384 #else
385 BUF_DATA(PERF_TI_DISPDATA_32, UPPER_32(thdi->kpthdi_dq_serialno),
386 LOWER_32(thdi->kpthdi_dq_serialno));
387 #endif /* defined(__LP64__) */
388
389 if (thdi->kpthdi_dq_label[0] != '\0') {
390 kernel_debug_string_simple(PERF_TI_DISPLABEL, thdi->kpthdi_dq_label);
391 }
392 }
393
394 /*
395 * A bit different from other samplers -- since logging disables interrupts,
396 * it's a fine place to sample the thread counters.
397 */
398 void
kperf_thread_inscyc_log(struct kperf_context * context)399 kperf_thread_inscyc_log(struct kperf_context *context)
400 {
401 #if CONFIG_PERVASIVE_CPI
402 thread_t cur_thread = current_thread();
403
404 if (context->cur_thread != cur_thread) {
405 /* can't safely access another thread's counters */
406 return;
407 }
408
409 struct recount_usage usage = { 0 };
410 struct recount_usage perf_only = { 0 };
411 recount_current_thread_usage_perf_only(&usage, &perf_only);
412 BUF_DATA(PERF_TI_INSCYCDATA, usage.ru_instructions, usage.ru_cycles,
413 perf_only.ru_instructions, perf_only.ru_cycles);
414 #else /* CONFIG_PERVASIVE_CPI */
415 #pragma unused(context)
416 #endif /* !CONFIG_PERVASIVE_CPI */
417 }
418