1 /*
2 * Copyright (c) 2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <machine/machine_routines.h>
30 #include <sys/sysproto.h>
31 #include <sys/malloc.h>
32 #include <sys/systm.h>
33 #include <sys/pgo.h>
34 #include <sys/kauth.h>
35 #include <security/mac_framework.h>
36 #include <libkern/OSKextLib.h>
37
38
39 #ifdef PROFILE
40
41 /* These __llvm functions are defined in InstrProfiling.h in compiler_rt. That
42 * is a internal header, so we need to re-prototype them here. */
43
44 uint64_t __llvm_profile_get_size_for_buffer(void);
45 int __llvm_profile_write_buffer(char *Buffer);
46 uint64_t __llvm_profile_get_size_for_buffer_internal(const char *DataBegin,
47 const char *DataEnd,
48 const char *CountersBegin,
49 const char *CountersEnd,
50 const char *NamesBegin,
51 const char *NamesEnd);
52 int __llvm_profile_write_buffer_internal(char *Buffer,
53 const char *DataBegin,
54 const char *DataEnd,
55 const char *CountersBegin,
56 const char *CountersEnd,
57 const char *NamesBegin,
58 const char *NamesEnd);
59
60 extern char __pgo_hib_DataStart __asm("section$start$__HIB$__llvm_prf_data");
61 extern char __pgo_hib_DataEnd __asm("section$end$__HIB$__llvm_prf_data");
62 extern char __pgo_hib_NamesStart __asm("section$start$__HIB$__llvm_prf_names");
63 extern char __pgo_hib_NamesEnd __asm("section$end$__HIB$__llvm_prf_names");
64 extern char __pgo_hib_CountersStart __asm("section$start$__HIB$__llvm_prf_cnts");
65 extern char __pgo_hib_CountersEnd __asm("section$end$__HIB$__llvm_prf_cnts");
66
67
68 static uint64_t
get_size_for_buffer(int flags)69 get_size_for_buffer(int flags)
70 {
71 if (flags & PGO_HIB) {
72 return __llvm_profile_get_size_for_buffer_internal(
73 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
74 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
75 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
76 } else {
77 return __llvm_profile_get_size_for_buffer();
78 }
79 }
80
81
82 static int
write_buffer(int flags,char * buffer)83 write_buffer(int flags, char *buffer)
84 {
85 if (flags & PGO_HIB) {
86 return __llvm_profile_write_buffer_internal(
87 buffer,
88 &__pgo_hib_DataStart, &__pgo_hib_DataEnd,
89 &__pgo_hib_CountersStart, &__pgo_hib_CountersEnd,
90 &__pgo_hib_NamesStart, &__pgo_hib_NamesEnd);
91 } else {
92 return __llvm_profile_write_buffer(buffer);
93 }
94 }
95
96
97 #endif
98
99 /* this variable is used to signal to the debugger that we'd like it to reset
100 * the counters */
101 int kdp_pgo_reset_counters = 0;
102
103 /* called in debugger context */
104 kern_return_t
do_pgo_reset_counters()105 do_pgo_reset_counters()
106 {
107 #ifdef PROFILE
108 memset(&__pgo_hib_CountersStart, 0,
109 ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart)));
110 #endif
111 OSKextResetPgoCounters();
112 kdp_pgo_reset_counters = 0;
113 return KERN_SUCCESS;
114 }
115
116 static kern_return_t
kextpgo_trap()117 kextpgo_trap()
118 {
119 return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, NULL, FALSE, 0);
120 }
121
122 static kern_return_t
pgo_reset_counters()123 pgo_reset_counters()
124 {
125 kern_return_t r;
126 boolean_t istate;
127
128 OSKextResetPgoCountersLock();
129
130 istate = ml_set_interrupts_enabled(FALSE);
131
132 kdp_pgo_reset_counters = 1;
133 r = kextpgo_trap();
134
135 ml_set_interrupts_enabled(istate);
136
137 OSKextResetPgoCountersUnlock();
138 return r;
139 }
140
141
142 /*
143 * returns:
144 * EPERM unless you are root
145 * EINVAL for invalid args.
146 * ENOSYS for not implemented
147 * ERANGE for integer overflow
148 * ENOENT if kext not found
149 * ENOTSUP kext does not support PGO
150 * EIO llvm returned an error. shouldn't ever happen.
151 */
152
153 int
grab_pgo_data(struct proc * p,struct grab_pgo_data_args * uap,register_t * retval)154 grab_pgo_data(struct proc *p,
155 struct grab_pgo_data_args *uap,
156 register_t *retval)
157 {
158 char *buffer = NULL;
159 uint64_t size64 = 0;
160 int err = 0;
161
162 (void) p;
163
164 if (!kauth_cred_issuser(kauth_cred_get())) {
165 err = EPERM;
166 goto out;
167 }
168
169 #if CONFIG_MACF
170 err = mac_system_check_info(kauth_cred_get(), "kern.profiling_data");
171 if (err) {
172 goto out;
173 }
174 #endif
175
176 if (uap->flags & ~PGO_ALL_FLAGS ||
177 uap->size < 0 ||
178 (uap->size > 0 && uap->buffer == 0)) {
179 err = EINVAL;
180 goto out;
181 }
182
183 if (uap->flags & PGO_RESET_ALL) {
184 if (uap->flags != PGO_RESET_ALL || uap->uuid || uap->buffer || uap->size) {
185 err = EINVAL;
186 } else {
187 kern_return_t r = pgo_reset_counters();
188 switch (r) {
189 case KERN_SUCCESS:
190 err = 0;
191 break;
192 case KERN_OPERATION_TIMED_OUT:
193 err = ETIMEDOUT;
194 break;
195 default:
196 err = EIO;
197 break;
198 }
199 }
200 goto out;
201 }
202
203 *retval = 0;
204
205 if (uap->uuid) {
206 uuid_t uuid;
207 err = copyin(uap->uuid, &uuid, sizeof(uuid));
208 if (err) {
209 goto out;
210 }
211
212 if (uap->buffer == 0 && uap->size == 0) {
213 if (uap->flags & PGO_WAIT_FOR_UNLOAD) {
214 err = EINVAL;
215 goto out;
216 }
217
218 err = OSKextGrabPgoData(uuid, &size64, NULL, 0, 0, !!(uap->flags & PGO_METADATA));
219 if (size64 == 0 && err == 0) {
220 err = EIO;
221 }
222 if (err) {
223 goto out;
224 }
225
226 ssize_t size = size64;
227 if (((uint64_t) size) != size64 ||
228 size < 0) {
229 err = ERANGE;
230 goto out;
231 }
232
233 *retval = size;
234 err = 0;
235 goto out;
236 } else if (!uap->buffer || uap->size <= 0) {
237 err = EINVAL;
238 goto out;
239 } else {
240 err = OSKextGrabPgoData(uuid, &size64, NULL, 0,
241 false,
242 !!(uap->flags & PGO_METADATA));
243
244 if (size64 == 0 && err == 0) {
245 err = EIO;
246 }
247 if (err) {
248 goto out;
249 }
250
251 if (uap->size < 0 || (uint64_t)uap->size < size64) {
252 err = EINVAL;
253 goto out;
254 }
255
256 buffer = kalloc_data(size64, Z_WAITOK | Z_ZERO);
257 if (!buffer) {
258 err = ENOMEM;
259 goto out;
260 }
261
262 err = OSKextGrabPgoData(uuid, &size64, buffer, size64,
263 !!(uap->flags & PGO_WAIT_FOR_UNLOAD),
264 !!(uap->flags & PGO_METADATA));
265 if (err) {
266 goto out;
267 }
268
269 ssize_t size = size64;
270 if (((uint64_t) size) != size64 ||
271 size < 0) {
272 err = ERANGE;
273 goto out;
274 }
275
276 err = copyout(buffer, uap->buffer, size);
277 if (err) {
278 goto out;
279 }
280
281 *retval = size;
282 goto out;
283 }
284 }
285
286
287 #ifdef PROFILE
288
289 size64 = get_size_for_buffer(uap->flags);
290 ssize_t size = size64;
291
292 if (uap->flags & (PGO_WAIT_FOR_UNLOAD | PGO_METADATA)) {
293 err = EINVAL;
294 goto out;
295 }
296
297 if (((uint64_t) size) != size64 ||
298 size < 0) {
299 err = ERANGE;
300 goto out;
301 }
302
303
304 if (uap->buffer == 0 && uap->size == 0) {
305 *retval = size;
306 err = 0;
307 goto out;
308 } else if (uap->size < size) {
309 err = EINVAL;
310 goto out;
311 } else {
312 buffer = kalloc_data(size, Z_WAITOK | Z_ZERO);
313 if (!buffer) {
314 err = ENOMEM;
315 goto out;
316 }
317
318 err = write_buffer(uap->flags, buffer);
319 if (err) {
320 err = EIO;
321 goto out;
322 }
323
324 err = copyout(buffer, uap->buffer, size);
325 if (err) {
326 goto out;
327 }
328
329 *retval = size;
330 goto out;
331 }
332
333 #else
334
335 *retval = -1;
336 err = ENOSYS;
337 goto out;
338
339 #endif
340
341 out:
342 if (buffer) {
343 kfree_data(buffer, size64);
344 }
345 if (err) {
346 *retval = -1;
347 }
348 return err;
349 }
350