1 /*
2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 #include <sys/sysctl.h>
31 extern "C" {
32 #include <vm/vm_kern.h>
33 #include <kern/task.h>
34 #include <kern/debug.h>
35 }
36
37 #include <libkern/c++/OSContainers.h>
38 #include <libkern/OSDebug.h>
39 #include <libkern/c++/OSCPPDebug.h>
40 #include <kern/backtrace.h>
41
42 #include <IOKit/IOKitDebug.h>
43 #include <IOKit/IOLib.h>
44 #include <IOKit/assert.h>
45 #include <IOKit/IODeviceTreeSupport.h>
46 #include <IOKit/IOService.h>
47
48 #include "IOKitKernelInternal.h"
49
50 TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE);
51 TUNABLE(SInt64, gIOKitTrace, "iotrace", 0);
52
53 #if DEVELOPMENT || DEBUG
54 #define IODEBUG_CTLFLAGS CTLFLAG_RW
55 #else
56 #define IODEBUG_CTLFLAGS CTLFLAG_RD
57 #endif
58
59 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
60
61 static int
sysctl_debug_iokit(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)62 sysctl_debug_iokit
63 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
64 {
65 SInt64 newValue;
66 int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed);
67 if (changed) {
68 gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions));
69 }
70 return error;
71 }
72
73 SYSCTL_PROC(_debug, OID_AUTO, iokit,
74 CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED,
75 &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io");
76
77 size_t debug_malloc_size;
78 size_t debug_iomalloc_size;
79
80 vm_size_t debug_iomallocpageable_size;
81 size_t debug_container_malloc_size;
82 // int debug_ivars_size; // in OSObject.cpp
83
84 extern "C" {
85 #if 0
86 #define DEBG(fmt, args...) { kprintf(fmt, ## args); }
87 #else
88 #define DEBG(fmt, args...) { IOLog(fmt, ## args); }
89 #endif
90
91 void
IOPrintPlane(const IORegistryPlane * plane)92 IOPrintPlane( const IORegistryPlane * plane )
93 {
94 IORegistryEntry * next;
95 IORegistryIterator * iter;
96 OSOrderedSet * all;
97 char format[] = "%xxxs";
98 IOService * service;
99
100 iter = IORegistryIterator::iterateOver( plane );
101 assert( iter );
102 all = iter->iterateAll();
103 if (all) {
104 DEBG("Count %d\n", all->getCount());
105 all->release();
106 } else {
107 DEBG("Empty\n");
108 }
109
110 iter->reset();
111 while ((next = iter->getNextObjectRecursive())) {
112 snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane ));
113 DEBG( format, "");
114 DEBG( "\033[33m%s", next->getName( plane ));
115 if ((next->getLocation( plane ))) {
116 DEBG("@%s", next->getLocation( plane ));
117 }
118 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName());
119 if ((service = OSDynamicCast(IOService, next))) {
120 DEBG(", busy %ld", (long) service->getBusyState());
121 }
122 DEBG( ">\n");
123 // IOSleep(250);
124 }
125 iter->release();
126 }
127
128 void
db_piokjunk(void)129 db_piokjunk(void)
130 {
131 }
132
133 void
db_dumpiojunk(const IORegistryPlane * plane __unused)134 db_dumpiojunk( const IORegistryPlane * plane __unused )
135 {
136 }
137
138 void
IOPrintMemory(void)139 IOPrintMemory( void )
140 {
141 // OSMetaClass::printInstanceCounts();
142
143 IOLog("\n"
144 "ivar kalloc() 0x%08lx\n"
145 "malloc() 0x%08lx\n"
146 "containers kalloc() 0x%08lx\n"
147 "IOMalloc() 0x%08lx\n"
148 "----------------------------------------\n",
149 debug_ivars_size,
150 debug_malloc_size,
151 debug_container_malloc_size,
152 debug_iomalloc_size
153 );
154 }
155 } /* extern "C" */
156
157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
158
159 #define super OSObject
OSDefineMetaClassAndStructors(IOKitDiagnostics,OSObject)160 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject)
161
162 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
163
164 OSObject * IOKitDiagnostics::diagnostics( void )
165 {
166 IOKitDiagnostics * diags;
167
168 diags = new IOKitDiagnostics;
169 if (diags && !diags->init()) {
170 diags->release();
171 diags = NULL;
172 }
173
174 return diags;
175 }
176
177 void
updateOffset(OSDictionary * dict,UInt64 value,const char * name)178 IOKitDiagnostics::updateOffset( OSDictionary * dict,
179 UInt64 value, const char * name )
180 {
181 OSNumber * off;
182
183 off = OSNumber::withNumber( value, 64 );
184 if (!off) {
185 return;
186 }
187
188 dict->setObject( name, off );
189 off->release();
190 }
191
192 bool
serialize(OSSerialize * s) const193 IOKitDiagnostics::serialize(OSSerialize *s) const
194 {
195 OSDictionary * dict;
196 bool ok;
197
198 dict = OSDictionary::withCapacity( 5 );
199 if (!dict) {
200 return false;
201 }
202
203 updateOffset( dict, debug_ivars_size, "Instance allocation" );
204 updateOffset( dict, debug_container_malloc_size, "Container allocation" );
205 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" );
206 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" );
207
208 OSMetaClass::serializeClassDictionary(dict);
209
210 ok = dict->serialize( s );
211
212 dict->release();
213
214 return ok;
215 }
216
217 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
218
219 #if IOTRACKING
220
221 #include <libkern/c++/OSCPPDebug.h>
222 #include <libkern/c++/OSKext.h>
223 #include <kern/zalloc.h>
224
225 __private_extern__ "C" void qsort(
226 void * array,
227 size_t nmembers,
228 size_t member_size,
229 int (*)(const void *, const void *));
230
231 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
232 extern "C" ppnum_t pmap_valid_page(ppnum_t pn);
233
234 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
235
236 struct IOTRecursiveLock {
237 lck_mtx_t * mutex;
238 thread_t thread;
239 UInt32 count;
240 };
241
242 struct IOTrackingQueue {
243 queue_chain_t link;
244 IOTRecursiveLock lock;
245 const char * name;
246 uintptr_t btEntry;
247 size_t allocSize;
248 size_t minCaptureSize;
249 uint32_t siteCount;
250 uint32_t type;
251 uint32_t numSiteQs;
252 uint8_t captureOn;
253 queue_head_t sites[];
254 };
255
256
257 struct IOTrackingCallSiteUser {
258 pid_t pid;
259 uint8_t user32;
260 uint8_t userCount;
261 uintptr_t bt[kIOTrackingCallSiteBTs];
262 };
263
264 struct IOTrackingCallSite {
265 queue_chain_t link;
266 queue_head_t instances;
267 IOTrackingQueue * queue;
268 IOTracking * addresses;
269 size_t size[2];
270 uint32_t crc;
271 uint32_t count;
272
273 vm_tag_t tag;
274 uint8_t user32;
275 uint8_t userCount;
276 pid_t btPID;
277
278 uintptr_t bt[kIOTrackingCallSiteBTs];
279 IOTrackingCallSiteUser user[0];
280 };
281
282 struct IOTrackingCallSiteWithUser {
283 struct IOTrackingCallSite site;
284 struct IOTrackingCallSiteUser user;
285 };
286
287 struct IOTrackingLeaksRef {
288 uintptr_t * instances;
289 uint32_t zoneSize;
290 uint32_t count;
291 uint32_t found;
292 uint32_t foundzlen;
293 size_t bytes;
294 };
295
296 lck_mtx_t * gIOTrackingLock;
297 queue_head_t gIOTrackingQ;
298
299 enum{
300 kTrackingAddressFlagAllocated = 0x00000001
301 };
302
303 #if defined(__LP64__)
304 #define IOTrackingAddressFlags(ptr) (ptr->flags)
305 #else
306 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags)
307 #endif
308
309 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
310
311 static void
IOTRecursiveLockLock(IOTRecursiveLock * lock)312 IOTRecursiveLockLock(IOTRecursiveLock * lock)
313 {
314 if (lock->thread == current_thread()) {
315 lock->count++;
316 } else {
317 lck_mtx_lock(lock->mutex);
318 assert(lock->thread == NULL);
319 assert(lock->count == 0);
320 lock->thread = current_thread();
321 lock->count = 1;
322 }
323 }
324
325 static void
IOTRecursiveLockUnlock(IOTRecursiveLock * lock)326 IOTRecursiveLockUnlock(IOTRecursiveLock * lock)
327 {
328 assert(lock->thread == current_thread());
329 if (0 == (--lock->count)) {
330 lock->thread = NULL;
331 lck_mtx_unlock(lock->mutex);
332 }
333 }
334
335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
336
337 void
IOTrackingInit(void)338 IOTrackingInit(void)
339 {
340 queue_init(&gIOTrackingQ);
341 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
342 }
343
344 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
345
346 IOTrackingQueue *
IOTrackingQueueAlloc(const char * name,uintptr_t btEntry,size_t allocSize,size_t minCaptureSize,uint32_t type,uint32_t numSiteQs)347 IOTrackingQueueAlloc(const char * name, uintptr_t btEntry,
348 size_t allocSize, size_t minCaptureSize,
349 uint32_t type, uint32_t numSiteQs)
350 {
351 IOTrackingQueue * queue;
352 uint32_t idx;
353
354 if (!numSiteQs) {
355 numSiteQs = 1;
356 }
357 queue = kalloc_type(IOTrackingQueue, queue_head_t, numSiteQs, Z_WAITOK_ZERO);
358 queue->name = name;
359 queue->btEntry = btEntry;
360 queue->allocSize = allocSize;
361 queue->minCaptureSize = minCaptureSize;
362 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
363 queue->numSiteQs = numSiteQs;
364 queue->type = type;
365 enum { kFlags = (kIOTracking | kIOTrackingBoot) };
366 queue->captureOn = (kFlags == (kFlags & gIOKitDebug))
367 || (kIOTrackingQueueTypeDefaultOn & type);
368
369 for (idx = 0; idx < numSiteQs; idx++) {
370 queue_init(&queue->sites[idx]);
371 }
372
373 lck_mtx_lock(gIOTrackingLock);
374 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link);
375 lck_mtx_unlock(gIOTrackingLock);
376
377 return queue;
378 };
379
380 void
IOTrackingQueueCollectUser(IOTrackingQueue * queue)381 IOTrackingQueueCollectUser(IOTrackingQueue * queue)
382 {
383 assert(0 == queue->siteCount);
384 queue->type |= kIOTrackingQueueTypeUser;
385 }
386
387 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
388
389 void
IOTrackingQueueFree(IOTrackingQueue * queue)390 IOTrackingQueueFree(IOTrackingQueue * queue)
391 {
392 lck_mtx_lock(gIOTrackingLock);
393 IOTrackingReset(queue);
394 remque(&queue->link);
395 lck_mtx_unlock(gIOTrackingLock);
396
397 lck_mtx_free(queue->lock.mutex, IOLockGroup);
398
399 kfree_type(IOTrackingQueue, queue_head_t, queue->numSiteQs, queue);
400 };
401
402 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
403
404 /* fasthash
405 * The MIT License
406 *
407 * Copyright (C) 2012 Zilong Tan ([email protected])
408 *
409 * Permission is hereby granted, free of charge, to any person
410 * obtaining a copy of this software and associated documentation
411 * files (the "Software"), to deal in the Software without
412 * restriction, including without limitation the rights to use, copy,
413 * modify, merge, publish, distribute, sublicense, and/or sell copies
414 * of the Software, and to permit persons to whom the Software is
415 * furnished to do so, subject to the following conditions:
416 *
417 * The above copyright notice and this permission notice shall be
418 * included in all copies or substantial portions of the Software.
419 *
420 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
421 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
422 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
423 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
424 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
425 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
426 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
427 * SOFTWARE.
428 */
429
430
431 // Compression function for Merkle-Damgard construction.
432 // This function is generated using the framework provided.
433 #define mix(h) ({ \
434 (h) ^= (h) >> 23; \
435 (h) *= 0x2127599bf4325c37ULL; \
436 (h) ^= (h) >> 47; })
437
438 static uint64_t
fasthash64(const void * buf,size_t len,uint64_t seed)439 fasthash64(const void *buf, size_t len, uint64_t seed)
440 {
441 const uint64_t m = 0x880355f21e6d1965ULL;
442 const uint64_t *pos = (const uint64_t *)buf;
443 const uint64_t *end = pos + (len / 8);
444 const unsigned char *pos2;
445 uint64_t h = seed ^ (len * m);
446 uint64_t v;
447
448 while (pos != end) {
449 v = *pos++;
450 h ^= mix(v);
451 h *= m;
452 }
453
454 pos2 = (const unsigned char*)pos;
455 v = 0;
456
457 switch (len & 7) {
458 case 7: v ^= (uint64_t)pos2[6] << 48;
459 [[clang::fallthrough]];
460 case 6: v ^= (uint64_t)pos2[5] << 40;
461 [[clang::fallthrough]];
462 case 5: v ^= (uint64_t)pos2[4] << 32;
463 [[clang::fallthrough]];
464 case 4: v ^= (uint64_t)pos2[3] << 24;
465 [[clang::fallthrough]];
466 case 3: v ^= (uint64_t)pos2[2] << 16;
467 [[clang::fallthrough]];
468 case 2: v ^= (uint64_t)pos2[1] << 8;
469 [[clang::fallthrough]];
470 case 1: v ^= (uint64_t)pos2[0];
471 h ^= mix(v);
472 h *= m;
473 }
474
475 return mix(h);
476 }
477
478 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
479
480 static uint32_t
fasthash32(const void * buf,size_t len,uint32_t seed)481 fasthash32(const void *buf, size_t len, uint32_t seed)
482 {
483 // the following trick converts the 64-bit hashcode to Fermat
484 // residue, which shall retain information from both the higher
485 // and lower parts of hashcode.
486 uint64_t h = fasthash64(buf, len, seed);
487 return (uint32_t) (h - (h >> 32));
488 }
489
490 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
491
492 void
IOTrackingAddUser(IOTrackingQueue * queue,IOTrackingUser * mem,vm_size_t size)493 IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size)
494 {
495 uint32_t num;
496 int pid;
497
498 if (!queue->captureOn) {
499 return;
500 }
501 if (size < queue->minCaptureSize) {
502 return;
503 }
504
505 assert(!mem->link.next);
506
507 num = backtrace(&mem->bt[0], kIOTrackingCallSiteBTs, NULL, NULL);
508 num = 0;
509 if ((kernel_task != current_task()) && (pid = proc_selfpid())) {
510 struct backtrace_user_info btinfo = BTUINFO_INIT;
511 mem->btPID = pid;
512 num = backtrace_user(&mem->btUser[0], kIOTrackingCallSiteBTs - 1,
513 NULL, &btinfo);
514 mem->user32 = !(btinfo.btui_info & BTI_64_BIT);
515 }
516 assert(num <= kIOTrackingCallSiteBTs);
517 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX);
518 mem->userCount = ((uint8_t) num);
519
520 IOTRecursiveLockLock(&queue->lock);
521 queue_enter/*last*/ (&queue->sites[0], mem, IOTrackingUser *, link);
522 queue->siteCount++;
523 IOTRecursiveLockUnlock(&queue->lock);
524 }
525
526 void
IOTrackingRemoveUser(IOTrackingQueue * queue,IOTrackingUser * mem)527 IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem)
528 {
529 if (!mem->link.next) {
530 return;
531 }
532
533 IOTRecursiveLockLock(&queue->lock);
534 if (mem->link.next) {
535 remque(&mem->link);
536 assert(queue->siteCount);
537 queue->siteCount--;
538 }
539 IOTRecursiveLockUnlock(&queue->lock);
540 }
541
542 uint64_t gIOTrackingAddTime;
543
544 void
IOTrackingAdd(IOTrackingQueue * queue,IOTracking * mem,size_t size,bool address,vm_tag_t tag)545 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag)
546 {
547 IOTrackingCallSite * site;
548 uint32_t crc, num;
549 uintptr_t bt[kIOTrackingCallSiteBTs + 1];
550 uintptr_t btUser[kIOTrackingCallSiteBTs];
551 queue_head_t * que;
552 bool user;
553 int pid;
554 int userCount;
555
556 if (mem->site) {
557 return;
558 }
559 if (!queue->captureOn) {
560 return;
561 }
562 if (size < queue->minCaptureSize) {
563 return;
564 }
565
566 user = (0 != (kIOTrackingQueueTypeUser & queue->type));
567
568 assert(!mem->link.next);
569
570 num = backtrace(&bt[0], kIOTrackingCallSiteBTs + 1, NULL, NULL);
571 if (!num) {
572 return;
573 }
574 num--;
575 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7);
576
577 userCount = 0;
578 pid = 0;
579 backtrace_info_t btinfo = BTI_NONE;
580 if (user) {
581 if ((kernel_task != current_task()) && (pid = proc_selfpid())) {
582 struct backtrace_user_info btuinfo = BTUINFO_INIT;
583 userCount = backtrace_user(&btUser[0], kIOTrackingCallSiteBTs,
584 NULL, &btuinfo);
585 assert(userCount <= kIOTrackingCallSiteBTs);
586 btinfo = btuinfo.btui_info;
587 crc = fasthash32(&btUser[0], userCount * sizeof(bt[0]), crc);
588 }
589 }
590
591 IOTRecursiveLockLock(&queue->lock);
592 que = &queue->sites[crc % queue->numSiteQs];
593 queue_iterate(que, site, IOTrackingCallSite *, link)
594 {
595 if (tag != site->tag) {
596 continue;
597 }
598 if (user && (pid != site->user[0].pid)) {
599 continue;
600 }
601 if (crc == site->crc) {
602 break;
603 }
604 }
605
606 if (queue_end(que, (queue_entry_t) site)) {
607 if (user) {
608 site = &kalloc_type(IOTrackingCallSiteWithUser,
609 Z_WAITOK_ZERO_NOFAIL)->site;
610 } else {
611 site = kalloc_type(IOTrackingCallSite,
612 Z_WAITOK_ZERO_NOFAIL);
613 }
614
615 queue_init(&site->instances);
616 site->addresses = (IOTracking *) &site->instances;
617 site->queue = queue;
618 site->crc = crc;
619 site->count = 0;
620 site->tag = tag;
621 memset(&site->size[0], 0, sizeof(site->size));
622 bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0]));
623 assert(num <= kIOTrackingCallSiteBTs);
624 bzero(&site->bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->bt[0]));
625 if (user) {
626 bcopy(&btUser[0], &site->user[0].bt[0], userCount * sizeof(site->user[0].bt[0]));
627 assert(userCount <= kIOTrackingCallSiteBTs);
628 bzero(&site->user[0].bt[userCount], (kIOTrackingCallSiteBTs - userCount) * sizeof(site->user[0].bt[0]));
629 site->user[0].pid = pid;
630 site->user[0].user32 = !(btinfo & BTI_64_BIT);
631 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX);
632 site->user[0].userCount = ((uint8_t) userCount);
633 }
634 queue_enter_first(que, site, IOTrackingCallSite *, link);
635 queue->siteCount++;
636 }
637
638 if (address) {
639 queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
640 if (queue_end(&site->instances, (queue_entry_t)site->addresses)) {
641 site->addresses = mem;
642 }
643 } else {
644 queue_enter_first(&site->instances, mem, IOTracking *, link);
645 }
646
647 mem->site = site;
648 site->size[0] += size;
649 site->count++;
650
651 IOTRecursiveLockUnlock(&queue->lock);
652 }
653
654 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
655
656 void
IOTrackingRemove(IOTrackingQueue * queue,IOTracking * mem,size_t size)657 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
658 {
659 if (!mem->link.next) {
660 return;
661 }
662
663 IOTRecursiveLockLock(&queue->lock);
664 if (mem->link.next) {
665 assert(mem->site);
666
667 if (mem == mem->site->addresses) {
668 mem->site->addresses = (IOTracking *) queue_next(&mem->link);
669 }
670 remque(&mem->link);
671
672 assert(mem->site->count);
673 mem->site->count--;
674 assert(mem->site->size[0] >= size);
675 mem->site->size[0] -= size;
676 if (!mem->site->count) {
677 assert(queue_empty(&mem->site->instances));
678 assert(!mem->site->size[0]);
679 assert(!mem->site->size[1]);
680
681 remque(&mem->site->link);
682 assert(queue->siteCount);
683 queue->siteCount--;
684 if (kIOTrackingQueueTypeUser & queue->type) {
685 kfree_type(IOTrackingCallSiteWithUser, mem->site);
686 } else {
687 kfree_type(IOTrackingCallSite, mem->site);
688 }
689 }
690 mem->site = NULL;
691 }
692 IOTRecursiveLockUnlock(&queue->lock);
693 }
694
695 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
696
697 void
IOTrackingAlloc(IOTrackingQueue * queue,uintptr_t address,size_t size)698 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size)
699 {
700 IOTrackingAddress * tracking;
701
702 if (!queue->captureOn) {
703 return;
704 }
705 if (size < queue->minCaptureSize) {
706 return;
707 }
708
709 address = ~address;
710 tracking = kalloc_type(IOTrackingAddress, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
711 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated;
712 tracking->address = address;
713 tracking->size = size;
714
715 IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE);
716 }
717
718 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
719
720 void
IOTrackingFree(IOTrackingQueue * queue,uintptr_t address,size_t size)721 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size)
722 {
723 IOTrackingCallSite * site;
724 IOTrackingAddress * tracking;
725 uint32_t idx;
726 bool done;
727
728 address = ~address;
729 IOTRecursiveLockLock(&queue->lock);
730 done = false;
731 for (idx = 0; idx < queue->numSiteQs; idx++) {
732 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
733 {
734 tracking = (IOTrackingAddress *) site->addresses;
735 while (!queue_end(&site->instances, &tracking->tracking.link)) {
736 if ((done = (address == tracking->address))) {
737 IOTrackingRemove(queue, &tracking->tracking, size);
738 kfree_type(IOTrackingAddress, tracking);
739 break;
740 } else {
741 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
742 }
743 }
744 if (done) {
745 break;
746 }
747 }
748 if (done) {
749 break;
750 }
751 }
752 IOTRecursiveLockUnlock(&queue->lock);
753 }
754
755 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
756
757 void
IOTrackingAccumSize(IOTrackingQueue * queue,IOTracking * mem,size_t size)758 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size)
759 {
760 IOTRecursiveLockLock(&queue->lock);
761 if (mem->link.next) {
762 assert(mem->site);
763 assert((size > 0) || (mem->site->size[1] >= -size));
764 mem->site->size[1] += size;
765 }
766 ;
767 IOTRecursiveLockUnlock(&queue->lock);
768 }
769
770 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
771
772 void
IOTrackingReset(IOTrackingQueue * queue)773 IOTrackingReset(IOTrackingQueue * queue)
774 {
775 IOTrackingCallSite * site;
776 IOTrackingUser * user;
777 IOTracking * tracking;
778 IOTrackingAddress * trackingAddress;
779 uint32_t idx;
780 bool addresses;
781
782 IOTRecursiveLockLock(&queue->lock);
783 for (idx = 0; idx < queue->numSiteQs; idx++) {
784 while (!queue_empty(&queue->sites[idx])) {
785 if (kIOTrackingQueueTypeMap & queue->type) {
786 queue_remove_first(&queue->sites[idx], user, IOTrackingUser *, link);
787 user->link.next = user->link.prev = NULL;
788 } else {
789 queue_remove_first(&queue->sites[idx], site, IOTrackingCallSite *, link);
790 addresses = false;
791 while (!queue_empty(&site->instances)) {
792 queue_remove_first(&site->instances, tracking, IOTracking *, link);
793 if (tracking == site->addresses) {
794 addresses = true;
795 }
796 if (addresses) {
797 trackingAddress = (typeof(trackingAddress))tracking;
798 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) {
799 kfree_type(IOTrackingAddress, tracking);
800 }
801 }
802 }
803 if (kIOTrackingQueueTypeUser & queue->type) {
804 kfree_type(IOTrackingCallSiteWithUser, site);
805 } else {
806 kfree_type(IOTrackingCallSite, site);
807 }
808 }
809 }
810 }
811 queue->siteCount = 0;
812 IOTRecursiveLockUnlock(&queue->lock);
813 }
814
815 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
816
817 static int
IOTrackingCallSiteInfoCompare(const void * left,const void * right)818 IOTrackingCallSiteInfoCompare(const void * left, const void * right)
819 {
820 IOTrackingCallSiteInfo * l = (typeof(l))left;
821 IOTrackingCallSiteInfo * r = (typeof(r))right;
822 size_t lsize, rsize;
823
824 rsize = r->size[0] + r->size[1];
825 lsize = l->size[0] + l->size[1];
826
827 return (rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1);
828 }
829
830 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
831
832 static int
IOTrackingAddressCompare(const void * left,const void * right)833 IOTrackingAddressCompare(const void * left, const void * right)
834 {
835 IOTracking * instance;
836 uintptr_t inst, laddr, raddr;
837
838 inst = ((typeof(inst) *)left)[0];
839 instance = (typeof(instance))INSTANCE_GET(inst);
840 if (kInstanceFlagAddress & inst) {
841 laddr = ~((IOTrackingAddress *)instance)->address;
842 } else {
843 laddr = (uintptr_t) (instance + 1);
844 }
845
846 inst = ((typeof(inst) *)right)[0];
847 instance = (typeof(instance))(inst & ~kInstanceFlags);
848 if (kInstanceFlagAddress & inst) {
849 raddr = ~((IOTrackingAddress *)instance)->address;
850 } else {
851 raddr = (uintptr_t) (instance + 1);
852 }
853
854 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1);
855 }
856
857
858 static int
IOTrackingZoneElementCompare(const void * left,const void * right)859 IOTrackingZoneElementCompare(const void * left, const void * right)
860 {
861 uintptr_t inst, laddr, raddr;
862
863 inst = ((typeof(inst) *)left)[0];
864 laddr = INSTANCE_PUT(inst);
865 inst = ((typeof(inst) *)right)[0];
866 raddr = INSTANCE_PUT(inst);
867
868 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1);
869 }
870
871 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
872
873 static void
CopyOutBacktraces(IOTrackingCallSite * site,IOTrackingCallSiteInfo * siteInfo)874 CopyOutBacktraces(IOTrackingCallSite * site, IOTrackingCallSiteInfo * siteInfo)
875 {
876 uint32_t j;
877 mach_vm_address_t bt, btEntry;
878
879 btEntry = site->queue->btEntry;
880 for (j = 0; j < kIOTrackingCallSiteBTs; j++) {
881 bt = site->bt[j];
882 if (btEntry
883 && (!bt || (j == (kIOTrackingCallSiteBTs - 1)))) {
884 bt = btEntry;
885 btEntry = 0;
886 }
887 siteInfo->bt[0][j] = VM_KERNEL_UNSLIDE(bt);
888 }
889
890 siteInfo->btPID = 0;
891 if (kIOTrackingQueueTypeUser & site->queue->type) {
892 siteInfo->btPID = site->user[0].pid;
893 uint32_t * bt32 = (typeof(bt32))((void *) &site->user[0].bt[0]);
894 uint64_t * bt64 = (typeof(bt64))((void *) &site->user[0].bt[0]);
895 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
896 if (j >= site->user[0].userCount) {
897 siteInfo->bt[1][j] = 0;
898 } else if (site->user[0].user32) {
899 siteInfo->bt[1][j] = bt32[j];
900 } else {
901 siteInfo->bt[1][j] = bt64[j];
902 }
903 }
904 }
905 }
906
907 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
908
909 static void
IOTrackingLeakScan(void * refcon)910 IOTrackingLeakScan(void * refcon)
911 {
912 IOTrackingLeaksRef * ref = (typeof(ref))refcon;
913 uintptr_t * instances;
914 IOTracking * instance;
915 uint64_t vaddr, vincr;
916 ppnum_t ppn;
917 uintptr_t ptr, addr, vphysaddr, inst;
918 size_t size, origsize;
919 uint32_t baseIdx, lim, ptrIdx, count;
920 boolean_t is;
921 AbsoluteTime deadline;
922
923 instances = ref->instances;
924 count = ref->count;
925 size = origsize = ref->zoneSize;
926
927 for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
928 ;
929 vaddr += vincr) {
930 if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) {
931 if (deadline) {
932 ml_set_interrupts_enabled(is);
933 IODelay(10);
934 }
935 if (vaddr >= VM_MAX_KERNEL_ADDRESS) {
936 break;
937 }
938 is = ml_set_interrupts_enabled(false);
939 clock_interval_to_deadline(10, kMillisecondScale, &deadline);
940 }
941
942 ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr);
943 // check noencrypt to avoid VM structs (map entries) with pointers
944 if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) {
945 ppn = 0;
946 }
947 if (!ppn) {
948 continue;
949 }
950
951 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) {
952 ptr = ((uintptr_t *)vphysaddr)[ptrIdx];
953 #if defined(HAS_APPLE_PAC)
954 // strip possible ptrauth signature from candidate data pointer
955 ptr = (uintptr_t)ptrauth_strip((void*)ptr, ptrauth_key_process_independent_data);
956 #endif /* defined(HAS_APPLE_PAC) */
957
958 for (lim = count, baseIdx = 0; lim; lim >>= 1) {
959 inst = instances[baseIdx + (lim >> 1)];
960 instance = (typeof(instance))INSTANCE_GET(inst);
961
962 if (ref->zoneSize) {
963 addr = INSTANCE_PUT(inst) & ~kInstanceFlags;
964 } else if (kInstanceFlagAddress & inst) {
965 addr = ~((IOTrackingAddress *)instance)->address;
966 origsize = size = ((IOTrackingAddress *)instance)->size;
967 if (!size) {
968 size = 1;
969 }
970 } else {
971 addr = (uintptr_t) (instance + 1);
972 origsize = size = instance->site->queue->allocSize;
973 }
974 if ((ptr >= addr) && (ptr < (addr + size))
975
976 && (((vaddr + ptrIdx * sizeof(uintptr_t)) < addr)
977 || ((vaddr + ptrIdx * sizeof(uintptr_t)) >= (addr + size)))) {
978 if (!(kInstanceFlagReferenced & inst)) {
979 inst |= kInstanceFlagReferenced;
980 instances[baseIdx + (lim >> 1)] = inst;
981 ref->found++;
982 if (!origsize) {
983 ref->foundzlen++;
984 }
985 }
986 break;
987 }
988 if (ptr > addr) {
989 // move right
990 baseIdx += (lim >> 1) + 1;
991 lim--;
992 }
993 // else move left
994 }
995 }
996 ref->bytes += page_size;
997 }
998 }
999
1000 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1001
1002 extern "C" void
zone_leaks_scan(uintptr_t * instances,uint32_t count,uint32_t zoneSize,uint32_t * found)1003 zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found)
1004 {
1005 IOTrackingLeaksRef ref;
1006 IOTrackingCallSiteInfo siteInfo;
1007 uint32_t idx;
1008
1009 qsort(instances, count, sizeof(*instances), &IOTrackingZoneElementCompare);
1010
1011 bzero(&siteInfo, sizeof(siteInfo));
1012 bzero(&ref, sizeof(ref));
1013 ref.instances = instances;
1014 ref.count = count;
1015 ref.zoneSize = zoneSize;
1016
1017 for (idx = 0; idx < 2; idx++) {
1018 ref.bytes = 0;
1019 IOTrackingLeakScan(&ref);
1020 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found);
1021 if (count <= ref.found) {
1022 break;
1023 }
1024 }
1025
1026 *found = ref.found;
1027 }
1028
1029 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1030
1031 static void
ZoneSiteProc(void * refCon,uint32_t siteCount,uint32_t zoneSize,uintptr_t * btrace,uint32_t btCount)1032 ZoneSiteProc(void * refCon, uint32_t siteCount, uint32_t zoneSize,
1033 uintptr_t * btrace, uint32_t btCount)
1034 {
1035 IOTrackingCallSiteInfo siteInfo;
1036 OSData * leakData;
1037 uint32_t idx;
1038
1039 leakData = (typeof(leakData))refCon;
1040
1041 bzero(&siteInfo, sizeof(siteInfo));
1042 siteInfo.count = siteCount;
1043 siteInfo.size[0] = zoneSize * siteCount;
1044
1045 for (idx = 0; (idx < btCount) && (idx < kIOTrackingCallSiteBTs); idx++) {
1046 siteInfo.bt[0][idx] = VM_KERNEL_UNSLIDE(btrace[idx]);
1047 }
1048
1049 leakData->appendBytes(&siteInfo, sizeof(siteInfo));
1050 }
1051
1052
1053 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1054
1055 static OSData *
IOTrackingLeaks(LIBKERN_CONSUMED OSData * data)1056 IOTrackingLeaks(LIBKERN_CONSUMED OSData * data)
1057 {
1058 IOTrackingLeaksRef ref;
1059 IOTrackingCallSiteInfo siteInfo;
1060 IOTrackingCallSite * site;
1061 OSData * leakData;
1062 uintptr_t * instances;
1063 IOTracking * instance;
1064 uintptr_t inst;
1065 uint32_t count, idx, numSites, dups, siteCount;
1066
1067 instances = (typeof(instances))data->getBytesNoCopy();
1068 count = (data->getLength() / sizeof(*instances));
1069 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare);
1070
1071 bzero(&siteInfo, sizeof(siteInfo));
1072 bzero(&ref, sizeof(ref));
1073 ref.instances = instances;
1074 ref.count = count;
1075 for (idx = 0; idx < 2; idx++) {
1076 ref.bytes = 0;
1077 IOTrackingLeakScan(&ref);
1078 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen);
1079 if (count <= ref.found) {
1080 break;
1081 }
1082 }
1083
1084 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
1085
1086 for (numSites = 0, idx = 0; idx < count; idx++) {
1087 inst = instances[idx];
1088 if (kInstanceFlagReferenced & inst) {
1089 continue;
1090 }
1091 instance = (typeof(instance))INSTANCE_GET(inst);
1092 site = instance->site;
1093 instances[numSites] = (uintptr_t) site;
1094 numSites++;
1095 }
1096
1097 for (idx = 0; idx < numSites; idx++) {
1098 inst = instances[idx];
1099 if (!inst) {
1100 continue;
1101 }
1102 site = (typeof(site))inst;
1103 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) {
1104 if (instances[dups] == (uintptr_t) site) {
1105 siteCount++;
1106 instances[dups] = 0;
1107 }
1108 }
1109 siteInfo.count = siteCount;
1110 siteInfo.size[0] = (site->size[0] * site->count) / siteCount;
1111 siteInfo.size[1] = (site->size[1] * site->count) / siteCount;
1112 CopyOutBacktraces(site, &siteInfo);
1113 leakData->appendBytes(&siteInfo, sizeof(siteInfo));
1114 }
1115 data->release();
1116
1117 return leakData;
1118 }
1119
1120 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1121
1122 static bool
SkipName(uint32_t options,const char * name,size_t namesLen,const char * names)1123 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names)
1124 {
1125 const char * scan;
1126 const char * next;
1127 bool exclude, found;
1128 size_t qLen, sLen;
1129
1130 if (!namesLen || !names) {
1131 return false;
1132 }
1133 // <len><name>...<len><name><0>
1134 exclude = (0 != (kIOTrackingExcludeNames & options));
1135 qLen = strlen(name);
1136 scan = names;
1137 found = false;
1138 do{
1139 sLen = scan[0];
1140 scan++;
1141 next = scan + sLen;
1142 if (next >= (names + namesLen)) {
1143 break;
1144 }
1145 found = ((sLen == qLen) && !strncmp(scan, name, sLen));
1146 scan = next;
1147 }while (!found && (scan < (names + namesLen)));
1148
1149 return !(exclude ^ found);
1150 }
1151
1152 #endif /* IOTRACKING */
1153
1154 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1155
1156 static kern_return_t
IOTrackingDebug(uint32_t selector,uint32_t options,uint64_t value,uint32_t intag,uint32_t inzsize,const char * names,size_t namesLen,size_t size,OSObject ** result)1157 IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value,
1158 uint32_t intag, uint32_t inzsize,
1159 const char * names, size_t namesLen,
1160 size_t size, OSObject ** result)
1161 {
1162 kern_return_t ret;
1163 OSData * data;
1164
1165 if (result) {
1166 *result = NULL;
1167 }
1168 data = NULL;
1169 ret = kIOReturnNotReady;
1170
1171 #if IOTRACKING
1172
1173 kern_return_t kr;
1174 IOTrackingQueue * queue;
1175 IOTracking * instance;
1176 IOTrackingCallSite * site;
1177 IOTrackingCallSiteInfo siteInfo;
1178 IOTrackingUser * user;
1179 task_t mapTask;
1180 mach_vm_address_t mapAddress;
1181 mach_vm_size_t mapSize;
1182 uint32_t num, idx, qIdx;
1183 uintptr_t instFlags;
1184 proc_t proc;
1185 bool addresses;
1186
1187 ret = kIOReturnNotFound;
1188 proc = NULL;
1189 if (kIOTrackingGetMappings == selector) {
1190 if (value != -1ULL) {
1191 proc = proc_find((pid_t) value);
1192 if (!proc) {
1193 return kIOReturnNotFound;
1194 }
1195 }
1196 }
1197
1198 bzero(&siteInfo, sizeof(siteInfo));
1199 lck_mtx_lock(gIOTrackingLock);
1200 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
1201 {
1202 if (SkipName(options, queue->name, namesLen, names)) {
1203 continue;
1204 }
1205
1206 if (!(kIOTracking & gIOKitDebug) && (kIOTrackingQueueTypeAlloc & queue->type)) {
1207 continue;
1208 }
1209
1210 switch (selector) {
1211 case kIOTrackingResetTracking:
1212 {
1213 IOTrackingReset(queue);
1214 ret = kIOReturnSuccess;
1215 break;
1216 }
1217
1218 case kIOTrackingStartCapture:
1219 case kIOTrackingStopCapture:
1220 {
1221 queue->captureOn = (kIOTrackingStartCapture == selector);
1222 ret = kIOReturnSuccess;
1223 break;
1224 }
1225
1226 case kIOTrackingSetMinCaptureSize:
1227 {
1228 queue->minCaptureSize = size;
1229 ret = kIOReturnSuccess;
1230 break;
1231 }
1232
1233 case kIOTrackingLeaks:
1234 {
1235 if (!(kIOTrackingQueueTypeAlloc & queue->type)) {
1236 break;
1237 }
1238
1239 if (!data) {
1240 data = OSData::withCapacity(1024 * sizeof(uintptr_t));
1241 }
1242
1243 IOTRecursiveLockLock(&queue->lock);
1244 for (idx = 0; idx < queue->numSiteQs; idx++) {
1245 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
1246 {
1247 addresses = false;
1248 queue_iterate(&site->instances, instance, IOTracking *, link)
1249 {
1250 if (instance == site->addresses) {
1251 addresses = true;
1252 }
1253 instFlags = (typeof(instFlags))instance;
1254 if (addresses) {
1255 instFlags |= kInstanceFlagAddress;
1256 }
1257 data->appendBytes(&instFlags, sizeof(instFlags));
1258 }
1259 }
1260 }
1261 // queue is locked
1262 ret = kIOReturnSuccess;
1263 break;
1264 }
1265
1266
1267 case kIOTrackingGetTracking:
1268 {
1269 if (kIOTrackingQueueTypeMap & queue->type) {
1270 break;
1271 }
1272
1273 if (!data) {
1274 data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
1275 }
1276
1277 IOTRecursiveLockLock(&queue->lock);
1278 num = queue->siteCount;
1279 idx = 0;
1280 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) {
1281 queue_iterate(&queue->sites[qIdx], site, IOTrackingCallSite *, link)
1282 {
1283 assert(idx < num);
1284 idx++;
1285
1286 size_t tsize[2];
1287 uint32_t count = site->count;
1288 tsize[0] = site->size[0];
1289 tsize[1] = site->size[1];
1290
1291 if (intag || inzsize) {
1292 uintptr_t addr;
1293 vm_size_t size, zoneSize;
1294 vm_tag_t tag;
1295
1296 if (kIOTrackingQueueTypeAlloc & queue->type) {
1297 addresses = false;
1298 count = 0;
1299 tsize[0] = tsize[1] = 0;
1300 queue_iterate(&site->instances, instance, IOTracking *, link)
1301 {
1302 if (instance == site->addresses) {
1303 addresses = true;
1304 }
1305
1306 if (addresses) {
1307 addr = ~((IOTrackingAddress *)instance)->address;
1308 } else {
1309 addr = (uintptr_t) (instance + 1);
1310 }
1311
1312 kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize);
1313 if (KERN_SUCCESS != kr) {
1314 continue;
1315 }
1316
1317 if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) {
1318 continue;
1319 }
1320 if (inzsize && (inzsize != zoneSize)) {
1321 continue;
1322 }
1323
1324 count++;
1325 tsize[0] += size;
1326 }
1327 } else {
1328 if (!intag || inzsize || (intag != site->tag)) {
1329 continue;
1330 }
1331 }
1332 }
1333
1334 if (!count) {
1335 continue;
1336 }
1337 if (size && ((tsize[0] + tsize[1]) < size)) {
1338 continue;
1339 }
1340 siteInfo.count = count;
1341 siteInfo.size[0] = tsize[0];
1342 siteInfo.size[1] = tsize[1];
1343 CopyOutBacktraces(site, &siteInfo);
1344 data->appendBytes(&siteInfo, sizeof(siteInfo));
1345 }
1346 }
1347 assert(idx == num);
1348 IOTRecursiveLockUnlock(&queue->lock);
1349 ret = kIOReturnSuccess;
1350 break;
1351 }
1352
1353 case kIOTrackingGetMappings:
1354 {
1355 if (!(kIOTrackingQueueTypeMap & queue->type)) {
1356 break;
1357 }
1358 if (!data) {
1359 data = OSData::withCapacity((unsigned int) page_size);
1360 }
1361
1362 IOTRecursiveLockLock(&queue->lock);
1363 num = queue->siteCount;
1364 idx = 0;
1365 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) {
1366 queue_iterate(&queue->sites[qIdx], user, IOTrackingUser *, link)
1367 {
1368 assert(idx < num);
1369 idx++;
1370
1371 kr = IOMemoryMapTracking(user, &mapTask, &mapAddress, &mapSize);
1372 if (kIOReturnSuccess != kr) {
1373 continue;
1374 }
1375 if (proc && (mapTask != proc_task(proc))) {
1376 continue;
1377 }
1378 if (size && (mapSize < size)) {
1379 continue;
1380 }
1381
1382 siteInfo.count = 1;
1383 siteInfo.size[0] = mapSize;
1384 siteInfo.address = mapAddress;
1385 siteInfo.addressPID = task_pid(mapTask);
1386 siteInfo.btPID = user->btPID;
1387
1388 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
1389 siteInfo.bt[0][j] = VM_KERNEL_UNSLIDE(user->bt[j]);
1390 }
1391 uint32_t * bt32 = (typeof(bt32)) & user->btUser[0];
1392 uint64_t * bt64 = (typeof(bt64))((void *) &user->btUser[0]);
1393 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
1394 if (j >= user->userCount) {
1395 siteInfo.bt[1][j] = 0;
1396 } else if (user->user32) {
1397 siteInfo.bt[1][j] = bt32[j];
1398 } else {
1399 siteInfo.bt[1][j] = bt64[j];
1400 }
1401 }
1402 data->appendBytes(&siteInfo, sizeof(siteInfo));
1403 }
1404 }
1405 assert(idx == num);
1406 IOTRecursiveLockUnlock(&queue->lock);
1407 ret = kIOReturnSuccess;
1408 break;
1409 }
1410
1411 default:
1412 ret = kIOReturnUnsupported;
1413 break;
1414 }
1415 }
1416
1417 if ((kIOTrackingLeaks == selector) && data) {
1418 data = IOTrackingLeaks(data);
1419 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
1420 {
1421 if (SkipName(options, queue->name, namesLen, names)) {
1422 continue;
1423 }
1424 if (!(kIOTrackingQueueTypeAlloc & queue->type)) {
1425 continue;
1426 }
1427 IOTRecursiveLockUnlock(&queue->lock);
1428 }
1429 }
1430
1431 lck_mtx_unlock(gIOTrackingLock);
1432
1433 if ((kIOTrackingLeaks == selector) && namesLen && names) {
1434 const char * scan;
1435 const char * next;
1436 uint8_t sLen;
1437
1438 if (!data) {
1439 data = OSData::withCapacity(4096 * sizeof(uintptr_t));
1440 }
1441
1442 // <len><name>...<len><name><0>
1443 scan = names;
1444 do{
1445 sLen = ((uint8_t) scan[0]);
1446 scan++;
1447 next = scan + sLen;
1448 if (next >= (names + namesLen)) {
1449 break;
1450 }
1451 kr = zone_leaks(scan, sLen, &ZoneSiteProc, data);
1452 if (KERN_SUCCESS == kr) {
1453 ret = kIOReturnSuccess;
1454 } else if (KERN_INVALID_NAME != kr) {
1455 ret = kIOReturnVMError;
1456 }
1457 scan = next;
1458 }while (scan < (names + namesLen));
1459 }
1460
1461 if (data) {
1462 switch (selector) {
1463 case kIOTrackingLeaks:
1464 case kIOTrackingGetTracking:
1465 case kIOTrackingGetMappings:
1466 {
1467 IOTrackingCallSiteInfo * siteInfos;
1468 siteInfos = (typeof(siteInfos))data->getBytesNoCopy();
1469 num = (data->getLength() / sizeof(*siteInfos));
1470 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare);
1471 break;
1472 }
1473 default: assert(false); break;
1474 }
1475 }
1476
1477 *result = data;
1478 if (proc) {
1479 proc_rele(proc);
1480 }
1481
1482 #endif /* IOTRACKING */
1483
1484 return ret;
1485 }
1486
1487 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1488
1489 #include <IOKit/IOKitDiagnosticsUserClient.h>
1490
1491 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1492
1493 #undef super
1494 #define super IOUserClient
1495
OSDefineMetaClassAndStructors(IOKitDiagnosticsClient,IOUserClient)1496 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient)
1497
1498 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1499
1500 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask)
1501 {
1502 IOKitDiagnosticsClient * inst;
1503
1504 inst = new IOKitDiagnosticsClient;
1505 if (inst && !inst->init()) {
1506 inst->release();
1507 inst = NULL;
1508 }
1509
1510 return inst;
1511 }
1512
1513 IOReturn
clientClose(void)1514 IOKitDiagnosticsClient::clientClose(void)
1515 {
1516 terminate();
1517 return kIOReturnSuccess;
1518 }
1519
1520 IOReturn
setProperties(OSObject * properties)1521 IOKitDiagnosticsClient::setProperties(OSObject * properties)
1522 {
1523 IOReturn kr = kIOReturnUnsupported;
1524 return kr;
1525 }
1526
1527 IOReturn
externalMethod(uint32_t selector,IOExternalMethodArguments * args,IOExternalMethodDispatch * dispatch,OSObject * target,void * reference)1528 IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
1529 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
1530 {
1531 IOReturn ret = kIOReturnBadArgument;
1532 const IOKitDiagnosticsParameters * params;
1533 const char * names;
1534 size_t namesLen;
1535 OSObject * result;
1536
1537 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) {
1538 return kIOReturnBadArgument;
1539 }
1540 params = (typeof(params))args->structureInput;
1541 if (!params) {
1542 return kIOReturnBadArgument;
1543 }
1544
1545 names = NULL;
1546 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters);
1547 if (namesLen) {
1548 names = (typeof(names))(params + 1);
1549 }
1550
1551 ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
1552
1553 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) {
1554 *args->structureVariableOutputData = result;
1555 } else if (result) {
1556 result->release();
1557 }
1558
1559 return ret;
1560 }
1561
1562 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1563