1 /*
2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30 #include <sys/sysctl.h>
31 extern "C" {
32 #include <vm/vm_kern.h>
33 #include <kern/task.h>
34 #include <kern/debug.h>
35 }
36
37 #include <libkern/c++/OSContainers.h>
38 #include <libkern/OSDebug.h>
39 #include <libkern/c++/OSCPPDebug.h>
40 #include <kern/backtrace.h>
41 #include <kern/btlog.h>
42
43 #include <IOKit/IOKitDebug.h>
44 #include <IOKit/IOLib.h>
45 #include <IOKit/assert.h>
46 #include <IOKit/IODeviceTreeSupport.h>
47 #include <IOKit/IOService.h>
48
49 #include "IOKitKernelInternal.h"
50
51 TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE);
52 TUNABLE(SInt64, gIOKitTrace, "iotrace", 0);
53
54 #if DEVELOPMENT || DEBUG
55 #define IODEBUG_CTLFLAGS CTLFLAG_RW
56 #else
57 #define IODEBUG_CTLFLAGS CTLFLAG_RD
58 #endif
59
60 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
61
62 static int
sysctl_debug_iokit(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)63 sysctl_debug_iokit
64 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
65 {
66 SInt64 newValue;
67 int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed);
68 if (changed) {
69 gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions));
70 }
71 return error;
72 }
73
74 SYSCTL_PROC(_debug, OID_AUTO, iokit,
75 CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED,
76 &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io");
77
78 size_t debug_malloc_size;
79 size_t debug_iomalloc_size;
80
81 vm_size_t debug_iomallocpageable_size;
82 size_t debug_container_malloc_size;
83 // int debug_ivars_size; // in OSObject.cpp
84
85 extern "C" {
86 #if 0
87 #define DEBG(fmt, args...) { kprintf(fmt, ## args); }
88 #else
89 #define DEBG(fmt, args...) { IOLog(fmt, ## args); }
90 #endif
91
92 void
IOPrintPlane(const IORegistryPlane * plane)93 IOPrintPlane( const IORegistryPlane * plane )
94 {
95 IORegistryEntry * next;
96 IORegistryIterator * iter;
97 OSOrderedSet * all;
98 IOService * service;
99
100 iter = IORegistryIterator::iterateOver( plane );
101 assert( iter );
102 all = iter->iterateAll();
103 if (all) {
104 DEBG("Count %d\n", all->getCount());
105 all->release();
106 } else {
107 DEBG("Empty\n");
108 }
109
110 iter->reset();
111 while ((next = iter->getNextObjectRecursive())) {
112 DEBG( "%*s\033[33m%s", 2 * next->getDepth( plane ), "", next->getName( plane ));
113 if ((next->getLocation( plane ))) {
114 DEBG("@%s", next->getLocation( plane ));
115 }
116 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName());
117 if ((service = OSDynamicCast(IOService, next))) {
118 DEBG(", busy %ld", (long) service->getBusyState());
119 }
120 DEBG( ">\n");
121 // IOSleep(250);
122 }
123 iter->release();
124
125 #undef IOPrintPlaneFormat
126 }
127
128 void
db_piokjunk(void)129 db_piokjunk(void)
130 {
131 }
132
133 void
db_dumpiojunk(const IORegistryPlane * plane __unused)134 db_dumpiojunk( const IORegistryPlane * plane __unused )
135 {
136 }
137
138 void
IOPrintMemory(void)139 IOPrintMemory( void )
140 {
141 // OSMetaClass::printInstanceCounts();
142
143 IOLog("\n"
144 "ivar kalloc() 0x%08lx\n"
145 "malloc() 0x%08lx\n"
146 "containers kalloc() 0x%08lx\n"
147 "IOMalloc() 0x%08lx\n"
148 "----------------------------------------\n",
149 debug_ivars_size,
150 debug_malloc_size,
151 debug_container_malloc_size,
152 debug_iomalloc_size
153 );
154 }
155 } /* extern "C" */
156
157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
158
159 #define super OSObject
OSDefineMetaClassAndStructors(IOKitDiagnostics,OSObject)160 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject)
161
162 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
163
164 OSObject * IOKitDiagnostics::diagnostics( void )
165 {
166 IOKitDiagnostics * diags;
167
168 diags = new IOKitDiagnostics;
169 if (diags && !diags->init()) {
170 diags->release();
171 diags = NULL;
172 }
173
174 return diags;
175 }
176
177 void
updateOffset(OSDictionary * dict,UInt64 value,const char * name)178 IOKitDiagnostics::updateOffset( OSDictionary * dict,
179 UInt64 value, const char * name )
180 {
181 OSNumber * off;
182
183 off = OSNumber::withNumber( value, 64 );
184 if (!off) {
185 return;
186 }
187
188 dict->setObject( name, off );
189 off->release();
190 }
191
192 bool
serialize(OSSerialize * s) const193 IOKitDiagnostics::serialize(OSSerialize *s) const
194 {
195 OSDictionary * dict;
196 bool ok;
197
198 dict = OSDictionary::withCapacity( 5 );
199 if (!dict) {
200 return false;
201 }
202
203 updateOffset( dict, debug_ivars_size, "Instance allocation" );
204 updateOffset( dict, debug_container_malloc_size, "Container allocation" );
205 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" );
206 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" );
207
208 OSMetaClass::serializeClassDictionary(dict);
209
210 ok = dict->serialize( s );
211
212 dict->release();
213
214 return ok;
215 }
216
217 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
218
219 #if IOTRACKING
220
221 #include <libkern/c++/OSCPPDebug.h>
222 #include <libkern/c++/OSKext.h>
223 #include <kern/zalloc.h>
224
225 __private_extern__ "C" void qsort(
226 void * array,
227 size_t nmembers,
228 size_t member_size,
229 int (*)(const void *, const void *));
230
231 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
232 extern "C" ppnum_t pmap_valid_page(ppnum_t pn);
233
234 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
235
236 struct IOTRecursiveLock {
237 lck_mtx_t * mutex;
238 thread_t thread;
239 UInt32 count;
240 };
241
242 struct IOTrackingQueue {
243 queue_chain_t link;
244 IOTRecursiveLock lock;
245 const char * name;
246 uintptr_t btEntry;
247 size_t allocSize;
248 size_t minCaptureSize;
249 uint32_t siteCount;
250 uint32_t type;
251 uint32_t numSiteQs;
252 uint8_t captureOn;
253 queue_head_t sites[];
254 };
255
256
257 struct IOTrackingCallSiteUser {
258 pid_t pid;
259 uint8_t user32;
260 uint8_t userCount;
261 uintptr_t bt[kIOTrackingCallSiteBTs];
262 };
263
264 struct IOTrackingCallSite {
265 queue_chain_t link;
266 queue_head_t instances;
267 IOTrackingQueue * queue;
268 IOTracking * addresses;
269 size_t size[2];
270 uint32_t crc;
271 uint32_t count;
272
273 vm_tag_t tag;
274 uint8_t user32;
275 uint8_t userCount;
276 pid_t btPID;
277
278 uintptr_t bt[kIOTrackingCallSiteBTs];
279 IOTrackingCallSiteUser user[0];
280 };
281
282 struct IOTrackingCallSiteWithUser {
283 struct IOTrackingCallSite site;
284 struct IOTrackingCallSiteUser user;
285 };
286
287 static void IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site);
288
289 struct IOTrackingLeaksRef {
290 uintptr_t * instances;
291 uint32_t zoneSize;
292 uint32_t count;
293 uint32_t found;
294 uint32_t foundzlen;
295 size_t bytes;
296 };
297
298 lck_mtx_t * gIOTrackingLock;
299 queue_head_t gIOTrackingQ;
300
301 enum{
302 kTrackingAddressFlagAllocated = 0x00000001
303 };
304
305 #if defined(__LP64__)
306 #define IOTrackingAddressFlags(ptr) (ptr->flags)
307 #else
308 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags)
309 #endif
310
311 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
312
313 static void
IOTRecursiveLockLock(IOTRecursiveLock * lock)314 IOTRecursiveLockLock(IOTRecursiveLock * lock)
315 {
316 if (lock->thread == current_thread()) {
317 lock->count++;
318 } else {
319 lck_mtx_lock(lock->mutex);
320 assert(lock->thread == NULL);
321 assert(lock->count == 0);
322 lock->thread = current_thread();
323 lock->count = 1;
324 }
325 }
326
327 static void
IOTRecursiveLockUnlock(IOTRecursiveLock * lock)328 IOTRecursiveLockUnlock(IOTRecursiveLock * lock)
329 {
330 assert(lock->thread == current_thread());
331 if (0 == (--lock->count)) {
332 lock->thread = NULL;
333 lck_mtx_unlock(lock->mutex);
334 }
335 }
336
337 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
338
339 void
IOTrackingInit(void)340 IOTrackingInit(void)
341 {
342 queue_init(&gIOTrackingQ);
343 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
344 }
345
346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348 IOTrackingQueue *
IOTrackingQueueAlloc(const char * name,uintptr_t btEntry,size_t allocSize,size_t minCaptureSize,uint32_t type,uint32_t numSiteQs)349 IOTrackingQueueAlloc(const char * name, uintptr_t btEntry,
350 size_t allocSize, size_t minCaptureSize,
351 uint32_t type, uint32_t numSiteQs)
352 {
353 IOTrackingQueue * queue;
354 uint32_t idx;
355
356 if (!numSiteQs) {
357 numSiteQs = 1;
358 }
359 queue = kalloc_type(IOTrackingQueue, queue_head_t, numSiteQs, Z_WAITOK_ZERO);
360 queue->name = name;
361 queue->btEntry = btEntry;
362 queue->allocSize = allocSize;
363 queue->minCaptureSize = minCaptureSize;
364 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
365 queue->numSiteQs = numSiteQs;
366 queue->type = type;
367 enum { kFlags = (kIOTracking | kIOTrackingBoot) };
368 queue->captureOn = (kFlags == (kFlags & gIOKitDebug))
369 || (kIOTrackingQueueTypeDefaultOn & type);
370
371 for (idx = 0; idx < numSiteQs; idx++) {
372 queue_init(&queue->sites[idx]);
373 }
374
375 lck_mtx_lock(gIOTrackingLock);
376 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link);
377 lck_mtx_unlock(gIOTrackingLock);
378
379 return queue;
380 };
381
382 void
IOTrackingQueueCollectUser(IOTrackingQueue * queue)383 IOTrackingQueueCollectUser(IOTrackingQueue * queue)
384 {
385 assert(0 == queue->siteCount);
386 queue->type |= kIOTrackingQueueTypeUser;
387 }
388
389 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
390
391 void
IOTrackingQueueFree(IOTrackingQueue * queue)392 IOTrackingQueueFree(IOTrackingQueue * queue)
393 {
394 lck_mtx_lock(gIOTrackingLock);
395 IOTrackingReset(queue);
396 remque(&queue->link);
397 lck_mtx_unlock(gIOTrackingLock);
398
399 lck_mtx_free(queue->lock.mutex, IOLockGroup);
400
401 kfree_type(IOTrackingQueue, queue_head_t, queue->numSiteQs, queue);
402 };
403
404 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
405
406 /* fasthash
407 * The MIT License
408 *
409 * Copyright (C) 2012 Zilong Tan ([email protected])
410 *
411 * Permission is hereby granted, free of charge, to any person
412 * obtaining a copy of this software and associated documentation
413 * files (the "Software"), to deal in the Software without
414 * restriction, including without limitation the rights to use, copy,
415 * modify, merge, publish, distribute, sublicense, and/or sell copies
416 * of the Software, and to permit persons to whom the Software is
417 * furnished to do so, subject to the following conditions:
418 *
419 * The above copyright notice and this permission notice shall be
420 * included in all copies or substantial portions of the Software.
421 *
422 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
423 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
424 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
425 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
426 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
427 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
428 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
429 * SOFTWARE.
430 */
431
432
433 // Compression function for Merkle-Damgard construction.
434 // This function is generated using the framework provided.
435 #define mix(h) ({ \
436 (h) ^= (h) >> 23; \
437 (h) *= 0x2127599bf4325c37ULL; \
438 (h) ^= (h) >> 47; })
439
440 static uint64_t
fasthash64(const void * buf,size_t len,uint64_t seed)441 fasthash64(const void *buf, size_t len, uint64_t seed)
442 {
443 const uint64_t m = 0x880355f21e6d1965ULL;
444 const uint64_t *pos = (const uint64_t *)buf;
445 const uint64_t *end = pos + (len / 8);
446 const unsigned char *pos2;
447 uint64_t h = seed ^ (len * m);
448 uint64_t v;
449
450 while (pos != end) {
451 v = *pos++;
452 h ^= mix(v);
453 h *= m;
454 }
455
456 pos2 = (const unsigned char*)pos;
457 v = 0;
458
459 switch (len & 7) {
460 case 7: v ^= (uint64_t)pos2[6] << 48;
461 [[clang::fallthrough]];
462 case 6: v ^= (uint64_t)pos2[5] << 40;
463 [[clang::fallthrough]];
464 case 5: v ^= (uint64_t)pos2[4] << 32;
465 [[clang::fallthrough]];
466 case 4: v ^= (uint64_t)pos2[3] << 24;
467 [[clang::fallthrough]];
468 case 3: v ^= (uint64_t)pos2[2] << 16;
469 [[clang::fallthrough]];
470 case 2: v ^= (uint64_t)pos2[1] << 8;
471 [[clang::fallthrough]];
472 case 1: v ^= (uint64_t)pos2[0];
473 h ^= mix(v);
474 h *= m;
475 }
476
477 return mix(h);
478 }
479
480 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
481
482 static uint32_t
fasthash32(const void * buf,size_t len,uint32_t seed)483 fasthash32(const void *buf, size_t len, uint32_t seed)
484 {
485 // the following trick converts the 64-bit hashcode to Fermat
486 // residue, which shall retain information from both the higher
487 // and lower parts of hashcode.
488 uint64_t h = fasthash64(buf, len, seed);
489 return (uint32_t) (h - (h >> 32));
490 }
491
492 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
493
494 void
IOTrackingAddUser(IOTrackingQueue * queue,IOTrackingUser * mem,vm_size_t size)495 IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size)
496 {
497 uint32_t num;
498 int pid;
499
500 if (!queue->captureOn) {
501 return;
502 }
503 if (size < queue->minCaptureSize) {
504 return;
505 }
506
507 assert(!mem->link.next);
508
509 num = backtrace(&mem->bt[0], kIOTrackingCallSiteBTs, NULL, NULL);
510 num = 0;
511 if ((kernel_task != current_task()) && (pid = proc_selfpid())) {
512 struct backtrace_user_info btinfo = BTUINFO_INIT;
513 mem->btPID = pid;
514 num = backtrace_user(&mem->btUser[0], kIOTrackingCallSiteBTs - 1,
515 NULL, &btinfo);
516 mem->user32 = !(btinfo.btui_info & BTI_64_BIT);
517 }
518 assert(num <= kIOTrackingCallSiteBTs);
519 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX);
520 mem->userCount = ((uint8_t) num);
521
522 IOTRecursiveLockLock(&queue->lock);
523 queue_enter/*last*/ (&queue->sites[0], mem, IOTrackingUser *, link);
524 queue->siteCount++;
525 IOTRecursiveLockUnlock(&queue->lock);
526 }
527
528 void
IOTrackingRemoveUser(IOTrackingQueue * queue,IOTrackingUser * mem)529 IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem)
530 {
531 if (!mem->link.next) {
532 return;
533 }
534
535 IOTRecursiveLockLock(&queue->lock);
536 if (mem->link.next) {
537 remque(&mem->link);
538 assert(queue->siteCount);
539 queue->siteCount--;
540 }
541 IOTRecursiveLockUnlock(&queue->lock);
542 }
543
544 uint64_t gIOTrackingAddTime;
545
546 void
IOTrackingAdd(IOTrackingQueue * queue,IOTracking * mem,size_t size,bool address,vm_tag_t tag)547 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag)
548 {
549 IOTrackingCallSite * site;
550 uint32_t crc, num;
551 uintptr_t bt[kIOTrackingCallSiteBTs + 1];
552 uintptr_t btUser[kIOTrackingCallSiteBTs];
553 queue_head_t * que;
554 bool user;
555 int pid;
556 int userCount;
557
558 if (mem->site) {
559 return;
560 }
561 if (!queue->captureOn) {
562 return;
563 }
564 if (size < queue->minCaptureSize) {
565 return;
566 }
567
568 user = (0 != (kIOTrackingQueueTypeUser & queue->type));
569
570 assert(!mem->link.next);
571
572 num = backtrace(&bt[0], kIOTrackingCallSiteBTs + 1, NULL, NULL);
573 if (!num) {
574 return;
575 }
576 num--;
577 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7);
578
579 userCount = 0;
580 pid = 0;
581 backtrace_info_t btinfo = BTI_NONE;
582 if (user) {
583 if ((kernel_task != current_task()) && (pid = proc_selfpid())) {
584 struct backtrace_user_info btuinfo = BTUINFO_INIT;
585 userCount = backtrace_user(&btUser[0], kIOTrackingCallSiteBTs,
586 NULL, &btuinfo);
587 assert(userCount <= kIOTrackingCallSiteBTs);
588 btinfo = btuinfo.btui_info;
589 crc = fasthash32(&btUser[0], userCount * sizeof(bt[0]), crc);
590 }
591 }
592
593 IOTRecursiveLockLock(&queue->lock);
594 que = &queue->sites[crc % queue->numSiteQs];
595 queue_iterate(que, site, IOTrackingCallSite *, link)
596 {
597 if (tag != site->tag) {
598 continue;
599 }
600 if (user && (pid != site->user[0].pid)) {
601 continue;
602 }
603 if (crc == site->crc) {
604 break;
605 }
606 }
607
608 if (queue_end(que, (queue_entry_t) site)) {
609 if (user) {
610 site = &kalloc_type(IOTrackingCallSiteWithUser,
611 Z_WAITOK_ZERO_NOFAIL)->site;
612 } else {
613 site = kalloc_type(IOTrackingCallSite,
614 Z_WAITOK_ZERO_NOFAIL);
615 }
616
617 queue_init(&site->instances);
618 site->addresses = (IOTracking *) &site->instances;
619 site->queue = queue;
620 site->crc = crc;
621 site->count = 0;
622 site->tag = tag;
623 memset(&site->size[0], 0, sizeof(site->size));
624 bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0]));
625 assert(num <= kIOTrackingCallSiteBTs);
626 bzero(&site->bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->bt[0]));
627 if (user) {
628 bcopy(&btUser[0], &site->user[0].bt[0], userCount * sizeof(site->user[0].bt[0]));
629 assert(userCount <= kIOTrackingCallSiteBTs);
630 bzero(&site->user[0].bt[userCount], (kIOTrackingCallSiteBTs - userCount) * sizeof(site->user[0].bt[0]));
631 site->user[0].pid = pid;
632 site->user[0].user32 = !(btinfo & BTI_64_BIT);
633 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX);
634 site->user[0].userCount = ((uint8_t) userCount);
635 }
636 queue_enter_first(que, site, IOTrackingCallSite *, link);
637 queue->siteCount++;
638 }
639
640 if (address) {
641 queue_enter/*last*/ (&site->instances, mem, IOTracking *, link);
642 if (queue_end(&site->instances, (queue_entry_t)site->addresses)) {
643 site->addresses = mem;
644 }
645 } else {
646 queue_enter_first(&site->instances, mem, IOTracking *, link);
647 }
648
649 mem->site = site;
650 site->size[0] += size;
651 site->count++;
652
653 IOTRecursiveLockUnlock(&queue->lock);
654 }
655
656 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
657
658 void
IOTrackingRemove(IOTrackingQueue * queue,IOTracking * mem,size_t size)659 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
660 {
661 if (!mem->link.next) {
662 return;
663 }
664
665 IOTRecursiveLockLock(&queue->lock);
666 if (mem->link.next) {
667 assert(mem->site);
668
669 if (mem == mem->site->addresses) {
670 mem->site->addresses = (IOTracking *) queue_next(&mem->link);
671 }
672 remque(&mem->link);
673
674 assert(mem->site->count);
675 mem->site->count--;
676 assert(mem->site->size[0] >= size);
677 mem->site->size[0] -= size;
678 if (!mem->site->count) {
679 assert(queue_empty(&mem->site->instances));
680 assert(!mem->site->size[0]);
681 assert(!mem->site->size[1]);
682
683 remque(&mem->site->link);
684 assert(queue->siteCount);
685 queue->siteCount--;
686 IOTrackingFreeCallSite(queue->type, &mem->site);
687 }
688 mem->site = NULL;
689 }
690 IOTRecursiveLockUnlock(&queue->lock);
691 }
692
693 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
694
695 void
IOTrackingAlloc(IOTrackingQueue * queue,uintptr_t address,size_t size)696 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size)
697 {
698 IOTrackingAddress * tracking;
699
700 if (!queue->captureOn) {
701 return;
702 }
703 if (size < queue->minCaptureSize) {
704 return;
705 }
706
707 address = ~address;
708 tracking = kalloc_type(IOTrackingAddress, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
709 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated;
710 tracking->address = address;
711 tracking->size = size;
712
713 IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE);
714 }
715
716 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
717
718 void
IOTrackingFree(IOTrackingQueue * queue,uintptr_t address,size_t size)719 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size)
720 {
721 IOTrackingCallSite * site;
722 IOTrackingAddress * tracking;
723 uint32_t idx;
724 bool done;
725
726 address = ~address;
727 IOTRecursiveLockLock(&queue->lock);
728 done = false;
729 for (idx = 0; idx < queue->numSiteQs; idx++) {
730 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
731 {
732 tracking = (IOTrackingAddress *) site->addresses;
733 while (!queue_end(&site->instances, &tracking->tracking.link)) {
734 if ((done = (address == tracking->address))) {
735 IOTrackingRemove(queue, &tracking->tracking, size);
736 kfree_type(IOTrackingAddress, tracking);
737 break;
738 } else {
739 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link);
740 }
741 }
742 if (done) {
743 break;
744 }
745 }
746 if (done) {
747 break;
748 }
749 }
750 IOTRecursiveLockUnlock(&queue->lock);
751 }
752
753 static void
IOTrackingFreeCallSite(uint32_t type,IOTrackingCallSite ** site)754 IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site)
755 {
756 void ** ptr;
757
758 ptr = reinterpret_cast<void **>(site);
759 if (kIOTrackingQueueTypeUser & type) {
760 kfree_type(IOTrackingCallSiteWithUser, *ptr);
761 } else {
762 kfree_type(IOTrackingCallSite, *ptr);
763 }
764 }
765
766 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
767
768 void
IOTrackingAccumSize(IOTrackingQueue * queue,IOTracking * mem,size_t size)769 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size)
770 {
771 IOTRecursiveLockLock(&queue->lock);
772 if (mem->link.next) {
773 assert(mem->site);
774 assert((size > 0) || (mem->site->size[1] >= -size));
775 mem->site->size[1] += size;
776 }
777 ;
778 IOTRecursiveLockUnlock(&queue->lock);
779 }
780
781 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
782
783 void
IOTrackingReset(IOTrackingQueue * queue)784 IOTrackingReset(IOTrackingQueue * queue)
785 {
786 IOTrackingCallSite * site;
787 IOTrackingUser * user;
788 IOTracking * tracking;
789 IOTrackingAddress * trackingAddress;
790 uint32_t idx;
791 bool addresses;
792
793 IOTRecursiveLockLock(&queue->lock);
794 for (idx = 0; idx < queue->numSiteQs; idx++) {
795 while (!queue_empty(&queue->sites[idx])) {
796 if (kIOTrackingQueueTypeMap & queue->type) {
797 queue_remove_first(&queue->sites[idx], user, IOTrackingUser *, link);
798 user->link.next = user->link.prev = NULL;
799 } else {
800 queue_remove_first(&queue->sites[idx], site, IOTrackingCallSite *, link);
801 addresses = false;
802 while (!queue_empty(&site->instances)) {
803 queue_remove_first(&site->instances, tracking, IOTracking *, link);
804 if (tracking == site->addresses) {
805 addresses = true;
806 }
807 if (addresses) {
808 trackingAddress = (typeof(trackingAddress))tracking;
809 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) {
810 kfree_type(IOTrackingAddress, trackingAddress);
811 }
812 }
813 }
814 IOTrackingFreeCallSite(queue->type, &site);
815 }
816 }
817 }
818 queue->siteCount = 0;
819 IOTRecursiveLockUnlock(&queue->lock);
820 }
821
822 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
823
824 static int
IOTrackingCallSiteInfoCompare(const void * left,const void * right)825 IOTrackingCallSiteInfoCompare(const void * left, const void * right)
826 {
827 IOTrackingCallSiteInfo * l = (typeof(l))left;
828 IOTrackingCallSiteInfo * r = (typeof(r))right;
829 size_t lsize, rsize;
830
831 rsize = r->size[0] + r->size[1];
832 lsize = l->size[0] + l->size[1];
833
834 return (rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1);
835 }
836
837 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
838
839 static int
IOTrackingAddressCompare(const void * left,const void * right)840 IOTrackingAddressCompare(const void * left, const void * right)
841 {
842 IOTracking * instance;
843 uintptr_t inst, laddr, raddr;
844
845 inst = ((typeof(inst) *)left)[0];
846 instance = (typeof(instance))INSTANCE_GET(inst);
847 if (kInstanceFlagAddress & inst) {
848 laddr = ~((IOTrackingAddress *)instance)->address;
849 } else {
850 laddr = (uintptr_t) (instance + 1);
851 }
852
853 inst = ((typeof(inst) *)right)[0];
854 instance = (typeof(instance))(inst & ~kInstanceFlags);
855 if (kInstanceFlagAddress & inst) {
856 raddr = ~((IOTrackingAddress *)instance)->address;
857 } else {
858 raddr = (uintptr_t) (instance + 1);
859 }
860
861 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1);
862 }
863
864
865 static int
IOTrackingZoneElementCompare(const void * left,const void * right)866 IOTrackingZoneElementCompare(const void * left, const void * right)
867 {
868 uintptr_t inst, laddr, raddr;
869
870 inst = ((typeof(inst) *)left)[0];
871 laddr = INSTANCE_PUT(inst);
872 inst = ((typeof(inst) *)right)[0];
873 raddr = INSTANCE_PUT(inst);
874
875 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1);
876 }
877
878 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
879
880 static void
CopyOutBacktraces(IOTrackingCallSite * site,IOTrackingCallSiteInfo * siteInfo)881 CopyOutBacktraces(IOTrackingCallSite * site, IOTrackingCallSiteInfo * siteInfo)
882 {
883 uint32_t j;
884 mach_vm_address_t bt, btEntry;
885
886 btEntry = site->queue->btEntry;
887 for (j = 0; j < kIOTrackingCallSiteBTs; j++) {
888 bt = site->bt[j];
889 if (btEntry
890 && (!bt || (j == (kIOTrackingCallSiteBTs - 1)))) {
891 bt = btEntry;
892 btEntry = 0;
893 }
894 siteInfo->bt[0][j] = VM_KERNEL_UNSLIDE(bt);
895 }
896
897 siteInfo->btPID = 0;
898 if (kIOTrackingQueueTypeUser & site->queue->type) {
899 siteInfo->btPID = site->user[0].pid;
900 uint32_t * bt32 = (typeof(bt32))((void *) &site->user[0].bt[0]);
901 uint64_t * bt64 = (typeof(bt64))((void *) &site->user[0].bt[0]);
902 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
903 if (j >= site->user[0].userCount) {
904 siteInfo->bt[1][j] = 0;
905 } else if (site->user[0].user32) {
906 siteInfo->bt[1][j] = bt32[j];
907 } else {
908 siteInfo->bt[1][j] = bt64[j];
909 }
910 }
911 }
912 }
913
914 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
915
916 static void
IOTrackingLeakScan(void * refcon)917 IOTrackingLeakScan(void * refcon)
918 {
919 IOTrackingLeaksRef * ref = (typeof(ref))refcon;
920 uintptr_t * instances;
921 IOTracking * instance;
922 uint64_t vaddr, vincr;
923 ppnum_t ppn;
924 uintptr_t ptr, addr, vphysaddr, inst;
925 size_t size, origsize;
926 uint32_t baseIdx, lim, ptrIdx, count;
927 boolean_t is;
928 AbsoluteTime deadline;
929
930 instances = ref->instances;
931 count = ref->count;
932 size = origsize = ref->zoneSize;
933
934 for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
935 ;
936 vaddr += vincr) {
937 if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) {
938 if (deadline) {
939 ml_set_interrupts_enabled(is);
940 IODelay(10);
941 }
942 if (vaddr >= VM_MAX_KERNEL_ADDRESS) {
943 break;
944 }
945 is = ml_set_interrupts_enabled(false);
946 clock_interval_to_deadline(10, kMillisecondScale, &deadline);
947 }
948
949 ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr);
950 // check noencrypt to avoid VM structs (map entries) with pointers
951 if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) {
952 ppn = 0;
953 }
954 if (!ppn) {
955 continue;
956 }
957
958 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) {
959 ptr = ((uintptr_t *)vphysaddr)[ptrIdx];
960 #if defined(HAS_APPLE_PAC)
961 // strip possible ptrauth signature from candidate data pointer
962 ptr = (uintptr_t)ptrauth_strip((void*)ptr, ptrauth_key_process_independent_data);
963 #endif /* defined(HAS_APPLE_PAC) */
964
965 for (lim = count, baseIdx = 0; lim; lim >>= 1) {
966 inst = instances[baseIdx + (lim >> 1)];
967 instance = (typeof(instance))INSTANCE_GET(inst);
968
969 if (ref->zoneSize) {
970 addr = INSTANCE_PUT(inst) & ~kInstanceFlags;
971 } else if (kInstanceFlagAddress & inst) {
972 addr = ~((IOTrackingAddress *)instance)->address;
973 origsize = size = ((IOTrackingAddress *)instance)->size;
974 if (!size) {
975 size = 1;
976 }
977 } else {
978 addr = (uintptr_t) (instance + 1);
979 origsize = size = instance->site->queue->allocSize;
980 }
981 if ((ptr >= addr) && (ptr < (addr + size))
982
983 && (((vaddr + ptrIdx * sizeof(uintptr_t)) < addr)
984 || ((vaddr + ptrIdx * sizeof(uintptr_t)) >= (addr + size)))) {
985 if (!(kInstanceFlagReferenced & inst)) {
986 inst |= kInstanceFlagReferenced;
987 instances[baseIdx + (lim >> 1)] = inst;
988 ref->found++;
989 if (!origsize) {
990 ref->foundzlen++;
991 }
992 }
993 break;
994 }
995 if (ptr > addr) {
996 // move right
997 baseIdx += (lim >> 1) + 1;
998 lim--;
999 }
1000 // else move left
1001 }
1002 }
1003 ref->bytes += page_size;
1004 }
1005 }
1006
1007 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1008
1009 extern "C" void
zone_leaks_scan(uintptr_t * instances,uint32_t count,uint32_t zoneSize,uint32_t * found)1010 zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found)
1011 {
1012 IOTrackingLeaksRef ref;
1013 IOTrackingCallSiteInfo siteInfo;
1014 uint32_t idx;
1015
1016 qsort(instances, count, sizeof(*instances), &IOTrackingZoneElementCompare);
1017
1018 bzero(&siteInfo, sizeof(siteInfo));
1019 bzero(&ref, sizeof(ref));
1020 ref.instances = instances;
1021 ref.count = count;
1022 ref.zoneSize = zoneSize;
1023
1024 for (idx = 0; idx < 2; idx++) {
1025 ref.bytes = 0;
1026 IOTrackingLeakScan(&ref);
1027 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found);
1028 if (count <= ref.found) {
1029 break;
1030 }
1031 }
1032
1033 *found = ref.found;
1034 }
1035
1036 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1037
1038 static OSData *
IOTrackingLeaks(LIBKERN_CONSUMED OSData * data)1039 IOTrackingLeaks(LIBKERN_CONSUMED OSData * data)
1040 {
1041 IOTrackingLeaksRef ref;
1042 IOTrackingCallSiteInfo siteInfo;
1043 IOTrackingCallSite * site;
1044 OSData * leakData;
1045 uintptr_t * instances;
1046 IOTracking * instance;
1047 uintptr_t inst;
1048 uint32_t count, idx, numSites, dups, siteCount;
1049
1050 instances = (typeof(instances))data->getBytesNoCopy();
1051 count = (data->getLength() / sizeof(*instances));
1052 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare);
1053
1054 bzero(&siteInfo, sizeof(siteInfo));
1055 bzero(&ref, sizeof(ref));
1056 ref.instances = instances;
1057 ref.count = count;
1058 for (idx = 0; idx < 2; idx++) {
1059 ref.bytes = 0;
1060 IOTrackingLeakScan(&ref);
1061 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen);
1062 if (count <= ref.found) {
1063 break;
1064 }
1065 }
1066
1067 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
1068
1069 for (numSites = 0, idx = 0; idx < count; idx++) {
1070 inst = instances[idx];
1071 if (kInstanceFlagReferenced & inst) {
1072 continue;
1073 }
1074 instance = (typeof(instance))INSTANCE_GET(inst);
1075 site = instance->site;
1076 instances[numSites] = (uintptr_t) site;
1077 numSites++;
1078 }
1079
1080 for (idx = 0; idx < numSites; idx++) {
1081 inst = instances[idx];
1082 if (!inst) {
1083 continue;
1084 }
1085 site = (typeof(site))inst;
1086 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) {
1087 if (instances[dups] == (uintptr_t) site) {
1088 siteCount++;
1089 instances[dups] = 0;
1090 }
1091 }
1092 siteInfo.count = siteCount;
1093 siteInfo.size[0] = (site->size[0] * site->count) / siteCount;
1094 siteInfo.size[1] = (site->size[1] * site->count) / siteCount;
1095 CopyOutBacktraces(site, &siteInfo);
1096 leakData->appendBytes(&siteInfo, sizeof(siteInfo));
1097 }
1098 data->release();
1099
1100 return leakData;
1101 }
1102
1103 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1104
1105 static bool
SkipName(uint32_t options,const char * name,size_t namesLen,const char * names)1106 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names)
1107 {
1108 const char * scan;
1109 const char * next;
1110 bool exclude, found;
1111 size_t qLen, sLen;
1112
1113 if (!namesLen || !names) {
1114 return false;
1115 }
1116 // <len><name>...<len><name><0>
1117 exclude = (0 != (kIOTrackingExcludeNames & options));
1118 qLen = strlen(name);
1119 scan = names;
1120 found = false;
1121 do{
1122 sLen = scan[0];
1123 scan++;
1124 next = scan + sLen;
1125 if (next >= (names + namesLen)) {
1126 break;
1127 }
1128 found = ((sLen == qLen) && !strncmp(scan, name, sLen));
1129 scan = next;
1130 }while (!found && (scan < (names + namesLen)));
1131
1132 return !(exclude ^ found);
1133 }
1134
1135 #endif /* IOTRACKING */
1136
1137 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1138
1139 static kern_return_t
IOTrackingDebug(uint32_t selector,uint32_t options,uint64_t value,uint32_t intag,uint32_t inzsize,const char * names,size_t namesLen,size_t size,OSObject ** result)1140 IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value,
1141 uint32_t intag, uint32_t inzsize,
1142 const char * names, size_t namesLen,
1143 size_t size, OSObject ** result)
1144 {
1145 kern_return_t ret;
1146 OSData * data;
1147
1148 if (result) {
1149 *result = NULL;
1150 }
1151 data = NULL;
1152 ret = kIOReturnNotReady;
1153
1154 #if IOTRACKING
1155
1156 kern_return_t kr;
1157 IOTrackingQueue * queue;
1158 IOTracking * instance;
1159 IOTrackingCallSite * site;
1160 IOTrackingCallSiteInfo siteInfo;
1161 IOTrackingUser * user;
1162 task_t mapTask;
1163 mach_vm_address_t mapAddress;
1164 mach_vm_size_t mapSize;
1165 uint32_t num, idx, qIdx;
1166 uintptr_t instFlags;
1167 proc_t proc;
1168 bool addresses;
1169
1170 ret = kIOReturnNotFound;
1171 proc = NULL;
1172 if (kIOTrackingGetMappings == selector) {
1173 if (value != -1ULL) {
1174 proc = proc_find((pid_t) value);
1175 if (!proc) {
1176 return kIOReturnNotFound;
1177 }
1178 }
1179 }
1180
1181 bzero(&siteInfo, sizeof(siteInfo));
1182 lck_mtx_lock(gIOTrackingLock);
1183 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
1184 {
1185 if (SkipName(options, queue->name, namesLen, names)) {
1186 continue;
1187 }
1188
1189 if (!(kIOTracking & gIOKitDebug) && (kIOTrackingQueueTypeAlloc & queue->type)) {
1190 continue;
1191 }
1192
1193 switch (selector) {
1194 case kIOTrackingResetTracking:
1195 {
1196 IOTrackingReset(queue);
1197 ret = kIOReturnSuccess;
1198 break;
1199 }
1200
1201 case kIOTrackingStartCapture:
1202 case kIOTrackingStopCapture:
1203 {
1204 queue->captureOn = (kIOTrackingStartCapture == selector);
1205 ret = kIOReturnSuccess;
1206 break;
1207 }
1208
1209 case kIOTrackingSetMinCaptureSize:
1210 {
1211 queue->minCaptureSize = size;
1212 ret = kIOReturnSuccess;
1213 break;
1214 }
1215
1216 case kIOTrackingLeaks:
1217 {
1218 if (!(kIOTrackingQueueTypeAlloc & queue->type)) {
1219 break;
1220 }
1221
1222 if (!data) {
1223 data = OSData::withCapacity(1024 * sizeof(uintptr_t));
1224 }
1225
1226 IOTRecursiveLockLock(&queue->lock);
1227 for (idx = 0; idx < queue->numSiteQs; idx++) {
1228 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link)
1229 {
1230 addresses = false;
1231 queue_iterate(&site->instances, instance, IOTracking *, link)
1232 {
1233 if (instance == site->addresses) {
1234 addresses = true;
1235 }
1236 instFlags = (typeof(instFlags))instance;
1237 if (addresses) {
1238 instFlags |= kInstanceFlagAddress;
1239 }
1240 data->appendValue(instFlags);
1241 }
1242 }
1243 }
1244 // queue is locked
1245 ret = kIOReturnSuccess;
1246 break;
1247 }
1248
1249
1250 case kIOTrackingGetTracking:
1251 {
1252 if (kIOTrackingQueueTypeMap & queue->type) {
1253 break;
1254 }
1255
1256 if (!data) {
1257 data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
1258 }
1259
1260 IOTRecursiveLockLock(&queue->lock);
1261 num = queue->siteCount;
1262 idx = 0;
1263 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) {
1264 queue_iterate(&queue->sites[qIdx], site, IOTrackingCallSite *, link)
1265 {
1266 assert(idx < num);
1267 idx++;
1268
1269 size_t tsize[2];
1270 uint32_t count = site->count;
1271 tsize[0] = site->size[0];
1272 tsize[1] = site->size[1];
1273
1274 if (intag || inzsize) {
1275 uintptr_t addr;
1276 vm_size_t size, zoneSize;
1277 vm_tag_t tag;
1278
1279 if (kIOTrackingQueueTypeAlloc & queue->type) {
1280 addresses = false;
1281 count = 0;
1282 tsize[0] = tsize[1] = 0;
1283 queue_iterate(&site->instances, instance, IOTracking *, link)
1284 {
1285 if (instance == site->addresses) {
1286 addresses = true;
1287 }
1288
1289 if (addresses) {
1290 addr = ~((IOTrackingAddress *)instance)->address;
1291 } else {
1292 addr = (uintptr_t) (instance + 1);
1293 }
1294
1295 kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize);
1296 if (KERN_SUCCESS != kr) {
1297 continue;
1298 }
1299
1300 if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) {
1301 continue;
1302 }
1303 if (inzsize && (inzsize != zoneSize)) {
1304 continue;
1305 }
1306
1307 count++;
1308 tsize[0] += size;
1309 }
1310 } else {
1311 if (!intag || inzsize || (intag != site->tag)) {
1312 continue;
1313 }
1314 }
1315 }
1316
1317 if (!count) {
1318 continue;
1319 }
1320 if (size && ((tsize[0] + tsize[1]) < size)) {
1321 continue;
1322 }
1323 siteInfo.count = count;
1324 siteInfo.size[0] = tsize[0];
1325 siteInfo.size[1] = tsize[1];
1326 CopyOutBacktraces(site, &siteInfo);
1327 data->appendBytes(&siteInfo, sizeof(siteInfo));
1328 }
1329 }
1330 assert(idx == num);
1331 IOTRecursiveLockUnlock(&queue->lock);
1332 ret = kIOReturnSuccess;
1333 break;
1334 }
1335
1336 case kIOTrackingGetMappings:
1337 {
1338 if (!(kIOTrackingQueueTypeMap & queue->type)) {
1339 break;
1340 }
1341 if (!data) {
1342 data = OSData::withCapacity((unsigned int) page_size);
1343 }
1344
1345 IOTRecursiveLockLock(&queue->lock);
1346 num = queue->siteCount;
1347 idx = 0;
1348 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) {
1349 queue_iterate(&queue->sites[qIdx], user, IOTrackingUser *, link)
1350 {
1351 assert(idx < num);
1352 idx++;
1353
1354 kr = IOMemoryMapTracking(user, &mapTask, &mapAddress, &mapSize);
1355 if (kIOReturnSuccess != kr) {
1356 continue;
1357 }
1358 if (proc && (mapTask != proc_task(proc))) {
1359 continue;
1360 }
1361 if (size && (mapSize < size)) {
1362 continue;
1363 }
1364
1365 siteInfo.count = 1;
1366 siteInfo.size[0] = mapSize;
1367 siteInfo.address = mapAddress;
1368 siteInfo.addressPID = task_pid(mapTask);
1369 siteInfo.btPID = user->btPID;
1370
1371 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
1372 siteInfo.bt[0][j] = VM_KERNEL_UNSLIDE(user->bt[j]);
1373 }
1374 uint32_t * bt32 = (typeof(bt32)) & user->btUser[0];
1375 uint64_t * bt64 = (typeof(bt64))((void *) &user->btUser[0]);
1376 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) {
1377 if (j >= user->userCount) {
1378 siteInfo.bt[1][j] = 0;
1379 } else if (user->user32) {
1380 siteInfo.bt[1][j] = bt32[j];
1381 } else {
1382 siteInfo.bt[1][j] = bt64[j];
1383 }
1384 }
1385 data->appendBytes(&siteInfo, sizeof(siteInfo));
1386 }
1387 }
1388 assert(idx == num);
1389 IOTRecursiveLockUnlock(&queue->lock);
1390 ret = kIOReturnSuccess;
1391 break;
1392 }
1393
1394 default:
1395 ret = kIOReturnUnsupported;
1396 break;
1397 }
1398 }
1399
1400 if ((kIOTrackingLeaks == selector) && data) {
1401 data = IOTrackingLeaks(data);
1402 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
1403 {
1404 if (SkipName(options, queue->name, namesLen, names)) {
1405 continue;
1406 }
1407 if (!(kIOTrackingQueueTypeAlloc & queue->type)) {
1408 continue;
1409 }
1410 IOTRecursiveLockUnlock(&queue->lock);
1411 }
1412 }
1413
1414 lck_mtx_unlock(gIOTrackingLock);
1415
1416 if ((kIOTrackingLeaks == selector) && namesLen && names) {
1417 const char * scan;
1418 const char * next;
1419 uint8_t sLen;
1420
1421 if (!data) {
1422 data = OSData::withCapacity(4096 * sizeof(uintptr_t));
1423 }
1424
1425 // <len><name>...<len><name><0>
1426 scan = names;
1427 do{
1428 sLen = ((uint8_t) scan[0]);
1429 scan++;
1430 next = scan + sLen;
1431 if (next >= (names + namesLen)) {
1432 break;
1433 }
1434 kr = zone_leaks(scan, sLen, ^(uint32_t count, uint32_t eSize, btref_t ref) {
1435 IOTrackingCallSiteInfo siteInfo = {
1436 .count = count,
1437 .size[0] = eSize * count,
1438 };
1439
1440 btref_decode_unslide(ref, siteInfo.bt[0]);
1441
1442 data->appendBytes(&siteInfo, sizeof(siteInfo));
1443 });
1444 if (KERN_SUCCESS == kr) {
1445 ret = kIOReturnSuccess;
1446 } else if (KERN_INVALID_NAME != kr) {
1447 ret = kIOReturnVMError;
1448 }
1449 scan = next;
1450 }while (scan < (names + namesLen));
1451 }
1452
1453 if (data) {
1454 switch (selector) {
1455 case kIOTrackingLeaks:
1456 case kIOTrackingGetTracking:
1457 case kIOTrackingGetMappings:
1458 {
1459 IOTrackingCallSiteInfo * siteInfos;
1460 siteInfos = (typeof(siteInfos))data->getBytesNoCopy();
1461 num = (data->getLength() / sizeof(*siteInfos));
1462 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare);
1463 break;
1464 }
1465 default: assert(false); break;
1466 }
1467 }
1468
1469 *result = data;
1470 if (proc) {
1471 proc_rele(proc);
1472 }
1473
1474 #endif /* IOTRACKING */
1475
1476 return ret;
1477 }
1478
1479 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1480
1481 #include <IOKit/IOKitDiagnosticsUserClient.h>
1482
1483 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1484
1485 #undef super
1486 #define super IOUserClient
1487
OSDefineMetaClassAndStructors(IOKitDiagnosticsClient,IOUserClient)1488 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient)
1489
1490 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1491
1492 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask)
1493 {
1494 IOKitDiagnosticsClient * inst;
1495
1496 inst = new IOKitDiagnosticsClient;
1497 if (inst && !inst->init()) {
1498 inst->release();
1499 inst = NULL;
1500 }
1501
1502 return inst;
1503 }
1504
1505 IOReturn
clientClose(void)1506 IOKitDiagnosticsClient::clientClose(void)
1507 {
1508 terminate();
1509 return kIOReturnSuccess;
1510 }
1511
1512 IOReturn
setProperties(OSObject * properties)1513 IOKitDiagnosticsClient::setProperties(OSObject * properties)
1514 {
1515 IOReturn kr = kIOReturnUnsupported;
1516 return kr;
1517 }
1518
1519 IOReturn
externalMethod(uint32_t selector,IOExternalMethodArguments * args,IOExternalMethodDispatch * dispatch,OSObject * target,void * reference)1520 IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
1521 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
1522 {
1523 IOReturn ret = kIOReturnBadArgument;
1524 const IOKitDiagnosticsParameters * params;
1525 const char * names;
1526 size_t namesLen;
1527 OSObject * result;
1528
1529 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) {
1530 return kIOReturnBadArgument;
1531 }
1532 params = (typeof(params))args->structureInput;
1533 if (!params) {
1534 return kIOReturnBadArgument;
1535 }
1536
1537 names = NULL;
1538 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters);
1539 if (namesLen) {
1540 names = (typeof(names))(params + 1);
1541 }
1542
1543 ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result);
1544
1545 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) {
1546 *args->structureVariableOutputData = result;
1547 } else if (result) {
1548 result->release();
1549 }
1550
1551 return ret;
1552 }
1553
1554 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1555