1 /*
2 * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * HISTORY
30 *
31 * 17-Apr-91 Portions from libIO.m, Doug Mitchell at NeXT.
32 * 17-Nov-98 cpp
33 *
34 */
35
36 #include <IOKit/system.h>
37 #include <mach/sync_policy.h>
38 #include <machine/machine_routines.h>
39 #include <vm/vm_kern.h>
40 #include <libkern/c++/OSCPPDebug.h>
41
42 #include <IOKit/assert.h>
43
44 #include <IOKit/IOReturn.h>
45 #include <IOKit/IOLib.h>
46 #include <IOKit/IOLocks.h>
47 #include <IOKit/IOMapper.h>
48 #include <IOKit/IOBufferMemoryDescriptor.h>
49 #include <IOKit/IOKitDebug.h>
50
51 #include "IOKitKernelInternal.h"
52
53 #ifdef IOALLOCDEBUG
54 #include <libkern/OSDebug.h>
55 #include <sys/sysctl.h>
56 #endif
57
58 #include "libkern/OSAtomic.h"
59 #include <libkern/c++/OSKext.h>
60 #include <IOKit/IOStatisticsPrivate.h>
61 #include <os/log_private.h>
62 #include <sys/msgbuf.h>
63 #include <console/serial_protos.h>
64
65 #if IOKITSTATS
66
67 #define IOStatisticsAlloc(type, size) \
68 do { \
69 IOStatistics::countAlloc(type, size); \
70 } while (0)
71
72 #else
73
74 #define IOStatisticsAlloc(type, size)
75
76 #endif /* IOKITSTATS */
77
78
79 #define TRACK_ALLOC (IOTRACKING && (kIOTracking & gIOKitDebug))
80
81
82 extern "C"
83 {
84 mach_timespec_t IOZeroTvalspec = { 0, 0 };
85
86 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
87
88 extern int
89 __doprnt(
90 const char *fmt,
91 va_list argp,
92 void (*putc)(int, void *),
93 void *arg,
94 int radix,
95 int is_log);
96
97 extern bool bsd_log_lock(bool);
98 extern void bsd_log_unlock(void);
99
100
101 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
102
103 lck_grp_t *IOLockGroup;
104
105 /*
106 * Global variables for use by iLogger
107 * These symbols are for use only by Apple diagnostic code.
108 * Binary compatibility is not guaranteed for kexts that reference these symbols.
109 */
110
111 void *_giDebugLogInternal = NULL;
112 void *_giDebugLogDataInternal = NULL;
113 void *_giDebugReserved1 = NULL;
114 void *_giDebugReserved2 = NULL;
115
116 #if defined(__x86_64__)
117 iopa_t gIOBMDPageAllocator;
118 #endif /* defined(__x86_64__) */
119
120 /*
121 * Static variables for this module.
122 */
123
124 static queue_head_t gIOMallocContiguousEntries;
125 static lck_mtx_t * gIOMallocContiguousEntriesLock;
126
127 #if __x86_64__
128 enum { kIOMaxPageableMaps = 8 };
129 enum { kIOMaxFixedRanges = 4 };
130 enum { kIOPageableMapSize = 512 * 1024 * 1024 };
131 enum { kIOPageableMaxMapSize = 512 * 1024 * 1024 };
132 #else
133 enum { kIOMaxPageableMaps = 16 };
134 enum { kIOMaxFixedRanges = 4 };
135 enum { kIOPageableMapSize = 96 * 1024 * 1024 };
136 enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 };
137 #endif
138
139 typedef struct {
140 vm_map_t map;
141 vm_offset_t address;
142 vm_offset_t end;
143 } IOMapData;
144
145 static SECURITY_READ_ONLY_LATE(struct mach_vm_range)
146 gIOKitPageableFixedRanges[kIOMaxFixedRanges];
147
148 static struct {
149 UInt32 count;
150 UInt32 hint;
151 IOMapData maps[kIOMaxPageableMaps];
152 lck_mtx_t * lock;
153 } gIOKitPageableSpace;
154
155 #if defined(__x86_64__)
156 static iopa_t gIOPageablePageAllocator;
157
158 uint32_t gIOPageAllocChunkBytes;
159 #endif /* defined(__x86_64__) */
160
161 #if IOTRACKING
162 IOTrackingQueue * gIOMallocTracking;
163 IOTrackingQueue * gIOWireTracking;
164 IOTrackingQueue * gIOMapTracking;
165 #endif /* IOTRACKING */
166
167 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
168
169 KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed0,
170 &gIOKitPageableFixedRanges[0], kIOPageableMapSize);
171 KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed1,
172 &gIOKitPageableFixedRanges[1], kIOPageableMapSize);
173 KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed2,
174 &gIOKitPageableFixedRanges[2], kIOPageableMapSize);
175 KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed3,
176 &gIOKitPageableFixedRanges[3], kIOPageableMapSize);
177 void
IOLibInit(void)178 IOLibInit(void)
179 {
180 static bool libInitialized;
181
182 if (libInitialized) {
183 return;
184 }
185
186 IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL);
187
188 #if IOTRACKING
189 IOTrackingInit();
190 gIOMallocTracking = IOTrackingQueueAlloc(kIOMallocTrackingName, 0, 0, 0,
191 kIOTrackingQueueTypeAlloc,
192 37);
193 gIOWireTracking = IOTrackingQueueAlloc(kIOWireTrackingName, 0, 0, page_size, 0, 0);
194
195 size_t mapCaptureSize = (kIOTracking & gIOKitDebug) ? page_size : (1024 * 1024);
196 gIOMapTracking = IOTrackingQueueAlloc(kIOMapTrackingName, 0, 0, mapCaptureSize,
197 kIOTrackingQueueTypeDefaultOn
198 | kIOTrackingQueueTypeMap
199 | kIOTrackingQueueTypeUser,
200 0);
201 #endif
202
203 gIOKitPageableSpace.maps[0].map = kmem_suballoc(kernel_map,
204 &gIOKitPageableFixedRanges[0].min_address,
205 kIOPageableMapSize,
206 VM_MAP_CREATE_PAGEABLE,
207 VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
208 (kms_flags_t)(KMS_PERMANENT | KMS_DATA | KMS_NOFAIL),
209 VM_KERN_MEMORY_IOKIT).kmr_submap;
210
211 gIOKitPageableSpace.maps[0].address = gIOKitPageableFixedRanges[0].min_address;
212 gIOKitPageableSpace.maps[0].end = gIOKitPageableFixedRanges[0].max_address;
213 gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
214 gIOKitPageableSpace.hint = 0;
215 gIOKitPageableSpace.count = 1;
216
217 gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
218 queue_init( &gIOMallocContiguousEntries );
219
220 #if defined(__x86_64__)
221 gIOPageAllocChunkBytes = PAGE_SIZE / 64;
222
223 assert(sizeof(iopa_page_t) <= gIOPageAllocChunkBytes);
224 iopa_init(&gIOBMDPageAllocator);
225 iopa_init(&gIOPageablePageAllocator);
226 #endif /* defined(__x86_64__) */
227
228
229 libInitialized = true;
230 }
231
232 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
233
234 vm_size_t
log2up(vm_size_t size)235 log2up(vm_size_t size)
236 {
237 if (size <= 1) {
238 size = 0;
239 } else {
240 #if __LP64__
241 size = 64 - __builtin_clzl(size - 1);
242 #else
243 size = 32 - __builtin_clzl(size - 1);
244 #endif
245 }
246 return size;
247 }
248
249 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
250
251 IOThread
IOCreateThread(IOThreadFunc fcn,void * arg)252 IOCreateThread(IOThreadFunc fcn, void *arg)
253 {
254 kern_return_t result;
255 thread_t thread;
256
257 result = kernel_thread_start((thread_continue_t)(void (*)(void))fcn, arg, &thread);
258 if (result != KERN_SUCCESS) {
259 return NULL;
260 }
261
262 thread_deallocate(thread);
263
264 return thread;
265 }
266
267
268 void
IOExitThread(void)269 IOExitThread(void)
270 {
271 (void) thread_terminate(current_thread());
272 }
273
274 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
275
276 #if IOTRACKING
277 struct IOLibMallocHeader {
278 IOTrackingAddress tracking;
279 };
280 #endif
281
282 #if IOTRACKING
283 #define sizeofIOLibMallocHeader (sizeof(IOLibMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress)))
284 #else
285 #define sizeofIOLibMallocHeader (0)
286 #endif
287
288 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
289
290 __typed_allocators_ignore_push // allocator implementation
291
292 void *
293 (IOMalloc_internal)(struct kalloc_heap *kheap, vm_size_t size,
294 zalloc_flags_t flags)
295 {
296 void * address;
297 vm_size_t allocSize;
298
299 allocSize = size + sizeofIOLibMallocHeader;
300 #if IOTRACKING
301 if (sizeofIOLibMallocHeader && (allocSize <= size)) {
302 return NULL; // overflow
303 }
304 #endif
305 address = kheap_alloc(kheap, allocSize,
306 Z_VM_TAG(Z_WAITOK | flags, VM_KERN_MEMORY_IOKIT));
307
308 if (address) {
309 #if IOTRACKING
310 if (TRACK_ALLOC) {
311 IOLibMallocHeader * hdr;
312 hdr = (typeof(hdr))address;
313 bzero(&hdr->tracking, sizeof(hdr->tracking));
314 hdr->tracking.address = ~(((uintptr_t) address) + sizeofIOLibMallocHeader);
315 hdr->tracking.size = size;
316 IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE);
317 }
318 #endif
319 address = (typeof(address))(((uintptr_t) address) + sizeofIOLibMallocHeader);
320
321 #if IOALLOCDEBUG
322 OSAddAtomicLong(size, &debug_iomalloc_size);
323 #endif
324 IOStatisticsAlloc(kIOStatisticsMalloc, size);
325 }
326
327 return address;
328 }
329
330 void
IOFree_internal(struct kalloc_heap * kheap,void * inAddress,vm_size_t size)331 IOFree_internal(struct kalloc_heap *kheap, void * inAddress, vm_size_t size)
332 {
333 void * address;
334
335 if ((address = inAddress)) {
336 address = (typeof(address))(((uintptr_t) address) - sizeofIOLibMallocHeader);
337
338 #if IOTRACKING
339 if (TRACK_ALLOC) {
340 IOLibMallocHeader * hdr;
341 struct ptr_reference { void * ptr; };
342 volatile struct ptr_reference ptr;
343
344 // we're about to block in IOTrackingRemove(), make sure the original pointer
345 // exists in memory or a register for leak scanning to find
346 ptr.ptr = inAddress;
347
348 hdr = (typeof(hdr))address;
349 if (size != hdr->tracking.size) {
350 OSReportWithBacktrace("bad IOFree size 0x%zx should be 0x%zx",
351 (size_t)size, (size_t)hdr->tracking.size);
352 size = hdr->tracking.size;
353 }
354 IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size);
355 ptr.ptr = NULL;
356 }
357 #endif
358
359 kheap_free(kheap, address, size + sizeofIOLibMallocHeader);
360 #if IOALLOCDEBUG
361 OSAddAtomicLong(-size, &debug_iomalloc_size);
362 #endif
363 IOStatisticsAlloc(kIOStatisticsFree, size);
364 }
365 }
366
367 void *
368 IOMalloc_external(
369 vm_size_t size);
370 void *
IOMalloc_external(vm_size_t size)371 IOMalloc_external(
372 vm_size_t size)
373 {
374 return IOMalloc_internal(KHEAP_DEFAULT, size, Z_VM_TAG_BT_BIT);
375 }
376
377 void
IOFree(void * inAddress,vm_size_t size)378 IOFree(void * inAddress, vm_size_t size)
379 {
380 IOFree_internal(KHEAP_DEFAULT, inAddress, size);
381 }
382
383 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
384
385 void *
386 IOMallocZero_external(
387 vm_size_t size);
388 void *
IOMallocZero_external(vm_size_t size)389 IOMallocZero_external(
390 vm_size_t size)
391 {
392 return IOMalloc_internal(KHEAP_DEFAULT, size, Z_ZERO_VM_TAG_BT_BIT);
393 }
394
395 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
396
397 vm_tag_t
IOMemoryTag(vm_map_t map)398 IOMemoryTag(vm_map_t map)
399 {
400 vm_tag_t tag;
401
402 if (!vm_kernel_map_is_kernel(map)) {
403 return VM_MEMORY_IOKIT;
404 }
405
406 tag = vm_tag_bt();
407 if (tag == VM_KERN_MEMORY_NONE) {
408 tag = VM_KERN_MEMORY_IOKIT;
409 }
410
411 return tag;
412 }
413
414 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
415
416 struct IOLibPageMallocHeader {
417 mach_vm_size_t alignMask;
418 mach_vm_offset_t allocationOffset;
419 #if IOTRACKING
420 IOTrackingAddress tracking;
421 #endif
422 };
423
424 #if IOTRACKING
425 #define sizeofIOLibPageMallocHeader (sizeof(IOLibPageMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress)))
426 #else
427 #define sizeofIOLibPageMallocHeader (sizeof(IOLibPageMallocHeader))
428 #endif
429
430 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
431
432 static __header_always_inline void
IOMallocAlignedSetHdr(IOLibPageMallocHeader * hdr,mach_vm_size_t alignMask,mach_vm_address_t allocationStart,mach_vm_address_t alignedStart)433 IOMallocAlignedSetHdr(
434 IOLibPageMallocHeader *hdr,
435 mach_vm_size_t alignMask,
436 mach_vm_address_t allocationStart,
437 mach_vm_address_t alignedStart)
438 {
439 mach_vm_offset_t offset = alignedStart - allocationStart;
440 #if __has_feature(ptrauth_calls)
441 offset = (mach_vm_offset_t) ptrauth_sign_unauthenticated((void *)offset,
442 ptrauth_key_process_independent_data,
443 ptrauth_blend_discriminator((void *)(alignedStart | alignMask),
444 OS_PTRAUTH_DISCRIMINATOR("IOLibPageMallocHeader.allocationOffset")));
445 #endif /* __has_feature(ptrauth_calls) */
446 hdr->allocationOffset = offset;
447 hdr->alignMask = alignMask;
448 }
449
450 __abortlike
451 static void
IOMallocAlignedHdrCorruptionPanic(mach_vm_offset_t offset,mach_vm_size_t alignMask,mach_vm_address_t alignedStart,vm_size_t size)452 IOMallocAlignedHdrCorruptionPanic(
453 mach_vm_offset_t offset,
454 mach_vm_size_t alignMask,
455 mach_vm_address_t alignedStart,
456 vm_size_t size)
457 {
458 mach_vm_address_t address = 0;
459 mach_vm_address_t recalAlignedStart = 0;
460
461 if (os_sub_overflow(alignedStart, offset, &address)) {
462 panic("Invalid offset %p for aligned addr %p", (void *)offset,
463 (void *)alignedStart);
464 }
465 if (os_add3_overflow(address, sizeofIOLibPageMallocHeader, alignMask,
466 &recalAlignedStart)) {
467 panic("alignMask 0x%llx overflows recalAlignedStart %p for provided addr "
468 "%p", alignMask, (void *)recalAlignedStart, (void *)alignedStart);
469 }
470 if (((recalAlignedStart &= ~alignMask) != alignedStart) &&
471 (round_page(recalAlignedStart) != alignedStart)) {
472 panic("Recalculated aligned addr %p doesn't match provided addr %p",
473 (void *)recalAlignedStart, (void *)alignedStart);
474 }
475 if (offset < sizeofIOLibPageMallocHeader) {
476 panic("Offset %zd doesn't accomodate IOLibPageMallocHeader for aligned "
477 "addr %p", (size_t)offset, (void *)alignedStart);
478 }
479 panic("alignMask 0x%llx overflows adjusted size %zd for aligned addr %p",
480 alignMask, (size_t)size, (void *)alignedStart);
481 }
482
483 static __header_always_inline mach_vm_address_t
IOMallocAlignedGetAddress(IOLibPageMallocHeader * hdr,mach_vm_address_t alignedStart,vm_size_t * size)484 IOMallocAlignedGetAddress(
485 IOLibPageMallocHeader *hdr,
486 mach_vm_address_t alignedStart,
487 vm_size_t *size)
488 {
489 mach_vm_address_t address = 0;
490 mach_vm_address_t recalAlignedStart = 0;
491 mach_vm_offset_t offset = hdr->allocationOffset;
492 mach_vm_size_t alignMask = hdr->alignMask;
493 #if __has_feature(ptrauth_calls)
494 offset = (mach_vm_offset_t) ptrauth_auth_data((void *)offset,
495 ptrauth_key_process_independent_data,
496 ptrauth_blend_discriminator((void *)(alignedStart | alignMask),
497 OS_PTRAUTH_DISCRIMINATOR("IOLibPageMallocHeader.allocationOffset")));
498 #endif /* __has_feature(ptrauth_calls) */
499 if (os_sub_overflow(alignedStart, offset, &address) ||
500 os_add3_overflow(address, sizeofIOLibPageMallocHeader, alignMask,
501 &recalAlignedStart) ||
502 (((recalAlignedStart &= ~alignMask) != alignedStart) &&
503 (round_page(recalAlignedStart) != alignedStart)) ||
504 (offset < sizeofIOLibPageMallocHeader) ||
505 os_add_overflow(*size, alignMask, size)) {
506 IOMallocAlignedHdrCorruptionPanic(offset, alignMask, alignedStart, *size);
507 }
508 return address;
509 }
510
511 void *
512 (IOMallocAligned_internal)(struct kalloc_heap *kheap, vm_size_t size,
513 vm_size_t alignment, zalloc_flags_t flags)
514 {
515 kern_return_t kr;
516 vm_offset_t address;
517 vm_offset_t allocationAddress;
518 vm_size_t adjustedSize;
519 uintptr_t alignMask;
520 IOLibPageMallocHeader * hdr;
521 kma_flags_t kma_flags = KMA_NONE;
522
523 if (size == 0) {
524 return NULL;
525 }
526 if (((uint32_t) alignment) != alignment) {
527 return NULL;
528 }
529
530 if (flags & Z_ZERO) {
531 kma_flags = KMA_ZERO;
532 }
533
534 if (kheap == KHEAP_DATA_BUFFERS) {
535 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
536 }
537
538 alignment = (1UL << log2up((uint32_t) alignment));
539 alignMask = alignment - 1;
540 adjustedSize = size + sizeofIOLibPageMallocHeader;
541
542 if (size > adjustedSize) {
543 address = 0; /* overflow detected */
544 } else if (adjustedSize >= page_size) {
545 kr = kernel_memory_allocate(kernel_map, &address,
546 size, alignMask, kma_flags, IOMemoryTag(kernel_map));
547 if (KERN_SUCCESS != kr) {
548 address = 0;
549 }
550 #if IOTRACKING
551 else if (TRACK_ALLOC) {
552 IOTrackingAlloc(gIOMallocTracking, address, size);
553 }
554 #endif
555 } else {
556 adjustedSize += alignMask;
557
558 if (adjustedSize >= page_size) {
559 kr = kmem_alloc(kernel_map, &allocationAddress,
560 adjustedSize, kma_flags, IOMemoryTag(kernel_map));
561 if (KERN_SUCCESS != kr) {
562 allocationAddress = 0;
563 }
564 } else {
565 allocationAddress = (vm_address_t) kheap_alloc(kheap,
566 adjustedSize, Z_VM_TAG(Z_WAITOK | flags, VM_KERN_MEMORY_IOKIT));
567 }
568
569 if (allocationAddress) {
570 address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader)
571 & (~alignMask);
572
573 hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader);
574 IOMallocAlignedSetHdr(hdr, alignMask, allocationAddress, address);
575 #if IOTRACKING
576 if (TRACK_ALLOC) {
577 bzero(&hdr->tracking, sizeof(hdr->tracking));
578 hdr->tracking.address = ~address;
579 hdr->tracking.size = size;
580 IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE);
581 }
582 #endif
583 } else {
584 address = 0;
585 }
586 }
587
588 assert(0 == (address & alignMask));
589
590 if (address) {
591 #if IOALLOCDEBUG
592 OSAddAtomicLong(size, &debug_iomalloc_size);
593 #endif
594 IOStatisticsAlloc(kIOStatisticsMallocAligned, size);
595 }
596
597 return (void *) address;
598 }
599
600 void
IOFreeAligned_internal(kalloc_heap_t kheap,void * address,vm_size_t size)601 IOFreeAligned_internal(kalloc_heap_t kheap, void * address, vm_size_t size)
602 {
603 vm_address_t allocationAddress;
604 vm_size_t adjustedSize;
605 IOLibPageMallocHeader * hdr;
606
607 if (!address) {
608 return;
609 }
610
611 assert(size);
612
613 adjustedSize = size + sizeofIOLibPageMallocHeader;
614 if (adjustedSize >= page_size) {
615 #if IOTRACKING
616 if (TRACK_ALLOC) {
617 IOTrackingFree(gIOMallocTracking, (uintptr_t) address, size);
618 }
619 #endif
620 kmem_free(kernel_map, (vm_offset_t) address, size);
621 } else {
622 hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader);
623 allocationAddress = IOMallocAlignedGetAddress(hdr,
624 (mach_vm_address_t)address, &adjustedSize);
625
626 #if IOTRACKING
627 if (TRACK_ALLOC) {
628 if (size != hdr->tracking.size) {
629 OSReportWithBacktrace("bad IOFreeAligned size 0x%zx should be 0x%zx",
630 (size_t)size, (size_t)hdr->tracking.size);
631 size = hdr->tracking.size;
632 }
633 IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size);
634 }
635 #endif
636 if (adjustedSize >= page_size) {
637 kmem_free(kernel_map, allocationAddress, adjustedSize);
638 } else {
639 kheap_free(kheap, allocationAddress, adjustedSize);
640 }
641 }
642
643 #if IOALLOCDEBUG
644 OSAddAtomicLong(-size, &debug_iomalloc_size);
645 #endif
646
647 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
648 }
649
650 void *
651 IOMallocAligned_external(
652 vm_size_t size, vm_size_t alignment);
653 void *
IOMallocAligned_external(vm_size_t size,vm_size_t alignment)654 IOMallocAligned_external(
655 vm_size_t size, vm_size_t alignment)
656 {
657 return IOMallocAligned_internal(KHEAP_DATA_BUFFERS, size, alignment,
658 Z_VM_TAG_BT_BIT);
659 }
660
661 void
IOFreeAligned(void * address,vm_size_t size)662 IOFreeAligned(
663 void * address,
664 vm_size_t size)
665 {
666 IOFreeAligned_internal(KHEAP_DATA_BUFFERS, address, size);
667 }
668
669 __typed_allocators_ignore_pop
670
671 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
672
673 void
IOKernelFreePhysical(kalloc_heap_t kheap,mach_vm_address_t address,mach_vm_size_t size)674 IOKernelFreePhysical(
675 kalloc_heap_t kheap,
676 mach_vm_address_t address,
677 mach_vm_size_t size)
678 {
679 vm_address_t allocationAddress;
680 vm_size_t adjustedSize;
681 IOLibPageMallocHeader * hdr;
682
683 if (!address) {
684 return;
685 }
686
687 assert(size);
688
689 adjustedSize = (2 * size) + sizeofIOLibPageMallocHeader;
690 if (adjustedSize >= page_size) {
691 #if IOTRACKING
692 if (TRACK_ALLOC) {
693 IOTrackingFree(gIOMallocTracking, address, size);
694 }
695 #endif
696 kmem_free(kernel_map, (vm_offset_t) address, size);
697 } else {
698 hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader);
699 allocationAddress = IOMallocAlignedGetAddress(hdr, address, &adjustedSize);
700 #if IOTRACKING
701 if (TRACK_ALLOC) {
702 IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size);
703 }
704 #endif
705 __typed_allocators_ignore(kheap_free(kheap, allocationAddress, adjustedSize));
706 }
707
708 IOStatisticsAlloc(kIOStatisticsFreeContiguous, size);
709 #if IOALLOCDEBUG
710 OSAddAtomicLong(-size, &debug_iomalloc_size);
711 #endif
712 }
713
714 #if __arm64__
715 extern unsigned long gPhysBase, gPhysSize;
716 #endif
717
718 mach_vm_address_t
IOKernelAllocateWithPhysicalRestrict(kalloc_heap_t kheap,mach_vm_size_t size,mach_vm_address_t maxPhys,mach_vm_size_t alignment,bool contiguous)719 IOKernelAllocateWithPhysicalRestrict(
720 kalloc_heap_t kheap,
721 mach_vm_size_t size,
722 mach_vm_address_t maxPhys,
723 mach_vm_size_t alignment,
724 bool contiguous)
725 {
726 kern_return_t kr;
727 mach_vm_address_t address;
728 mach_vm_address_t allocationAddress;
729 mach_vm_size_t adjustedSize;
730 mach_vm_address_t alignMask;
731 IOLibPageMallocHeader * hdr;
732
733 if (size == 0) {
734 return 0;
735 }
736 if (alignment == 0) {
737 alignment = 1;
738 }
739
740 alignMask = alignment - 1;
741
742 if (os_mul_and_add_overflow(2, size, sizeofIOLibPageMallocHeader, &adjustedSize)) {
743 return 0;
744 }
745
746 contiguous = (contiguous && (adjustedSize > page_size))
747 || (alignment > page_size);
748
749 if (contiguous || maxPhys) {
750 kma_flags_t options = KMA_ZERO;
751 vm_offset_t virt;
752
753 if (kheap == KHEAP_DATA_BUFFERS) {
754 options = (kma_flags_t) (options | KMA_DATA);
755 }
756
757 adjustedSize = size;
758 contiguous = (contiguous && (adjustedSize > page_size))
759 || (alignment > page_size);
760
761 if (!contiguous) {
762 #if __arm64__
763 if (maxPhys >= (mach_vm_address_t)(gPhysBase + gPhysSize)) {
764 maxPhys = 0;
765 } else
766 #endif
767 if (maxPhys <= 0xFFFFFFFF) {
768 maxPhys = 0;
769 options = (kma_flags_t)(options | KMA_LOMEM);
770 } else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) {
771 maxPhys = 0;
772 }
773 }
774 if (contiguous || maxPhys) {
775 kr = kmem_alloc_contig(kernel_map, &virt, size,
776 alignMask, (ppnum_t) atop(maxPhys), (ppnum_t) atop(alignMask),
777 options, IOMemoryTag(kernel_map));
778 } else {
779 kr = kernel_memory_allocate(kernel_map, &virt,
780 size, alignMask, options, IOMemoryTag(kernel_map));
781 }
782 if (KERN_SUCCESS == kr) {
783 address = virt;
784 #if IOTRACKING
785 if (TRACK_ALLOC) {
786 IOTrackingAlloc(gIOMallocTracking, address, size);
787 }
788 #endif
789 } else {
790 address = 0;
791 }
792 } else {
793 adjustedSize += alignMask;
794 if (adjustedSize < size) {
795 return 0;
796 }
797 /* BEGIN IGNORE CODESTYLE */
798 __typed_allocators_ignore_push // allocator implementation
799 allocationAddress = (mach_vm_address_t) kheap_alloc(kheap,
800 adjustedSize, Z_VM_TAG_BT(Z_WAITOK, VM_KERN_MEMORY_IOKIT));
801 __typed_allocators_ignore_pop
802 /* END IGNORE CODESTYLE */
803
804 if (allocationAddress) {
805 address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader)
806 & (~alignMask);
807
808 if (atop_32(address) != atop_32(address + size - 1)) {
809 address = round_page(address);
810 }
811
812 hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader);
813 IOMallocAlignedSetHdr(hdr, alignMask, allocationAddress, address);
814 #if IOTRACKING
815 if (TRACK_ALLOC) {
816 bzero(&hdr->tracking, sizeof(hdr->tracking));
817 hdr->tracking.address = ~address;
818 hdr->tracking.size = size;
819 IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE);
820 }
821 #endif
822 } else {
823 address = 0;
824 }
825 }
826
827 if (address) {
828 IOStatisticsAlloc(kIOStatisticsMallocContiguous, size);
829 #if IOALLOCDEBUG
830 OSAddAtomicLong(size, &debug_iomalloc_size);
831 #endif
832 }
833
834 return address;
835 }
836
837
838 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
839
840 struct _IOMallocContiguousEntry {
841 mach_vm_address_t virtualAddr;
842 IOBufferMemoryDescriptor * md;
843 queue_chain_t link;
844 };
845 typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry;
846
847 void *
IOMallocContiguous(vm_size_t size,vm_size_t alignment,IOPhysicalAddress * physicalAddress)848 IOMallocContiguous(vm_size_t size, vm_size_t alignment,
849 IOPhysicalAddress * physicalAddress)
850 {
851 mach_vm_address_t address = 0;
852
853 if (size == 0) {
854 return NULL;
855 }
856 if (alignment == 0) {
857 alignment = 1;
858 }
859
860 /* Do we want a physical address? */
861 if (!physicalAddress) {
862 address = IOKernelAllocateWithPhysicalRestrict(KHEAP_DEFAULT,
863 size, 0 /*maxPhys*/, alignment, true);
864 } else {
865 do {
866 IOBufferMemoryDescriptor * bmd;
867 mach_vm_address_t physicalMask;
868 vm_offset_t alignMask;
869
870 alignMask = alignment - 1;
871 physicalMask = (0xFFFFFFFF ^ alignMask);
872
873 bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
874 kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask);
875 if (!bmd) {
876 break;
877 }
878
879 _IOMallocContiguousEntry *
880 entry = IOMallocType(_IOMallocContiguousEntry);
881 if (!entry) {
882 bmd->release();
883 break;
884 }
885 entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy();
886 entry->md = bmd;
887 lck_mtx_lock(gIOMallocContiguousEntriesLock);
888 queue_enter( &gIOMallocContiguousEntries, entry,
889 _IOMallocContiguousEntry *, link );
890 lck_mtx_unlock(gIOMallocContiguousEntriesLock);
891
892 address = (mach_vm_address_t) entry->virtualAddr;
893 *physicalAddress = bmd->getPhysicalAddress();
894 }while (false);
895 }
896
897 return (void *) address;
898 }
899
900 void
IOFreeContiguous(void * _address,vm_size_t size)901 IOFreeContiguous(void * _address, vm_size_t size)
902 {
903 _IOMallocContiguousEntry * entry;
904 IOMemoryDescriptor * md = NULL;
905
906 mach_vm_address_t address = (mach_vm_address_t) _address;
907
908 if (!address) {
909 return;
910 }
911
912 assert(size);
913
914 lck_mtx_lock(gIOMallocContiguousEntriesLock);
915 queue_iterate( &gIOMallocContiguousEntries, entry,
916 _IOMallocContiguousEntry *, link )
917 {
918 if (entry->virtualAddr == address) {
919 md = entry->md;
920 queue_remove( &gIOMallocContiguousEntries, entry,
921 _IOMallocContiguousEntry *, link );
922 break;
923 }
924 }
925 lck_mtx_unlock(gIOMallocContiguousEntriesLock);
926
927 if (md) {
928 md->release();
929 IOFreeType(entry, _IOMallocContiguousEntry);
930 } else {
931 IOKernelFreePhysical(KHEAP_DEFAULT, (mach_vm_address_t) address, size);
932 }
933 }
934
935 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
936
937 kern_return_t
IOIteratePageableMaps(vm_size_t size,IOIteratePageableMapsCallback callback,void * ref)938 IOIteratePageableMaps(vm_size_t size,
939 IOIteratePageableMapsCallback callback, void * ref)
940 {
941 kern_return_t kr = kIOReturnNotReady;
942 kmem_return_t kmr;
943 vm_size_t segSize;
944 UInt32 attempts;
945 UInt32 index;
946 mach_vm_offset_t min;
947 int flags;
948
949 if (size > kIOPageableMaxMapSize) {
950 return kIOReturnBadArgument;
951 }
952
953 do {
954 index = gIOKitPageableSpace.hint;
955 attempts = gIOKitPageableSpace.count;
956 while (attempts--) {
957 kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref);
958 if (KERN_SUCCESS == kr) {
959 gIOKitPageableSpace.hint = index;
960 break;
961 }
962 if (index) {
963 index--;
964 } else {
965 index = gIOKitPageableSpace.count - 1;
966 }
967 }
968 if (KERN_NO_SPACE != kr) {
969 break;
970 }
971
972 lck_mtx_lock( gIOKitPageableSpace.lock );
973
974 index = gIOKitPageableSpace.count;
975 if (index >= (kIOMaxPageableMaps - 1)) {
976 lck_mtx_unlock( gIOKitPageableSpace.lock );
977 break;
978 }
979
980 if (size < kIOPageableMapSize) {
981 segSize = kIOPageableMapSize;
982 } else {
983 segSize = size;
984 }
985
986 /*
987 * Use the predefine ranges if available, else default to data
988 */
989 if (index < kIOMaxFixedRanges) {
990 min = gIOKitPageableFixedRanges[index].min_address;
991 flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE;
992 } else {
993 min = 0;
994 flags = VM_FLAGS_ANYWHERE;
995 }
996 kmr = kmem_suballoc(kernel_map,
997 &min,
998 segSize,
999 VM_MAP_CREATE_PAGEABLE,
1000 flags,
1001 (kms_flags_t)(KMS_PERMANENT | KMS_DATA),
1002 VM_KERN_MEMORY_IOKIT);
1003 if (kmr.kmr_return != KERN_SUCCESS) {
1004 kr = kmr.kmr_return;
1005 lck_mtx_unlock( gIOKitPageableSpace.lock );
1006 break;
1007 }
1008
1009 gIOKitPageableSpace.maps[index].map = kmr.kmr_submap;
1010 gIOKitPageableSpace.maps[index].address = min;
1011 gIOKitPageableSpace.maps[index].end = min + segSize;
1012 gIOKitPageableSpace.hint = index;
1013 gIOKitPageableSpace.count = index + 1;
1014
1015 lck_mtx_unlock( gIOKitPageableSpace.lock );
1016 } while (true);
1017
1018 return kr;
1019 }
1020
1021 struct IOMallocPageableRef {
1022 vm_offset_t address;
1023 vm_size_t size;
1024 vm_tag_t tag;
1025 };
1026
1027 static kern_return_t
IOMallocPageableCallback(vm_map_t map,void * _ref)1028 IOMallocPageableCallback(vm_map_t map, void * _ref)
1029 {
1030 struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref;
1031 kma_flags_t flags = (kma_flags_t)(KMA_PAGEABLE | KMA_DATA);
1032
1033 return kmem_alloc( map, &ref->address, ref->size, flags, ref->tag );
1034 }
1035
1036 static void *
IOMallocPageablePages(vm_size_t size,vm_size_t alignment,vm_tag_t tag)1037 IOMallocPageablePages(vm_size_t size, vm_size_t alignment, vm_tag_t tag)
1038 {
1039 kern_return_t kr = kIOReturnNotReady;
1040 struct IOMallocPageableRef ref;
1041
1042 if (alignment > page_size) {
1043 return NULL;
1044 }
1045 if (size > kIOPageableMaxMapSize) {
1046 return NULL;
1047 }
1048
1049 ref.size = size;
1050 ref.tag = tag;
1051 kr = IOIteratePageableMaps( size, &IOMallocPageableCallback, &ref );
1052 if (kIOReturnSuccess != kr) {
1053 ref.address = 0;
1054 }
1055
1056 return (void *) ref.address;
1057 }
1058
1059 vm_map_t
IOPageableMapForAddress(uintptr_t address)1060 IOPageableMapForAddress( uintptr_t address )
1061 {
1062 vm_map_t map = NULL;
1063 UInt32 index;
1064
1065 for (index = 0; index < gIOKitPageableSpace.count; index++) {
1066 if ((address >= gIOKitPageableSpace.maps[index].address)
1067 && (address < gIOKitPageableSpace.maps[index].end)) {
1068 map = gIOKitPageableSpace.maps[index].map;
1069 break;
1070 }
1071 }
1072 if (!map) {
1073 panic("IOPageableMapForAddress: null");
1074 }
1075
1076 return map;
1077 }
1078
1079 static void
IOFreePageablePages(void * address,vm_size_t size)1080 IOFreePageablePages(void * address, vm_size_t size)
1081 {
1082 vm_map_t map;
1083
1084 map = IOPageableMapForAddress((vm_address_t) address);
1085 if (map) {
1086 kmem_free( map, (vm_offset_t) address, size);
1087 }
1088 }
1089
1090 #if defined(__x86_64__)
1091 static uintptr_t
IOMallocOnePageablePage(kalloc_heap_t kheap __unused,iopa_t * a)1092 IOMallocOnePageablePage(kalloc_heap_t kheap __unused, iopa_t * a)
1093 {
1094 return (uintptr_t) IOMallocPageablePages(page_size, page_size, VM_KERN_MEMORY_IOKIT);
1095 }
1096 #endif /* defined(__x86_64__) */
1097
1098 static void *
IOMallocPageableInternal(vm_size_t size,vm_size_t alignment,bool zeroed)1099 IOMallocPageableInternal(vm_size_t size, vm_size_t alignment, bool zeroed)
1100 {
1101 void * addr;
1102
1103 if (((uint32_t) alignment) != alignment) {
1104 return NULL;
1105 }
1106 #if defined(__x86_64__)
1107 if (size >= (page_size - 4 * gIOPageAllocChunkBytes) ||
1108 alignment > page_size) {
1109 addr = IOMallocPageablePages(size, alignment, IOMemoryTag(kernel_map));
1110 /* Memory allocated this way will already be zeroed. */
1111 } else {
1112 addr = ((void *) iopa_alloc(&gIOPageablePageAllocator,
1113 &IOMallocOnePageablePage, KHEAP_DEFAULT, size, (uint32_t) alignment));
1114 if (addr && zeroed) {
1115 bzero(addr, size);
1116 }
1117 }
1118 #else /* !defined(__x86_64__) */
1119 vm_size_t allocSize = size;
1120 if (allocSize == 0) {
1121 allocSize = 1;
1122 }
1123 addr = IOMallocPageablePages(allocSize, alignment, IOMemoryTag(kernel_map));
1124 /* already zeroed */
1125 #endif /* defined(__x86_64__) */
1126
1127 if (addr) {
1128 #if IOALLOCDEBUG
1129 OSAddAtomicLong(size, &debug_iomallocpageable_size);
1130 #endif
1131 IOStatisticsAlloc(kIOStatisticsMallocPageable, size);
1132 }
1133
1134 return addr;
1135 }
1136
1137 void *
IOMallocPageable(vm_size_t size,vm_size_t alignment)1138 IOMallocPageable(vm_size_t size, vm_size_t alignment)
1139 {
1140 return IOMallocPageableInternal(size, alignment, /*zeroed*/ false);
1141 }
1142
1143 void *
IOMallocPageableZero(vm_size_t size,vm_size_t alignment)1144 IOMallocPageableZero(vm_size_t size, vm_size_t alignment)
1145 {
1146 return IOMallocPageableInternal(size, alignment, /*zeroed*/ true);
1147 }
1148
1149 void
IOFreePageable(void * address,vm_size_t size)1150 IOFreePageable(void * address, vm_size_t size)
1151 {
1152 #if IOALLOCDEBUG
1153 OSAddAtomicLong(-size, &debug_iomallocpageable_size);
1154 #endif
1155 IOStatisticsAlloc(kIOStatisticsFreePageable, size);
1156
1157 #if defined(__x86_64__)
1158 if (size < (page_size - 4 * gIOPageAllocChunkBytes)) {
1159 address = (void *) iopa_free(&gIOPageablePageAllocator, (uintptr_t) address, size);
1160 size = page_size;
1161 }
1162 if (address) {
1163 IOFreePageablePages(address, size);
1164 }
1165 #else /* !defined(__x86_64__) */
1166 if (size == 0) {
1167 size = 1;
1168 }
1169 if (address) {
1170 IOFreePageablePages(address, size);
1171 }
1172 #endif /* defined(__x86_64__) */
1173 }
1174
1175 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1176
1177 void *
1178 IOMallocData_external(
1179 vm_size_t size);
1180 void *
IOMallocData_external(vm_size_t size)1181 IOMallocData_external(vm_size_t size)
1182 {
1183 return IOMalloc_internal(KHEAP_DATA_BUFFERS, size, Z_VM_TAG_BT_BIT);
1184 }
1185
1186 void *
1187 IOMallocZeroData_external(
1188 vm_size_t size);
1189 void *
IOMallocZeroData_external(vm_size_t size)1190 IOMallocZeroData_external(vm_size_t size)
1191 {
1192 return IOMalloc_internal(KHEAP_DATA_BUFFERS, size, Z_ZERO_VM_TAG_BT_BIT);
1193 }
1194
1195 void
IOFreeData(void * address,vm_size_t size)1196 IOFreeData(void * address, vm_size_t size)
1197 {
1198 return IOFree_internal(KHEAP_DATA_BUFFERS, address, size);
1199 }
1200
1201 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1202
1203 __typed_allocators_ignore_push // allocator implementation
1204
1205 void *
IOMallocTypeImpl(kalloc_type_view_t kt_view)1206 IOMallocTypeImpl(kalloc_type_view_t kt_view)
1207 {
1208 #if IOTRACKING
1209 /*
1210 * When leak detection is on default to using IOMalloc as kalloc
1211 * type infrastructure isn't aware of needing additional space for
1212 * the header.
1213 */
1214 if (TRACK_ALLOC) {
1215 uint32_t kt_size = kalloc_type_get_size(kt_view->kt_size);
1216 void *mem = IOMalloc_internal(KHEAP_DEFAULT, kt_size, Z_ZERO);
1217 if (!IOMallocType_from_vm(kt_view)) {
1218 assert(mem);
1219 }
1220 return mem;
1221 }
1222 #endif
1223 zalloc_flags_t kt_flags = (zalloc_flags_t) (Z_WAITOK | Z_ZERO);
1224 if (!IOMallocType_from_vm(kt_view)) {
1225 kt_flags = (zalloc_flags_t) (kt_flags | Z_NOFAIL);
1226 }
1227 /*
1228 * Use external symbol for kalloc_type_impl as
1229 * kalloc_type_views generated at some external callsites
1230 * many not have been processed during boot.
1231 */
1232 return kalloc_type_impl_external(kt_view, kt_flags);
1233 }
1234
1235 void
IOFreeTypeImpl(kalloc_type_view_t kt_view,void * address)1236 IOFreeTypeImpl(kalloc_type_view_t kt_view, void * address)
1237 {
1238 #if IOTRACKING
1239 if (TRACK_ALLOC) {
1240 return IOFree_internal(KHEAP_DEFAULT, address,
1241 kalloc_type_get_size(kt_view->kt_size));
1242 }
1243 #endif
1244 /*
1245 * Use external symbol for kalloc_type_impl as
1246 * kalloc_type_views generated at some external callsites
1247 * many not have been processed during boot.
1248 */
1249 return kfree_type_impl_external(kt_view, address);
1250 }
1251
1252 void *
IOMallocTypeVarImpl(kalloc_type_var_view_t kt_view,vm_size_t size)1253 IOMallocTypeVarImpl(kalloc_type_var_view_t kt_view, vm_size_t size)
1254 {
1255 #if IOTRACKING
1256 /*
1257 * When leak detection is on default to using IOMalloc as kalloc
1258 * type infrastructure isn't aware of needing additional space for
1259 * the header.
1260 */
1261 if (TRACK_ALLOC) {
1262 return IOMalloc_internal(KHEAP_DEFAULT, size, Z_ZERO);
1263 }
1264 #endif
1265 zalloc_flags_t kt_flags = (zalloc_flags_t) (Z_WAITOK | Z_ZERO);
1266
1267 kt_flags = Z_VM_TAG_BT(kt_flags, VM_KERN_MEMORY_KALLOC_TYPE);
1268 return kalloc_type_var_impl(kt_view, size, kt_flags, NULL);
1269 }
1270
1271 void
IOFreeTypeVarImpl(kalloc_type_var_view_t kt_view,void * address,vm_size_t size)1272 IOFreeTypeVarImpl(kalloc_type_var_view_t kt_view, void * address,
1273 vm_size_t size)
1274 {
1275 #if IOTRACKING
1276 if (TRACK_ALLOC) {
1277 return IOFree_internal(KHEAP_DEFAULT, address, size);
1278 }
1279 #endif
1280
1281 return kfree_type_var_impl(kt_view, address, size);
1282 }
1283
1284 __typed_allocators_ignore_pop
1285
1286 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1287
1288 #if defined(__x86_64__)
1289
1290
1291 extern "C" void
iopa_init(iopa_t * a)1292 iopa_init(iopa_t * a)
1293 {
1294 bzero(a, sizeof(*a));
1295 a->lock = IOLockAlloc();
1296 queue_init(&a->list);
1297 }
1298
1299 static uintptr_t
iopa_allocinpage(iopa_page_t * pa,uint32_t count,uint64_t align)1300 iopa_allocinpage(iopa_page_t * pa, uint32_t count, uint64_t align)
1301 {
1302 uint32_t n, s;
1303 uint64_t avail = pa->avail;
1304
1305 assert(avail);
1306
1307 // find strings of count 1 bits in avail
1308 for (n = count; n > 1; n -= s) {
1309 s = n >> 1;
1310 avail = avail & (avail << s);
1311 }
1312 // and aligned
1313 avail &= align;
1314
1315 if (avail) {
1316 n = __builtin_clzll(avail);
1317 pa->avail &= ~((-1ULL << (64 - count)) >> n);
1318 if (!pa->avail && pa->link.next) {
1319 remque(&pa->link);
1320 pa->link.next = NULL;
1321 }
1322 return n * gIOPageAllocChunkBytes + trunc_page((uintptr_t) pa);
1323 }
1324
1325 return 0;
1326 }
1327
1328 uintptr_t
iopa_alloc(iopa_t * a,iopa_proc_t alloc,kalloc_heap_t kheap,vm_size_t bytes,vm_size_t balign)1329 iopa_alloc(
1330 iopa_t * a,
1331 iopa_proc_t alloc,
1332 kalloc_heap_t kheap,
1333 vm_size_t bytes,
1334 vm_size_t balign)
1335 {
1336 static const uint64_t align_masks[] = {
1337 0xFFFFFFFFFFFFFFFF,
1338 0xAAAAAAAAAAAAAAAA,
1339 0x8888888888888888,
1340 0x8080808080808080,
1341 0x8000800080008000,
1342 0x8000000080000000,
1343 0x8000000000000000,
1344 };
1345 iopa_page_t * pa;
1346 uintptr_t addr = 0;
1347 uint32_t count;
1348 uint64_t align;
1349 vm_size_t align_masks_idx;
1350
1351 if (((uint32_t) bytes) != bytes) {
1352 return 0;
1353 }
1354 if (!bytes) {
1355 bytes = 1;
1356 }
1357 count = (((uint32_t) bytes) + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes;
1358
1359 align_masks_idx = log2up((balign + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes);
1360 assert(align_masks_idx < sizeof(align_masks) / sizeof(*align_masks));
1361 align = align_masks[align_masks_idx];
1362
1363 IOLockLock(a->lock);
1364 __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_first(&a->list));
1365 while (!queue_end(&a->list, &pa->link)) {
1366 addr = iopa_allocinpage(pa, count, align);
1367 if (addr) {
1368 a->bytecount += bytes;
1369 break;
1370 }
1371 __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_next(&pa->link));
1372 }
1373 IOLockUnlock(a->lock);
1374
1375 if (!addr) {
1376 addr = alloc(kheap, a);
1377 if (addr) {
1378 pa = (typeof(pa))(addr + page_size - gIOPageAllocChunkBytes);
1379 pa->signature = kIOPageAllocSignature;
1380 pa->avail = -2ULL;
1381
1382 addr = iopa_allocinpage(pa, count, align);
1383 IOLockLock(a->lock);
1384 if (pa->avail) {
1385 enqueue_head(&a->list, &pa->link);
1386 }
1387 a->pagecount++;
1388 if (addr) {
1389 a->bytecount += bytes;
1390 }
1391 IOLockUnlock(a->lock);
1392 }
1393 }
1394
1395 assert((addr & ((1 << log2up(balign)) - 1)) == 0);
1396 return addr;
1397 }
1398
1399 uintptr_t
iopa_free(iopa_t * a,uintptr_t addr,vm_size_t bytes)1400 iopa_free(iopa_t * a, uintptr_t addr, vm_size_t bytes)
1401 {
1402 iopa_page_t * pa;
1403 uint32_t count;
1404 uintptr_t chunk;
1405
1406 if (((uint32_t) bytes) != bytes) {
1407 return 0;
1408 }
1409 if (!bytes) {
1410 bytes = 1;
1411 }
1412
1413 chunk = (addr & page_mask);
1414 assert(0 == (chunk & (gIOPageAllocChunkBytes - 1)));
1415
1416 pa = (typeof(pa))(addr | (page_size - gIOPageAllocChunkBytes));
1417 assert(kIOPageAllocSignature == pa->signature);
1418
1419 count = (((uint32_t) bytes) + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes;
1420 chunk /= gIOPageAllocChunkBytes;
1421
1422 IOLockLock(a->lock);
1423 if (!pa->avail) {
1424 assert(!pa->link.next);
1425 enqueue_tail(&a->list, &pa->link);
1426 }
1427 pa->avail |= ((-1ULL << (64 - count)) >> chunk);
1428 if (pa->avail != -2ULL) {
1429 pa = NULL;
1430 } else {
1431 remque(&pa->link);
1432 pa->link.next = NULL;
1433 pa->signature = 0;
1434 a->pagecount--;
1435 // page to free
1436 pa = (typeof(pa))trunc_page(pa);
1437 }
1438 a->bytecount -= bytes;
1439 IOLockUnlock(a->lock);
1440
1441 return (uintptr_t) pa;
1442 }
1443
1444 #endif /* defined(__x86_64__) */
1445
1446 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1447
1448 IOReturn
IOSetProcessorCacheMode(task_t task,IOVirtualAddress address,IOByteCount length,IOOptionBits cacheMode)1449 IOSetProcessorCacheMode( task_t task, IOVirtualAddress address,
1450 IOByteCount length, IOOptionBits cacheMode )
1451 {
1452 IOReturn ret = kIOReturnSuccess;
1453 ppnum_t pagenum;
1454
1455 if (task != kernel_task) {
1456 return kIOReturnUnsupported;
1457 }
1458 if ((address | length) & PAGE_MASK) {
1459 // OSReportWithBacktrace("IOSetProcessorCacheMode(0x%x, 0x%x, 0x%x) fails\n", address, length, cacheMode);
1460 return kIOReturnUnsupported;
1461 }
1462 length = round_page(address + length) - trunc_page( address );
1463 address = trunc_page( address );
1464
1465 // make map mode
1466 cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask;
1467
1468 while ((kIOReturnSuccess == ret) && (length > 0)) {
1469 // Get the physical page number
1470 pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address);
1471 if (pagenum) {
1472 ret = IOUnmapPages( get_task_map(task), address, page_size );
1473 ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode );
1474 } else {
1475 ret = kIOReturnVMError;
1476 }
1477
1478 address += page_size;
1479 length -= page_size;
1480 }
1481
1482 return ret;
1483 }
1484
1485
1486 IOReturn
IOFlushProcessorCache(task_t task,IOVirtualAddress address,IOByteCount length)1487 IOFlushProcessorCache( task_t task, IOVirtualAddress address,
1488 IOByteCount length )
1489 {
1490 if (task != kernel_task) {
1491 return kIOReturnUnsupported;
1492 }
1493
1494 flush_dcache64((addr64_t) address, (unsigned) length, false );
1495
1496 return kIOReturnSuccess;
1497 }
1498
1499 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1500
1501 vm_offset_t
OSKernelStackRemaining(void)1502 OSKernelStackRemaining( void )
1503 {
1504 return ml_stack_remaining();
1505 }
1506
1507 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1508
1509 /*
1510 * Spin for indicated number of milliseconds.
1511 */
1512 void
IOSleep(unsigned milliseconds)1513 IOSleep(unsigned milliseconds)
1514 {
1515 delay_for_interval(milliseconds, kMillisecondScale);
1516 }
1517
1518 /*
1519 * Spin for indicated number of milliseconds, and potentially an
1520 * additional number of milliseconds up to the leeway values.
1521 */
1522 void
IOSleepWithLeeway(unsigned intervalMilliseconds,unsigned leewayMilliseconds)1523 IOSleepWithLeeway(unsigned intervalMilliseconds, unsigned leewayMilliseconds)
1524 {
1525 delay_for_interval_with_leeway(intervalMilliseconds, leewayMilliseconds, kMillisecondScale);
1526 }
1527
1528 /*
1529 * Spin for indicated number of microseconds.
1530 */
1531 void
IODelay(unsigned microseconds)1532 IODelay(unsigned microseconds)
1533 {
1534 delay_for_interval(microseconds, kMicrosecondScale);
1535 }
1536
1537 /*
1538 * Spin for indicated number of nanoseconds.
1539 */
1540 void
IOPause(unsigned nanoseconds)1541 IOPause(unsigned nanoseconds)
1542 {
1543 delay_for_interval(nanoseconds, kNanosecondScale);
1544 }
1545
1546 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1547
1548 static void _IOLogv(const char *format, va_list ap, void *caller) __printflike(1, 0);
1549
1550 __attribute__((noinline, not_tail_called))
1551 void
IOLog(const char * format,...)1552 IOLog(const char *format, ...)
1553 {
1554 void *caller = __builtin_return_address(0);
1555 va_list ap;
1556
1557 va_start(ap, format);
1558 _IOLogv(format, ap, caller);
1559 va_end(ap);
1560 }
1561
1562 __attribute__((noinline, not_tail_called))
1563 void
IOLogv(const char * format,va_list ap)1564 IOLogv(const char *format, va_list ap)
1565 {
1566 void *caller = __builtin_return_address(0);
1567 _IOLogv(format, ap, caller);
1568 }
1569
1570 void
_IOLogv(const char * format,va_list ap,void * caller)1571 _IOLogv(const char *format, va_list ap, void *caller)
1572 {
1573 va_list ap2;
1574 struct console_printbuf_state info_data;
1575 console_printbuf_state_init(&info_data, TRUE, TRUE);
1576
1577 va_copy(ap2, ap);
1578
1579 #pragma clang diagnostic push
1580 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1581 os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, format, ap, caller);
1582 #pragma clang diagnostic pop
1583
1584 if (!disable_iolog_serial_output) {
1585 __doprnt(format, ap2, console_printbuf_putc, &info_data, 16, TRUE);
1586 console_printbuf_clear(&info_data);
1587 }
1588 va_end(ap2);
1589
1590 assertf(ml_get_interrupts_enabled() || ml_is_quiescing() ||
1591 debug_mode_active() || !gCPUsRunning,
1592 "IOLog called with interrupts disabled");
1593 }
1594
1595 #if !__LP64__
1596 void
IOPanic(const char * reason)1597 IOPanic(const char *reason)
1598 {
1599 panic("%s", reason);
1600 }
1601 #endif
1602
1603 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1604
1605 void
IOKitKernelLogBuffer(const char * title,const void * buffer,size_t size,void (* output)(const char * format,...))1606 IOKitKernelLogBuffer(const char * title, const void * buffer, size_t size,
1607 void (*output)(const char *format, ...))
1608 {
1609 size_t idx, linestart;
1610 enum { bytelen = (sizeof("0xZZ, ") - 1) };
1611 char hex[(bytelen * 16) + 1];
1612 uint8_t c, chars[17];
1613
1614 output("%s(0x%lx):\n", title, size);
1615 output(" 0 1 2 3 4 5 6 7 8 9 A B C D E F\n");
1616 if (size > 4096) {
1617 size = 4096;
1618 }
1619 chars[16] = 0;
1620 for (idx = 0, linestart = 0; idx < size;) {
1621 c = ((char *)buffer)[idx];
1622 snprintf(&hex[bytelen * (idx & 15)], bytelen + 1, "0x%02x, ", c);
1623 chars[idx & 15] = ((c >= 0x20) && (c <= 0x7f)) ? c : ' ';
1624 idx++;
1625 if ((idx == size) || !(idx & 15)) {
1626 if (idx & 15) {
1627 chars[idx & 15] = 0;
1628 }
1629 output("/* %04lx: */ %-96s /* |%-16s| */\n", linestart, hex, chars);
1630 linestart += 16;
1631 }
1632 }
1633 }
1634
1635 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1636
1637 /*
1638 * Convert a integer constant (typically a #define or enum) to a string.
1639 */
1640 static char noValue[80]; // that's pretty
1641
1642 const char *
IOFindNameForValue(int value,const IONamedValue * regValueArray)1643 IOFindNameForValue(int value, const IONamedValue *regValueArray)
1644 {
1645 for (; regValueArray->name; regValueArray++) {
1646 if (regValueArray->value == value) {
1647 return regValueArray->name;
1648 }
1649 }
1650 snprintf(noValue, sizeof(noValue), "0x%x (UNDEFINED)", value);
1651 return (const char *)noValue;
1652 }
1653
1654 IOReturn
IOFindValueForName(const char * string,const IONamedValue * regValueArray,int * value)1655 IOFindValueForName(const char *string,
1656 const IONamedValue *regValueArray,
1657 int *value)
1658 {
1659 for (; regValueArray->name; regValueArray++) {
1660 if (!strcmp(regValueArray->name, string)) {
1661 *value = regValueArray->value;
1662 return kIOReturnSuccess;
1663 }
1664 }
1665 return kIOReturnBadArgument;
1666 }
1667
1668 OSString *
IOCopyLogNameForPID(int pid)1669 IOCopyLogNameForPID(int pid)
1670 {
1671 char buf[128];
1672 size_t len;
1673 snprintf(buf, sizeof(buf), "pid %d, ", pid);
1674 len = strlen(buf);
1675 proc_name(pid, buf + len, (int) (sizeof(buf) - len));
1676 return OSString::withCString(buf);
1677 }
1678
1679 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1680
1681 IOAlignment
IOSizeToAlignment(unsigned int size)1682 IOSizeToAlignment(unsigned int size)
1683 {
1684 int shift;
1685 const int intsize = sizeof(unsigned int) * 8;
1686
1687 for (shift = 1; shift < intsize; shift++) {
1688 if (size & 0x80000000) {
1689 return (IOAlignment)(intsize - shift);
1690 }
1691 size <<= 1;
1692 }
1693 return 0;
1694 }
1695
1696 unsigned int
IOAlignmentToSize(IOAlignment align)1697 IOAlignmentToSize(IOAlignment align)
1698 {
1699 unsigned int size;
1700
1701 for (size = 1; align; align--) {
1702 size <<= 1;
1703 }
1704 return size;
1705 }
1706 } /* extern "C" */
1707