1 /*
2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #define IOKIT_ENABLE_SHARED_PTR
29
30 #define _IOMEMORYDESCRIPTOR_INTERNAL_
31
32 #include <IOKit/assert.h>
33 #include <IOKit/system.h>
34
35 #include <IOKit/IOLib.h>
36 #include <IOKit/IOMapper.h>
37 #include <IOKit/IOBufferMemoryDescriptor.h>
38 #include <libkern/OSDebug.h>
39 #include <mach/mach_vm.h>
40
41 #include "IOKitKernelInternal.h"
42
43 #ifdef IOALLOCDEBUG
44 #include <libkern/c++/OSCPPDebug.h>
45 #endif
46 #include <IOKit/IOStatisticsPrivate.h>
47
48 #if IOKITSTATS
49 #define IOStatisticsAlloc(type, size) \
50 do { \
51 IOStatistics::countAlloc(type, size); \
52 } while (0)
53 #else
54 #define IOStatisticsAlloc(type, size)
55 #endif /* IOKITSTATS */
56
57
58 __BEGIN_DECLS
59 void ipc_port_release_send(ipc_port_t port);
60 #include <vm/pmap.h>
61
62 __END_DECLS
63
64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
65
66 enum{
67 kInternalFlagPhysical = 0x00000001,
68 kInternalFlagPageSized = 0x00000002,
69 kInternalFlagPageAllocated = 0x00000004,
70 kInternalFlagInit = 0x00000008,
71 kInternalFlagHasPointers = 0x00000010,
72 kInternalFlagGuardPages = 0x00000020,
73 };
74
75 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
76
77 #define super IOGeneralMemoryDescriptor
78 OSDefineMetaClassAndStructorsWithZone(IOBufferMemoryDescriptor,
79 IOGeneralMemoryDescriptor, ZC_ZFREE_CLEARMEM);
80
81 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
82
83 #if defined(__x86_64__)
84 static uintptr_t
IOBMDPageProc(kalloc_heap_t kheap,iopa_t * a)85 IOBMDPageProc(kalloc_heap_t kheap, iopa_t * a)
86 {
87 kern_return_t kr;
88 vm_address_t vmaddr = 0;
89 kma_flags_t kma_flags = KMA_ZERO;
90
91 if (kheap == KHEAP_DATA_BUFFERS) {
92 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
93 }
94 kr = kmem_alloc(kernel_map, &vmaddr, page_size,
95 kma_flags, VM_KERN_MEMORY_IOKIT);
96
97 if (KERN_SUCCESS != kr) {
98 vmaddr = 0;
99 }
100
101 return (uintptr_t) vmaddr;
102 }
103 #endif /* defined(__x86_64__) */
104
105 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
106
107 #ifndef __LP64__
108 bool
initWithOptions(IOOptionBits options,vm_size_t capacity,vm_offset_t alignment,task_t inTask)109 IOBufferMemoryDescriptor::initWithOptions(
110 IOOptionBits options,
111 vm_size_t capacity,
112 vm_offset_t alignment,
113 task_t inTask)
114 {
115 mach_vm_address_t physicalMask = 0;
116 return initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask);
117 }
118 #endif /* !__LP64__ */
119
120 OSSharedPtr<IOBufferMemoryDescriptor>
withCopy(task_t inTask,IOOptionBits options,vm_map_t sourceMap,mach_vm_address_t source,mach_vm_size_t size)121 IOBufferMemoryDescriptor::withCopy(
122 task_t inTask,
123 IOOptionBits options,
124 vm_map_t sourceMap,
125 mach_vm_address_t source,
126 mach_vm_size_t size)
127 {
128 OSSharedPtr<IOBufferMemoryDescriptor> inst;
129 kern_return_t err;
130 vm_map_copy_t copy;
131 vm_map_address_t address;
132
133 copy = NULL;
134 do {
135 err = kIOReturnNoMemory;
136 inst = OSMakeShared<IOBufferMemoryDescriptor>();
137 if (!inst) {
138 break;
139 }
140 inst->_ranges.v64 = IOMallocType(IOAddressRange);
141
142 err = vm_map_copyin(sourceMap, source, size,
143 false /* src_destroy */, ©);
144 if (KERN_SUCCESS != err) {
145 break;
146 }
147
148 err = vm_map_copyout(get_task_map(inTask), &address, copy);
149 if (KERN_SUCCESS != err) {
150 break;
151 }
152 copy = NULL;
153
154 inst->_ranges.v64->address = address;
155 inst->_ranges.v64->length = size;
156
157 if (!inst->initWithPhysicalMask(inTask, options, size, page_size, 0)) {
158 err = kIOReturnError;
159 }
160 } while (false);
161
162 if (KERN_SUCCESS == err) {
163 return inst;
164 }
165
166 if (copy) {
167 vm_map_copy_discard(copy);
168 }
169
170 return nullptr;
171 }
172
173
174 bool
initWithPhysicalMask(task_t inTask,IOOptionBits options,mach_vm_size_t capacity,mach_vm_address_t alignment,mach_vm_address_t physicalMask)175 IOBufferMemoryDescriptor::initWithPhysicalMask(
176 task_t inTask,
177 IOOptionBits options,
178 mach_vm_size_t capacity,
179 mach_vm_address_t alignment,
180 mach_vm_address_t physicalMask)
181 {
182 task_t mapTask = NULL;
183 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS;
184 mach_vm_address_t highestMask = 0;
185 IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
186 IODMAMapSpecification mapSpec;
187 bool mapped = false;
188 bool withCopy = false;
189 bool mappedOrShared = false;
190
191 if (!capacity) {
192 return false;
193 }
194
195 /*
196 * The IOKit constructor requests the allocator for zeroed memory
197 * so the members of the class do not need to be explicitly zeroed.
198 */
199 _options = options;
200 _capacity = capacity;
201
202 if (!_ranges.v64) {
203 _ranges.v64 = IOMallocType(IOAddressRange);
204 _ranges.v64->address = 0;
205 _ranges.v64->length = 0;
206 } else {
207 if (!_ranges.v64->address) {
208 return false;
209 }
210 if (!(kIOMemoryPageable & options)) {
211 return false;
212 }
213 if (!inTask) {
214 return false;
215 }
216 _buffer = (void *) _ranges.v64->address;
217 withCopy = true;
218 }
219
220 /*
221 * Set kalloc_heap to default if allocation contains pointers
222 */
223 if (kInternalFlagHasPointers & _internalFlags) {
224 kheap = KHEAP_DEFAULT;
225 }
226
227 // make sure super::free doesn't dealloc _ranges before super::init
228 _flags = kIOMemoryAsReference;
229
230 // Grab IOMD bits from the Buffer MD options
231 iomdOptions |= (options & kIOBufferDescriptorMemoryFlags);
232
233 if (!(kIOMemoryMapperNone & options)) {
234 IOMapper::checkForSystemMapper();
235 mapped = (NULL != IOMapper::gSystem);
236 }
237
238 if (physicalMask && (alignment <= 1)) {
239 alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
240 highestMask = (physicalMask | alignment);
241 alignment++;
242 if (alignment < page_size) {
243 alignment = page_size;
244 }
245 }
246
247 if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) {
248 alignment = page_size;
249 }
250
251 if (alignment >= page_size) {
252 if (round_page_overflow(capacity, &capacity)) {
253 return false;
254 }
255 }
256
257 if (alignment > page_size) {
258 options |= kIOMemoryPhysicallyContiguous;
259 }
260
261 _alignment = alignment;
262
263 if ((capacity + alignment) < _capacity) {
264 return false;
265 }
266
267 if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) {
268 return false;
269 }
270
271 bzero(&mapSpec, sizeof(mapSpec));
272 mapSpec.alignment = _alignment;
273 mapSpec.numAddressBits = 64;
274 if (highestMask && mapped) {
275 if (highestMask <= 0xFFFFFFFF) {
276 mapSpec.numAddressBits = (uint8_t)(32 - __builtin_clz((unsigned int) highestMask));
277 } else {
278 mapSpec.numAddressBits = (uint8_t)(64 - __builtin_clz((unsigned int) (highestMask >> 32)));
279 }
280 highestMask = 0;
281 }
282
283 // set memory entry cache mode, pageable, purgeable
284 iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
285 if (options & kIOMemoryPageable) {
286 if (_internalFlags & kInternalFlagGuardPages) {
287 printf("IOBMD: Unsupported use of guard pages with pageable memory.\n");
288 return false;
289 }
290 iomdOptions |= kIOMemoryBufferPageable;
291 if (options & kIOMemoryPurgeable) {
292 iomdOptions |= kIOMemoryBufferPurgeable;
293 }
294 } else {
295 // Buffer shouldn't auto prepare they should be prepared explicitly
296 // But it never was enforced so what are you going to do?
297 iomdOptions |= kIOMemoryAutoPrepare;
298
299 /* Allocate a wired-down buffer inside kernel space. */
300
301 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));
302
303 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) {
304 contig |= (!mapped);
305 contig |= (0 != (kIOMemoryMapperNone & options));
306 #if 0
307 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
308 contig |= true;
309 #endif
310 }
311
312 mappedOrShared = (mapped || (0 != (kIOMemorySharingTypeMask & options)));
313 if (contig || highestMask || (alignment > page_size)) {
314 if (_internalFlags & kInternalFlagGuardPages) {
315 printf("IOBMD: Unsupported use of guard pages with physical mask or contiguous memory.\n");
316 return false;
317 }
318 _internalFlags |= kInternalFlagPhysical;
319 if (highestMask) {
320 _internalFlags |= kInternalFlagPageSized;
321 if (round_page_overflow(capacity, &capacity)) {
322 return false;
323 }
324 }
325 _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(kheap,
326 capacity, highestMask, alignment, contig);
327 } else if (_internalFlags & kInternalFlagGuardPages) {
328 vm_offset_t address = 0;
329 kern_return_t kr;
330 uintptr_t alignMask;
331 kma_flags_t kma_flags = (kma_flags_t) (KMA_GUARD_FIRST |
332 KMA_GUARD_LAST | KMA_ZERO);
333
334 if (((uint32_t) alignment) != alignment) {
335 return NULL;
336 }
337 if (kheap == KHEAP_DATA_BUFFERS) {
338 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
339 }
340
341 alignMask = (1UL << log2up((uint32_t) alignment)) - 1;
342 kr = kernel_memory_allocate(kernel_map, &address,
343 capacity + page_size * 2, alignMask, kma_flags,
344 IOMemoryTag(kernel_map));
345 if (kr != KERN_SUCCESS || address == 0) {
346 return false;
347 }
348 #if IOALLOCDEBUG
349 OSAddAtomicLong(capacity, &debug_iomalloc_size);
350 #endif
351 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
352 _buffer = (void *)(address + page_size);
353 #if defined(__x86_64__)
354 } else if (mappedOrShared
355 && (capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)) {
356 _internalFlags |= kInternalFlagPageAllocated;
357 _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator,
358 &IOBMDPageProc, kheap, capacity, alignment);
359 if (_buffer) {
360 bzero(_buffer, capacity);
361 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
362 #if IOALLOCDEBUG
363 OSAddAtomicLong(capacity, &debug_iomalloc_size);
364 #endif
365 }
366 #endif /* defined(__x86_64__) */
367 } else if (alignment > 1) {
368 _buffer = IOMallocAligned_internal(kheap, capacity, alignment,
369 Z_ZERO_VM_TAG_BT_BIT);
370 } else {
371 _buffer = IOMalloc_internal(kheap, capacity, Z_ZERO_VM_TAG_BT_BIT);
372 }
373 if (!_buffer) {
374 return false;
375 }
376 }
377
378 if ((options & (kIOMemoryPageable | kIOMapCacheMask))) {
379 vm_size_t size = round_page(capacity);
380
381 // initWithOptions will create memory entry
382 if (!withCopy) {
383 iomdOptions |= kIOMemoryPersistent;
384 }
385
386 if (options & kIOMemoryPageable) {
387 #if IOALLOCDEBUG
388 OSAddAtomicLong(size, &debug_iomallocpageable_size);
389 #endif
390 if (!withCopy) {
391 mapTask = inTask;
392 }
393 if (NULL == inTask) {
394 inTask = kernel_task;
395 }
396 } else if (options & kIOMapCacheMask) {
397 // Prefetch each page to put entries into the pmap
398 volatile UInt8 * startAddr = (UInt8 *)_buffer;
399 volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity;
400
401 while (startAddr < endAddr) {
402 UInt8 dummyVar = *startAddr;
403 (void) dummyVar;
404 startAddr += page_size;
405 }
406 }
407 }
408
409 _ranges.v64->address = (mach_vm_address_t) pgz_decode(_buffer, _capacity);
410 _ranges.v64->length = _capacity;
411
412 if (!super::initWithOptions(_ranges.v64, 1, 0,
413 inTask, iomdOptions, /* System mapper */ NULL)) {
414 return false;
415 }
416
417 _internalFlags |= kInternalFlagInit;
418 #if IOTRACKING
419 if (!(options & kIOMemoryPageable)) {
420 trackingAccumSize(capacity);
421 }
422 #endif /* IOTRACKING */
423
424 // give any system mapper the allocation params
425 if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec,
426 &mapSpec, sizeof(mapSpec))) {
427 return false;
428 }
429
430 if (mapTask) {
431 if (!reserved) {
432 reserved = IOMallocType(ExpansionData);
433 if (!reserved) {
434 return false;
435 }
436 }
437 reserved->map = createMappingInTask(mapTask, 0,
438 kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0).detach();
439 if (!reserved->map) {
440 _buffer = NULL;
441 return false;
442 }
443 release(); // map took a retain on this
444 reserved->map->retain();
445 removeMapping(reserved->map);
446 mach_vm_address_t buffer = reserved->map->getAddress();
447 _buffer = (void *) buffer;
448 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) {
449 _ranges.v64->address = buffer;
450 }
451 }
452
453 setLength(_capacity);
454
455 return true;
456 }
457
458 bool
initControlWithPhysicalMask(task_t inTask,IOOptionBits options,mach_vm_size_t capacity,mach_vm_address_t alignment,mach_vm_address_t physicalMask)459 IOBufferMemoryDescriptor::initControlWithPhysicalMask(
460 task_t inTask,
461 IOOptionBits options,
462 mach_vm_size_t capacity,
463 mach_vm_address_t alignment,
464 mach_vm_address_t physicalMask)
465 {
466 _internalFlags = kInternalFlagHasPointers;
467 return initWithPhysicalMask(inTask, options, capacity, alignment,
468 physicalMask);
469 }
470
471 bool
initWithGuardPages(task_t inTask,IOOptionBits options,mach_vm_size_t capacity)472 IOBufferMemoryDescriptor::initWithGuardPages(
473 task_t inTask,
474 IOOptionBits options,
475 mach_vm_size_t capacity)
476 {
477 mach_vm_size_t roundedCapacity;
478
479 _internalFlags = kInternalFlagGuardPages;
480
481 if (round_page_overflow(capacity, &roundedCapacity)) {
482 return false;
483 }
484
485 return initWithPhysicalMask(inTask, options, roundedCapacity, page_size,
486 (mach_vm_address_t)0);
487 }
488
489 OSSharedPtr<IOBufferMemoryDescriptor>
inTaskWithOptions(task_t inTask,IOOptionBits options,vm_size_t capacity,vm_offset_t alignment)490 IOBufferMemoryDescriptor::inTaskWithOptions(
491 task_t inTask,
492 IOOptionBits options,
493 vm_size_t capacity,
494 vm_offset_t alignment)
495 {
496 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
497
498 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
499 me.reset();
500 }
501 return me;
502 }
503
504 OSSharedPtr<IOBufferMemoryDescriptor>
inTaskWithOptions(task_t inTask,IOOptionBits options,vm_size_t capacity,vm_offset_t alignment,uint32_t kernTag,uint32_t userTag)505 IOBufferMemoryDescriptor::inTaskWithOptions(
506 task_t inTask,
507 IOOptionBits options,
508 vm_size_t capacity,
509 vm_offset_t alignment,
510 uint32_t kernTag,
511 uint32_t userTag)
512 {
513 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
514
515 if (me) {
516 me->setVMTags(kernTag, userTag);
517
518 if (!me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
519 me.reset();
520 }
521 }
522 return me;
523 }
524
525 OSSharedPtr<IOBufferMemoryDescriptor>
inTaskWithPhysicalMask(task_t inTask,IOOptionBits options,mach_vm_size_t capacity,mach_vm_address_t physicalMask)526 IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
527 task_t inTask,
528 IOOptionBits options,
529 mach_vm_size_t capacity,
530 mach_vm_address_t physicalMask)
531 {
532 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
533
534 if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) {
535 me.reset();
536 }
537 return me;
538 }
539
540 OSSharedPtr<IOBufferMemoryDescriptor>
inTaskWithGuardPages(task_t inTask,IOOptionBits options,mach_vm_size_t capacity)541 IOBufferMemoryDescriptor::inTaskWithGuardPages(
542 task_t inTask,
543 IOOptionBits options,
544 mach_vm_size_t capacity)
545 {
546 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
547
548 if (me && !me->initWithGuardPages(inTask, options, capacity)) {
549 me.reset();
550 }
551 return me;
552 }
553
554 #ifndef __LP64__
555 bool
initWithOptions(IOOptionBits options,vm_size_t capacity,vm_offset_t alignment)556 IOBufferMemoryDescriptor::initWithOptions(
557 IOOptionBits options,
558 vm_size_t capacity,
559 vm_offset_t alignment)
560 {
561 return initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0);
562 }
563 #endif /* !__LP64__ */
564
565 OSSharedPtr<IOBufferMemoryDescriptor>
withOptions(IOOptionBits options,vm_size_t capacity,vm_offset_t alignment)566 IOBufferMemoryDescriptor::withOptions(
567 IOOptionBits options,
568 vm_size_t capacity,
569 vm_offset_t alignment)
570 {
571 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
572
573 if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
574 me.reset();
575 }
576 return me;
577 }
578
579
580 /*
581 * withCapacity:
582 *
583 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
584 * hold capacity bytes. The descriptor's length is initially set to the capacity.
585 */
586 OSSharedPtr<IOBufferMemoryDescriptor>
withCapacity(vm_size_t inCapacity,IODirection inDirection,bool inContiguous)587 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity,
588 IODirection inDirection,
589 bool inContiguous)
590 {
591 return IOBufferMemoryDescriptor::withOptions(
592 inDirection | kIOMemoryUnshared
593 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
594 inCapacity, inContiguous ? inCapacity : 1 );
595 }
596
597 #ifndef __LP64__
598 /*
599 * initWithBytes:
600 *
601 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
602 * The descriptor's length and capacity are set to the input buffer's size.
603 */
604 bool
initWithBytes(const void * inBytes,vm_size_t inLength,IODirection inDirection,bool inContiguous)605 IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
606 vm_size_t inLength,
607 IODirection inDirection,
608 bool inContiguous)
609 {
610 if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
611 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
612 inLength, inLength, (mach_vm_address_t)0)) {
613 return false;
614 }
615
616 // start out with no data
617 setLength(0);
618
619 if (!appendBytes(inBytes, inLength)) {
620 return false;
621 }
622
623 return true;
624 }
625 #endif /* !__LP64__ */
626
627 /*
628 * withBytes:
629 *
630 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
631 * The descriptor's length and capacity are set to the input buffer's size.
632 */
633 OSSharedPtr<IOBufferMemoryDescriptor>
withBytes(const void * inBytes,vm_size_t inLength,IODirection inDirection,bool inContiguous)634 IOBufferMemoryDescriptor::withBytes(const void * inBytes,
635 vm_size_t inLength,
636 IODirection inDirection,
637 bool inContiguous)
638 {
639 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
640
641 if (me && !me->initWithPhysicalMask(
642 kernel_task, inDirection | kIOMemoryUnshared
643 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
644 inLength, inLength, 0 )) {
645 me.reset();
646 }
647
648 if (me) {
649 // start out with no data
650 me->setLength(0);
651
652 if (!me->appendBytes(inBytes, inLength)) {
653 me.reset();
654 }
655 }
656 return me;
657 }
658
659 /*
660 * free:
661 *
662 * Free resources
663 */
664 void
free()665 IOBufferMemoryDescriptor::free()
666 {
667 // Cache all of the relevant information on the stack for use
668 // after we call super::free()!
669 IOOptionBits flags = _flags;
670 IOOptionBits internalFlags = _internalFlags;
671 IOOptionBits options = _options;
672 vm_size_t size = _capacity;
673 void * buffer = _buffer;
674 IOMemoryMap * map = NULL;
675 IOAddressRange * range = _ranges.v64;
676 vm_offset_t alignment = _alignment;
677 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS;
678
679 if (alignment >= page_size) {
680 size = round_page(size);
681 }
682
683 if (reserved) {
684 map = reserved->map;
685 IOFreeType(reserved, ExpansionData);
686 if (map) {
687 map->release();
688 }
689 }
690
691 if ((options & kIOMemoryPageable)
692 || (kInternalFlagPageSized & internalFlags)) {
693 size = round_page(size);
694 }
695
696 if (internalFlags & kInternalFlagHasPointers) {
697 kheap = KHEAP_DEFAULT;
698 }
699
700 #if IOTRACKING
701 if (!(options & kIOMemoryPageable)
702 && buffer
703 && (kInternalFlagInit & _internalFlags)) {
704 trackingAccumSize(-size);
705 }
706 #endif /* IOTRACKING */
707
708 /* super::free may unwire - deallocate buffer afterwards */
709 super::free();
710
711 if (options & kIOMemoryPageable) {
712 #if IOALLOCDEBUG
713 OSAddAtomicLong(-size, &debug_iomallocpageable_size);
714 #endif
715 } else if (buffer) {
716 if (kInternalFlagPhysical & internalFlags) {
717 IOKernelFreePhysical(kheap, (mach_vm_address_t) buffer, size);
718 } else if (kInternalFlagPageAllocated & internalFlags) {
719 #if defined(__x86_64__)
720 uintptr_t page;
721 page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
722 if (page) {
723 kmem_free(kernel_map, page, page_size);
724 }
725 #if IOALLOCDEBUG
726 OSAddAtomicLong(-size, &debug_iomalloc_size);
727 #endif
728 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
729 #else /* !defined(__x86_64__) */
730 /* should be unreachable */
731 panic("Attempting to free IOBMD with page allocated flag");
732 #endif /* defined(__x86_64__) */
733 } else if (kInternalFlagGuardPages & internalFlags) {
734 vm_offset_t allocation = (vm_offset_t)buffer - page_size;
735 kmem_free(kernel_map, allocation, size + page_size * 2);
736 #if IOALLOCDEBUG
737 OSAddAtomicLong(-size, &debug_iomalloc_size);
738 #endif
739 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
740 } else if (alignment > 1) {
741 IOFreeAligned_internal(kheap, buffer, size);
742 } else {
743 IOFree_internal(kheap, buffer, size);
744 }
745 }
746 if (range && (kIOMemoryAsReference & flags)) {
747 IOFreeType(range, IOAddressRange);
748 }
749 }
750
751 /*
752 * getCapacity:
753 *
754 * Get the buffer capacity
755 */
756 vm_size_t
getCapacity() const757 IOBufferMemoryDescriptor::getCapacity() const
758 {
759 return _capacity;
760 }
761
762 /*
763 * setLength:
764 *
765 * Change the buffer length of the memory descriptor. When a new buffer
766 * is created, the initial length of the buffer is set to be the same as
767 * the capacity. The length can be adjusted via setLength for a shorter
768 * transfer (there is no need to create more buffer descriptors when you
769 * can reuse an existing one, even for different transfer sizes). Note
770 * that the specified length must not exceed the capacity of the buffer.
771 */
772 void
setLength(vm_size_t length)773 IOBufferMemoryDescriptor::setLength(vm_size_t length)
774 {
775 assert(length <= _capacity);
776 if (length > _capacity) {
777 return;
778 }
779
780 _length = length;
781 _ranges.v64->length = length;
782 }
783
784 /*
785 * setDirection:
786 *
787 * Change the direction of the transfer. This method allows one to redirect
788 * the descriptor's transfer direction. This eliminates the need to destroy
789 * and create new buffers when different transfer directions are needed.
790 */
791 void
setDirection(IODirection direction)792 IOBufferMemoryDescriptor::setDirection(IODirection direction)
793 {
794 _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
795 #ifndef __LP64__
796 _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
797 #endif /* !__LP64__ */
798 }
799
800 /*
801 * appendBytes:
802 *
803 * Add some data to the end of the buffer. This method automatically
804 * maintains the memory descriptor buffer length. Note that appendBytes
805 * will not copy past the end of the memory descriptor's current capacity.
806 */
807 bool
appendBytes(const void * bytes,vm_size_t withLength)808 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
809 {
810 vm_size_t actualBytesToCopy = min(withLength, _capacity - _length);
811 IOByteCount offset;
812
813 assert(_length <= _capacity);
814
815 offset = _length;
816 _length += actualBytesToCopy;
817 _ranges.v64->length += actualBytesToCopy;
818
819 if (_task == kernel_task) {
820 bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
821 actualBytesToCopy);
822 } else {
823 writeBytes(offset, bytes, actualBytesToCopy);
824 }
825
826 return true;
827 }
828
829 /*
830 * getBytesNoCopy:
831 *
832 * Return the virtual address of the beginning of the buffer
833 */
834 void *
getBytesNoCopy()835 IOBufferMemoryDescriptor::getBytesNoCopy()
836 {
837 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
838 return _buffer;
839 } else {
840 return (void *)_ranges.v64->address;
841 }
842 }
843
844
845 /*
846 * getBytesNoCopy:
847 *
848 * Return the virtual address of an offset from the beginning of the buffer
849 */
850 void *
getBytesNoCopy(vm_size_t start,vm_size_t withLength)851 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
852 {
853 IOVirtualAddress address;
854
855 if ((start + withLength) < start) {
856 return NULL;
857 }
858
859 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
860 address = (IOVirtualAddress) _buffer;
861 } else {
862 address = _ranges.v64->address;
863 }
864
865 if (start < _length && (start + withLength) <= _length) {
866 return (void *)(address + start);
867 }
868 return NULL;
869 }
870
871 #ifndef __LP64__
872 void *
getVirtualSegment(IOByteCount offset,IOByteCount * lengthOfSegment)873 IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
874 IOByteCount * lengthOfSegment)
875 {
876 void * bytes = getBytesNoCopy(offset, 0);
877
878 if (bytes && lengthOfSegment) {
879 *lengthOfSegment = _length - offset;
880 }
881
882 return bytes;
883 }
884 #endif /* !__LP64__ */
885
886 #ifdef __LP64__
887 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
888 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
889 #else /* !__LP64__ */
890 OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 0);
891 OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 1);
892 #endif /* !__LP64__ */
893 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
894 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
895 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
896 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
897 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
898 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
899 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
900 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
901 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
902 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
903 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
904 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
905 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
906 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);
907