1 /*
2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/uio.h>
30 #include <sys/conf.h>
31
32 #include <IOKit/IOLib.h>
33 #include <IOKit/IOBSD.h>
34 #include <IOKit/IOService.h>
35 #include <IOKit/IOPlatformExpert.h>
36 #include <IOKit/IOPolledInterface.h>
37 #include <IOKit/IOHibernatePrivate.h>
38 #include <IOKit/IOBufferMemoryDescriptor.h>
39 #include <IOKit/AppleKeyStoreInterface.h>
40 #include <libkern/c++/OSSharedPtr.h>
41 #include "IOKitKernelInternal.h"
42
43 #if defined(__arm64__)
44 #include <pexpert/arm64/board_config.h>
45 #endif /* defined(__arm64__) */
46
47 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
48
49 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
50
51 OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
52 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
53 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
54 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
55 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
56 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
57 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
58 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
59 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
60 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
61 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
62 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
63 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
64 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
65 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
66 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
67
68 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
69
70 #ifndef kIOMediaPreferredBlockSizeKey
71 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
72 #endif
73
74 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
75
76 class IOPolledFilePollers : public OSObject
77 {
78 OSDeclareDefaultStructors(IOPolledFilePollers);
79
80 public:
81 IOService * media;
82 OSArray * pollers;
83 IOBufferMemoryDescriptor * ioBuffer;
84 bool abortable;
85 bool io;
86 IOReturn ioStatus;
87 uint32_t openCount;
88
89 static IOPolledFilePollers * copyPollers(IOService * media);
90 };
91
OSDefineMetaClassAndStructors(IOPolledFilePollers,OSObject)92 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
93
94 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
95
96 IOPolledFilePollers *
97 IOPolledFilePollers::copyPollers(IOService * media)
98 {
99 IOPolledFilePollers * vars;
100 IOReturn err;
101 IOService * service;
102 OSObject * obj;
103 IORegistryEntry * next;
104 IORegistryEntry * child;
105
106 if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
107 IOPolledFilePollers * ioPFPObj = OSDynamicCast(IOPolledFilePollers, obj);
108 if (!ioPFPObj) {
109 OSSafeReleaseNULL(obj);
110 }
111 return ioPFPObj;
112 }
113
114 do{
115 vars = OSTypeAlloc(IOPolledFilePollers);
116 vars->init();
117
118 vars->pollers = OSArray::withCapacity(4);
119 if (!vars->pollers) {
120 err = kIOReturnNoMemory;
121 break;
122 }
123
124 next = vars->media = media;
125 do{
126 IOPolledInterface * poller;
127 OSObject * obj;
128
129 obj = next->getProperty(kIOPolledInterfaceSupportKey);
130 if (kOSBooleanFalse == obj) {
131 vars->pollers->flushCollection();
132 break;
133 } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
134 vars->pollers->setObject(poller);
135 }
136
137 if ((service = OSDynamicCast(IOService, next))
138 && service->getDeviceMemory()
139 && !vars->pollers->getCount()) {
140 break;
141 }
142
143 child = next;
144 }while ((next = child->getParentEntry(gIOServicePlane))
145 && child->isParent(next, gIOServicePlane, true));
146
147 if (!vars->pollers->getCount()) {
148 err = kIOReturnUnsupported;
149 break;
150 }
151 }while (false);
152
153 media->setProperty(kIOPolledInterfaceStackKey, vars);
154
155 return vars;
156 }
157
158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
159
160 static IOReturn
161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
162
163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164
165 static IOReturn
IOPolledFilePollersProbe(IOPolledFilePollers * vars)166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
167 {
168 IOReturn err = kIOReturnError;
169 int32_t idx;
170 IOPolledInterface * poller;
171
172 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
173 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
174 err = poller->probe(vars->media);
175 if (err) {
176 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
177 break;
178 }
179 }
180
181 return err;
182 }
183
184 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
185
186 IOReturn
IOPolledFilePollersOpen(IOPolledFileIOVars * filevars,uint32_t state,bool abortable)187 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
188 {
189 IOPolledFilePollers * vars = filevars->pollers;
190 IOBufferMemoryDescriptor * ioBuffer;
191 IOPolledInterface * poller;
192 IOService * next;
193 IOReturn err = kIOReturnError;
194 int32_t idx;
195
196 vars->abortable = abortable;
197 ioBuffer = NULL;
198
199 if (kIOPolledAfterSleepState == state) {
200 vars->ioStatus = 0;
201 vars->io = false;
202 }
203 (void) IOPolledFilePollersIODone(vars, false);
204
205 if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
206 ioBuffer = vars->ioBuffer;
207 if (!ioBuffer) {
208 vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
209 kDefaultIONumBuffers * kDefaultIOSize, page_size);
210 if (!ioBuffer) {
211 return kIOReturnNoMemory;
212 }
213 }
214 }
215
216 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
217 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
218 err = poller->open(state, ioBuffer);
219 if (kIOReturnSuccess != err) {
220 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
221 break;
222 }
223 }
224 if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
225 next = vars->media;
226 while (next) {
227 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
228 next = next->getProvider();
229 }
230 }
231
232 return err;
233 }
234
235 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
236
237 IOReturn
IOPolledFilePollersClose(IOPolledFileIOVars * filevars,uint32_t state)238 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
239 {
240 IOPolledFilePollers * vars = filevars->pollers;
241 IOPolledInterface * poller;
242 IORegistryEntry * next;
243 IOReturn err;
244 int32_t idx;
245
246 (void) IOPolledFilePollersIODone(vars, false);
247
248 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
249 vars->openCount--;
250 }
251
252 for (idx = 0, err = kIOReturnSuccess;
253 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
254 idx++) {
255 err = poller->close(state);
256 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
257 err = poller->close(kIOPolledBeforeSleepState);
258 }
259 if (err) {
260 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
261 }
262 }
263
264 if (kIOPolledPostflightState == state) {
265 next = vars->media;
266 while (next) {
267 next->removeProperty(kIOPolledInterfaceActiveKey);
268 next = next->getParentEntry(gIOServicePlane);
269 }
270 }
271
272 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
273 do{
274 if (vars->openCount) {
275 break;
276 }
277 if (vars->ioBuffer) {
278 vars->ioBuffer->release();
279 vars->ioBuffer = NULL;
280 }
281 }while (false);
282 }
283
284 return err;
285 }
286
287 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
288
289 IOReturn
setEncryptionKey(const uint8_t * key,size_t keySize)290 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
291 {
292 return kIOReturnUnsupported;
293 }
294
295 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
296
297 IOReturn
IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,const uint8_t * key,size_t keySize)298 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
299 const uint8_t * key, size_t keySize)
300 {
301 IOReturn ret = kIOReturnUnsupported;
302 IOReturn err;
303 int32_t idx;
304 IOPolledFilePollers * vars = filevars->pollers;
305 IOPolledInterface * poller;
306
307 for (idx = 0;
308 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
309 idx++) {
310 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
311 err = poller->setEncryptionKey(key, keySize);
312 if (kIOReturnSuccess == err) {
313 ret = err;
314 }
315 }
316
317 return ret;
318 }
319
320 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
321
322 IOMemoryDescriptor *
IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)323 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
324 {
325 return vars->pollers->ioBuffer;
326 }
327
328 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
329
330 static void
IOPolledIOComplete(void * target,void * parameter,IOReturn status,UInt64 actualByteCount)331 IOPolledIOComplete(void * target,
332 void * parameter,
333 IOReturn status,
334 UInt64 actualByteCount)
335 {
336 IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
337
338 vars->ioStatus = status;
339 }
340
341 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
342
343 static IOReturn
IOStartPolledIO(IOPolledFilePollers * vars,uint32_t operation,uint32_t bufferOffset,uint64_t deviceOffset,uint64_t length)344 IOStartPolledIO(IOPolledFilePollers * vars,
345 uint32_t operation, uint32_t bufferOffset,
346 uint64_t deviceOffset, uint64_t length)
347 {
348 IOReturn err;
349 IOPolledInterface * poller;
350 IOPolledCompletion completion;
351
352 err = vars->ioStatus;
353 if (kIOReturnSuccess != err) {
354 return err;
355 }
356
357 completion.target = NULL;
358 completion.action = &IOPolledIOComplete;
359 completion.parameter = vars;
360
361 vars->ioStatus = -1;
362
363 poller = (IOPolledInterface *) vars->pollers->getObject(0);
364 err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
365 if (err) {
366 if (kernel_debugger_entry_count) {
367 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
368 } else {
369 HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
370 vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
371 }
372 }
373 return err;
374 }
375
376 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
377
378 static IOReturn
IOPolledFilePollersIODone(IOPolledFilePollers * vars,bool abortable)379 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
380 {
381 IOReturn err = kIOReturnSuccess;
382 int32_t idx = 0;
383 IOPolledInterface * poller;
384 AbsoluteTime deadline;
385
386 if (!vars->io) {
387 return kIOReturnSuccess;
388 }
389
390 abortable &= vars->abortable;
391
392 clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
393
394 while (-1 == vars->ioStatus) {
395 for (idx = 0;
396 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
397 idx++) {
398 IOReturn newErr;
399 newErr = poller->checkForWork();
400 if ((newErr == kIOReturnAborted) && !abortable) {
401 newErr = kIOReturnSuccess;
402 }
403 if (kIOReturnSuccess == err) {
404 err = newErr;
405 }
406 }
407 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
408 HIBLOG("IOPolledInterface::forced timeout\n");
409 vars->ioStatus = kIOReturnTimeout;
410 }
411 }
412 vars->io = false;
413
414 #if HIBERNATION
415 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
416 err = kIOReturnAborted;
417 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
418 }
419 #endif
420
421 if (err) {
422 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
423 } else {
424 err = vars->ioStatus;
425 if (kIOReturnSuccess != err) {
426 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
427 }
428 }
429
430 return err;
431 }
432
433 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435
436 struct _OpenFileContext {
437 OSData * extents;
438 uint64_t size;
439 };
440
441 static void
file_extent_callback(void * ref,uint64_t start,uint64_t length)442 file_extent_callback(void * ref, uint64_t start, uint64_t length)
443 {
444 _OpenFileContext * ctx = (_OpenFileContext *) ref;
445 IOPolledFileExtent extent;
446
447 extent.start = start;
448 extent.length = length;
449 ctx->extents->appendValue(extent);
450 ctx->size += length;
451 }
452
453 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
454
455 static IOService *
IOCopyMediaForDev(dev_t device)456 IOCopyMediaForDev(dev_t device)
457 {
458 OSDictionary * matching;
459 OSNumber * num;
460 OSIterator * iter;
461 IOService * result = NULL;
462
463 matching = IOService::serviceMatching("IOMedia");
464 if (!matching) {
465 return NULL;
466 }
467 do{
468 num = OSNumber::withNumber(major(device), 32);
469 if (!num) {
470 break;
471 }
472 matching->setObject(kIOBSDMajorKey, num);
473 num->release();
474 num = OSNumber::withNumber(minor(device), 32);
475 if (!num) {
476 break;
477 }
478 matching->setObject(kIOBSDMinorKey, num);
479 num->release();
480 if (!num) {
481 break;
482 }
483 iter = IOService::getMatchingServices(matching);
484 if (iter) {
485 result = (IOService *) iter->getNextObject();
486 result->retain();
487 iter->release();
488 }
489 }while (false);
490 matching->release();
491
492 return result;
493 }
494
495 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
496
497 #if defined(__i386__) || defined(__x86_64__)
498 #define APFSMEDIA_GETHIBERKEY "getHiberKey"
499
500 static IOReturn
IOGetVolumeCryptKey(dev_t block_dev,LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,uint8_t * volumeCryptKey,size_t * keySize)501 IOGetVolumeCryptKey(dev_t block_dev,
502 LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
503 uint8_t * volumeCryptKey,
504 size_t * keySize)
505 {
506 IOReturn err;
507 IOService * part;
508 OSString * keyUUID = NULL;
509 OSString * keyStoreUUID = NULL;
510 uuid_t volumeKeyUUID;
511 aks_volume_key_t vek;
512 size_t callerKeySize;
513
514 static IOService * sKeyStore;
515
516 part = IOCopyMediaForDev(block_dev);
517 if (!part) {
518 return kIOReturnNotFound;
519 }
520
521 callerKeySize = *keySize;
522 // Try APFS first
523 {
524 uuid_t volUuid = {0};
525 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
526 if (kIOReturnBadArgument == err) {
527 // apfs fails on buffer size >32
528 *keySize = 32;
529 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
530 }
531 if (err != kIOReturnSuccess) {
532 *keySize = 0;
533 } else {
534 // No need to create uuid string if it's not requested
535 if (pKeyUUID) {
536 uuid_string_t volUuidStr;
537 uuid_unparse(volUuid, volUuidStr);
538 *pKeyUUID = OSString::withCString(volUuidStr);
539 }
540
541 part->release();
542 return kIOReturnSuccess;
543 }
544 }
545
546 // Then old CS path
547 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
548 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
549 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
550 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
551
552 if (!sKeyStore) {
553 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
554 }
555 if (sKeyStore) {
556 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
557 } else {
558 err = kIOReturnNoResources;
559 }
560 if (kIOReturnSuccess == err) {
561 err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
562 }
563 if (kIOReturnSuccess != err) {
564 IOLog("volume key err 0x%x\n", err);
565 } else {
566 if (vek.key.keybytecount <= callerKeySize) {
567 *keySize = vek.key.keybytecount;
568 }
569 bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
570 }
571 bzero(&vek, sizeof(vek));
572
573 if (pKeyUUID) {
574 // Create a copy because the caller would release it
575 *pKeyUUID = OSString::withString(keyUUID);
576 }
577 }
578
579 part->release();
580 return err;
581 }
582 #endif /* defined(__i386__) || defined(__x86_64__) */
583
584 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
585
586 #if defined(__arm64__)
587 static IOReturn
IOGetHibernationCryptKey(uint8_t * hibernationKey,size_t * keySize,uint32_t * swSeed)588 IOGetHibernationCryptKey(uint8_t * hibernationKey,
589 size_t * keySize,
590 uint32_t *swSeed
591 )
592 {
593 return kIOReturnNotFound;
594 }
595 #endif /* defined(__arm64__) */
596
597 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
598
599 IOReturn
IOPolledFileOpen(const char * filename,uint32_t flags,uint64_t setFileSizeMin,uint64_t setFileSizeMax,uint64_t fsFreeSize,void * write_file_addr,size_t write_file_len,IOPolledFileIOVars ** fileVars,OSData ** imagePath,uint8_t * volumeCryptKey,size_t * keySize)600 IOPolledFileOpen(const char * filename,
601 uint32_t flags,
602 uint64_t setFileSizeMin, uint64_t setFileSizeMax, uint64_t fsFreeSize,
603 void * write_file_addr, size_t write_file_len,
604 IOPolledFileIOVars ** fileVars,
605 OSData ** imagePath,
606 uint8_t * volumeCryptKey, size_t * keySize)
607 {
608 IOReturn err = kIOReturnSuccess;
609 IOPolledFileIOVars * vars;
610 _OpenFileContext ctx;
611 OSData * extentsData = NULL;
612 OSNumber * num;
613 IOService * part = NULL;
614 dev_t block_dev;
615 dev_t image_dev;
616 AbsoluteTime startTime, endTime;
617 uint64_t nsec;
618
619 vars = IOMallocType(IOPolledFileIOVars);
620 vars->allocated = true;
621
622 do{
623 extentsData = OSData::withCapacity(32);
624 ctx.extents = extentsData;
625 ctx.size = 0;
626 clock_get_uptime(&startTime);
627
628 vars->fileRef = kern_open_file_for_direct_io(filename,
629 flags,
630 &file_extent_callback, &ctx,
631 setFileSizeMin,
632 setFileSizeMax,
633 fsFreeSize,
634 // write file:
635 0, write_file_addr, write_file_len,
636 // results
637 &block_dev,
638 &image_dev,
639 &vars->block0,
640 &vars->maxiobytes,
641 &vars->flags);
642 #if 0
643 uint32_t msDelay = (131071 & random());
644 HIBLOG("sleep %d\n", msDelay);
645 IOSleep(msDelay);
646 #endif
647 clock_get_uptime(&endTime);
648 SUB_ABSOLUTETIME(&endTime, &startTime);
649 absolutetime_to_nanoseconds(endTime, &nsec);
650
651 if (!vars->fileRef) {
652 err = kIOReturnNoSpace;
653 }
654
655 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
656 if (kIOReturnSuccess != err) {
657 break;
658 }
659
660 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
661 (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
662 vars->maxiobytes, kIOPolledFileSSD & vars->flags);
663 assert(!vars->block0);
664 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
665 err = kIOReturnNoSpace;
666 break;
667 }
668
669 vars->fileSizeMin = setFileSizeMin;
670 vars->fileSizeMax = setFileSizeMax;
671 vars->fileSize = ctx.size;
672 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
673
674 part = IOCopyMediaForDev(image_dev);
675 if (!part) {
676 err = kIOReturnNotFound;
677 break;
678 }
679
680 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
681 break;
682 }
683
684 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
685 vars->blockSize = num->unsigned32BitValue();
686 }
687 if (vars->blockSize < 4096) {
688 vars->blockSize = 4096;
689 }
690
691 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
692 major(image_dev), minor(image_dev), (long)vars->blockSize,
693 vars->pollers->pollers->getCount());
694
695 OSString * keyUUID = NULL;
696 #if defined(__i386__) || defined(__x86_64__)
697 if (volumeCryptKey) {
698 err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
699 }
700 #elif defined(__arm64__)
701 uint32_t swSeed = 0;
702 if (volumeCryptKey) {
703 if (flags & kIOPolledFileHibernate) {
704 err = IOGetHibernationCryptKey(volumeCryptKey, keySize, &swSeed);
705 if (kIOReturnSuccess != err) {
706 HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
707 break;
708 }
709 } else {
710 *keySize = 0;
711 }
712 }
713 #else
714 if (volumeCryptKey) {
715 HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
716 err = kIOReturnNotFound;
717 break;
718 }
719 #endif
720
721 *fileVars = vars;
722 vars->fileExtents = extentsData;
723
724 // make imagePath
725 OSData * data = NULL;
726 if (imagePath) {
727 #if defined(__i386__) || defined(__x86_64__)
728 char str2[24 + sizeof(uuid_string_t) + 2];
729
730 if (keyUUID) {
731 snprintf(str2, sizeof(str2), "%qx:%s",
732 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
733 } else {
734 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
735 }
736
737 err = IOService::getPlatform()->callPlatformFunction(
738 gIOCreateEFIDevicePathSymbol, false,
739 (void *) part, (void *) str2,
740 (void *) (uintptr_t) true, (void *) &data);
741 #elif defined(__arm64__)
742 char str2[26];
743 snprintf(str2, sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
744 data = OSData::withBytes(str2, (unsigned int) strlen(str2));
745 err = kIOReturnSuccess;
746 #else
747 err = kIOReturnNotFound;
748 #endif
749 if (kIOReturnSuccess != err) {
750 HIBLOG("error 0x%x getting path\n", err);
751 OSSafeReleaseNULL(keyUUID);
752 break;
753 }
754 *imagePath = data;
755 }
756
757 // Release key UUID if we have one
758 OSSafeReleaseNULL(keyUUID);
759 }while (false);
760
761 if (kIOReturnSuccess != err) {
762 HIBLOG("error 0x%x opening polled file\n", err);
763 IOPolledFileClose(&vars, 0, NULL, 0, 0, 0, false);
764 if (extentsData) {
765 extentsData->release();
766 }
767 }
768
769 if (part) {
770 part->release();
771 }
772
773 return err;
774 }
775
776 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
777
778 IOReturn
IOPolledFileClose(IOPolledFileIOVars ** pVars,off_t write_offset,void * addr,size_t write_length,off_t discard_offset,off_t discard_end,bool unlink)779 IOPolledFileClose(IOPolledFileIOVars ** pVars,
780 off_t write_offset, void * addr, size_t write_length,
781 off_t discard_offset, off_t discard_end, bool unlink)
782 {
783 IOPolledFileIOVars * vars;
784
785 vars = *pVars;
786 if (!vars) {
787 return kIOReturnSuccess;
788 }
789
790 if (vars->fileRef) {
791 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
792 discard_offset, discard_end, vars->fileSizeMin, unlink);
793 vars->fileRef = NULL;
794 }
795 if (vars->fileExtents) {
796 vars->fileExtents->release();
797 vars->fileExtents = NULL;
798 }
799 if (vars->pollers) {
800 vars->pollers->release();
801 vars->pollers = NULL;
802 }
803
804 if (vars->allocated) {
805 IOFreeType(vars, IOPolledFileIOVars);
806 } else {
807 bzero(vars, sizeof(IOPolledFileIOVars));
808 }
809 *pVars = NULL;
810
811 return kIOReturnSuccess;
812 }
813
814 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
815
816 IOReturn
IOPolledFilePollersSetup(IOPolledFileIOVars * vars,uint32_t openState)817 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
818 uint32_t openState)
819 {
820 IOReturn err;
821
822 err = kIOReturnSuccess;
823 do{
824 if (!vars->pollers->openCount) {
825 err = IOPolledFilePollersProbe(vars->pollers);
826 if (kIOReturnSuccess != err) {
827 break;
828 }
829 }
830 err = IOPolledFilePollersOpen(vars, openState, false);
831 if (kIOReturnSuccess != err) {
832 break;
833 }
834 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
835 vars->pollers->openCount++;
836 }
837 vars->pollers->io = false;
838 vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
839 vars->bufferHalf = 0;
840 vars->bufferOffset = 0;
841 assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
842 vars->bufferSize = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
843
844 if (vars->maxiobytes < vars->bufferSize) {
845 vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
846 }
847 }while (false);
848
849 if (kIOReturnSuccess != err) {
850 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
851 }
852
853 return err;
854 }
855
856
857 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
858
859 IOReturn
IOPolledFileSeek(IOPolledFileIOVars * vars,uint64_t position)860 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
861 {
862 IOPolledFileExtent * extentMap;
863
864 extentMap = vars->extentMap;
865
866 vars->position = position;
867
868 if (position > vars->fileSize) {
869 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
870 return kIOReturnNoSpace;
871 }
872
873 while (position >= extentMap->length) {
874 position -= extentMap->length;
875 extentMap++;
876 }
877
878 vars->currentExtent = extentMap;
879 vars->extentRemaining = extentMap->length - position;
880 vars->extentPosition = vars->position - position;
881
882 if (vars->bufferSize <= vars->extentRemaining) {
883 vars->bufferLimit = vars->bufferSize;
884 } else {
885 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
886 }
887
888 return kIOReturnSuccess;
889 }
890
891 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
892
893 IOReturn
IOPolledFileWrite(IOPolledFileIOVars * vars,const uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)894 IOPolledFileWrite(IOPolledFileIOVars * vars,
895 const uint8_t * bytes, IOByteCount size,
896 IOPolledFileCryptVars * cryptvars)
897 {
898 IOReturn err = kIOReturnSuccess;
899 IOByteCount copy, original_size = size;
900 bool flush = false;
901
902 do{
903 if (!bytes && !size) {
904 // seek to end of block & flush
905 size = vars->position & (vars->blockSize - 1);
906 if (size) {
907 size = vars->blockSize - size;
908 }
909 flush = true;
910 }
911
912 copy = vars->bufferLimit - vars->bufferOffset;
913 if (copy > size) {
914 copy = size;
915 } else {
916 flush = true;
917 }
918
919 if (bytes) {
920 #if KASAN
921 /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
922 * avoid triggering global redzone sanitizer violations when accessing
923 * interstices between 'C' structures
924 */
925 __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
926 #else
927 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
928 #endif
929 bytes += copy;
930 } else {
931 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
932 }
933
934 size -= copy;
935 vars->bufferOffset += copy;
936 vars->position += copy;
937
938 if (flush && vars->bufferOffset) {
939 uint64_t offset = (vars->position - vars->bufferOffset
940 - vars->extentPosition + vars->currentExtent->start);
941 uint32_t length = (vars->bufferOffset);
942
943 #if CRYPTO
944 if (cryptvars && vars->encryptStart
945 && (vars->position > vars->encryptStart)
946 && ((vars->position - length) < vars->encryptEnd)) {
947 AbsoluteTime startTime, endTime;
948
949 uint64_t encryptLen, encryptStart;
950 encryptLen = vars->position - vars->encryptStart;
951 if (encryptLen > length) {
952 encryptLen = length;
953 }
954 encryptStart = length - encryptLen;
955 if (vars->position > vars->encryptEnd) {
956 encryptLen -= (vars->position - vars->encryptEnd);
957 }
958
959 clock_get_uptime(&startTime);
960
961 assert(encryptLen <= UINT_MAX);
962 // encrypt the buffer
963 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
964 &cryptvars->aes_iv[0],
965 (unsigned int) (encryptLen / AES_BLOCK_SIZE),
966 vars->buffer + vars->bufferHalf + encryptStart,
967 &cryptvars->ctx.encrypt);
968
969 clock_get_uptime(&endTime);
970 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
971 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
972 vars->cryptBytes += encryptLen;
973
974 // save initial vector for following encrypts
975 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
976 &cryptvars->aes_iv[0],
977 AES_BLOCK_SIZE);
978 }
979 #endif /* CRYPTO */
980
981 err = IOPolledFilePollersIODone(vars->pollers, true);
982 if (kIOReturnSuccess != err) {
983 break;
984 }
985
986 if (vars->position & (vars->blockSize - 1)) {
987 HIBLOG("misaligned file pos %qx\n", vars->position);
988 }
989 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
990
991 err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
992 if (kIOReturnSuccess != err) {
993 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
994 vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
995 break;
996 }
997 vars->pollers->io = true;
998
999 vars->extentRemaining -= vars->bufferOffset;
1000 if (!vars->extentRemaining) {
1001 vars->currentExtent++;
1002 vars->extentRemaining = vars->currentExtent->length;
1003 vars->extentPosition = vars->position;
1004 }
1005
1006 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1007 vars->bufferOffset = 0;
1008 if (vars->bufferSize <= vars->extentRemaining) {
1009 vars->bufferLimit = vars->bufferSize;
1010 } else {
1011 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
1012 }
1013
1014 if (!vars->extentRemaining) {
1015 err = kIOReturnOverrun;
1016 break;
1017 }
1018
1019 flush = false;
1020 }
1021 }while (size);
1022
1023 return err;
1024 }
1025
1026 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1027
1028 IOReturn
IOPolledFileFlush(IOPolledFileIOVars * vars)1029 IOPolledFileFlush(IOPolledFileIOVars * vars)
1030 {
1031 // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1032 IOReturn err = kIOReturnSuccess;
1033
1034 err = IOPolledFilePollersIODone(vars->pollers, true);
1035 if (kIOReturnSuccess != err) {
1036 return err;
1037 }
1038
1039 err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
1040 if (kIOReturnSuccess != err) {
1041 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1042 vars, vars->pollers, err);
1043 return err;
1044 }
1045 vars->pollers->io = true;
1046
1047 return err;
1048 }
1049
1050 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1051
1052 IOReturn
IOPolledFileRead(IOPolledFileIOVars * vars,uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)1053 IOPolledFileRead(IOPolledFileIOVars * vars,
1054 uint8_t * bytes, IOByteCount size,
1055 IOPolledFileCryptVars * cryptvars)
1056 {
1057 IOReturn err = kIOReturnSuccess;
1058 IOByteCount copy;
1059
1060 // bytesWritten += size;
1061
1062 do{
1063 copy = vars->bufferLimit - vars->bufferOffset;
1064 if (copy > size) {
1065 copy = size;
1066 }
1067
1068 if (bytes) {
1069 #if KASAN
1070 __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1071 #else
1072 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1073 #endif
1074 bytes += copy;
1075 }
1076 size -= copy;
1077 vars->bufferOffset += copy;
1078 // vars->position += copy;
1079
1080 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1081 if (!vars->pollers->io) {
1082 cryptvars = NULL;
1083 }
1084 err = IOPolledFilePollersIODone(vars->pollers, true);
1085 if (kIOReturnSuccess != err) {
1086 break;
1087 }
1088
1089 if (vars->position & (vars->blockSize - 1)) {
1090 HIBLOG("misaligned file pos %qx\n", vars->position);
1091 }
1092
1093 vars->position += vars->lastRead;
1094 vars->extentRemaining -= vars->lastRead;
1095 vars->bufferLimit = vars->lastRead;
1096
1097 if (!vars->extentRemaining) {
1098 vars->currentExtent++;
1099 vars->extentRemaining = vars->currentExtent->length;
1100 vars->extentPosition = vars->position;
1101 if (!vars->extentRemaining) {
1102 err = kIOReturnOverrun;
1103 break;
1104 }
1105 }
1106
1107 uint32_t length;
1108 uint32_t lastReadLength = vars->lastRead;
1109 uint64_t offset = (vars->position
1110 - vars->extentPosition + vars->currentExtent->start);
1111 if (vars->extentRemaining <= vars->bufferSize) {
1112 length = ((uint32_t) vars->extentRemaining);
1113 } else {
1114 length = vars->bufferSize;
1115 }
1116 if ((length + vars->position) > vars->readEnd) {
1117 length = ((uint32_t) (vars->readEnd - vars->position));
1118 }
1119
1120 vars->lastRead = length;
1121 if (length) {
1122 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1123 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1124 if (kIOReturnSuccess != err) {
1125 break;
1126 }
1127 vars->pollers->io = true;
1128 }
1129
1130 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1131 vars->bufferOffset = 0;
1132
1133 #if CRYPTO
1134 if (cryptvars) {
1135 uint8_t thisVector[AES_BLOCK_SIZE];
1136 AbsoluteTime startTime, endTime;
1137
1138 // save initial vector for following decrypts
1139 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1140 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1141 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1142
1143 // decrypt the buffer
1144 clock_get_uptime(&startTime);
1145
1146 assert(lastReadLength <= UINT_MAX);
1147 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1148 &thisVector[0],
1149 (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1150 vars->buffer + vars->bufferHalf,
1151 &cryptvars->ctx.decrypt);
1152
1153 clock_get_uptime(&endTime);
1154 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1155 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1156 vars->cryptBytes += lastReadLength;
1157 }
1158 #endif /* CRYPTO */
1159 }
1160 }while (size);
1161
1162 return err;
1163 }
1164
1165 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1166