1 /*
2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/uio.h>
30 #include <sys/conf.h>
31
32 #include <IOKit/IOLib.h>
33 #include <IOKit/IOBSD.h>
34 #include <IOKit/IOService.h>
35 #include <IOKit/IOPlatformExpert.h>
36 #include <IOKit/IOPolledInterface.h>
37 #include <IOKit/IOHibernatePrivate.h>
38 #include <IOKit/IOBufferMemoryDescriptor.h>
39 #include <IOKit/AppleKeyStoreInterface.h>
40 #include <libkern/c++/OSSharedPtr.h>
41 #include "IOKitKernelInternal.h"
42
43 #if defined(__arm64__)
44 #include <pexpert/arm64/board_config.h>
45 #if XNU_MONITOR_PPL_HIB
46 #include <IOKit/SEPHibernator.h>
47 #endif /* XNU_MONITOR_PPL_HIB */
48 #endif /* defined(__arm64__) */
49
50 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
51
52 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
53
54 OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
55 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
56 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
57 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
58 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
59 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
60 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
61 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
62 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
63 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
64 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
65 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
66 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
67 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
68 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
69 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
70
71 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
72
73 #ifndef kIOMediaPreferredBlockSizeKey
74 #define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
75 #endif
76
77 enum { kDefaultIOSize = 128 * 1024 };
78
79 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80
81 class IOPolledFilePollers : public OSObject
82 {
83 OSDeclareDefaultStructors(IOPolledFilePollers);
84
85 public:
86 IOService * media;
87 OSArray * pollers;
88 IOBufferMemoryDescriptor * ioBuffer;
89 bool abortable;
90 bool io;
91 IOReturn ioStatus;
92 uint32_t openCount;
93
94 static IOPolledFilePollers * copyPollers(IOService * media);
95 };
96
OSDefineMetaClassAndStructors(IOPolledFilePollers,OSObject)97 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
98
99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
100
101 IOPolledFilePollers *
102 IOPolledFilePollers::copyPollers(IOService * media)
103 {
104 IOPolledFilePollers * vars;
105 IOReturn err;
106 IOService * service;
107 OSObject * obj;
108 IORegistryEntry * next;
109 IORegistryEntry * child;
110
111 if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
112 IOPolledFilePollers * ioPFPObj = OSDynamicCast(IOPolledFilePollers, obj);
113 if (!ioPFPObj) {
114 OSSafeReleaseNULL(obj);
115 }
116 return ioPFPObj;
117 }
118
119 do{
120 vars = OSTypeAlloc(IOPolledFilePollers);
121 vars->init();
122
123 vars->pollers = OSArray::withCapacity(4);
124 if (!vars->pollers) {
125 err = kIOReturnNoMemory;
126 break;
127 }
128
129 next = vars->media = media;
130 do{
131 IOPolledInterface * poller;
132 OSObject * obj;
133
134 obj = next->getProperty(kIOPolledInterfaceSupportKey);
135 if (kOSBooleanFalse == obj) {
136 vars->pollers->flushCollection();
137 break;
138 } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
139 vars->pollers->setObject(poller);
140 }
141
142 if ((service = OSDynamicCast(IOService, next))
143 && service->getDeviceMemory()
144 && !vars->pollers->getCount()) {
145 break;
146 }
147
148 child = next;
149 }while ((next = child->getParentEntry(gIOServicePlane))
150 && child->isParent(next, gIOServicePlane, true));
151
152 if (!vars->pollers->getCount()) {
153 err = kIOReturnUnsupported;
154 break;
155 }
156 }while (false);
157
158 media->setProperty(kIOPolledInterfaceStackKey, vars);
159
160 return vars;
161 }
162
163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164
165 static IOReturn
166 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
167
168 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
169
170 static IOReturn
IOPolledFilePollersProbe(IOPolledFilePollers * vars)171 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
172 {
173 IOReturn err = kIOReturnError;
174 int32_t idx;
175 IOPolledInterface * poller;
176
177 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
178 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
179 err = poller->probe(vars->media);
180 if (err) {
181 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
182 break;
183 }
184 }
185
186 return err;
187 }
188
189 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
190
191 IOReturn
IOPolledFilePollersOpen(IOPolledFileIOVars * filevars,uint32_t state,bool abortable)192 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
193 {
194 IOPolledFilePollers * vars = filevars->pollers;
195 IOBufferMemoryDescriptor * ioBuffer;
196 IOPolledInterface * poller;
197 IOService * next;
198 IOReturn err = kIOReturnError;
199 int32_t idx;
200
201 vars->abortable = abortable;
202 ioBuffer = NULL;
203
204 if (kIOPolledAfterSleepState == state) {
205 vars->ioStatus = 0;
206 vars->io = false;
207 }
208 (void) IOPolledFilePollersIODone(vars, false);
209
210 if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
211 ioBuffer = vars->ioBuffer;
212 if (!ioBuffer) {
213 vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
214 2 * kDefaultIOSize, page_size);
215 if (!ioBuffer) {
216 return kIOReturnNoMemory;
217 }
218 }
219 }
220
221 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
222 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
223 err = poller->open(state, ioBuffer);
224 if (kIOReturnSuccess != err) {
225 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
226 break;
227 }
228 }
229 if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
230 next = vars->media;
231 while (next) {
232 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
233 next = next->getProvider();
234 }
235 }
236
237 return err;
238 }
239
240 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241
242 IOReturn
IOPolledFilePollersClose(IOPolledFileIOVars * filevars,uint32_t state)243 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
244 {
245 IOPolledFilePollers * vars = filevars->pollers;
246 IOPolledInterface * poller;
247 IORegistryEntry * next;
248 IOReturn err;
249 int32_t idx;
250
251 (void) IOPolledFilePollersIODone(vars, false);
252
253 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
254 vars->openCount--;
255 }
256
257 for (idx = 0, err = kIOReturnSuccess;
258 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
259 idx++) {
260 err = poller->close(state);
261 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
262 err = poller->close(kIOPolledBeforeSleepState);
263 }
264 if (err) {
265 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
266 }
267 }
268
269 if (kIOPolledPostflightState == state) {
270 next = vars->media;
271 while (next) {
272 next->removeProperty(kIOPolledInterfaceActiveKey);
273 next = next->getParentEntry(gIOServicePlane);
274 }
275 }
276
277 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
278 do{
279 if (vars->openCount) {
280 break;
281 }
282 if (vars->ioBuffer) {
283 vars->ioBuffer->release();
284 vars->ioBuffer = NULL;
285 }
286 }while (false);
287 }
288
289 return err;
290 }
291
292 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
293
294 IOReturn
setEncryptionKey(const uint8_t * key,size_t keySize)295 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
296 {
297 return kIOReturnUnsupported;
298 }
299
300 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
301
302 IOReturn
IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,const uint8_t * key,size_t keySize)303 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
304 const uint8_t * key, size_t keySize)
305 {
306 IOReturn ret = kIOReturnUnsupported;
307 IOReturn err;
308 int32_t idx;
309 IOPolledFilePollers * vars = filevars->pollers;
310 IOPolledInterface * poller;
311
312 for (idx = 0;
313 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
314 idx++) {
315 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
316 err = poller->setEncryptionKey(key, keySize);
317 if (kIOReturnSuccess == err) {
318 ret = err;
319 }
320 }
321
322 return ret;
323 }
324
325 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
326
327 IOMemoryDescriptor *
IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)328 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
329 {
330 return vars->pollers->ioBuffer;
331 }
332
333 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
334
335 static void
IOPolledIOComplete(void * target,void * parameter,IOReturn status,UInt64 actualByteCount)336 IOPolledIOComplete(void * target,
337 void * parameter,
338 IOReturn status,
339 UInt64 actualByteCount)
340 {
341 IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
342
343 vars->ioStatus = status;
344 }
345
346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348 static IOReturn
IOStartPolledIO(IOPolledFilePollers * vars,uint32_t operation,uint32_t bufferOffset,uint64_t deviceOffset,uint64_t length)349 IOStartPolledIO(IOPolledFilePollers * vars,
350 uint32_t operation, uint32_t bufferOffset,
351 uint64_t deviceOffset, uint64_t length)
352 {
353 IOReturn err;
354 IOPolledInterface * poller;
355 IOPolledCompletion completion;
356
357 err = vars->ioStatus;
358 if (kIOReturnSuccess != err) {
359 return err;
360 }
361
362 completion.target = NULL;
363 completion.action = &IOPolledIOComplete;
364 completion.parameter = vars;
365
366 vars->ioStatus = -1;
367
368 poller = (IOPolledInterface *) vars->pollers->getObject(0);
369 err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
370 if (err) {
371 if (kernel_debugger_entry_count) {
372 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
373 } else {
374 HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
375 vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
376 }
377 }
378 return err;
379 }
380
381 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
382
383 static IOReturn
IOPolledFilePollersIODone(IOPolledFilePollers * vars,bool abortable)384 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
385 {
386 IOReturn err = kIOReturnSuccess;
387 int32_t idx = 0;
388 IOPolledInterface * poller;
389 AbsoluteTime deadline;
390
391 if (!vars->io) {
392 return kIOReturnSuccess;
393 }
394
395 abortable &= vars->abortable;
396
397 clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
398
399 while (-1 == vars->ioStatus) {
400 for (idx = 0;
401 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
402 idx++) {
403 IOReturn newErr;
404 newErr = poller->checkForWork();
405 if ((newErr == kIOReturnAborted) && !abortable) {
406 newErr = kIOReturnSuccess;
407 }
408 if (kIOReturnSuccess == err) {
409 err = newErr;
410 }
411 }
412 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
413 HIBLOG("IOPolledInterface::forced timeout\n");
414 vars->ioStatus = kIOReturnTimeout;
415 }
416 }
417 vars->io = false;
418
419 #if HIBERNATION
420 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
421 err = kIOReturnAborted;
422 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
423 }
424 #endif
425
426 if (err) {
427 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
428 } else {
429 err = vars->ioStatus;
430 if (kIOReturnSuccess != err) {
431 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
432 }
433 }
434
435 return err;
436 }
437
438 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
439 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
440
441 struct _OpenFileContext {
442 OSData * extents;
443 uint64_t size;
444 };
445
446 static void
file_extent_callback(void * ref,uint64_t start,uint64_t length)447 file_extent_callback(void * ref, uint64_t start, uint64_t length)
448 {
449 _OpenFileContext * ctx = (_OpenFileContext *) ref;
450 IOPolledFileExtent extent;
451
452 extent.start = start;
453 extent.length = length;
454 ctx->extents->appendBytes(&extent, sizeof(extent));
455 ctx->size += length;
456 }
457
458 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
459
460 static IOService *
IOCopyMediaForDev(dev_t device)461 IOCopyMediaForDev(dev_t device)
462 {
463 OSDictionary * matching;
464 OSNumber * num;
465 OSIterator * iter;
466 IOService * result = NULL;
467
468 matching = IOService::serviceMatching("IOMedia");
469 if (!matching) {
470 return NULL;
471 }
472 do{
473 num = OSNumber::withNumber(major(device), 32);
474 if (!num) {
475 break;
476 }
477 matching->setObject(kIOBSDMajorKey, num);
478 num->release();
479 num = OSNumber::withNumber(minor(device), 32);
480 if (!num) {
481 break;
482 }
483 matching->setObject(kIOBSDMinorKey, num);
484 num->release();
485 if (!num) {
486 break;
487 }
488 iter = IOService::getMatchingServices(matching);
489 if (iter) {
490 result = (IOService *) iter->getNextObject();
491 result->retain();
492 iter->release();
493 }
494 }while (false);
495 matching->release();
496
497 return result;
498 }
499
500 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
501
502 #if defined(__i386__) || defined(__x86_64__)
503 #define APFSMEDIA_GETHIBERKEY "getHiberKey"
504
505 static IOReturn
IOGetVolumeCryptKey(dev_t block_dev,LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,uint8_t * volumeCryptKey,size_t * keySize)506 IOGetVolumeCryptKey(dev_t block_dev,
507 LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
508 uint8_t * volumeCryptKey,
509 size_t * keySize)
510 {
511 IOReturn err;
512 IOService * part;
513 OSString * keyUUID = NULL;
514 OSString * keyStoreUUID = NULL;
515 uuid_t volumeKeyUUID;
516 aks_volume_key_t vek;
517 size_t callerKeySize;
518
519 static IOService * sKeyStore;
520
521 part = IOCopyMediaForDev(block_dev);
522 if (!part) {
523 return kIOReturnNotFound;
524 }
525
526 callerKeySize = *keySize;
527 // Try APFS first
528 {
529 uuid_t volUuid = {0};
530 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
531 if (kIOReturnBadArgument == err) {
532 // apfs fails on buffer size >32
533 *keySize = 32;
534 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
535 }
536 if (err != kIOReturnSuccess) {
537 *keySize = 0;
538 } else {
539 // No need to create uuid string if it's not requested
540 if (pKeyUUID) {
541 uuid_string_t volUuidStr;
542 uuid_unparse(volUuid, volUuidStr);
543 *pKeyUUID = OSString::withCString(volUuidStr);
544 }
545
546 part->release();
547 return kIOReturnSuccess;
548 }
549 }
550
551 // Then old CS path
552 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
553 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
554 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
555 // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
556
557 if (!sKeyStore) {
558 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
559 }
560 if (sKeyStore) {
561 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
562 } else {
563 err = kIOReturnNoResources;
564 }
565 if (kIOReturnSuccess == err) {
566 err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
567 }
568 if (kIOReturnSuccess != err) {
569 IOLog("volume key err 0x%x\n", err);
570 } else {
571 if (vek.key.keybytecount <= callerKeySize) {
572 *keySize = vek.key.keybytecount;
573 }
574 bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
575 }
576 bzero(&vek, sizeof(vek));
577
578 if (pKeyUUID) {
579 // Create a copy because the caller would release it
580 *pKeyUUID = OSString::withString(keyUUID);
581 }
582 }
583
584 part->release();
585 return err;
586 }
587 #endif /* defined(__i386__) || defined(__x86_64__) */
588
589 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
590
591 #if defined(__arm64__)
592 static IOReturn
IOGetHibernationCryptKey(uint8_t * hibernationKey,size_t * keySize,uint32_t * swSeed)593 IOGetHibernationCryptKey(uint8_t * hibernationKey,
594 size_t * keySize,
595 uint32_t *swSeed
596 )
597 {
598 return kIOReturnNotFound;
599 }
600 #endif /* defined(__arm64__) */
601
602 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
603
604 IOReturn
IOPolledFileOpen(const char * filename,uint32_t flags,uint64_t setFileSize,uint64_t fsFreeSize,void * write_file_addr,size_t write_file_len,IOPolledFileIOVars ** fileVars,OSData ** imagePath,uint8_t * volumeCryptKey,size_t * keySize)605 IOPolledFileOpen(const char * filename,
606 uint32_t flags,
607 uint64_t setFileSize, uint64_t fsFreeSize,
608 void * write_file_addr, size_t write_file_len,
609 IOPolledFileIOVars ** fileVars,
610 OSData ** imagePath,
611 uint8_t * volumeCryptKey, size_t * keySize)
612 {
613 IOReturn err = kIOReturnSuccess;
614 IOPolledFileIOVars * vars;
615 _OpenFileContext ctx;
616 OSData * extentsData = NULL;
617 OSNumber * num;
618 IOService * part = NULL;
619 dev_t block_dev;
620 dev_t image_dev;
621 AbsoluteTime startTime, endTime;
622 uint64_t nsec;
623
624 vars = IOMallocType(IOPolledFileIOVars);
625 vars->allocated = true;
626
627 do{
628 extentsData = OSData::withCapacity(32);
629 ctx.extents = extentsData;
630 ctx.size = 0;
631 clock_get_uptime(&startTime);
632
633 vars->fileRef = kern_open_file_for_direct_io(filename,
634 flags,
635 &file_extent_callback, &ctx,
636 setFileSize,
637 fsFreeSize,
638 // write file:
639 0, write_file_addr, write_file_len,
640 // results
641 &block_dev,
642 &image_dev,
643 &vars->block0,
644 &vars->maxiobytes,
645 &vars->flags);
646 #if 0
647 uint32_t msDelay = (131071 & random());
648 HIBLOG("sleep %d\n", msDelay);
649 IOSleep(msDelay);
650 #endif
651 clock_get_uptime(&endTime);
652 SUB_ABSOLUTETIME(&endTime, &startTime);
653 absolutetime_to_nanoseconds(endTime, &nsec);
654
655 if (!vars->fileRef) {
656 err = kIOReturnNoSpace;
657 }
658
659 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
660 if (kIOReturnSuccess != err) {
661 break;
662 }
663
664 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
665 (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
666 vars->maxiobytes, kIOPolledFileSSD & vars->flags);
667 assert(!vars->block0);
668 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
669 err = kIOReturnNoSpace;
670 break;
671 }
672
673 vars->fileSize = ctx.size;
674 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
675
676 part = IOCopyMediaForDev(image_dev);
677 if (!part) {
678 err = kIOReturnNotFound;
679 break;
680 }
681
682 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
683 break;
684 }
685
686 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
687 vars->blockSize = num->unsigned32BitValue();
688 }
689 if (vars->blockSize < 4096) {
690 vars->blockSize = 4096;
691 }
692
693 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
694 major(image_dev), minor(image_dev), (long)vars->blockSize,
695 vars->pollers->pollers->getCount());
696
697 OSString * keyUUID = NULL;
698 #if defined(__i386__) || defined(__x86_64__)
699 if (volumeCryptKey) {
700 err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
701 }
702 #elif defined(__arm64__)
703 uint32_t swSeed = 0;
704 if (volumeCryptKey) {
705 if (flags & kIOPolledFileHibernate) {
706 err = IOGetHibernationCryptKey(volumeCryptKey, keySize, &swSeed);
707 if (kIOReturnSuccess != err) {
708 HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
709 break;
710 }
711 } else {
712 *keySize = 0;
713 }
714 }
715 #else
716 if (volumeCryptKey) {
717 HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
718 err = kIOReturnNotFound;
719 break;
720 }
721 #endif
722
723 *fileVars = vars;
724 vars->fileExtents = extentsData;
725
726 // make imagePath
727 OSData * data = NULL;
728 if (imagePath) {
729 #if defined(__i386__) || defined(__x86_64__)
730 char str2[24 + sizeof(uuid_string_t) + 2];
731
732 if (keyUUID) {
733 snprintf(str2, sizeof(str2), "%qx:%s",
734 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
735 } else {
736 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
737 }
738
739 err = IOService::getPlatform()->callPlatformFunction(
740 gIOCreateEFIDevicePathSymbol, false,
741 (void *) part, (void *) str2,
742 (void *) (uintptr_t) true, (void *) &data);
743 #elif defined(__arm64__)
744 char str2[26];
745 snprintf(str2, sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
746 data = OSData::withBytes(str2, (unsigned int) strlen(str2));
747 err = kIOReturnSuccess;
748 #else
749 err = kIOReturnNotFound;
750 #endif
751 if (kIOReturnSuccess != err) {
752 HIBLOG("error 0x%x getting path\n", err);
753 OSSafeReleaseNULL(keyUUID);
754 break;
755 }
756 *imagePath = data;
757 }
758
759 // Release key UUID if we have one
760 OSSafeReleaseNULL(keyUUID);
761 }while (false);
762
763 if (kIOReturnSuccess != err) {
764 HIBLOG("error 0x%x opening polled file\n", err);
765 IOPolledFileClose(&vars, 0, NULL, 0, 0, 0);
766 if (extentsData) {
767 extentsData->release();
768 }
769 }
770
771 if (part) {
772 part->release();
773 }
774
775 return err;
776 }
777
778 IOReturn
IOPolledFileOpen(const char * filename,uint32_t flags,uint64_t setFileSize,uint64_t fsFreeSize,void * write_file_addr,size_t write_file_len,IOPolledFileIOVars ** fileVars,OSSharedPtr<OSData> & imagePath,uint8_t * volumeCryptKey,size_t * keySize)779 IOPolledFileOpen(const char * filename,
780 uint32_t flags,
781 uint64_t setFileSize, uint64_t fsFreeSize,
782 void * write_file_addr, size_t write_file_len,
783 IOPolledFileIOVars ** fileVars,
784 OSSharedPtr<OSData>& imagePath,
785 uint8_t * volumeCryptKey, size_t * keySize)
786 {
787 OSData* imagePathRaw = NULL;
788 IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len,
789 fileVars, &imagePathRaw, volumeCryptKey, keySize);
790 imagePath.reset(imagePathRaw, OSNoRetain);
791 return result;
792 }
793
794 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
795
796 IOReturn
IOPolledFileClose(IOPolledFileIOVars ** pVars,off_t write_offset,void * addr,size_t write_length,off_t discard_offset,off_t discard_end)797 IOPolledFileClose(IOPolledFileIOVars ** pVars,
798 off_t write_offset, void * addr, size_t write_length,
799 off_t discard_offset, off_t discard_end)
800 {
801 IOPolledFileIOVars * vars;
802
803 vars = *pVars;
804 if (!vars) {
805 return kIOReturnSuccess;
806 }
807
808 if (vars->fileRef) {
809 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
810 discard_offset, discard_end);
811 vars->fileRef = NULL;
812 }
813 if (vars->fileExtents) {
814 vars->fileExtents->release();
815 vars->fileExtents = NULL;
816 }
817 if (vars->pollers) {
818 vars->pollers->release();
819 vars->pollers = NULL;
820 }
821
822 if (vars->allocated) {
823 IOFreeType(vars, IOPolledFileIOVars);
824 } else {
825 bzero(vars, sizeof(IOPolledFileIOVars));
826 }
827 *pVars = NULL;
828
829 return kIOReturnSuccess;
830 }
831
832 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
833
834 IOReturn
IOPolledFilePollersSetup(IOPolledFileIOVars * vars,uint32_t openState)835 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
836 uint32_t openState)
837 {
838 IOReturn err;
839
840 err = kIOReturnSuccess;
841 do{
842 if (!vars->pollers->openCount) {
843 err = IOPolledFilePollersProbe(vars->pollers);
844 if (kIOReturnSuccess != err) {
845 break;
846 }
847 }
848 err = IOPolledFilePollersOpen(vars, openState, false);
849 if (kIOReturnSuccess != err) {
850 break;
851 }
852 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
853 vars->pollers->openCount++;
854 }
855 vars->pollers->io = false;
856 vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
857 vars->bufferHalf = 0;
858 vars->bufferOffset = 0;
859 assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
860 vars->bufferSize = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
861
862 if (vars->maxiobytes < vars->bufferSize) {
863 vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
864 }
865 }while (false);
866
867 if (kIOReturnSuccess != err) {
868 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
869 }
870
871 return err;
872 }
873
874
875 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
876
877 IOReturn
IOPolledFileSeek(IOPolledFileIOVars * vars,uint64_t position)878 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
879 {
880 IOPolledFileExtent * extentMap;
881
882 extentMap = vars->extentMap;
883
884 vars->position = position;
885
886 if (position > vars->fileSize) {
887 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
888 return kIOReturnNoSpace;
889 }
890
891 while (position >= extentMap->length) {
892 position -= extentMap->length;
893 extentMap++;
894 }
895
896 vars->currentExtent = extentMap;
897 vars->extentRemaining = extentMap->length - position;
898 vars->extentPosition = vars->position - position;
899
900 if (vars->bufferSize <= vars->extentRemaining) {
901 vars->bufferLimit = vars->bufferSize;
902 } else {
903 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
904 }
905
906 return kIOReturnSuccess;
907 }
908
909 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
910
911 IOReturn
IOPolledFileWrite(IOPolledFileIOVars * vars,const uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)912 IOPolledFileWrite(IOPolledFileIOVars * vars,
913 const uint8_t * bytes, IOByteCount size,
914 IOPolledFileCryptVars * cryptvars)
915 {
916 IOReturn err = kIOReturnSuccess;
917 IOByteCount copy, original_size = size;
918 bool flush = false;
919
920 do{
921 if (!bytes && !size) {
922 // seek to end of block & flush
923 size = vars->position & (vars->blockSize - 1);
924 if (size) {
925 size = vars->blockSize - size;
926 }
927 flush = true;
928 }
929
930 copy = vars->bufferLimit - vars->bufferOffset;
931 if (copy > size) {
932 copy = size;
933 } else {
934 flush = true;
935 }
936
937 if (bytes) {
938 #if KASAN
939 /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
940 * avoid triggering global redzone sanitizer violations when accessing
941 * interstices between 'C' structures
942 */
943 __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
944 #else
945 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
946 #endif
947 bytes += copy;
948 } else {
949 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
950 }
951
952 size -= copy;
953 vars->bufferOffset += copy;
954 vars->position += copy;
955
956 if (flush && vars->bufferOffset) {
957 uint64_t offset = (vars->position - vars->bufferOffset
958 - vars->extentPosition + vars->currentExtent->start);
959 uint32_t length = (vars->bufferOffset);
960
961 #if CRYPTO
962 if (cryptvars && vars->encryptStart
963 && (vars->position > vars->encryptStart)
964 && ((vars->position - length) < vars->encryptEnd)) {
965 AbsoluteTime startTime, endTime;
966
967 uint64_t encryptLen, encryptStart;
968 encryptLen = vars->position - vars->encryptStart;
969 if (encryptLen > length) {
970 encryptLen = length;
971 }
972 encryptStart = length - encryptLen;
973 if (vars->position > vars->encryptEnd) {
974 encryptLen -= (vars->position - vars->encryptEnd);
975 }
976
977 clock_get_uptime(&startTime);
978
979 assert(encryptLen <= UINT_MAX);
980 // encrypt the buffer
981 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
982 &cryptvars->aes_iv[0],
983 (unsigned int) (encryptLen / AES_BLOCK_SIZE),
984 vars->buffer + vars->bufferHalf + encryptStart,
985 &cryptvars->ctx.encrypt);
986
987 clock_get_uptime(&endTime);
988 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
989 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
990 vars->cryptBytes += encryptLen;
991
992 // save initial vector for following encrypts
993 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
994 &cryptvars->aes_iv[0],
995 AES_BLOCK_SIZE);
996 }
997 #endif /* CRYPTO */
998
999 err = IOPolledFilePollersIODone(vars->pollers, true);
1000 if (kIOReturnSuccess != err) {
1001 break;
1002 }
1003
1004 if (vars->position & (vars->blockSize - 1)) {
1005 HIBLOG("misaligned file pos %qx\n", vars->position);
1006 }
1007 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1008
1009 err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
1010 if (kIOReturnSuccess != err) {
1011 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
1012 vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
1013 break;
1014 }
1015 vars->pollers->io = true;
1016
1017 vars->extentRemaining -= vars->bufferOffset;
1018 if (!vars->extentRemaining) {
1019 vars->currentExtent++;
1020 vars->extentRemaining = vars->currentExtent->length;
1021 vars->extentPosition = vars->position;
1022 }
1023
1024 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1025 vars->bufferOffset = 0;
1026 if (vars->bufferSize <= vars->extentRemaining) {
1027 vars->bufferLimit = vars->bufferSize;
1028 } else {
1029 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
1030 }
1031
1032 if (!vars->extentRemaining) {
1033 err = kIOReturnOverrun;
1034 break;
1035 }
1036
1037 flush = false;
1038 }
1039 }while (size);
1040
1041 return err;
1042 }
1043
1044 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1045
1046 IOReturn
IOPolledFileFlush(IOPolledFileIOVars * vars)1047 IOPolledFileFlush(IOPolledFileIOVars * vars)
1048 {
1049 // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1050 IOReturn err = kIOReturnSuccess;
1051
1052 err = IOPolledFilePollersIODone(vars->pollers, true);
1053 if (kIOReturnSuccess != err) {
1054 return err;
1055 }
1056
1057 err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
1058 if (kIOReturnSuccess != err) {
1059 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1060 vars, vars->pollers, err);
1061 return err;
1062 }
1063 vars->pollers->io = true;
1064
1065 return err;
1066 }
1067
1068 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1069
1070 IOReturn
IOPolledFileRead(IOPolledFileIOVars * vars,uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)1071 IOPolledFileRead(IOPolledFileIOVars * vars,
1072 uint8_t * bytes, IOByteCount size,
1073 IOPolledFileCryptVars * cryptvars)
1074 {
1075 IOReturn err = kIOReturnSuccess;
1076 IOByteCount copy;
1077
1078 // bytesWritten += size;
1079
1080 do{
1081 copy = vars->bufferLimit - vars->bufferOffset;
1082 if (copy > size) {
1083 copy = size;
1084 }
1085
1086 if (bytes) {
1087 #if KASAN
1088 __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1089 #else
1090 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1091 #endif
1092 bytes += copy;
1093 }
1094 size -= copy;
1095 vars->bufferOffset += copy;
1096 // vars->position += copy;
1097
1098 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1099 if (!vars->pollers->io) {
1100 cryptvars = NULL;
1101 }
1102 err = IOPolledFilePollersIODone(vars->pollers, true);
1103 if (kIOReturnSuccess != err) {
1104 break;
1105 }
1106
1107 if (vars->position & (vars->blockSize - 1)) {
1108 HIBLOG("misaligned file pos %qx\n", vars->position);
1109 }
1110
1111 vars->position += vars->lastRead;
1112 vars->extentRemaining -= vars->lastRead;
1113 vars->bufferLimit = vars->lastRead;
1114
1115 if (!vars->extentRemaining) {
1116 vars->currentExtent++;
1117 vars->extentRemaining = vars->currentExtent->length;
1118 vars->extentPosition = vars->position;
1119 if (!vars->extentRemaining) {
1120 err = kIOReturnOverrun;
1121 break;
1122 }
1123 }
1124
1125 uint32_t length;
1126 uint32_t lastReadLength = vars->lastRead;
1127 uint64_t offset = (vars->position
1128 - vars->extentPosition + vars->currentExtent->start);
1129 if (vars->extentRemaining <= vars->bufferSize) {
1130 length = ((uint32_t) vars->extentRemaining);
1131 } else {
1132 length = vars->bufferSize;
1133 }
1134 if ((length + vars->position) > vars->readEnd) {
1135 length = ((uint32_t) (vars->readEnd - vars->position));
1136 }
1137
1138 vars->lastRead = length;
1139 if (length) {
1140 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1141 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1142 if (kIOReturnSuccess != err) {
1143 break;
1144 }
1145 vars->pollers->io = true;
1146 }
1147
1148 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1149 vars->bufferOffset = 0;
1150
1151 #if CRYPTO
1152 if (cryptvars) {
1153 uint8_t thisVector[AES_BLOCK_SIZE];
1154 AbsoluteTime startTime, endTime;
1155
1156 // save initial vector for following decrypts
1157 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1158 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1159 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1160
1161 // decrypt the buffer
1162 clock_get_uptime(&startTime);
1163
1164 assert(lastReadLength <= UINT_MAX);
1165 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1166 &thisVector[0],
1167 (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1168 vars->buffer + vars->bufferHalf,
1169 &cryptvars->ctx.decrypt);
1170
1171 clock_get_uptime(&endTime);
1172 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1173 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1174 vars->cryptBytes += lastReadLength;
1175 }
1176 #endif /* CRYPTO */
1177 }
1178 }while (size);
1179
1180 return err;
1181 }
1182
1183 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1184