1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #if defined(__i386__) || defined(__x86_64__)
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <vm/WKdm_new.h>
179 #elif defined(__arm64__)
180 #include <arm64/amcc_rorgn.h>
181 #include <kern/ecc.h>
182 #endif /* defined(__i386__) || defined(__x86_64__) */
183 #include <san/kasan.h>
184
185
186 extern "C" addr64_t kvtophys(vm_offset_t va);
187 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
188
189 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
190
191 #define DISABLE_TRIM 0
192 #define TRIM_DELAY 25000
193
194 extern unsigned int save_kdebug_enable;
195 extern uint32_t gIOHibernateState;
196 uint32_t gIOHibernateMode;
197 static char gIOHibernateBootSignature[256 + 1];
198 static char gIOHibernateFilename[MAXPATHLEN + 1];
199 uint32_t gIOHibernateCount;
200
201 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
202
203 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
204 uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
205
206 enum {
207 HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
208 HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
209 };
210
211 #if defined(__arm64__)
212 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
213 #else
214 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
215 #endif /* __arm64__ */
216 boolean_t gIOHibernateStandbyDisabled;
217
218 static IODTNVRAM * gIOOptionsEntry;
219 static IORegistryEntry * gIOChosenEntry;
220
221 static const OSSymbol * gIOHibernateBootImageKey;
222 static const OSSymbol * gIOHibernateBootSignatureKey;
223 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
224
225 #if defined(__i386__) || defined(__x86_64__)
226
227 static const OSSymbol * gIOHibernateRTCVariablesKey;
228 static const OSSymbol * gIOHibernateBoot0082Key;
229 static const OSSymbol * gIOHibernateBootNextKey;
230 static OSData * gIOHibernateBoot0082Data;
231 static OSData * gIOHibernateBootNextData;
232 static OSObject * gIOHibernateBootNextSave;
233
234 #endif /* defined(__i386__) || defined(__x86_64__) */
235
236 static IOLock * gFSLock;
237 uint32_t gFSState;
238 static thread_call_t gIOHibernateTrimCalloutEntry;
239 static IOPolledFileIOVars gFileVars;
240 static IOHibernateVars gIOHibernateVars;
241 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
242 static hibernate_graphics_t _hibernateGraphics;
243 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
244 static hibernate_statistics_t _hibernateStats;
245 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
246
247 enum{
248 kFSIdle = 0,
249 kFSOpening = 2,
250 kFSOpened = 3,
251 kFSTimedOut = 4,
252 kFSTrimDelay = 5
253 };
254
255 static IOReturn IOHibernateDone(IOHibernateVars * vars);
256 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
257 static void IOSetBootImageNVRAM(OSData * data);
258 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
259
260 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
261
262 enum { kDefaultIOSize = 128 * 1024 };
263 enum { kVideoMapSize = 80 * 1024 * 1024 };
264
265 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
266
267 // copy from phys addr to MD
268
269 static IOReturn
IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)270 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
271 IOByteCount offset, addr64_t bytes, IOByteCount length)
272 {
273 addr64_t srcAddr = bytes;
274 IOByteCount remaining;
275
276 remaining = length = min(length, md->getLength() - offset);
277 while (remaining) { // (process another target segment?)
278 addr64_t dstAddr64;
279 IOByteCount dstLen;
280
281 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
282 if (!dstAddr64) {
283 break;
284 }
285
286 // Clip segment length to remaining
287 if (dstLen > remaining) {
288 dstLen = remaining;
289 }
290
291 #if 1
292 bcopy_phys(srcAddr, dstAddr64, dstLen);
293 #else
294 copypv(srcAddr, dstAddr64, dstLen,
295 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
296 #endif
297 srcAddr += dstLen;
298 offset += dstLen;
299 remaining -= dstLen;
300 }
301
302 assert(!remaining);
303
304 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
305 }
306
307 // copy from MD to phys addr
308
309 static IOReturn
IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)310 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
311 IOByteCount offset, addr64_t bytes, IOByteCount length)
312 {
313 addr64_t dstAddr = bytes;
314 IOByteCount remaining;
315
316 remaining = length = min(length, md->getLength() - offset);
317 while (remaining) { // (process another target segment?)
318 addr64_t srcAddr64;
319 IOByteCount dstLen;
320
321 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
322 if (!srcAddr64) {
323 break;
324 }
325
326 // Clip segment length to remaining
327 if (dstLen > remaining) {
328 dstLen = remaining;
329 }
330
331 #if 1
332 bcopy_phys(srcAddr64, dstAddr, dstLen);
333 #else
334 copypv(srcAddr, dstAddr64, dstLen,
335 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
336 #endif
337 dstAddr += dstLen;
338 offset += dstLen;
339 remaining -= dstLen;
340 }
341
342 assert(!remaining);
343
344 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
345 }
346
347 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
348
349 void
hibernate_set_page_state(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired,vm_offset_t ppnum,vm_offset_t count,uint32_t kind)350 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
351 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
352 {
353 count += ppnum;
354
355 if (count > UINT_MAX) {
356 panic("hibernate_set_page_state ppnum");
357 }
358
359 switch (kind) {
360 case kIOHibernatePageStateUnwiredSave:
361 // unwired save
362 for (; ppnum < count; ppnum++) {
363 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
364 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
365 }
366 break;
367 case kIOHibernatePageStateWiredSave:
368 // wired save
369 for (; ppnum < count; ppnum++) {
370 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
371 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
372 }
373 break;
374 case kIOHibernatePageStateFree:
375 // free page
376 for (; ppnum < count; ppnum++) {
377 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
378 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
379 }
380 break;
381 default:
382 panic("hibernate_set_page_state");
383 }
384 }
385
386 static void
hibernate_set_descriptor_page_state(IOHibernateVars * vars,IOMemoryDescriptor * descriptor,uint32_t kind,uint32_t * pageCount)387 hibernate_set_descriptor_page_state(IOHibernateVars *vars,
388 IOMemoryDescriptor *descriptor,
389 uint32_t kind,
390 uint32_t *pageCount)
391 {
392 IOItemCount count;
393 addr64_t phys64;
394 IOByteCount segLen;
395 if (descriptor) {
396 for (count = 0;
397 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
398 count += segLen) {
399 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
400 atop_64(phys64), atop_32(segLen),
401 kind);
402 *pageCount -= atop_32(segLen);
403 }
404 }
405 }
406
407 static vm_offset_t
hibernate_page_list_iterate(hibernate_page_list_t * list,ppnum_t * pPage)408 hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
409 {
410 uint32_t page = ((typeof(page)) * pPage);
411 uint32_t count;
412 hibernate_bitmap_t * bitmap;
413
414 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
415 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
416 if (!count) {
417 break;
418 }
419 page += count;
420 if (page <= bitmap->last_page) {
421 break;
422 }
423 }
424
425 *pPage = page;
426 if (bitmap) {
427 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
428 } else {
429 count = 0;
430 }
431
432 return count;
433 }
434
435 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
436
437 IOReturn
IOHibernateSystemSleep(void)438 IOHibernateSystemSleep(void)
439 {
440 IOReturn err;
441 OSData * nvramData;
442 OSObject * obj;
443 OSString * str;
444 OSNumber * num;
445 bool dsSSD, vmflush, swapPinned;
446 IOHibernateVars * vars;
447 uint64_t setFileSize = 0;
448
449 gIOHibernateState = kIOHibernateStateInactive;
450
451 #if defined(__arm64__)
452 #endif /* __arm64__ */
453
454 gIOHibernateDebugFlags = 0;
455 if (kIOLogHibernate & gIOKitDebug) {
456 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
457 }
458
459 if (IOService::getPMRootDomain()->getHibernateSettings(
460 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
461 if (kIOHibernateModeSleep & gIOHibernateMode) {
462 // default to discard clean for safe sleep
463 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
464 | kIOHibernateModeDiscardCleanActive);
465 }
466 }
467
468 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
469 if ((str = OSDynamicCast(OSString, obj))) {
470 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
471 sizeof(gIOHibernateFilename));
472 }
473 obj->release();
474 }
475
476 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
477 return kIOReturnUnsupported;
478 }
479
480 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
481
482 vars = IOMallocType(IOHibernateVars);
483
484 IOLockLock(gFSLock);
485 if (!gIOHibernateTrimCalloutEntry) {
486 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
487 }
488 IOHibernateSystemPostWakeTrim(NULL, NULL);
489 thread_call_cancel(gIOHibernateTrimCalloutEntry);
490 if (kFSIdle != gFSState) {
491 HIBLOG("hibernate file busy\n");
492 IOLockUnlock(gFSLock);
493 IOFreeType(vars, IOHibernateVars);
494 return kIOReturnBusy;
495 }
496 gFSState = kFSOpening;
497 IOLockUnlock(gFSLock);
498
499 swapPinned = false;
500 do{
501 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
502 HIBERNATION_SRC_BUFFER_SIZE, page_size);
503
504 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
505 ptoa_64(gIOHibernateHandoffPageCount), page_size);
506
507 if (!vars->srcBuffer || !vars->handoffBuffer) {
508 err = kIOReturnNoMemory;
509 break;
510 }
511
512 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
513 if ((num = OSDynamicCast(OSNumber, obj))) {
514 vars->fileMinSize = num->unsigned64BitValue();
515 }
516 obj->release();
517 }
518 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
519 if ((num = OSDynamicCast(OSNumber, obj))) {
520 vars->fileMaxSize = num->unsigned64BitValue();
521 }
522 obj->release();
523 }
524
525 boolean_t encryptedswap = true;
526 uint32_t pageCount;
527 AbsoluteTime startTime, endTime;
528 uint64_t nsec;
529
530 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
531 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
532 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
533
534 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
535 err = hibernate_alloc_page_lists(&vars->page_list,
536 &vars->page_list_wired,
537 &vars->page_list_pal);
538 if (KERN_SUCCESS != err) {
539 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
540 break;
541 }
542
543 err = hibernate_pin_swap(TRUE);
544 if (KERN_SUCCESS != err) {
545 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
546 break;
547 }
548 swapPinned = true;
549
550 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
551 hibernate_page_list_setall(vars->page_list,
552 vars->page_list_wired,
553 vars->page_list_pal,
554 true /* preflight */,
555 vmflush /* discard */,
556 &pageCount);
557 PE_Video consoleInfo;
558 bzero(&consoleInfo, sizeof(consoleInfo));
559 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
560
561 // estimate: 6% increase in pages compressed
562 // screen preview 2 images compressed 0%
563 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
564 + vars->page_list->list_size
565 + (consoleInfo.v_width * consoleInfo.v_height * 8);
566 enum { setFileRound = 1024 * 1024ULL };
567 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
568
569 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
570 pageCount, (100ULL * gIOHibernateCompression) >> 8,
571 setFileSize, vars->fileMinSize);
572
573 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
574 && (setFileSize < vars->fileMinSize)) {
575 setFileSize = vars->fileMinSize;
576 }
577 }
578
579 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
580 err = IOPolledFileOpen(gIOHibernateFilename,
581 (kIOPolledFileCreate | kIOPolledFileHibernate),
582 setFileSize, 0,
583 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
584 &vars->fileVars, &nvramData,
585 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
586
587 if (KERN_SUCCESS != err) {
588 IOLockLock(gFSLock);
589 if (kFSOpening != gFSState) {
590 err = kIOReturnTimeout;
591 }
592 IOLockUnlock(gFSLock);
593 }
594
595 if (KERN_SUCCESS != err) {
596 HIBLOG("IOPolledFileOpen(%x)\n", err);
597 OSSafeReleaseNULL(nvramData);
598 break;
599 }
600
601 // write extents for debug data usage in EFI
602 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
603
604 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
605 if (KERN_SUCCESS != err) {
606 OSSafeReleaseNULL(nvramData);
607 break;
608 }
609
610 clock_get_uptime(&startTime);
611 err = hibernate_setup(gIOHibernateCurrentHeader,
612 vmflush,
613 vars->page_list, vars->page_list_wired, vars->page_list_pal);
614 clock_get_uptime(&endTime);
615 SUB_ABSOLUTETIME(&endTime, &startTime);
616 absolutetime_to_nanoseconds(endTime, &nsec);
617
618 boolean_t haveSwapPin, hibFileSSD;
619 haveSwapPin = vm_swap_files_pinned();
620
621 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
622
623 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
624 err, nsec / 1000000ULL,
625 haveSwapPin, hibFileSSD);
626 if (KERN_SUCCESS != err) {
627 OSSafeReleaseNULL(nvramData);
628 break;
629 }
630
631 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
632
633 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
634 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
635
636 if (dsSSD) {
637 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
638 } else {
639 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
640 }
641
642
643 #if defined(__i386__) || defined(__x86_64__)
644 if (vars->volumeCryptKeySize &&
645 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
646 uintptr_t smcVars[2];
647 smcVars[0] = vars->volumeCryptKeySize;
648 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
649
650 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
651 bzero(smcVars, sizeof(smcVars));
652 }
653 #endif
654
655
656 if (encryptedswap || vars->volumeCryptKeySize) {
657 gIOHibernateMode ^= kIOHibernateModeEncrypt;
658 }
659
660 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
661 vars->videoAllocSize = kVideoMapSize;
662 if (KERN_SUCCESS != kmem_alloc(kernel_map, &vars->videoMapping, vars->videoAllocSize,
663 (kma_flags_t)(KMA_PAGEABLE | KMA_DATA), VM_KERN_MEMORY_IOKIT)) {
664 vars->videoMapping = 0;
665 }
666 }
667
668 // generate crypt keys
669 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
670 vars->wiredCryptKey[i] = ((uint8_t) random());
671 }
672 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
673 vars->cryptKey[i] = ((uint8_t) random());
674 }
675
676 // set nvram
677
678 IOSetBootImageNVRAM(nvramData);
679 OSSafeReleaseNULL(nvramData);
680
681 #if defined(__i386__) || defined(__x86_64__)
682 {
683 struct AppleRTCHibernateVars {
684 uint8_t signature[4];
685 uint32_t revision;
686 uint8_t booterSignature[20];
687 uint8_t wiredCryptKey[16];
688 };
689 AppleRTCHibernateVars rtcVars;
690 OSData * data;
691
692 rtcVars.signature[0] = 'A';
693 rtcVars.signature[1] = 'A';
694 rtcVars.signature[2] = 'P';
695 rtcVars.signature[3] = 'L';
696 rtcVars.revision = 1;
697 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
698
699 if (gIOChosenEntry
700 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
701 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
702 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
703 } else if (gIOHibernateBootSignature[0]) {
704 char c;
705 uint8_t value = 0;
706 uint32_t in, out, digits;
707 for (in = out = digits = 0;
708 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
709 in++) {
710 if ((c >= 'a') && (c <= 'f')) {
711 c -= 'a' - 10;
712 } else if ((c >= 'A') && (c <= 'F')) {
713 c -= 'A' - 10;
714 } else if ((c >= '0') && (c <= '9')) {
715 c -= '0';
716 } else {
717 if (c == '=') {
718 out = digits = value = 0;
719 }
720 continue;
721 }
722 value = ((uint8_t) ((value << 4) | c));
723 if (digits & 1) {
724 rtcVars.booterSignature[out++] = value;
725 if (out >= sizeof(rtcVars.booterSignature)) {
726 break;
727 }
728 }
729 digits++;
730 }
731 }
732 #if DEBUG || DEVELOPMENT
733 if (kIOLogHibernate & gIOKitDebug) {
734 IOKitKernelLogBuffer("H> rtc:",
735 &rtcVars, sizeof(rtcVars), &kprintf);
736 }
737 #endif /* DEBUG || DEVELOPMENT */
738
739 data = OSData::withValue(rtcVars);
740 if (data) {
741 if (gIOHibernateRTCVariablesKey) {
742 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
743 }
744 data->release();
745 }
746 if (gIOChosenEntry && gIOOptionsEntry) {
747 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
748 if (data) {
749 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
750 }
751 // set BootNext
752 if (!gIOHibernateBoot0082Data) {
753 OSData * fileData = NULL;
754 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
755 if (data && data->getLength() >= 4) {
756 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
757 }
758 if (data && (data->getLength() <= UINT16_MAX)) {
759 // AppleNVRAM_EFI_LOAD_OPTION
760 struct {
761 uint32_t Attributes;
762 uint16_t FilePathLength;
763 uint16_t Desc;
764 } loadOptionHeader;
765 loadOptionHeader.Attributes = 1;
766 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
767 loadOptionHeader.Desc = 0;
768 if (fileData) {
769 loadOptionHeader.FilePathLength -= 4;
770 loadOptionHeader.FilePathLength += fileData->getLength();
771 }
772 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
773 if (gIOHibernateBoot0082Data) {
774 gIOHibernateBoot0082Data->appendValue(loadOptionHeader);
775 if (fileData) {
776 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
777 gIOHibernateBoot0082Data->appendBytes(fileData);
778 } else {
779 gIOHibernateBoot0082Data->appendBytes(data);
780 }
781 }
782 }
783 }
784 if (!gIOHibernateBootNextData) {
785 uint16_t bits = 0x0082;
786 gIOHibernateBootNextData = OSData::withValue(bits);
787 }
788
789 #if DEBUG || DEVELOPMENT
790 if (kIOLogHibernate & gIOKitDebug) {
791 IOKitKernelLogBuffer("H> bootnext:",
792 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
793 }
794 #endif /* DEBUG || DEVELOPMENT */
795 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
796 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
797 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
798 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
799 }
800 // BootNext
801 }
802 }
803 #endif /* !i386 && !x86_64 */
804 }while (false);
805
806 if (swapPinned) {
807 hibernate_pin_swap(FALSE);
808 }
809
810 IOLockLock(gFSLock);
811 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
812 HIBLOG("hibernate file close due timeout\n");
813 err = kIOReturnTimeout;
814 }
815 if (kIOReturnSuccess == err) {
816 gFSState = kFSOpened;
817 gIOHibernateVars = *vars;
818 gFileVars = *vars->fileVars;
819 gFileVars.allocated = false;
820 gIOHibernateVars.fileVars = &gFileVars;
821 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
822 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
823 gIOHibernateState = kIOHibernateStateHibernating;
824
825 #if DEBUG || DEVELOPMENT
826 if (kIOLogHibernate & gIOKitDebug) {
827 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
828 if (data) {
829 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
830 IOKitKernelLogBuffer("H> smc:",
831 (const void *)smcVars[1], smcVars[0], &kprintf);
832 }
833 }
834 #endif /* DEBUG || DEVELOPMENT */
835 } else {
836 IOPolledFileIOVars * fileVars = vars->fileVars;
837 IOHibernateDone(vars);
838 IOPolledFileClose(&fileVars,
839 #if DISABLE_TRIM
840 0, NULL, 0, 0, 0, false);
841 #else
842 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize, false);
843 #endif
844 gFSState = kFSIdle;
845 }
846 IOLockUnlock(gFSLock);
847
848 if (vars->fileVars) {
849 IOFreeType(vars->fileVars, IOPolledFileIOVars);
850 }
851 IOFreeType(vars, IOHibernateVars);
852
853 return err;
854 }
855
856 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
857
858 static void
IOSetBootImageNVRAM(OSData * data)859 IOSetBootImageNVRAM(OSData * data)
860 {
861 IORegistryEntry * regEntry;
862
863 if (!gIOOptionsEntry) {
864 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
865 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
866 if (regEntry && !gIOOptionsEntry) {
867 regEntry->release();
868 }
869 }
870 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
871 if (data) {
872 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
873 #if DEBUG || DEVELOPMENT
874 if (kIOLogHibernate & gIOKitDebug) {
875 IOKitKernelLogBuffer("H> boot-image:",
876 data->getBytesNoCopy(), data->getLength(), &kprintf);
877 }
878 #endif /* DEBUG || DEVELOPMENT */
879 } else {
880 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
881 #if __x86_64__
882 gIOOptionsEntry->sync();
883 #else
884 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
885 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
886 gIOOptionsEntry->sync();
887 }
888 #endif
889 }
890 }
891 }
892
893 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
894 /*
895 * Writes header to disk with signature, block size and file extents data.
896 * If there are more than 2 extents, then they are written on second block.
897 */
898 static IOReturn
IOWriteExtentsToFile(IOPolledFileIOVars * vars,uint32_t signature)899 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
900 {
901 IOHibernateImageHeader hdr;
902 IOItemCount count;
903 IOReturn err = kIOReturnSuccess;
904 int rc;
905 IOPolledFileExtent * fileExtents;
906
907 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
908
909 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
910 count = vars->fileExtents->getLength();
911 if (count > sizeof(hdr.fileExtentMap)) {
912 hdr.fileExtentMapSize = count;
913 count = sizeof(hdr.fileExtentMap);
914 } else {
915 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
916 }
917
918 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
919
920 // copy file block extent list if larger than header
921 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
922 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
923 rc = kern_write_file(vars->fileRef, vars->blockSize,
924 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
925 count, IO_SKIP_ENCRYPTION);
926 if (rc != 0) {
927 HIBLOG("kern_write_file returned %d\n", rc);
928 err = kIOReturnIOError;
929 goto exit;
930 }
931 }
932 hdr.signature = signature;
933 hdr.deviceBlockSize = vars->blockSize;
934
935 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
936 if (rc != 0) {
937 HIBLOG("kern_write_file returned %d\n", rc);
938 err = kIOReturnIOError;
939 goto exit;
940 }
941
942 exit:
943 return err;
944 }
945
946 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
947
948 DECLARE_IOHIBERNATEPROGRESSALPHA
949
950 static void
ProgressInit(hibernate_graphics_t * display,uint8_t * screen,uint8_t * saveunder,uint32_t savelen)951 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
952 {
953 uint32_t rowBytes, pixelShift;
954 uint32_t x, y;
955 int32_t blob;
956 uint32_t alpha, color, result;
957 uint8_t * out, in;
958 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
959
960 rowBytes = display->rowBytes;
961 pixelShift = display->depth >> 4;
962 if (pixelShift < 1) {
963 return;
964 }
965
966 screen += ((display->width
967 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
968 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
969
970 for (y = 0; y < kIOHibernateProgressHeight; y++) {
971 out = screen + y * rowBytes;
972 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
973 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
974 for (x = 0; x < kIOHibernateProgressWidth; x++) {
975 alpha = gIOHibernateProgressAlpha[y][x];
976 result = color;
977 if (alpha) {
978 if (0xff != alpha) {
979 if (1 == pixelShift) {
980 in = *((uint16_t *)out) & 0x1f; // 16
981 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
982 } else {
983 in = *((uint32_t *)out) & 0xff; // 32
984 }
985 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
986 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
987 }
988 if (1 == pixelShift) {
989 result >>= 3;
990 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
991 } else {
992 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
993 }
994 }
995 out += (1 << pixelShift);
996 }
997 out += (kIOHibernateProgressSpacing << pixelShift);
998 }
999 }
1000 }
1001
1002
1003 static void
ProgressUpdate(hibernate_graphics_t * display,uint8_t * screen,int32_t firstBlob,int32_t select)1004 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1005 {
1006 uint32_t rowBytes, pixelShift;
1007 uint32_t x, y;
1008 int32_t blob, lastBlob;
1009 uint32_t alpha, in, color, result;
1010 uint8_t * out;
1011 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1012
1013 pixelShift = display->depth >> 4;
1014 if (pixelShift < 1) {
1015 return;
1016 }
1017
1018 rowBytes = display->rowBytes;
1019
1020 screen += ((display->width
1021 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1022 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1023
1024 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1025
1026 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1027
1028 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1029 out = screen + y * rowBytes;
1030 for (blob = firstBlob; blob <= lastBlob; blob++) {
1031 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1032 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1033 alpha = gIOHibernateProgressAlpha[y][x];
1034 result = color;
1035 if (alpha) {
1036 if (0xff != alpha) {
1037 in = display->progressSaveUnder[blob][saveindex[blob]++];
1038 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1039 }
1040 if (1 == pixelShift) {
1041 result >>= 3;
1042 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
1043 } else {
1044 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1045 }
1046 }
1047 out += (1 << pixelShift);
1048 }
1049 out += (kIOHibernateProgressSpacing << pixelShift);
1050 }
1051 }
1052 }
1053
1054 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1055
1056 IOReturn
IOHibernateIOKitSleep(void)1057 IOHibernateIOKitSleep(void)
1058 {
1059 IOReturn ret = kIOReturnSuccess;
1060 IOLockLock(gFSLock);
1061 if (kFSOpening == gFSState) {
1062 gFSState = kFSTimedOut;
1063 HIBLOG("hibernate file open timed out\n");
1064 ret = kIOReturnTimeout;
1065 }
1066 IOLockUnlock(gFSLock);
1067 return ret;
1068 }
1069
1070 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1071
1072 IOReturn
IOHibernateSystemHasSlept(void)1073 IOHibernateSystemHasSlept(void)
1074 {
1075 IOReturn ret = kIOReturnSuccess;
1076 IOHibernateVars * vars = &gIOHibernateVars;
1077 OSObject * obj = NULL;
1078 OSData * data;
1079
1080 IOLockLock(gFSLock);
1081 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1082 ret = kIOReturnTimeout;
1083 }
1084 IOLockUnlock(gFSLock);
1085 if (kIOReturnSuccess != ret) {
1086 return ret;
1087 }
1088
1089 if (gIOHibernateMode) {
1090 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1091 }
1092 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1093 if (obj && !vars->previewBuffer) {
1094 obj->release();
1095 }
1096 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1097 OSSafeReleaseNULL(vars->previewBuffer);
1098 }
1099
1100 vars->consoleMapping = NULL;
1101 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1102 vars->previewBuffer->release();
1103 vars->previewBuffer = NULL;
1104 }
1105
1106 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1107 && vars->previewBuffer
1108 && (data = OSDynamicCast(OSData,
1109 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1110 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1111 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1112
1113 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1114
1115 if (kIOHibernatePreviewUpdates & flags) {
1116 PE_Video consoleInfo;
1117 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1118
1119 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1120
1121 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1122 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1123 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1124 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
1125 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1126
1127 HIBPRINT("video %p %d %d %d\n",
1128 vars->consoleMapping, graphicsInfo->depth,
1129 graphicsInfo->width, graphicsInfo->height);
1130 if (vars->consoleMapping) {
1131 ProgressInit(graphicsInfo, vars->consoleMapping,
1132 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1133 }
1134 }
1135 }
1136
1137 if (gIOOptionsEntry) {
1138 #if __x86_64__
1139 gIOOptionsEntry->sync();
1140 #else
1141 if (gIOHibernateMode) {
1142 gIOOptionsEntry->sync();
1143 }
1144 #endif
1145 }
1146
1147 return ret;
1148 }
1149
1150 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1151
1152 static const DeviceTreeNode *
MergeDeviceTree(const DeviceTreeNode * entry,IORegistryEntry * regEntry,OSSet * entriesToUpdate,vm_offset_t region_start,vm_size_t region_size)1153 MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
1154 {
1155 DeviceTreeNodeProperty * prop;
1156 const DeviceTreeNode * child;
1157 IORegistryEntry * childRegEntry;
1158 const char * nameProp;
1159 unsigned int propLen, idx;
1160
1161 bool updateEntry = true;
1162 if (!regEntry) {
1163 updateEntry = false;
1164 } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
1165 updateEntry = false;
1166 }
1167
1168 prop = (DeviceTreeNodeProperty *) (entry + 1);
1169 for (idx = 0; idx < entry->nProperties; idx++) {
1170 if (updateEntry && (0 != strcmp("name", prop->name))) {
1171 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1172 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1173 }
1174 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1175 }
1176
1177 if (entriesToUpdate) {
1178 entriesToUpdate->removeObject(regEntry);
1179 if (entriesToUpdate->getCount() == 0) {
1180 // we've updated all the entries we care about so we can stop
1181 return NULL;
1182 }
1183 }
1184
1185 child = (const DeviceTreeNode *) prop;
1186 for (idx = 0; idx < entry->nChildren; idx++) {
1187 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1188 region_start, region_size)) {
1189 panic("no name");
1190 }
1191 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1192 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1193 child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
1194 OSSafeReleaseNULL(childRegEntry);
1195 if (!child) {
1196 // the recursive call updated the last entry we cared about, so we can stop
1197 break;
1198 }
1199 }
1200 return child;
1201 }
1202
1203 IOReturn
IOHibernateSystemWake(void)1204 IOHibernateSystemWake(void)
1205 {
1206 if (kFSOpened == gFSState) {
1207 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1208 IOHibernateDone(&gIOHibernateVars);
1209 } else {
1210 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1211 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1212 }
1213
1214 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1215 // if we got this far, clear boot-image
1216 // we don't need to sync immediately; the booter should have already removed this entry
1217 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1218 // a stale boot-image value
1219 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1220 }
1221
1222 return kIOReturnSuccess;
1223 }
1224
1225 static IOReturn
IOHibernateDone(IOHibernateVars * vars)1226 IOHibernateDone(IOHibernateVars * vars)
1227 {
1228 IOReturn err;
1229 OSData * data;
1230
1231 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1232
1233 if (vars->videoMapping) {
1234 if (vars->videoMapSize) {
1235 // remove mappings
1236 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1237 }
1238 if (vars->videoAllocSize) {
1239 // dealloc range
1240 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1241 }
1242 }
1243
1244 if (vars->previewBuffer) {
1245 vars->previewBuffer->release();
1246 vars->previewBuffer = NULL;
1247 }
1248
1249 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1250 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1251 gIOHibernateCurrentHeader->options, 32);
1252 } else {
1253 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1254 }
1255
1256 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1257 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1258 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1259 &gIOHibernateGraphicsInfo->gfxStatus,
1260 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1261 } else {
1262 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1263 }
1264
1265 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1266
1267 #if defined(__i386__) || defined(__x86_64__)
1268 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1269 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1270
1271 /*
1272 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1273 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1274 */
1275 if (gIOOptionsEntry) {
1276 if (gIOHibernateRTCVariablesKey) {
1277 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1278 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1279 }
1280 }
1281
1282 if (gIOHibernateBootNextKey) {
1283 if (gIOHibernateBootNextSave) {
1284 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1285 gIOHibernateBootNextSave->release();
1286 gIOHibernateBootNextSave = NULL;
1287 } else {
1288 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1289 }
1290 }
1291 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1292 gIOOptionsEntry->sync();
1293 }
1294 }
1295 #endif
1296
1297 if (vars->srcBuffer) {
1298 vars->srcBuffer->release();
1299 }
1300
1301
1302 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1303 if (vars->handoffBuffer) {
1304 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1305 IOHibernateHandoff * handoff;
1306 bool done = false;
1307 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1308 !done;
1309 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1310 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1311 uint8_t * __unused data = &handoff->data[0];
1312 switch (handoff->type) {
1313 case kIOHibernateHandoffTypeEnd:
1314 done = true;
1315 break;
1316
1317 case kIOHibernateHandoffTypeDeviceTree:
1318 {
1319 #if defined(__i386__) || defined(__x86_64__)
1320 // On Intel, process the entirety of the passed in device tree
1321 OSSet * entriesToUpdate = NULL;
1322 #elif defined(__arm64__)
1323 // On ARM, only allow hibernation to update specific entries
1324 const char *mergePaths[] = {
1325 kIODeviceTreePlane ":/chosen/boot-object-manifests",
1326 kIODeviceTreePlane ":/chosen/secure-boot-hashes",
1327 };
1328 const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
1329 OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
1330 for (size_t i = 0; i < mergePathCount; i++) {
1331 IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
1332 if (!entry) {
1333 panic("failed to find %s in IORegistry", mergePaths[i]);
1334 }
1335 entriesToUpdate->setObject(entry);
1336 OSSafeReleaseNULL(entry);
1337 }
1338 #endif
1339 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
1340 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1341 OSSafeReleaseNULL(entriesToUpdate);
1342 break;
1343 }
1344
1345 case kIOHibernateHandoffTypeKeyStore:
1346 #if defined(__i386__) || defined(__x86_64__)
1347 {
1348 IOBufferMemoryDescriptor *
1349 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1350 if (md) {
1351 IOSetKeyStoreData(md);
1352 }
1353 }
1354 #endif
1355 break;
1356
1357 default:
1358 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1359 break;
1360 }
1361 }
1362 #if defined(__i386__) || defined(__x86_64__)
1363 if (vars->volumeCryptKeySize) {
1364 IOBufferMemoryDescriptor *
1365 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1366 vars->volumeCryptKeySize, kIODirectionOutIn);
1367 if (!bmd) {
1368 panic("IOBufferMemoryDescriptor");
1369 }
1370 IOSetAPFSKeyStoreData(bmd);
1371 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1372 }
1373 #endif
1374 }
1375 vars->handoffBuffer->release();
1376 }
1377
1378 if (gIOChosenEntry
1379 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1380 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1381 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1382 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1383 }
1384
1385 if (vars->hwEncrypt) {
1386 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1387 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1388 }
1389
1390 bzero(vars, sizeof(*vars));
1391
1392 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1393 gIOHibernateCount++;
1394
1395 return kIOReturnSuccess;
1396 }
1397
1398 static void
IOHibernateSystemPostWakeTrim(void * p1,void * p2)1399 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1400 {
1401 // invalidate & close the image file
1402 if (p1) {
1403 IOLockLock(gFSLock);
1404 }
1405 if (kFSTrimDelay == gFSState) {
1406 IOPolledFileIOVars * vars = &gFileVars;
1407 IOPolledFileClose(&vars,
1408 #if DISABLE_TRIM
1409 0, NULL, 0, 0, 0, false);
1410 #else
1411 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1412 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize, false);
1413 #endif
1414 gFSState = kFSIdle;
1415 }
1416 if (p1) {
1417 IOLockUnlock(gFSLock);
1418 }
1419 }
1420
1421 IOReturn
IOHibernateSystemPostWake(bool now)1422 IOHibernateSystemPostWake(bool now)
1423 {
1424 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1425 IOSetBootImageNVRAM(NULL);
1426
1427 IOLockLock(gFSLock);
1428 if (kFSTrimDelay == gFSState) {
1429 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1430 IOHibernateSystemPostWakeTrim(NULL, NULL);
1431 } else if (kFSOpened != gFSState) {
1432 gFSState = kFSIdle;
1433 } else {
1434 gFSState = kFSTrimDelay;
1435 if (now) {
1436 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1437 IOHibernateSystemPostWakeTrim(NULL, NULL);
1438 } else {
1439 AbsoluteTime deadline;
1440 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1441 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1442 }
1443 }
1444 IOLockUnlock(gFSLock);
1445
1446 return kIOReturnSuccess;
1447 }
1448
1449 uint32_t
IOHibernateWasScreenLocked(void)1450 IOHibernateWasScreenLocked(void)
1451 {
1452 uint32_t ret = 0;
1453 if (gIOChosenEntry) {
1454 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1455 OSData *
1456 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1457 if (data) {
1458 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1459 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1460 }
1461 } else {
1462 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1463 }
1464 }
1465
1466 return ret;
1467 }
1468
1469 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1470
1471 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1472 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1473 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1474 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1475 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1476 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1477 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1478 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1479 &gIOHibernateMode, 0, "");
1480 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1481 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
1482 &_hibernateStats, hibernate_statistics_t, "");
1483 SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
1484 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1485 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
1486 sysctl_handle_string, "A", "");
1487
1488 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1489 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1490 &_hibernateStats.graphicsReadyTime, 0, "");
1491 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1492 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1493 &_hibernateStats.wakeNotificationTime, 0, "");
1494 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1495 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1496 &_hibernateStats.lockScreenReadyTime, 0, "");
1497 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1498 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1499 &_hibernateStats.hidReadyTime, 0, "");
1500
1501 SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
1502 CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1503 &gIOHibernateCount, 0, "");
1504
1505 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1506
1507 static int
1508 hibernate_set_preview SYSCTL_HANDLER_ARGS
1509 {
1510 #pragma unused(oidp, arg1, arg2)
1511
1512 if (!IOCurrentTaskHasEntitlement(kIOHibernateSetPreviewEntitlementKey)) {
1513 return EPERM;
1514 }
1515
1516 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1517 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1518 return 0;
1519 }
1520 size_t rounded_size;
1521 if (round_page_overflow(req->newlen, &rounded_size)) {
1522 return ENOMEM;
1523 }
1524 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1525 if (!md) {
1526 return ENOMEM;
1527 }
1528
1529 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1530 int error = SYSCTL_IN(req, bytes, req->newlen);
1531 if (error) {
1532 md->release();
1533 return error;
1534 }
1535
1536 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1537 md->release();
1538
1539 return 0;
1540 }
1541
1542 SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1543 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1544 hibernate_set_preview, "S", "");
1545
1546 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1547
1548 void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)1549 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1550 {
1551 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1552 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1553 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1554
1555 #if defined(__i386__) || defined(__x86_64__)
1556 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1557 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1558 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1559 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1560 #endif /* defined(__i386__) || defined(__x86_64__) */
1561
1562 OSData * data = OSData::withValueNoCopy(gIOHibernateState);
1563 if (data) {
1564 rootDomain->setProperty(kIOHibernateStateKey, data);
1565 data->release();
1566 }
1567
1568 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1569 gIOHibernateMode = kIOHibernateModeOn;
1570 } else {
1571 gIOHibernateFilename[0] = 0;
1572 }
1573
1574 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1575
1576 if (gIOChosenEntry
1577 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1578 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1579 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1580 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1581 }
1582
1583 gFSLock = IOLockAlloc();
1584 gIOHibernateCount = 0;
1585 }
1586
1587 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1588
1589 static IOReturn
IOHibernatePolledFileWrite(IOHibernateVars * vars,const uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)1590 IOHibernatePolledFileWrite(IOHibernateVars * vars,
1591 const uint8_t * bytes, IOByteCount size,
1592 IOPolledFileCryptVars * cryptvars)
1593 {
1594 IOReturn err;
1595
1596
1597 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
1598 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1599 err = kIOReturnAborted;
1600 }
1601
1602
1603 return err;
1604 }
1605
1606 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1607
1608 extern "C" uint32_t
hibernate_write_image(void)1609 hibernate_write_image(void)
1610 {
1611 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1612 IOHibernateVars * vars = &gIOHibernateVars;
1613 IOPolledFileExtent * fileExtents;
1614
1615 #if !defined(__arm64__)
1616 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1617 #endif /* !defined(__arm64__) */
1618
1619 uint32_t pageCount, pagesDone;
1620 IOReturn err;
1621 ppnum_t ppnum, page;
1622 vm_offset_t count;
1623 uint8_t * src;
1624 uint8_t * data;
1625 uint8_t * compressed;
1626 uint8_t * scratch;
1627 IOByteCount pageCompressedSize;
1628 uint64_t compressedSize, uncompressedSize;
1629 uint64_t image1Size = 0;
1630 uint32_t bitmap_size;
1631 bool iterDone, pollerOpen, needEncrypt;
1632 int wkresult;
1633 uint32_t tag;
1634 uint32_t pageType;
1635 uint32_t pageAndCount[2];
1636 addr64_t phys64;
1637 IOByteCount segLen;
1638 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
1639 uintptr_t hibernateBase;
1640 uintptr_t hibernateEnd;
1641
1642 AbsoluteTime startTime, endTime;
1643 AbsoluteTime allTime, compTime;
1644 uint64_t compBytes;
1645 uint64_t nsec;
1646 uint64_t lastProgressStamp = 0;
1647 uint64_t progressStamp;
1648 uint32_t blob, lastBlob = (uint32_t) -1L;
1649
1650 uint32_t wiredPagesEncrypted;
1651 uint32_t dirtyPagesEncrypted;
1652 uint32_t wiredPagesClear;
1653 uint32_t svPageCount;
1654 uint32_t zvPageCount;
1655
1656 IOPolledFileCryptVars _cryptvars;
1657 IOPolledFileCryptVars * cryptvars = NULL;
1658
1659 wiredPagesEncrypted = 0;
1660 dirtyPagesEncrypted = 0;
1661 wiredPagesClear = 0;
1662 svPageCount = 0;
1663 zvPageCount = 0;
1664
1665 if (!vars->fileVars
1666 || !vars->fileVars->pollers
1667 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1668 return kIOHibernatePostWriteSleep;
1669 }
1670
1671
1672 #if !defined(__arm64__)
1673 if (kIOHibernateModeSleep & gIOHibernateMode) {
1674 kdebug_enable = save_kdebug_enable;
1675 }
1676 #endif /* !defined(__arm64__) */
1677
1678 pal_hib_write_hook();
1679
1680 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1681 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1682
1683 #if CRYPTO
1684 // encryption data. "iv" is the "initial vector".
1685 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1686 static const unsigned char first_iv[AES_BLOCK_SIZE]
1687 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1688 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1689
1690 cryptvars = &gIOHibernateCryptWakeContext;
1691 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1692 aes_encrypt_key(vars->cryptKey,
1693 kIOHibernateAESKeySize,
1694 &cryptvars->ctx.encrypt);
1695 aes_decrypt_key(vars->cryptKey,
1696 kIOHibernateAESKeySize,
1697 &cryptvars->ctx.decrypt);
1698
1699 cryptvars = &_cryptvars;
1700 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1701 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1702 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1703 }
1704 aes_encrypt_key(vars->wiredCryptKey,
1705 kIOHibernateAESKeySize,
1706 &cryptvars->ctx.encrypt);
1707
1708 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1709 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1710 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1711 }
1712 #endif /* CRYPTO */
1713
1714 hibernate_page_list_setall(vars->page_list,
1715 vars->page_list_wired,
1716 vars->page_list_pal,
1717 false /* !preflight */,
1718 /* discard_all */
1719 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1720 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1721 &pageCount);
1722
1723 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1724
1725 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1726
1727 #if 0
1728 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1729 for (page = 0; page < count; page++) {
1730 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1731 fileExtents[page].start, fileExtents[page].length,
1732 fileExtents[page].start + fileExtents[page].length);
1733 }
1734 #endif
1735
1736 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1737 AbsoluteTime_to_scalar(&compTime) = 0;
1738 compBytes = 0;
1739
1740 clock_get_uptime(&allTime);
1741 IOService::getPMRootDomain()->pmStatsRecordEvent(
1742 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1743 do{
1744 compressedSize = 0;
1745 uncompressedSize = 0;
1746 svPageCount = 0;
1747 zvPageCount = 0;
1748
1749 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1750
1751 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1752 ml_get_interrupts_enabled());
1753 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1754 // abortable if not low battery
1755 !IOService::getPMRootDomain()->mustHibernate());
1756 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1757 pollerOpen = (kIOReturnSuccess == err);
1758 if (!pollerOpen) {
1759 break;
1760 }
1761
1762 if (vars->volumeCryptKeySize) {
1763 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1764 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1765 vars->hwEncrypt = (kIOReturnSuccess == err);
1766 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1767 if (vars->hwEncrypt) {
1768 header->options |= kIOHibernateOptionHWEncrypt;
1769 }
1770 }
1771
1772 // copy file block extent list if larger than header
1773
1774 count = vars->fileVars->fileExtents->getLength();
1775 if (count > sizeof(header->fileExtentMap)) {
1776 count -= sizeof(header->fileExtentMap);
1777 err = IOHibernatePolledFileWrite(vars,
1778 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1779 if (kIOReturnSuccess != err) {
1780 break;
1781 }
1782 }
1783
1784 // copy out restore1 code
1785
1786 for (count = 0;
1787 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1788 count += segLen) {
1789 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1790 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
1791 }
1792 }
1793
1794 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1795 hibernateEnd = (segHIBB + segSizeHIB);
1796
1797 page = atop_32(kvtophys(hibernateBase));
1798 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1799 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1800 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1801 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1802 panic("malformed kernel layout");
1803 }
1804 header->restore1CodePhysPage = (ppnum_t) page;
1805 header->restore1CodeVirt = hibernateBase;
1806 header->restore1PageCount = (uint32_t) count;
1807 header->restore1CodeOffset = (uint32_t) entrypoint;
1808 header->restore1StackOffset = (uint32_t) stack;
1809
1810 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1811 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1812 }
1813
1814 // sum __HIB seg, with zeros for the stack
1815 src = (uint8_t *) trunc_page(hibernateBase);
1816 for (page = 0; page < count; page++) {
1817 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1818 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
1819 } else {
1820 restore1Sum += 0x00000000;
1821 }
1822 src += page_size;
1823 }
1824 sum1 = restore1Sum;
1825
1826 // write the __HIB seg, with zeros for the stack
1827
1828 src = (uint8_t *) trunc_page(hibernateBase);
1829 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1830 if (count) {
1831 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1832 if (kIOReturnSuccess != err) {
1833 break;
1834 }
1835 }
1836 err = IOHibernatePolledFileWrite(vars,
1837 (uint8_t *) NULL,
1838 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1839 cryptvars);
1840 if (kIOReturnSuccess != err) {
1841 break;
1842 }
1843 src = &gIOHibernateRestoreStackEnd[0];
1844 count = round_page(hibernateEnd) - ((uintptr_t) src);
1845 if (count) {
1846 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1847 if (kIOReturnSuccess != err) {
1848 break;
1849 }
1850 }
1851
1852 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1853 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1854 vars->fileVars->encryptEnd = UINT64_MAX;
1855 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1856 }
1857
1858 // write the preview buffer
1859
1860 if (vars->previewBuffer) {
1861 ppnum = 0;
1862 count = 0;
1863 do{
1864 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1865 pageAndCount[0] = atop_64_ppnum(phys64);
1866 pageAndCount[1] = atop_64_ppnum(segLen);
1867 err = IOHibernatePolledFileWrite(vars,
1868 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1869 cryptvars);
1870 if (kIOReturnSuccess != err) {
1871 break;
1872 }
1873 count += segLen;
1874 ppnum += sizeof(pageAndCount);
1875 }while (phys64);
1876 if (kIOReturnSuccess != err) {
1877 break;
1878 }
1879
1880 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1881
1882 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1883
1884 count = (uint32_t) vars->previewBuffer->getLength();
1885
1886 header->previewPageListSize = ((uint32_t) ppnum);
1887 header->previewSize = ((uint32_t) (count + ppnum));
1888
1889 for (page = 0; page < count; page += page_size) {
1890 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1891 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
1892 }
1893 if (kIOReturnSuccess != err) {
1894 break;
1895 }
1896 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1897 if (kIOReturnSuccess != err) {
1898 break;
1899 }
1900 }
1901
1902 // mark areas for no save
1903 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1904 kIOHibernatePageStateFree, &pageCount);
1905 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1906 kIOHibernatePageStateFree, &pageCount);
1907
1908 // copy out bitmap of pages available for trashing during restore
1909
1910 bitmap_size = vars->page_list_wired->list_size;
1911 src = (uint8_t *) vars->page_list_wired;
1912 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
1913 if (kIOReturnSuccess != err) {
1914 break;
1915 }
1916
1917 // mark more areas for no save, but these are not available
1918 // for trashing during restore
1919
1920 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1921
1922 #if defined(__i386__) || defined(__x86_64__)
1923 // __HIB is explicitly saved above so we don't have to save it again
1924 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1925 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1926 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1927 page, count,
1928 kIOHibernatePageStateFree);
1929 pageCount -= count;
1930 #elif defined(__arm64__)
1931 // the segments described in IOHibernateHibSegInfo are stored directly in the
1932 // hibernation file, so they don't need to be saved again
1933 extern unsigned long gPhysBase, gPhysSize;
1934 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1935 page = segInfo->segments[i].physPage;
1936 count = segInfo->segments[i].pageCount;
1937 uint64_t physAddr = ptoa_64(page);
1938 uint64_t size = ptoa_64(count);
1939 if (size &&
1940 (physAddr >= gPhysBase) &&
1941 (physAddr + size <= gPhysBase + gPhysSize)) {
1942 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1943 page, count,
1944 kIOHibernatePageStateFree);
1945 pageCount -= count;
1946 }
1947 }
1948 #else
1949 #error unimplemented
1950 #endif
1951
1952 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
1953 kIOHibernatePageStateFree, &pageCount);
1954 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
1955 kIOHibernatePageStateFree, &pageCount);
1956
1957 #if KASAN
1958 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1959
1960 /* no need to save unused shadow pages */
1961 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1962 atop_64(shadow_pnext),
1963 shadow_pages_free,
1964 kIOHibernatePageStateFree);
1965 #endif
1966
1967 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1968 compressed = src + page_size;
1969 scratch = compressed + page_size;
1970
1971 pagesDone = 0;
1972 lastBlob = 0;
1973
1974 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1975 bitmap_size, header->previewSize,
1976 pageCount, vars->fileVars->position);
1977
1978
1979 enum
1980 // pageType
1981 {
1982 kWired = 0x02,
1983 kEncrypt = 0x01,
1984 kWiredEncrypt = kWired | kEncrypt,
1985 kWiredClear = kWired,
1986 kUnwiredEncrypt = kEncrypt
1987 };
1988
1989 #if defined(__i386__) || defined(__x86_64__)
1990 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1991 #else /* defined(__i386__) || defined(__x86_64__) */
1992 static const bool cpuAES = true;
1993 #endif /* defined(__i386__) || defined(__x86_64__) */
1994
1995 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1996 if (kUnwiredEncrypt == pageType) {
1997 // start unwired image
1998 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1999 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2000 vars->fileVars->encryptEnd = UINT64_MAX;
2001 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2002 }
2003 bcopy(&cryptvars->aes_iv[0],
2004 &gIOHibernateCryptWakeContext.aes_iv[0],
2005 sizeof(cryptvars->aes_iv));
2006 cryptvars = &gIOHibernateCryptWakeContext;
2007 }
2008 for (iterDone = false, ppnum = 0; !iterDone;) {
2009 if (cpuAES && (pageType == kWiredClear)) {
2010 count = 0;
2011 } else {
2012 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2013 &ppnum);
2014 if (count > UINT_MAX) {
2015 count = UINT_MAX;
2016 }
2017 }
2018 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2019 iterDone = !count;
2020
2021 if (!cpuAES) {
2022 if (count && (kWired & pageType) && needEncrypt) {
2023 uint32_t checkIndex;
2024 for (checkIndex = 0;
2025 (checkIndex < count)
2026 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
2027 checkIndex++) {
2028 }
2029 if (!checkIndex) {
2030 ppnum++;
2031 continue;
2032 }
2033 count = checkIndex;
2034 }
2035 }
2036
2037 switch (pageType) {
2038 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2039 case kWiredClear: wiredPagesClear += count; break;
2040 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2041 }
2042
2043 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2044 } else {
2045 pageAndCount[0] = (uint32_t) ppnum;
2046 pageAndCount[1] = (uint32_t) count;
2047 err = IOHibernatePolledFileWrite(vars,
2048 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2049 cryptvars);
2050 if (kIOReturnSuccess != err) {
2051 break;
2052 }
2053 }
2054
2055 for (page = ppnum; page < (ppnum + count); page++) {
2056 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2057 if (err) {
2058 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2059 break;
2060 }
2061
2062 sum = hibernate_sum_page(src, (uint32_t) page);
2063 if (kWired & pageType) {
2064 sum1 += sum;
2065 } else {
2066 sum2 += sum;
2067 }
2068
2069 clock_get_uptime(&startTime);
2070 wkresult = WKdm_compress_new((const WK_word*) src,
2071 (WK_word*) compressed,
2072 (WK_word*) scratch,
2073 (uint32_t) (page_size - 4));
2074
2075 clock_get_uptime(&endTime);
2076 ADD_ABSOLUTETIME(&compTime, &endTime);
2077 SUB_ABSOLUTETIME(&compTime, &startTime);
2078
2079 compBytes += page_size;
2080 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2081
2082 if (pageCompressedSize == 0) {
2083 pageCompressedSize = 4;
2084 data = src;
2085
2086 if (*(uint32_t *)src) {
2087 svPageCount++;
2088 } else {
2089 zvPageCount++;
2090 }
2091 } else {
2092 if (pageCompressedSize != page_size) {
2093 data = compressed;
2094 } else {
2095 data = src;
2096 }
2097 }
2098
2099 assert(pageCompressedSize <= page_size);
2100 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2101 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2102 if (kIOReturnSuccess != err) {
2103 break;
2104 }
2105
2106 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2107 if (kIOReturnSuccess != err) {
2108 break;
2109 }
2110
2111 compressedSize += pageCompressedSize;
2112 uncompressedSize += page_size;
2113 pagesDone++;
2114
2115 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2116 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2117 if (blob != lastBlob) {
2118 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2119 lastBlob = blob;
2120 }
2121 }
2122 if (0 == (8191 & pagesDone)) {
2123 clock_get_uptime(&endTime);
2124 SUB_ABSOLUTETIME(&endTime, &allTime);
2125 absolutetime_to_nanoseconds(endTime, &nsec);
2126 progressStamp = nsec / 750000000ULL;
2127 if (progressStamp != lastProgressStamp) {
2128 lastProgressStamp = progressStamp;
2129 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2130 }
2131 }
2132 }
2133 if (kIOReturnSuccess != err) {
2134 break;
2135 }
2136 ppnum = page;
2137 }
2138
2139 if (kIOReturnSuccess != err) {
2140 break;
2141 }
2142
2143 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2144 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2145 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2146 }
2147
2148 if (kWiredEncrypt != pageType) {
2149 // end of image1/2 - fill to next block
2150 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2151 if (kIOReturnSuccess != err) {
2152 break;
2153 }
2154 }
2155 if (kWiredClear == pageType) {
2156 // enlarge wired image for test
2157 // err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
2158
2159 // end wired image
2160 header->encryptStart = vars->fileVars->encryptStart;
2161 header->encryptEnd = vars->fileVars->encryptEnd;
2162 image1Size = vars->fileVars->position;
2163 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2164 image1Size, header->encryptStart, header->encryptEnd);
2165 }
2166 }
2167 if (kIOReturnSuccess != err) {
2168 if (kIOReturnOverrun == err) {
2169 // update actual compression ratio on not enough space (for retry)
2170 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2171 }
2172
2173 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2174 header->imageSize = vars->fileVars->position;
2175 break;
2176 }
2177
2178
2179 // Header:
2180
2181 header->imageSize = vars->fileVars->position;
2182 header->image1Size = image1Size;
2183 header->bitmapSize = bitmap_size;
2184 header->pageCount = pageCount;
2185
2186 header->restore1Sum = restore1Sum;
2187 header->image1Sum = sum1;
2188 header->image2Sum = sum2;
2189 header->sleepTime = gIOLastSleepTime.tv_sec;
2190
2191 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
2192 #if defined(__arm64__)
2193 /*
2194 * We don't support retry on hibernation failure and so
2195 * we don't want to set this value to anything smaller
2196 * just because we may have been lucky this time around.
2197 * Though we'll let it go higher.
2198 */
2199 if (header->compression < HIB_COMPR_RATIO_ARM64) {
2200 header->compression = HIB_COMPR_RATIO_ARM64;
2201 }
2202 #endif /* __arm64__ */
2203
2204 gIOHibernateCompression = header->compression;
2205
2206 count = vars->fileVars->fileExtents->getLength();
2207 if (count > sizeof(header->fileExtentMap)) {
2208 header->fileExtentMapSize = ((uint32_t) count);
2209 count = sizeof(header->fileExtentMap);
2210 } else {
2211 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2212 }
2213 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2214
2215 header->deviceBase = vars->fileVars->block0;
2216 header->deviceBlockSize = vars->fileVars->blockSize;
2217 header->lastHibAbsTime = mach_absolute_time();
2218 header->lastHibContTime = mach_continuous_time();
2219
2220
2221 IOPolledFileSeek(vars->fileVars, 0);
2222 err = IOHibernatePolledFileWrite(vars,
2223 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2224 cryptvars);
2225 if (kIOReturnSuccess != err) {
2226 break;
2227 }
2228
2229 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2230 }while (false);
2231
2232 clock_get_uptime(&endTime);
2233
2234 IOService::getPMRootDomain()->pmStatsRecordEvent(
2235 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2236
2237 SUB_ABSOLUTETIME(&endTime, &allTime);
2238 absolutetime_to_nanoseconds(endTime, &nsec);
2239 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2240
2241 absolutetime_to_nanoseconds(compTime, &nsec);
2242 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2243 compBytes,
2244 nsec / 1000000ULL,
2245 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2246
2247 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2248 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2249 vars->fileVars->cryptBytes,
2250 nsec / 1000000ULL,
2251 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2252
2253 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
2254 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2255 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2256 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2257
2258 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
2259
2260 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2261 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2262
2263 if (pollerOpen) {
2264 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2265 }
2266
2267 if (vars->consoleMapping) {
2268 ProgressUpdate(gIOHibernateGraphicsInfo,
2269 vars->consoleMapping, 0, kIOHibernateProgressCount);
2270 }
2271
2272 HIBLOG("hibernate_write_image done(%x)\n", err);
2273
2274 // should we come back via regular wake, set the state in memory.
2275 gIOHibernateState = kIOHibernateStateInactive;
2276
2277 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2278 wiredPagesClear, dirtyPagesEncrypted);
2279
2280 #if defined(__arm64__)
2281 if (kIOReturnSuccess == err) {
2282 return kIOHibernatePostWriteHalt;
2283 } else {
2284 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2285 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2286 vm_panic_hibernate_write_image_failed(err);
2287 return err; //not coming here post panic
2288 }
2289 #else
2290 if (kIOReturnSuccess == err) {
2291 if (kIOHibernateModeSleep & gIOHibernateMode) {
2292 return kIOHibernatePostWriteSleep;
2293 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2294 return kIOHibernatePostWriteRestart;
2295 } else {
2296 /* by default, power down */
2297 return kIOHibernatePostWriteHalt;
2298 }
2299 } else if (kIOReturnAborted == err) {
2300 return kIOHibernatePostWriteWake;
2301 } else {
2302 /* on error, sleep */
2303 return kIOHibernatePostWriteSleep;
2304 }
2305 #endif
2306 }
2307
2308 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2309
2310 extern "C" void
hibernate_machine_init(void)2311 hibernate_machine_init(void)
2312 {
2313 IOReturn err;
2314 uint32_t sum;
2315 uint32_t pagesDone;
2316 uint32_t pagesRead = 0;
2317 AbsoluteTime startTime, compTime;
2318 AbsoluteTime allTime, endTime;
2319 AbsoluteTime startIOTime, endIOTime;
2320 uint64_t nsec, nsecIO;
2321 uint64_t compBytes;
2322 uint64_t lastProgressStamp = 0;
2323 uint64_t progressStamp;
2324 IOPolledFileCryptVars * cryptvars = NULL;
2325
2326 IOHibernateVars * vars = &gIOHibernateVars;
2327 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2328
2329 if (!vars->fileVars || !vars->fileVars->pollers) {
2330 return;
2331 }
2332
2333 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2334 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2335
2336 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2337 HIBLOG("regular wake\n");
2338 return;
2339 }
2340
2341 HIBPRINT("diag %x %x %x %x\n",
2342 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2343 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2344
2345 #if defined(__i386__) || defined(__x86_64__)
2346 #define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2347 #else /* defined(__i386__) || defined(__x86_64__) */
2348 #define t40ms(x) x
2349 #endif /* defined(__i386__) || defined(__x86_64__) */
2350 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2351 tStat(booterStart, booterStart);
2352 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2353 tStat(booterDuration0, booterTime0);
2354 tStat(booterDuration1, booterTime1);
2355 tStat(booterDuration2, booterTime2);
2356 tStat(booterDuration, booterTime);
2357 tStat(booterConnectDisplayDuration, connectDisplayTime);
2358 tStat(booterSplashDuration, splashTime);
2359 tStat(trampolineDuration, trampolineTime);
2360
2361 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2362 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2363 gIOHibernateStats->image1Pages = pagesDone;
2364
2365 /* HIBERNATE_stats */
2366 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2367 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2368 gIOHibernateStats->trampolineDuration);
2369
2370 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2371 gIOHibernateStats->booterStart,
2372 gIOHibernateStats->smcStart,
2373 gIOHibernateStats->booterDuration0,
2374 gIOHibernateStats->booterDuration1,
2375 gIOHibernateStats->booterDuration2,
2376 gIOHibernateStats->booterDuration,
2377 gIOHibernateStats->booterConnectDisplayDuration,
2378 gIOHibernateStats->booterSplashDuration,
2379 gIOHibernateStats->trampolineDuration);
2380
2381 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2382 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2383 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2384
2385 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2386 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2387 hibernate_page_list_discard(vars->page_list);
2388 }
2389
2390 if (vars->hwEncrypt) {
2391 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2392 // decryption key via IOPolledFilePollersSetEncryptionKey
2393 cryptvars = NULL;
2394 } else {
2395 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2396 }
2397
2398 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2399 panic("handoff overflow");
2400 }
2401
2402 IOHibernateHandoff * handoff;
2403 bool done = false;
2404 bool foundCryptData = false;
2405 bool foundVolumeEncryptData = false;
2406 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2407 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
2408
2409 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2410 !done;
2411 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
2412 if (((uint8_t*)handoff < handoffStart) ||
2413 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2414 panic("handoff out of range");
2415 }
2416 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2417 uint8_t * data = &handoff->data[0];
2418 switch (handoff->type) {
2419 case kIOHibernateHandoffTypeEnd:
2420 done = true;
2421 break;
2422
2423 case kIOHibernateHandoffTypeGraphicsInfo:
2424 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2425 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2426 }
2427 break;
2428
2429 case kIOHibernateHandoffTypeCryptVars:
2430 if (cryptvars) {
2431 hibernate_cryptwakevars_t *
2432 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2433 if (handoff->bytecount == sizeof(*wakevars)) {
2434 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2435 } else {
2436 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2437 }
2438 }
2439 foundCryptData = true;
2440 bzero(data, handoff->bytecount);
2441 break;
2442
2443 case kIOHibernateHandoffTypeVolumeCryptKey:
2444 if (handoff->bytecount == vars->volumeCryptKeySize) {
2445 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2446 foundVolumeEncryptData = true;
2447 } else {
2448 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2449 }
2450 break;
2451
2452 #if defined(__i386__) || defined(__x86_64__)
2453 case kIOHibernateHandoffTypeMemoryMap:
2454
2455 clock_get_uptime(&allTime);
2456
2457 hibernate_newruntime_map(data, handoff->bytecount,
2458 gIOHibernateCurrentHeader->systemTableOffset);
2459
2460 clock_get_uptime(&endTime);
2461
2462 SUB_ABSOLUTETIME(&endTime, &allTime);
2463 absolutetime_to_nanoseconds(endTime, &nsec);
2464
2465 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2466
2467 break;
2468
2469 case kIOHibernateHandoffTypeDeviceTree:
2470 {
2471 // DTEntry chosen = NULL;
2472 // HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2473 }
2474 break;
2475 #endif /* defined(__i386__) || defined(__x86_64__) */
2476
2477 default:
2478 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2479 break;
2480 }
2481 }
2482
2483 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2484 panic("no volumeCryptKey");
2485 } else if (cryptvars && !foundCryptData) {
2486 panic("hibernate handoff");
2487 }
2488
2489 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2490 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2491 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2492
2493 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2494 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2495 * gIOHibernateGraphicsInfo->rowBytes);
2496 if (vars->videoMapSize > vars->videoAllocSize) {
2497 vars->videoMapSize = 0;
2498 } else {
2499 IOMapPages(kernel_map,
2500 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2501 vars->videoMapSize, kIOMapInhibitCache );
2502 }
2503 }
2504
2505 if (vars->videoMapSize) {
2506 ProgressUpdate(gIOHibernateGraphicsInfo,
2507 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2508 }
2509
2510 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2511 uint8_t * compressed = src + page_size;
2512 uint8_t * scratch = compressed + page_size;
2513 uint32_t decoOffset;
2514
2515 clock_get_uptime(&allTime);
2516 AbsoluteTime_to_scalar(&compTime) = 0;
2517 compBytes = 0;
2518
2519 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2520 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2521 clock_get_uptime(&startIOTime);
2522 endTime = startIOTime;
2523 SUB_ABSOLUTETIME(&endTime, &allTime);
2524 absolutetime_to_nanoseconds(endTime, &nsec);
2525 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2526
2527 if (vars->hwEncrypt) {
2528 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2529 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2530 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2531 if (kIOReturnSuccess != err) {
2532 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2533 }
2534 cryptvars = NULL;
2535 }
2536
2537 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2538
2539 // kick off the read ahead
2540 vars->fileVars->bufferHalf = 0;
2541 vars->fileVars->bufferLimit = 0;
2542 vars->fileVars->lastRead = 0;
2543 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2544 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2545 vars->fileVars->cryptBytes = 0;
2546 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2547
2548 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
2549 if (kIOReturnSuccess != err) {
2550 panic("Hibernate restore error %x", err);
2551 }
2552 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2553 // --
2554
2555 HIBLOG("hibernate_machine_init reading\n");
2556
2557
2558 uint32_t * header = (uint32_t *) src;
2559 sum = 0;
2560
2561 while (kIOReturnSuccess == err) {
2562 unsigned int count;
2563 unsigned int page;
2564 uint32_t tag;
2565 vm_offset_t compressedSize;
2566 ppnum_t ppnum;
2567
2568 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2569 if (kIOReturnSuccess != err) {
2570 panic("Hibernate restore error %x", err);
2571 }
2572
2573 ppnum = header[0];
2574 count = header[1];
2575
2576 // HIBPRINT("(%x, %x)\n", ppnum, count);
2577
2578 if (!count) {
2579 break;
2580 }
2581
2582 for (page = 0; page < count; page++) {
2583 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2584 if (kIOReturnSuccess != err) {
2585 panic("Hibernate restore error %x", err);
2586 }
2587
2588 compressedSize = kIOHibernateTagLength & tag;
2589 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2590 err = kIOReturnIPCError;
2591 panic("Hibernate restore error %x", err);
2592 }
2593
2594 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2595 if (kIOReturnSuccess != err) {
2596 panic("Hibernate restore error %x", err);
2597 }
2598
2599 if (compressedSize < page_size) {
2600 decoOffset = ((uint32_t) page_size);
2601 clock_get_uptime(&startTime);
2602
2603 if (compressedSize == 4) {
2604 int i;
2605 uint32_t *s, *d;
2606
2607 s = (uint32_t *)src;
2608 d = (uint32_t *)(uintptr_t)compressed;
2609
2610 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2611 *d++ = *s;
2612 }
2613 } else {
2614 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
2615 }
2616 clock_get_uptime(&endTime);
2617 ADD_ABSOLUTETIME(&compTime, &endTime);
2618 SUB_ABSOLUTETIME(&compTime, &startTime);
2619 compBytes += page_size;
2620 } else {
2621 decoOffset = 0;
2622 }
2623
2624 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
2625 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2626 if (err) {
2627 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2628 panic("Hibernate restore error %x", err);
2629 }
2630
2631
2632 ppnum++;
2633 pagesDone++;
2634 pagesRead++;
2635
2636 if (0 == (8191 & pagesDone)) {
2637 clock_get_uptime(&endTime);
2638 SUB_ABSOLUTETIME(&endTime, &allTime);
2639 absolutetime_to_nanoseconds(endTime, &nsec);
2640 progressStamp = nsec / 750000000ULL;
2641 if (progressStamp != lastProgressStamp) {
2642 lastProgressStamp = progressStamp;
2643 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2644 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2645 }
2646 }
2647 }
2648 }
2649 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2650 err = kIOReturnLockedRead;
2651 }
2652
2653 if (kIOReturnSuccess != err) {
2654 panic("Hibernate restore error %x", err);
2655 }
2656
2657
2658 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2659 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2660
2661 clock_get_uptime(&endIOTime);
2662
2663 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2664
2665 clock_get_uptime(&endTime);
2666
2667 IOService::getPMRootDomain()->pmStatsRecordEvent(
2668 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2669 IOService::getPMRootDomain()->pmStatsRecordEvent(
2670 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2671
2672 SUB_ABSOLUTETIME(&endTime, &allTime);
2673 absolutetime_to_nanoseconds(endTime, &nsec);
2674
2675 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2676 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2677
2678 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
2679 gIOHibernateStats->imagePages = pagesDone;
2680
2681 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2682 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2683 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2684
2685 absolutetime_to_nanoseconds(compTime, &nsec);
2686 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2687 compBytes,
2688 nsec / 1000000ULL,
2689 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2690
2691 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2692 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2693 vars->fileVars->cryptBytes,
2694 nsec / 1000000ULL,
2695 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2696
2697 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2698 }
2699
2700 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2701
2702 void
IOHibernateSetWakeCapabilities(uint32_t capability)2703 IOHibernateSetWakeCapabilities(uint32_t capability)
2704 {
2705 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2706 gIOHibernateStats->wakeCapability = capability;
2707
2708 if (kIOPMSystemCapabilityGraphics & capability) {
2709 vm_compressor_do_warmup();
2710 }
2711 }
2712 }
2713
2714 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2715
2716 void
IOHibernateSystemRestart(void)2717 IOHibernateSystemRestart(void)
2718 {
2719 static uint8_t noteStore[32] __attribute__((aligned(32)));
2720 IORegistryEntry * regEntry;
2721 const OSSymbol * sym;
2722 OSData * noteProp;
2723 OSData * data;
2724 uintptr_t * smcVars;
2725 uint8_t * smcBytes;
2726 size_t len;
2727 addr64_t element;
2728
2729 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2730 if (!data) {
2731 return;
2732 }
2733
2734 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2735 smcBytes = (typeof(smcBytes))smcVars[1];
2736 len = smcVars[0];
2737 if (len > sizeof(noteStore)) {
2738 len = sizeof(noteStore);
2739 }
2740 noteProp = OSData::withCapacity(3 * sizeof(element));
2741 if (!noteProp) {
2742 return;
2743 }
2744 element = len;
2745 noteProp->appendValue(element);
2746 element = crc32(0, smcBytes, len);
2747 noteProp->appendValue(element);
2748
2749 bcopy(smcBytes, noteStore, len);
2750 element = (addr64_t) ¬eStore[0];
2751 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2752 noteProp->appendValue(element);
2753
2754 if (!gIOOptionsEntry) {
2755 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2756 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2757 if (regEntry && !gIOOptionsEntry) {
2758 regEntry->release();
2759 }
2760 }
2761
2762 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2763 if (gIOOptionsEntry && sym) {
2764 gIOOptionsEntry->setProperty(sym, noteProp);
2765 }
2766 if (noteProp) {
2767 noteProp->release();
2768 }
2769 if (sym) {
2770 sym->release();
2771 }
2772 }
2773