1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #if defined(__i386__) || defined(__x86_64__)
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <vm/WKdm_new.h>
179 #elif defined(__arm64__)
180 #include <arm64/amcc_rorgn.h>
181 #endif /* defined(__i386__) || defined(__x86_64__) */
182 #include <san/kasan.h>
183
184
185 extern "C" addr64_t kvtophys(vm_offset_t va);
186 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
187
188 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
189
190 #define DISABLE_TRIM 0
191 #define TRIM_DELAY 25000
192
193 extern unsigned int save_kdebug_enable;
194 extern uint32_t gIOHibernateState;
195 uint32_t gIOHibernateMode;
196 static char gIOHibernateBootSignature[256 + 1];
197 static char gIOHibernateFilename[MAXPATHLEN + 1];
198 uint32_t gIOHibernateCount;
199
200 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
201
202 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
203 uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
204
205 enum {
206 HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
207 HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
208 };
209
210 #if defined(__arm64__)
211 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
212 #else
213 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
214 #endif /* __arm64__ */
215 boolean_t gIOHibernateStandbyDisabled;
216
217 static IODTNVRAM * gIOOptionsEntry;
218 static IORegistryEntry * gIOChosenEntry;
219
220 static const OSSymbol * gIOHibernateBootImageKey;
221 static const OSSymbol * gIOHibernateBootSignatureKey;
222 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
223
224 #if defined(__i386__) || defined(__x86_64__)
225
226 static const OSSymbol * gIOHibernateRTCVariablesKey;
227 static const OSSymbol * gIOHibernateBoot0082Key;
228 static const OSSymbol * gIOHibernateBootNextKey;
229 static OSData * gIOHibernateBoot0082Data;
230 static OSData * gIOHibernateBootNextData;
231 static OSObject * gIOHibernateBootNextSave;
232
233 #endif /* defined(__i386__) || defined(__x86_64__) */
234
235 static IOLock * gFSLock;
236 uint32_t gFSState;
237 static thread_call_t gIOHibernateTrimCalloutEntry;
238 static IOPolledFileIOVars gFileVars;
239 static IOHibernateVars gIOHibernateVars;
240 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
241 static hibernate_graphics_t _hibernateGraphics;
242 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
243 static hibernate_statistics_t _hibernateStats;
244 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
245
246 enum{
247 kFSIdle = 0,
248 kFSOpening = 2,
249 kFSOpened = 3,
250 kFSTimedOut = 4,
251 kFSTrimDelay = 5
252 };
253
254 static IOReturn IOHibernateDone(IOHibernateVars * vars);
255 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
256 static void IOSetBootImageNVRAM(OSData * data);
257 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
258
259 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
260
261 enum { kDefaultIOSize = 128 * 1024 };
262 enum { kVideoMapSize = 80 * 1024 * 1024 };
263
264 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
265
266 // copy from phys addr to MD
267
268 static IOReturn
IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)269 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
270 IOByteCount offset, addr64_t bytes, IOByteCount length)
271 {
272 addr64_t srcAddr = bytes;
273 IOByteCount remaining;
274
275 remaining = length = min(length, md->getLength() - offset);
276 while (remaining) { // (process another target segment?)
277 addr64_t dstAddr64;
278 IOByteCount dstLen;
279
280 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
281 if (!dstAddr64) {
282 break;
283 }
284
285 // Clip segment length to remaining
286 if (dstLen > remaining) {
287 dstLen = remaining;
288 }
289
290 #if 1
291 bcopy_phys(srcAddr, dstAddr64, dstLen);
292 #else
293 copypv(srcAddr, dstAddr64, dstLen,
294 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
295 #endif
296 srcAddr += dstLen;
297 offset += dstLen;
298 remaining -= dstLen;
299 }
300
301 assert(!remaining);
302
303 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
304 }
305
306 // copy from MD to phys addr
307
308 static IOReturn
IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)309 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
310 IOByteCount offset, addr64_t bytes, IOByteCount length)
311 {
312 addr64_t dstAddr = bytes;
313 IOByteCount remaining;
314
315 remaining = length = min(length, md->getLength() - offset);
316 while (remaining) { // (process another target segment?)
317 addr64_t srcAddr64;
318 IOByteCount dstLen;
319
320 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
321 if (!srcAddr64) {
322 break;
323 }
324
325 // Clip segment length to remaining
326 if (dstLen > remaining) {
327 dstLen = remaining;
328 }
329
330 #if 1
331 bcopy_phys(srcAddr64, dstAddr, dstLen);
332 #else
333 copypv(srcAddr, dstAddr64, dstLen,
334 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
335 #endif
336 dstAddr += dstLen;
337 offset += dstLen;
338 remaining -= dstLen;
339 }
340
341 assert(!remaining);
342
343 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
344 }
345
346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348 void
hibernate_set_page_state(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired,vm_offset_t ppnum,vm_offset_t count,uint32_t kind)349 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
350 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
351 {
352 count += ppnum;
353
354 if (count > UINT_MAX) {
355 panic("hibernate_set_page_state ppnum");
356 }
357
358 switch (kind) {
359 case kIOHibernatePageStateUnwiredSave:
360 // unwired save
361 for (; ppnum < count; ppnum++) {
362 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
363 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
364 }
365 break;
366 case kIOHibernatePageStateWiredSave:
367 // wired save
368 for (; ppnum < count; ppnum++) {
369 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
370 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
371 }
372 break;
373 case kIOHibernatePageStateFree:
374 // free page
375 for (; ppnum < count; ppnum++) {
376 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
377 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
378 }
379 break;
380 default:
381 panic("hibernate_set_page_state");
382 }
383 }
384
385 static void
hibernate_set_descriptor_page_state(IOHibernateVars * vars,IOMemoryDescriptor * descriptor,uint32_t kind,uint32_t * pageCount)386 hibernate_set_descriptor_page_state(IOHibernateVars *vars,
387 IOMemoryDescriptor *descriptor,
388 uint32_t kind,
389 uint32_t *pageCount)
390 {
391 IOItemCount count;
392 addr64_t phys64;
393 IOByteCount segLen;
394 if (descriptor) {
395 for (count = 0;
396 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
397 count += segLen) {
398 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
399 atop_64(phys64), atop_32(segLen),
400 kind);
401 *pageCount -= atop_32(segLen);
402 }
403 }
404 }
405
406 static vm_offset_t
hibernate_page_list_iterate(hibernate_page_list_t * list,ppnum_t * pPage)407 hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
408 {
409 uint32_t page = ((typeof(page)) * pPage);
410 uint32_t count;
411 hibernate_bitmap_t * bitmap;
412
413 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
414 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
415 if (!count) {
416 break;
417 }
418 page += count;
419 if (page <= bitmap->last_page) {
420 break;
421 }
422 }
423
424 *pPage = page;
425 if (bitmap) {
426 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
427 } else {
428 count = 0;
429 }
430
431 return count;
432 }
433
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435
436 IOReturn
IOHibernateSystemSleep(void)437 IOHibernateSystemSleep(void)
438 {
439 IOReturn err;
440 OSData * nvramData;
441 OSObject * obj;
442 OSString * str;
443 OSNumber * num;
444 bool dsSSD, vmflush, swapPinned;
445 IOHibernateVars * vars;
446 uint64_t setFileSize = 0;
447
448 gIOHibernateState = kIOHibernateStateInactive;
449
450 gIOHibernateDebugFlags = 0;
451 if (kIOLogHibernate & gIOKitDebug) {
452 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
453 }
454
455 if (IOService::getPMRootDomain()->getHibernateSettings(
456 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
457 if (kIOHibernateModeSleep & gIOHibernateMode) {
458 // default to discard clean for safe sleep
459 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
460 | kIOHibernateModeDiscardCleanActive);
461 }
462 }
463
464 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
465 if ((str = OSDynamicCast(OSString, obj))) {
466 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
467 sizeof(gIOHibernateFilename));
468 }
469 obj->release();
470 }
471
472 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
473 return kIOReturnUnsupported;
474 }
475
476 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
477
478 vars = IOMallocType(IOHibernateVars);
479
480 IOLockLock(gFSLock);
481 if (!gIOHibernateTrimCalloutEntry) {
482 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
483 }
484 IOHibernateSystemPostWakeTrim(NULL, NULL);
485 thread_call_cancel(gIOHibernateTrimCalloutEntry);
486 if (kFSIdle != gFSState) {
487 HIBLOG("hibernate file busy\n");
488 IOLockUnlock(gFSLock);
489 IOFreeType(vars, IOHibernateVars);
490 return kIOReturnBusy;
491 }
492 gFSState = kFSOpening;
493 IOLockUnlock(gFSLock);
494
495 swapPinned = false;
496 do{
497 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
498 HIBERNATION_SRC_BUFFER_SIZE, page_size);
499
500 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
501 ptoa_64(gIOHibernateHandoffPageCount), page_size);
502
503 if (!vars->srcBuffer || !vars->handoffBuffer) {
504 err = kIOReturnNoMemory;
505 break;
506 }
507
508 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
509 if ((num = OSDynamicCast(OSNumber, obj))) {
510 vars->fileMinSize = num->unsigned64BitValue();
511 }
512 obj->release();
513 }
514 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
515 if ((num = OSDynamicCast(OSNumber, obj))) {
516 vars->fileMaxSize = num->unsigned64BitValue();
517 }
518 obj->release();
519 }
520
521 boolean_t encryptedswap = true;
522 uint32_t pageCount;
523 AbsoluteTime startTime, endTime;
524 uint64_t nsec;
525
526 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
527 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
528 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
529
530 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
531 err = hibernate_alloc_page_lists(&vars->page_list,
532 &vars->page_list_wired,
533 &vars->page_list_pal);
534 if (KERN_SUCCESS != err) {
535 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
536 break;
537 }
538
539 err = hibernate_pin_swap(TRUE);
540 if (KERN_SUCCESS != err) {
541 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
542 break;
543 }
544 swapPinned = true;
545
546 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
547 hibernate_page_list_setall(vars->page_list,
548 vars->page_list_wired,
549 vars->page_list_pal,
550 true /* preflight */,
551 vmflush /* discard */,
552 &pageCount);
553 PE_Video consoleInfo;
554 bzero(&consoleInfo, sizeof(consoleInfo));
555 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
556
557 // estimate: 6% increase in pages compressed
558 // screen preview 2 images compressed 0%
559 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
560 + vars->page_list->list_size
561 + (consoleInfo.v_width * consoleInfo.v_height * 8);
562 enum { setFileRound = 1024 * 1024ULL };
563 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
564
565 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
566 pageCount, (100ULL * gIOHibernateCompression) >> 8,
567 setFileSize, vars->fileMinSize);
568
569 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
570 && (setFileSize < vars->fileMinSize)) {
571 setFileSize = vars->fileMinSize;
572 }
573 }
574
575 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
576 err = IOPolledFileOpen(gIOHibernateFilename,
577 (kIOPolledFileCreate | kIOPolledFileHibernate),
578 setFileSize, 0,
579 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
580 &vars->fileVars, &nvramData,
581 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
582
583 if (KERN_SUCCESS != err) {
584 IOLockLock(gFSLock);
585 if (kFSOpening != gFSState) {
586 err = kIOReturnTimeout;
587 }
588 IOLockUnlock(gFSLock);
589 }
590
591 if (KERN_SUCCESS != err) {
592 HIBLOG("IOPolledFileOpen(%x)\n", err);
593 OSSafeReleaseNULL(nvramData);
594 break;
595 }
596
597 // write extents for debug data usage in EFI
598 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
599
600 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
601 if (KERN_SUCCESS != err) {
602 OSSafeReleaseNULL(nvramData);
603 break;
604 }
605
606 clock_get_uptime(&startTime);
607 err = hibernate_setup(gIOHibernateCurrentHeader,
608 vmflush,
609 vars->page_list, vars->page_list_wired, vars->page_list_pal);
610 clock_get_uptime(&endTime);
611 SUB_ABSOLUTETIME(&endTime, &startTime);
612 absolutetime_to_nanoseconds(endTime, &nsec);
613
614 boolean_t haveSwapPin, hibFileSSD;
615 haveSwapPin = vm_swap_files_pinned();
616
617 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
618
619 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
620 err, nsec / 1000000ULL,
621 haveSwapPin, hibFileSSD);
622 if (KERN_SUCCESS != err) {
623 OSSafeReleaseNULL(nvramData);
624 break;
625 }
626
627 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
628
629 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
630 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
631
632 if (dsSSD) {
633 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
634 } else {
635 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
636 }
637
638
639 #if defined(__i386__) || defined(__x86_64__)
640 if (vars->volumeCryptKeySize &&
641 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
642 uintptr_t smcVars[2];
643 smcVars[0] = vars->volumeCryptKeySize;
644 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
645
646 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
647 bzero(smcVars, sizeof(smcVars));
648 }
649 #endif
650
651
652 if (encryptedswap || vars->volumeCryptKeySize) {
653 gIOHibernateMode ^= kIOHibernateModeEncrypt;
654 }
655
656 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
657 vars->videoAllocSize = kVideoMapSize;
658 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
659 vars->videoMapping = 0;
660 }
661 }
662
663 // generate crypt keys
664 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
665 vars->wiredCryptKey[i] = ((uint8_t) random());
666 }
667 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
668 vars->cryptKey[i] = ((uint8_t) random());
669 }
670
671 // set nvram
672
673 IOSetBootImageNVRAM(nvramData);
674 OSSafeReleaseNULL(nvramData);
675
676 #if defined(__i386__) || defined(__x86_64__)
677 {
678 struct AppleRTCHibernateVars {
679 uint8_t signature[4];
680 uint32_t revision;
681 uint8_t booterSignature[20];
682 uint8_t wiredCryptKey[16];
683 };
684 AppleRTCHibernateVars rtcVars;
685 OSData * data;
686
687 rtcVars.signature[0] = 'A';
688 rtcVars.signature[1] = 'A';
689 rtcVars.signature[2] = 'P';
690 rtcVars.signature[3] = 'L';
691 rtcVars.revision = 1;
692 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
693
694 if (gIOChosenEntry
695 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
696 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
697 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
698 } else if (gIOHibernateBootSignature[0]) {
699 char c;
700 uint8_t value = 0;
701 uint32_t in, out, digits;
702 for (in = out = digits = 0;
703 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
704 in++) {
705 if ((c >= 'a') && (c <= 'f')) {
706 c -= 'a' - 10;
707 } else if ((c >= 'A') && (c <= 'F')) {
708 c -= 'A' - 10;
709 } else if ((c >= '0') && (c <= '9')) {
710 c -= '0';
711 } else {
712 if (c == '=') {
713 out = digits = value = 0;
714 }
715 continue;
716 }
717 value = ((uint8_t) ((value << 4) | c));
718 if (digits & 1) {
719 rtcVars.booterSignature[out++] = value;
720 if (out >= sizeof(rtcVars.booterSignature)) {
721 break;
722 }
723 }
724 digits++;
725 }
726 }
727 #if DEBUG || DEVELOPMENT
728 if (kIOLogHibernate & gIOKitDebug) {
729 IOKitKernelLogBuffer("H> rtc:",
730 &rtcVars, sizeof(rtcVars), &kprintf);
731 }
732 #endif /* DEBUG || DEVELOPMENT */
733
734 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
735 if (data) {
736 if (gIOHibernateRTCVariablesKey) {
737 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
738 }
739 data->release();
740 }
741 if (gIOChosenEntry && gIOOptionsEntry) {
742 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
743 if (data) {
744 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
745 }
746 // set BootNext
747 if (!gIOHibernateBoot0082Data) {
748 OSData * fileData = NULL;
749 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
750 if (data && data->getLength() >= 4) {
751 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
752 }
753 if (data && (data->getLength() <= UINT16_MAX)) {
754 // AppleNVRAM_EFI_LOAD_OPTION
755 struct {
756 uint32_t Attributes;
757 uint16_t FilePathLength;
758 uint16_t Desc;
759 } loadOptionHeader;
760 loadOptionHeader.Attributes = 1;
761 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
762 loadOptionHeader.Desc = 0;
763 if (fileData) {
764 loadOptionHeader.FilePathLength -= 4;
765 loadOptionHeader.FilePathLength += fileData->getLength();
766 }
767 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
768 if (gIOHibernateBoot0082Data) {
769 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
770 if (fileData) {
771 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
772 gIOHibernateBoot0082Data->appendBytes(fileData);
773 } else {
774 gIOHibernateBoot0082Data->appendBytes(data);
775 }
776 }
777 }
778 }
779 if (!gIOHibernateBootNextData) {
780 uint16_t bits = 0x0082;
781 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
782 }
783
784 #if DEBUG || DEVELOPMENT
785 if (kIOLogHibernate & gIOKitDebug) {
786 IOKitKernelLogBuffer("H> bootnext:",
787 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
788 }
789 #endif /* DEBUG || DEVELOPMENT */
790 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
791 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
792 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
793 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
794 }
795 // BootNext
796 }
797 }
798 #endif /* !i386 && !x86_64 */
799 }while (false);
800
801 if (swapPinned) {
802 hibernate_pin_swap(FALSE);
803 }
804
805 IOLockLock(gFSLock);
806 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
807 HIBLOG("hibernate file close due timeout\n");
808 err = kIOReturnTimeout;
809 }
810 if (kIOReturnSuccess == err) {
811 gFSState = kFSOpened;
812 gIOHibernateVars = *vars;
813 gFileVars = *vars->fileVars;
814 gFileVars.allocated = false;
815 gIOHibernateVars.fileVars = &gFileVars;
816 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
817 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
818 gIOHibernateState = kIOHibernateStateHibernating;
819
820 #if DEBUG || DEVELOPMENT
821 if (kIOLogHibernate & gIOKitDebug) {
822 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
823 if (data) {
824 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
825 IOKitKernelLogBuffer("H> smc:",
826 (const void *)smcVars[1], smcVars[0], &kprintf);
827 }
828 }
829 #endif /* DEBUG || DEVELOPMENT */
830 } else {
831 IOPolledFileIOVars * fileVars = vars->fileVars;
832 IOHibernateDone(vars);
833 IOPolledFileClose(&fileVars,
834 #if DISABLE_TRIM
835 0, NULL, 0, 0, 0);
836 #else
837 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
838 #endif
839 gFSState = kFSIdle;
840 }
841 IOLockUnlock(gFSLock);
842
843 if (vars->fileVars) {
844 IOFreeType(vars->fileVars, IOPolledFileIOVars);
845 }
846 IOFreeType(vars, IOHibernateVars);
847
848 return err;
849 }
850
851 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
852
853 static void
IOSetBootImageNVRAM(OSData * data)854 IOSetBootImageNVRAM(OSData * data)
855 {
856 IORegistryEntry * regEntry;
857
858 if (!gIOOptionsEntry) {
859 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
860 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
861 if (regEntry && !gIOOptionsEntry) {
862 regEntry->release();
863 }
864 }
865 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
866 if (data) {
867 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
868 #if DEBUG || DEVELOPMENT
869 if (kIOLogHibernate & gIOKitDebug) {
870 IOKitKernelLogBuffer("H> boot-image:",
871 data->getBytesNoCopy(), data->getLength(), &kprintf);
872 }
873 #endif /* DEBUG || DEVELOPMENT */
874 } else {
875 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
876 #if __x86_64__
877 gIOOptionsEntry->sync();
878 #else
879 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
880 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
881 gIOOptionsEntry->sync();
882 }
883 #endif
884 }
885 }
886 }
887
888 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
889 /*
890 * Writes header to disk with signature, block size and file extents data.
891 * If there are more than 2 extents, then they are written on second block.
892 */
893 static IOReturn
IOWriteExtentsToFile(IOPolledFileIOVars * vars,uint32_t signature)894 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
895 {
896 IOHibernateImageHeader hdr;
897 IOItemCount count;
898 IOReturn err = kIOReturnSuccess;
899 int rc;
900 IOPolledFileExtent * fileExtents;
901
902 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
903
904 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
905 count = vars->fileExtents->getLength();
906 if (count > sizeof(hdr.fileExtentMap)) {
907 hdr.fileExtentMapSize = count;
908 count = sizeof(hdr.fileExtentMap);
909 } else {
910 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
911 }
912
913 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
914
915 // copy file block extent list if larger than header
916 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
917 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
918 rc = kern_write_file(vars->fileRef, vars->blockSize,
919 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
920 count, IO_SKIP_ENCRYPTION);
921 if (rc != 0) {
922 HIBLOG("kern_write_file returned %d\n", rc);
923 err = kIOReturnIOError;
924 goto exit;
925 }
926 }
927 hdr.signature = signature;
928 hdr.deviceBlockSize = vars->blockSize;
929
930 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
931 if (rc != 0) {
932 HIBLOG("kern_write_file returned %d\n", rc);
933 err = kIOReturnIOError;
934 goto exit;
935 }
936
937 exit:
938 return err;
939 }
940
941 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
942
943 DECLARE_IOHIBERNATEPROGRESSALPHA
944
945 static void
ProgressInit(hibernate_graphics_t * display,uint8_t * screen,uint8_t * saveunder,uint32_t savelen)946 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
947 {
948 uint32_t rowBytes, pixelShift;
949 uint32_t x, y;
950 int32_t blob;
951 uint32_t alpha, color, result;
952 uint8_t * out, in;
953 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
954
955 rowBytes = display->rowBytes;
956 pixelShift = display->depth >> 4;
957 if (pixelShift < 1) {
958 return;
959 }
960
961 screen += ((display->width
962 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
963 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
964
965 for (y = 0; y < kIOHibernateProgressHeight; y++) {
966 out = screen + y * rowBytes;
967 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
968 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
969 for (x = 0; x < kIOHibernateProgressWidth; x++) {
970 alpha = gIOHibernateProgressAlpha[y][x];
971 result = color;
972 if (alpha) {
973 if (0xff != alpha) {
974 if (1 == pixelShift) {
975 in = *((uint16_t *)out) & 0x1f; // 16
976 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
977 } else {
978 in = *((uint32_t *)out) & 0xff; // 32
979 }
980 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
981 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
982 }
983 if (1 == pixelShift) {
984 result >>= 3;
985 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
986 } else {
987 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
988 }
989 }
990 out += (1 << pixelShift);
991 }
992 out += (kIOHibernateProgressSpacing << pixelShift);
993 }
994 }
995 }
996
997
998 static void
ProgressUpdate(hibernate_graphics_t * display,uint8_t * screen,int32_t firstBlob,int32_t select)999 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1000 {
1001 uint32_t rowBytes, pixelShift;
1002 uint32_t x, y;
1003 int32_t blob, lastBlob;
1004 uint32_t alpha, in, color, result;
1005 uint8_t * out;
1006 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1007
1008 pixelShift = display->depth >> 4;
1009 if (pixelShift < 1) {
1010 return;
1011 }
1012
1013 rowBytes = display->rowBytes;
1014
1015 screen += ((display->width
1016 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1017 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1018
1019 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1020
1021 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1022
1023 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1024 out = screen + y * rowBytes;
1025 for (blob = firstBlob; blob <= lastBlob; blob++) {
1026 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1027 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1028 alpha = gIOHibernateProgressAlpha[y][x];
1029 result = color;
1030 if (alpha) {
1031 if (0xff != alpha) {
1032 in = display->progressSaveUnder[blob][saveindex[blob]++];
1033 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1034 }
1035 if (1 == pixelShift) {
1036 result >>= 3;
1037 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
1038 } else {
1039 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1040 }
1041 }
1042 out += (1 << pixelShift);
1043 }
1044 out += (kIOHibernateProgressSpacing << pixelShift);
1045 }
1046 }
1047 }
1048
1049 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1050
1051 IOReturn
IOHibernateIOKitSleep(void)1052 IOHibernateIOKitSleep(void)
1053 {
1054 IOReturn ret = kIOReturnSuccess;
1055 IOLockLock(gFSLock);
1056 if (kFSOpening == gFSState) {
1057 gFSState = kFSTimedOut;
1058 HIBLOG("hibernate file open timed out\n");
1059 ret = kIOReturnTimeout;
1060 }
1061 IOLockUnlock(gFSLock);
1062 return ret;
1063 }
1064
1065 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1066
1067 IOReturn
IOHibernateSystemHasSlept(void)1068 IOHibernateSystemHasSlept(void)
1069 {
1070 IOReturn ret = kIOReturnSuccess;
1071 IOHibernateVars * vars = &gIOHibernateVars;
1072 OSObject * obj = NULL;
1073 OSData * data;
1074
1075 IOLockLock(gFSLock);
1076 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1077 ret = kIOReturnTimeout;
1078 }
1079 IOLockUnlock(gFSLock);
1080 if (kIOReturnSuccess != ret) {
1081 return ret;
1082 }
1083
1084 if (gIOHibernateMode) {
1085 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1086 }
1087 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1088 if (obj && !vars->previewBuffer) {
1089 obj->release();
1090 }
1091 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1092 OSSafeReleaseNULL(vars->previewBuffer);
1093 }
1094
1095 vars->consoleMapping = NULL;
1096 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1097 vars->previewBuffer->release();
1098 vars->previewBuffer = NULL;
1099 }
1100
1101 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1102 && vars->previewBuffer
1103 && (data = OSDynamicCast(OSData,
1104 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1105 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1106 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1107
1108 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1109
1110 if (kIOHibernatePreviewUpdates & flags) {
1111 PE_Video consoleInfo;
1112 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1113
1114 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1115
1116 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1117 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1118 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1119 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
1120 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1121
1122 HIBPRINT("video %p %d %d %d\n",
1123 vars->consoleMapping, graphicsInfo->depth,
1124 graphicsInfo->width, graphicsInfo->height);
1125 if (vars->consoleMapping) {
1126 ProgressInit(graphicsInfo, vars->consoleMapping,
1127 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1128 }
1129 }
1130 }
1131
1132 if (gIOOptionsEntry) {
1133 #if __x86_64__
1134 gIOOptionsEntry->sync();
1135 #else
1136 if (gIOHibernateMode) {
1137 gIOOptionsEntry->sync();
1138 }
1139 #endif
1140 }
1141
1142 return ret;
1143 }
1144
1145 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1146
1147 static const DeviceTreeNode *
MergeDeviceTree(const DeviceTreeNode * entry,IORegistryEntry * regEntry,OSSet * entriesToUpdate,vm_offset_t region_start,vm_size_t region_size)1148 MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
1149 {
1150 DeviceTreeNodeProperty * prop;
1151 const DeviceTreeNode * child;
1152 IORegistryEntry * childRegEntry;
1153 const char * nameProp;
1154 unsigned int propLen, idx;
1155
1156 bool updateEntry = true;
1157 if (!regEntry) {
1158 updateEntry = false;
1159 } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
1160 updateEntry = false;
1161 }
1162
1163 prop = (DeviceTreeNodeProperty *) (entry + 1);
1164 for (idx = 0; idx < entry->nProperties; idx++) {
1165 if (updateEntry && (0 != strcmp("name", prop->name))) {
1166 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1167 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1168 }
1169 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1170 }
1171
1172 if (entriesToUpdate) {
1173 entriesToUpdate->removeObject(regEntry);
1174 if (entriesToUpdate->getCount() == 0) {
1175 // we've updated all the entries we care about so we can stop
1176 return NULL;
1177 }
1178 }
1179
1180 child = (const DeviceTreeNode *) prop;
1181 for (idx = 0; idx < entry->nChildren; idx++) {
1182 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1183 region_start, region_size)) {
1184 panic("no name");
1185 }
1186 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1187 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1188 child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
1189 OSSafeReleaseNULL(childRegEntry);
1190 if (!child) {
1191 // the recursive call updated the last entry we cared about, so we can stop
1192 break;
1193 }
1194 }
1195 return child;
1196 }
1197
1198 IOReturn
IOHibernateSystemWake(void)1199 IOHibernateSystemWake(void)
1200 {
1201 if (kFSOpened == gFSState) {
1202 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1203 IOHibernateDone(&gIOHibernateVars);
1204 } else {
1205 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1207 }
1208
1209 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1210 // if we got this far, clear boot-image
1211 // we don't need to sync immediately; the booter should have already removed this entry
1212 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1213 // a stale boot-image value
1214 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1215 }
1216
1217 return kIOReturnSuccess;
1218 }
1219
1220 static IOReturn
IOHibernateDone(IOHibernateVars * vars)1221 IOHibernateDone(IOHibernateVars * vars)
1222 {
1223 IOReturn err;
1224 OSData * data;
1225
1226 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1227
1228 if (vars->videoMapping) {
1229 if (vars->videoMapSize) {
1230 // remove mappings
1231 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1232 }
1233 if (vars->videoAllocSize) {
1234 // dealloc range
1235 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1236 }
1237 }
1238
1239 if (vars->previewBuffer) {
1240 vars->previewBuffer->release();
1241 vars->previewBuffer = NULL;
1242 }
1243
1244 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1245 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1246 gIOHibernateCurrentHeader->options, 32);
1247 } else {
1248 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1249 }
1250
1251 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1252 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1253 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1254 &gIOHibernateGraphicsInfo->gfxStatus,
1255 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1256 } else {
1257 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1258 }
1259
1260 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1261
1262 #if defined(__i386__) || defined(__x86_64__)
1263 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1264 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1265
1266 /*
1267 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1268 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1269 */
1270 if (gIOOptionsEntry) {
1271 if (gIOHibernateRTCVariablesKey) {
1272 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1273 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1274 }
1275 }
1276
1277 if (gIOHibernateBootNextKey) {
1278 if (gIOHibernateBootNextSave) {
1279 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1280 gIOHibernateBootNextSave->release();
1281 gIOHibernateBootNextSave = NULL;
1282 } else {
1283 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1284 }
1285 }
1286 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1287 gIOOptionsEntry->sync();
1288 }
1289 }
1290 #endif
1291
1292 if (vars->srcBuffer) {
1293 vars->srcBuffer->release();
1294 }
1295
1296
1297 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1298 if (vars->handoffBuffer) {
1299 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1300 IOHibernateHandoff * handoff;
1301 bool done = false;
1302 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1303 !done;
1304 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1305 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1306 uint8_t * __unused data = &handoff->data[0];
1307 switch (handoff->type) {
1308 case kIOHibernateHandoffTypeEnd:
1309 done = true;
1310 break;
1311
1312 case kIOHibernateHandoffTypeDeviceTree:
1313 {
1314 #if defined(__i386__) || defined(__x86_64__)
1315 // On Intel, process the entirety of the passed in device tree
1316 OSSet * entriesToUpdate = NULL;
1317 #elif defined(__arm64__)
1318 // On ARM, only allow hibernation to update specific entries
1319 const char *mergePaths[] = {
1320 kIODeviceTreePlane ":/chosen/boot-object-manifests",
1321 kIODeviceTreePlane ":/chosen/secure-boot-hashes",
1322 };
1323 const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
1324 OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
1325 for (size_t i = 0; i < mergePathCount; i++) {
1326 IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
1327 if (!entry) {
1328 panic("failed to find %s in IORegistry", mergePaths[i]);
1329 }
1330 entriesToUpdate->setObject(entry);
1331 OSSafeReleaseNULL(entry);
1332 }
1333 #endif
1334 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
1335 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1336 OSSafeReleaseNULL(entriesToUpdate);
1337 break;
1338 }
1339
1340 case kIOHibernateHandoffTypeKeyStore:
1341 #if defined(__i386__) || defined(__x86_64__)
1342 {
1343 IOBufferMemoryDescriptor *
1344 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1345 if (md) {
1346 IOSetKeyStoreData(md);
1347 }
1348 }
1349 #endif
1350 break;
1351
1352 default:
1353 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1354 break;
1355 }
1356 }
1357 #if defined(__i386__) || defined(__x86_64__)
1358 if (vars->volumeCryptKeySize) {
1359 IOBufferMemoryDescriptor *
1360 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1361 vars->volumeCryptKeySize, kIODirectionOutIn);
1362 if (!bmd) {
1363 panic("IOBufferMemoryDescriptor");
1364 }
1365 IOSetAPFSKeyStoreData(bmd);
1366 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1367 }
1368 #endif
1369 }
1370 vars->handoffBuffer->release();
1371 }
1372
1373 if (gIOChosenEntry
1374 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1375 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1376 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1377 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1378 }
1379
1380 if (vars->hwEncrypt) {
1381 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1382 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1383 }
1384
1385 bzero(vars, sizeof(*vars));
1386
1387 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1388 gIOHibernateCount++;
1389
1390 return kIOReturnSuccess;
1391 }
1392
1393 static void
IOHibernateSystemPostWakeTrim(void * p1,void * p2)1394 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1395 {
1396 // invalidate & close the image file
1397 if (p1) {
1398 IOLockLock(gFSLock);
1399 }
1400 if (kFSTrimDelay == gFSState) {
1401 IOPolledFileIOVars * vars = &gFileVars;
1402 IOPolledFileClose(&vars,
1403 #if DISABLE_TRIM
1404 0, NULL, 0, 0, 0);
1405 #else
1406 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1407 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1408 #endif
1409 gFSState = kFSIdle;
1410 }
1411 if (p1) {
1412 IOLockUnlock(gFSLock);
1413 }
1414 }
1415
1416 IOReturn
IOHibernateSystemPostWake(bool now)1417 IOHibernateSystemPostWake(bool now)
1418 {
1419 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1420 IOSetBootImageNVRAM(NULL);
1421
1422 IOLockLock(gFSLock);
1423 if (kFSTrimDelay == gFSState) {
1424 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1425 IOHibernateSystemPostWakeTrim(NULL, NULL);
1426 } else if (kFSOpened != gFSState) {
1427 gFSState = kFSIdle;
1428 } else {
1429 gFSState = kFSTrimDelay;
1430 if (now) {
1431 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1432 IOHibernateSystemPostWakeTrim(NULL, NULL);
1433 } else {
1434 AbsoluteTime deadline;
1435 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1436 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1437 }
1438 }
1439 IOLockUnlock(gFSLock);
1440
1441 return kIOReturnSuccess;
1442 }
1443
1444 uint32_t
IOHibernateWasScreenLocked(void)1445 IOHibernateWasScreenLocked(void)
1446 {
1447 uint32_t ret = 0;
1448 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
1449 OSData *
1450 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1451 if (data) {
1452 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1453 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1454 }
1455 } else {
1456 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1457 }
1458
1459 return ret;
1460 }
1461
1462 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1463
1464 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1465 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1466 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1467 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1468 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1469 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1470 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1471 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1472 &gIOHibernateMode, 0, "");
1473 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1474 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
1475 &_hibernateStats, hibernate_statistics_t, "");
1476 SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
1477 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1478 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
1479 sysctl_handle_string, "A", "");
1480
1481 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1482 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1483 &_hibernateStats.graphicsReadyTime, 0, "");
1484 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1485 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1486 &_hibernateStats.wakeNotificationTime, 0, "");
1487 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1488 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1489 &_hibernateStats.lockScreenReadyTime, 0, "");
1490 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1491 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1492 &_hibernateStats.hidReadyTime, 0, "");
1493
1494 SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
1495 CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1496 &gIOHibernateCount, 0, "");
1497
1498 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1499
1500 static int
1501 hibernate_set_preview SYSCTL_HANDLER_ARGS
1502 {
1503 #pragma unused(oidp, arg1, arg2)
1504
1505 if (!IOCurrentTaskHasEntitlement(kIOHibernateSetPreviewEntitlementKey)) {
1506 return EPERM;
1507 }
1508
1509 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1510 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1511 return 0;
1512 }
1513
1514 size_t rounded_size = round_page(req->newlen);
1515 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1516 if (!md) {
1517 return ENOMEM;
1518 }
1519
1520 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1521 int error = SYSCTL_IN(req, bytes, req->newlen);
1522 if (error) {
1523 md->release();
1524 return error;
1525 }
1526
1527 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1528 md->release();
1529
1530 return 0;
1531 }
1532
1533 SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1534 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1535 hibernate_set_preview, "S", "");
1536
1537 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1538
1539 void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)1540 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1541 {
1542 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1543 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1544 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1545
1546 #if defined(__i386__) || defined(__x86_64__)
1547 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1548 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1549 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1550 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1551 #endif /* defined(__i386__) || defined(__x86_64__) */
1552
1553 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1554 if (data) {
1555 rootDomain->setProperty(kIOHibernateStateKey, data);
1556 data->release();
1557 }
1558
1559 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1560 gIOHibernateMode = kIOHibernateModeOn;
1561 } else {
1562 gIOHibernateFilename[0] = 0;
1563 }
1564
1565 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1566
1567 if (gIOChosenEntry
1568 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1569 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1570 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1571 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1572 }
1573
1574 gFSLock = IOLockAlloc();
1575 gIOHibernateCount = 0;
1576 }
1577
1578 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1579
1580 static IOReturn
IOHibernatePolledFileWrite(IOHibernateVars * vars,const uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)1581 IOHibernatePolledFileWrite(IOHibernateVars * vars,
1582 const uint8_t * bytes, IOByteCount size,
1583 IOPolledFileCryptVars * cryptvars)
1584 {
1585 IOReturn err;
1586
1587
1588 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
1589 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1590 err = kIOReturnAborted;
1591 }
1592
1593
1594 return err;
1595 }
1596
1597 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1598
1599 extern "C" uint32_t
hibernate_write_image(void)1600 hibernate_write_image(void)
1601 {
1602 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1603 IOHibernateVars * vars = &gIOHibernateVars;
1604 IOPolledFileExtent * fileExtents;
1605
1606 #if !defined(__arm64__)
1607 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1608 #endif /* !defined(__arm64__) */
1609
1610 uint32_t pageCount, pagesDone;
1611 IOReturn err;
1612 ppnum_t ppnum, page;
1613 vm_offset_t count;
1614 uint8_t * src;
1615 uint8_t * data;
1616 uint8_t * compressed;
1617 uint8_t * scratch;
1618 IOByteCount pageCompressedSize;
1619 uint64_t compressedSize, uncompressedSize;
1620 uint64_t image1Size = 0;
1621 uint32_t bitmap_size;
1622 bool iterDone, pollerOpen, needEncrypt;
1623 int wkresult;
1624 uint32_t tag;
1625 uint32_t pageType;
1626 uint32_t pageAndCount[2];
1627 addr64_t phys64;
1628 IOByteCount segLen;
1629 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
1630 uintptr_t hibernateBase;
1631 uintptr_t hibernateEnd;
1632
1633 AbsoluteTime startTime, endTime;
1634 AbsoluteTime allTime, compTime;
1635 uint64_t compBytes;
1636 uint64_t nsec;
1637 uint64_t lastProgressStamp = 0;
1638 uint64_t progressStamp;
1639 uint32_t blob, lastBlob = (uint32_t) -1L;
1640
1641 uint32_t wiredPagesEncrypted;
1642 uint32_t dirtyPagesEncrypted;
1643 uint32_t wiredPagesClear;
1644 uint32_t svPageCount;
1645 uint32_t zvPageCount;
1646
1647 IOPolledFileCryptVars _cryptvars;
1648 IOPolledFileCryptVars * cryptvars = NULL;
1649
1650 wiredPagesEncrypted = 0;
1651 dirtyPagesEncrypted = 0;
1652 wiredPagesClear = 0;
1653 svPageCount = 0;
1654 zvPageCount = 0;
1655
1656 if (!vars->fileVars
1657 || !vars->fileVars->pollers
1658 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1659 return kIOHibernatePostWriteSleep;
1660 }
1661
1662
1663 #if !defined(__arm64__)
1664 if (kIOHibernateModeSleep & gIOHibernateMode) {
1665 kdebug_enable = save_kdebug_enable;
1666 }
1667 #endif /* !defined(__arm64__) */
1668
1669 pal_hib_write_hook();
1670
1671 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1672 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1673
1674 #if CRYPTO
1675 // encryption data. "iv" is the "initial vector".
1676 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1677 static const unsigned char first_iv[AES_BLOCK_SIZE]
1678 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1679 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1680
1681 cryptvars = &gIOHibernateCryptWakeContext;
1682 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1683 aes_encrypt_key(vars->cryptKey,
1684 kIOHibernateAESKeySize,
1685 &cryptvars->ctx.encrypt);
1686 aes_decrypt_key(vars->cryptKey,
1687 kIOHibernateAESKeySize,
1688 &cryptvars->ctx.decrypt);
1689
1690 cryptvars = &_cryptvars;
1691 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1692 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1693 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1694 }
1695 aes_encrypt_key(vars->wiredCryptKey,
1696 kIOHibernateAESKeySize,
1697 &cryptvars->ctx.encrypt);
1698
1699 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1700 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1701 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1702 }
1703 #endif /* CRYPTO */
1704
1705 hibernate_page_list_setall(vars->page_list,
1706 vars->page_list_wired,
1707 vars->page_list_pal,
1708 false /* !preflight */,
1709 /* discard_all */
1710 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1711 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1712 &pageCount);
1713
1714 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1715
1716 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1717
1718 #if 0
1719 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1720 for (page = 0; page < count; page++) {
1721 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1722 fileExtents[page].start, fileExtents[page].length,
1723 fileExtents[page].start + fileExtents[page].length);
1724 }
1725 #endif
1726
1727 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1728 AbsoluteTime_to_scalar(&compTime) = 0;
1729 compBytes = 0;
1730
1731 clock_get_uptime(&allTime);
1732 IOService::getPMRootDomain()->pmStatsRecordEvent(
1733 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1734 do{
1735 compressedSize = 0;
1736 uncompressedSize = 0;
1737 svPageCount = 0;
1738 zvPageCount = 0;
1739
1740 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1741
1742 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1743 ml_get_interrupts_enabled());
1744 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1745 // abortable if not low battery
1746 !IOService::getPMRootDomain()->mustHibernate());
1747 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1748 pollerOpen = (kIOReturnSuccess == err);
1749 if (!pollerOpen) {
1750 break;
1751 }
1752
1753 if (vars->volumeCryptKeySize) {
1754 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1755 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1756 vars->hwEncrypt = (kIOReturnSuccess == err);
1757 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1758 if (vars->hwEncrypt) {
1759 header->options |= kIOHibernateOptionHWEncrypt;
1760 }
1761 }
1762
1763 // copy file block extent list if larger than header
1764
1765 count = vars->fileVars->fileExtents->getLength();
1766 if (count > sizeof(header->fileExtentMap)) {
1767 count -= sizeof(header->fileExtentMap);
1768 err = IOHibernatePolledFileWrite(vars,
1769 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1770 if (kIOReturnSuccess != err) {
1771 break;
1772 }
1773 }
1774
1775 // copy out restore1 code
1776
1777 for (count = 0;
1778 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1779 count += segLen) {
1780 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1781 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
1782 }
1783 }
1784
1785 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1786 hibernateEnd = (segHIBB + segSizeHIB);
1787
1788 page = atop_32(kvtophys(hibernateBase));
1789 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1790 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1791 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1792 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1793 panic("malformed kernel layout");
1794 }
1795 header->restore1CodePhysPage = (ppnum_t) page;
1796 header->restore1CodeVirt = hibernateBase;
1797 header->restore1PageCount = (uint32_t) count;
1798 header->restore1CodeOffset = (uint32_t) entrypoint;
1799 header->restore1StackOffset = (uint32_t) stack;
1800
1801 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1802 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1803 }
1804
1805 // sum __HIB seg, with zeros for the stack
1806 src = (uint8_t *) trunc_page(hibernateBase);
1807 for (page = 0; page < count; page++) {
1808 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1809 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
1810 } else {
1811 restore1Sum += 0x00000000;
1812 }
1813 src += page_size;
1814 }
1815 sum1 = restore1Sum;
1816
1817 // write the __HIB seg, with zeros for the stack
1818
1819 src = (uint8_t *) trunc_page(hibernateBase);
1820 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1821 if (count) {
1822 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1823 if (kIOReturnSuccess != err) {
1824 break;
1825 }
1826 }
1827 err = IOHibernatePolledFileWrite(vars,
1828 (uint8_t *) NULL,
1829 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1830 cryptvars);
1831 if (kIOReturnSuccess != err) {
1832 break;
1833 }
1834 src = &gIOHibernateRestoreStackEnd[0];
1835 count = round_page(hibernateEnd) - ((uintptr_t) src);
1836 if (count) {
1837 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1838 if (kIOReturnSuccess != err) {
1839 break;
1840 }
1841 }
1842
1843 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1844 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1845 vars->fileVars->encryptEnd = UINT64_MAX;
1846 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1847 }
1848
1849 // write the preview buffer
1850
1851 if (vars->previewBuffer) {
1852 ppnum = 0;
1853 count = 0;
1854 do{
1855 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1856 pageAndCount[0] = atop_64_ppnum(phys64);
1857 pageAndCount[1] = atop_64_ppnum(segLen);
1858 err = IOHibernatePolledFileWrite(vars,
1859 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1860 cryptvars);
1861 if (kIOReturnSuccess != err) {
1862 break;
1863 }
1864 count += segLen;
1865 ppnum += sizeof(pageAndCount);
1866 }while (phys64);
1867 if (kIOReturnSuccess != err) {
1868 break;
1869 }
1870
1871 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1872
1873 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1874
1875 count = (uint32_t) vars->previewBuffer->getLength();
1876
1877 header->previewPageListSize = ((uint32_t) ppnum);
1878 header->previewSize = ((uint32_t) (count + ppnum));
1879
1880 for (page = 0; page < count; page += page_size) {
1881 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1882 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
1883 }
1884 if (kIOReturnSuccess != err) {
1885 break;
1886 }
1887 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1888 if (kIOReturnSuccess != err) {
1889 break;
1890 }
1891 }
1892
1893 // mark areas for no save
1894 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1895 kIOHibernatePageStateFree, &pageCount);
1896 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1897 kIOHibernatePageStateFree, &pageCount);
1898
1899 // copy out bitmap of pages available for trashing during restore
1900
1901 bitmap_size = vars->page_list_wired->list_size;
1902 src = (uint8_t *) vars->page_list_wired;
1903 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
1904 if (kIOReturnSuccess != err) {
1905 break;
1906 }
1907
1908 // mark more areas for no save, but these are not available
1909 // for trashing during restore
1910
1911 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1912
1913 #if defined(__i386__) || defined(__x86_64__)
1914 // __HIB is explicitly saved above so we don't have to save it again
1915 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1916 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1917 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1918 page, count,
1919 kIOHibernatePageStateFree);
1920 pageCount -= count;
1921 #elif defined(__arm64__)
1922 // the segments described in IOHibernateHibSegInfo are stored directly in the
1923 // hibernation file, so they don't need to be saved again
1924 extern unsigned long gPhysBase, gPhysSize;
1925 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1926 page = segInfo->segments[i].physPage;
1927 count = segInfo->segments[i].pageCount;
1928 uint64_t physAddr = ptoa_64(page);
1929 uint64_t size = ptoa_64(count);
1930 if (size &&
1931 (physAddr >= gPhysBase) &&
1932 (physAddr + size <= gPhysBase + gPhysSize)) {
1933 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1934 page, count,
1935 kIOHibernatePageStateFree);
1936 pageCount -= count;
1937 }
1938 }
1939 #else
1940 #error unimplemented
1941 #endif
1942
1943 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
1944 kIOHibernatePageStateFree, &pageCount);
1945 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
1946 kIOHibernatePageStateFree, &pageCount);
1947
1948 #if KASAN
1949 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1950
1951 /* no need to save unused shadow pages */
1952 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1953 atop_64(shadow_pnext),
1954 shadow_pages_free,
1955 kIOHibernatePageStateFree);
1956 #endif
1957
1958 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1959 compressed = src + page_size;
1960 scratch = compressed + page_size;
1961
1962 pagesDone = 0;
1963 lastBlob = 0;
1964
1965 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1966 bitmap_size, header->previewSize,
1967 pageCount, vars->fileVars->position);
1968
1969
1970 enum
1971 // pageType
1972 {
1973 kWired = 0x02,
1974 kEncrypt = 0x01,
1975 kWiredEncrypt = kWired | kEncrypt,
1976 kWiredClear = kWired,
1977 kUnwiredEncrypt = kEncrypt
1978 };
1979
1980 #if defined(__i386__) || defined(__x86_64__)
1981 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1982 #else /* defined(__i386__) || defined(__x86_64__) */
1983 static const bool cpuAES = true;
1984 #endif /* defined(__i386__) || defined(__x86_64__) */
1985
1986 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1987 if (kUnwiredEncrypt == pageType) {
1988 // start unwired image
1989 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1990 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1991 vars->fileVars->encryptEnd = UINT64_MAX;
1992 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1993 }
1994 bcopy(&cryptvars->aes_iv[0],
1995 &gIOHibernateCryptWakeContext.aes_iv[0],
1996 sizeof(cryptvars->aes_iv));
1997 cryptvars = &gIOHibernateCryptWakeContext;
1998 }
1999 for (iterDone = false, ppnum = 0; !iterDone;) {
2000 if (cpuAES && (pageType == kWiredClear)) {
2001 count = 0;
2002 } else {
2003 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2004 &ppnum);
2005 if (count > UINT_MAX) {
2006 count = UINT_MAX;
2007 }
2008 }
2009 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2010 iterDone = !count;
2011
2012 if (!cpuAES) {
2013 if (count && (kWired & pageType) && needEncrypt) {
2014 uint32_t checkIndex;
2015 for (checkIndex = 0;
2016 (checkIndex < count)
2017 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
2018 checkIndex++) {
2019 }
2020 if (!checkIndex) {
2021 ppnum++;
2022 continue;
2023 }
2024 count = checkIndex;
2025 }
2026 }
2027
2028 switch (pageType) {
2029 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2030 case kWiredClear: wiredPagesClear += count; break;
2031 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2032 }
2033
2034 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2035 } else {
2036 pageAndCount[0] = (uint32_t) ppnum;
2037 pageAndCount[1] = (uint32_t) count;
2038 err = IOHibernatePolledFileWrite(vars,
2039 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2040 cryptvars);
2041 if (kIOReturnSuccess != err) {
2042 break;
2043 }
2044 }
2045
2046 for (page = ppnum; page < (ppnum + count); page++) {
2047 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2048 if (err) {
2049 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2050 break;
2051 }
2052
2053 sum = hibernate_sum_page(src, (uint32_t) page);
2054 if (kWired & pageType) {
2055 sum1 += sum;
2056 } else {
2057 sum2 += sum;
2058 }
2059
2060 clock_get_uptime(&startTime);
2061 wkresult = WKdm_compress_new((const WK_word*) src,
2062 (WK_word*) compressed,
2063 (WK_word*) scratch,
2064 (uint32_t) (page_size - 4));
2065
2066 clock_get_uptime(&endTime);
2067 ADD_ABSOLUTETIME(&compTime, &endTime);
2068 SUB_ABSOLUTETIME(&compTime, &startTime);
2069
2070 compBytes += page_size;
2071 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2072
2073 if (pageCompressedSize == 0) {
2074 pageCompressedSize = 4;
2075 data = src;
2076
2077 if (*(uint32_t *)src) {
2078 svPageCount++;
2079 } else {
2080 zvPageCount++;
2081 }
2082 } else {
2083 if (pageCompressedSize != page_size) {
2084 data = compressed;
2085 } else {
2086 data = src;
2087 }
2088 }
2089
2090 assert(pageCompressedSize <= page_size);
2091 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2092 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2093 if (kIOReturnSuccess != err) {
2094 break;
2095 }
2096
2097 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2098 if (kIOReturnSuccess != err) {
2099 break;
2100 }
2101
2102 compressedSize += pageCompressedSize;
2103 uncompressedSize += page_size;
2104 pagesDone++;
2105
2106 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2107 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2108 if (blob != lastBlob) {
2109 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2110 lastBlob = blob;
2111 }
2112 }
2113 if (0 == (8191 & pagesDone)) {
2114 clock_get_uptime(&endTime);
2115 SUB_ABSOLUTETIME(&endTime, &allTime);
2116 absolutetime_to_nanoseconds(endTime, &nsec);
2117 progressStamp = nsec / 750000000ULL;
2118 if (progressStamp != lastProgressStamp) {
2119 lastProgressStamp = progressStamp;
2120 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2121 }
2122 }
2123 }
2124 if (kIOReturnSuccess != err) {
2125 break;
2126 }
2127 ppnum = page;
2128 }
2129
2130 if (kIOReturnSuccess != err) {
2131 break;
2132 }
2133
2134 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2135 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2136 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2137 }
2138
2139 if (kWiredEncrypt != pageType) {
2140 // end of image1/2 - fill to next block
2141 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2142 if (kIOReturnSuccess != err) {
2143 break;
2144 }
2145 }
2146 if (kWiredClear == pageType) {
2147 // enlarge wired image for test
2148 // err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
2149
2150 // end wired image
2151 header->encryptStart = vars->fileVars->encryptStart;
2152 header->encryptEnd = vars->fileVars->encryptEnd;
2153 image1Size = vars->fileVars->position;
2154 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2155 image1Size, header->encryptStart, header->encryptEnd);
2156 }
2157 }
2158 if (kIOReturnSuccess != err) {
2159 if (kIOReturnOverrun == err) {
2160 // update actual compression ratio on not enough space (for retry)
2161 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2162 }
2163
2164 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2165 header->imageSize = vars->fileVars->position;
2166 break;
2167 }
2168
2169
2170 // Header:
2171
2172 header->imageSize = vars->fileVars->position;
2173 header->image1Size = image1Size;
2174 header->bitmapSize = bitmap_size;
2175 header->pageCount = pageCount;
2176
2177 header->restore1Sum = restore1Sum;
2178 header->image1Sum = sum1;
2179 header->image2Sum = sum2;
2180 header->sleepTime = gIOLastSleepTime.tv_sec;
2181
2182 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
2183 #if defined(__arm64__)
2184 /*
2185 * We don't support retry on hibernation failure and so
2186 * we don't want to set this value to anything smaller
2187 * just because we may have been lucky this time around.
2188 * Though we'll let it go higher.
2189 */
2190 if (header->compression < HIB_COMPR_RATIO_ARM64) {
2191 header->compression = HIB_COMPR_RATIO_ARM64;
2192 }
2193 #endif /* __arm64__ */
2194
2195 gIOHibernateCompression = header->compression;
2196
2197 count = vars->fileVars->fileExtents->getLength();
2198 if (count > sizeof(header->fileExtentMap)) {
2199 header->fileExtentMapSize = ((uint32_t) count);
2200 count = sizeof(header->fileExtentMap);
2201 } else {
2202 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2203 }
2204 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2205
2206 header->deviceBase = vars->fileVars->block0;
2207 header->deviceBlockSize = vars->fileVars->blockSize;
2208 header->lastHibAbsTime = mach_absolute_time();
2209 header->lastHibContTime = mach_continuous_time();
2210
2211
2212 IOPolledFileSeek(vars->fileVars, 0);
2213 err = IOHibernatePolledFileWrite(vars,
2214 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2215 cryptvars);
2216 if (kIOReturnSuccess != err) {
2217 break;
2218 }
2219
2220 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2221 }while (false);
2222
2223 clock_get_uptime(&endTime);
2224
2225 IOService::getPMRootDomain()->pmStatsRecordEvent(
2226 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2227
2228 SUB_ABSOLUTETIME(&endTime, &allTime);
2229 absolutetime_to_nanoseconds(endTime, &nsec);
2230 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2231
2232 absolutetime_to_nanoseconds(compTime, &nsec);
2233 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2234 compBytes,
2235 nsec / 1000000ULL,
2236 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2237
2238 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2239 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2240 vars->fileVars->cryptBytes,
2241 nsec / 1000000ULL,
2242 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2243
2244 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
2245 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2246 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2247 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2248
2249 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
2250
2251 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2252 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2253
2254 if (pollerOpen) {
2255 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2256 }
2257
2258 if (vars->consoleMapping) {
2259 ProgressUpdate(gIOHibernateGraphicsInfo,
2260 vars->consoleMapping, 0, kIOHibernateProgressCount);
2261 }
2262
2263 HIBLOG("hibernate_write_image done(%x)\n", err);
2264
2265 // should we come back via regular wake, set the state in memory.
2266 gIOHibernateState = kIOHibernateStateInactive;
2267
2268 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2269 wiredPagesClear, dirtyPagesEncrypted);
2270
2271 #if defined(__arm64__)
2272 if (kIOReturnSuccess == err) {
2273 return kIOHibernatePostWriteHalt;
2274 } else {
2275 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2276 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2277 vm_panic_hibernate_write_image_failed(err);
2278 return err; //not coming here post panic
2279 }
2280 #else
2281 if (kIOReturnSuccess == err) {
2282 if (kIOHibernateModeSleep & gIOHibernateMode) {
2283 return kIOHibernatePostWriteSleep;
2284 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2285 return kIOHibernatePostWriteRestart;
2286 } else {
2287 /* by default, power down */
2288 return kIOHibernatePostWriteHalt;
2289 }
2290 } else if (kIOReturnAborted == err) {
2291 return kIOHibernatePostWriteWake;
2292 } else {
2293 /* on error, sleep */
2294 return kIOHibernatePostWriteSleep;
2295 }
2296 #endif
2297 }
2298
2299 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2300
2301 extern "C" void
hibernate_machine_init(void)2302 hibernate_machine_init(void)
2303 {
2304 IOReturn err;
2305 uint32_t sum;
2306 uint32_t pagesDone;
2307 uint32_t pagesRead = 0;
2308 AbsoluteTime startTime, compTime;
2309 AbsoluteTime allTime, endTime;
2310 AbsoluteTime startIOTime, endIOTime;
2311 uint64_t nsec, nsecIO;
2312 uint64_t compBytes;
2313 uint64_t lastProgressStamp = 0;
2314 uint64_t progressStamp;
2315 IOPolledFileCryptVars * cryptvars = NULL;
2316
2317 IOHibernateVars * vars = &gIOHibernateVars;
2318 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2319
2320 if (!vars->fileVars || !vars->fileVars->pollers) {
2321 return;
2322 }
2323
2324 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2325 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2326
2327 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2328 HIBLOG("regular wake\n");
2329 return;
2330 }
2331
2332 HIBPRINT("diag %x %x %x %x\n",
2333 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2334 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2335
2336 #if defined(__i386__) || defined(__x86_64__)
2337 #define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2338 #else /* defined(__i386__) || defined(__x86_64__) */
2339 #define t40ms(x) x
2340 #endif /* defined(__i386__) || defined(__x86_64__) */
2341 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2342 tStat(booterStart, booterStart);
2343 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2344 tStat(booterDuration0, booterTime0);
2345 tStat(booterDuration1, booterTime1);
2346 tStat(booterDuration2, booterTime2);
2347 tStat(booterDuration, booterTime);
2348 tStat(booterConnectDisplayDuration, connectDisplayTime);
2349 tStat(booterSplashDuration, splashTime);
2350 tStat(trampolineDuration, trampolineTime);
2351
2352 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2353 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2354 gIOHibernateStats->image1Pages = pagesDone;
2355
2356 /* HIBERNATE_stats */
2357 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2358 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2359 gIOHibernateStats->trampolineDuration);
2360
2361 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2362 gIOHibernateStats->booterStart,
2363 gIOHibernateStats->smcStart,
2364 gIOHibernateStats->booterDuration0,
2365 gIOHibernateStats->booterDuration1,
2366 gIOHibernateStats->booterDuration2,
2367 gIOHibernateStats->booterDuration,
2368 gIOHibernateStats->booterConnectDisplayDuration,
2369 gIOHibernateStats->booterSplashDuration,
2370 gIOHibernateStats->trampolineDuration);
2371
2372 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2373 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2374 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2375
2376 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2377 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2378 hibernate_page_list_discard(vars->page_list);
2379 }
2380
2381 if (vars->hwEncrypt) {
2382 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2383 // decryption key via IOPolledFilePollersSetEncryptionKey
2384 cryptvars = NULL;
2385 } else {
2386 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2387 }
2388
2389 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2390 panic("handoff overflow");
2391 }
2392
2393 IOHibernateHandoff * handoff;
2394 bool done = false;
2395 bool foundCryptData = false;
2396 bool foundVolumeEncryptData = false;
2397 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2398 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
2399
2400 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2401 !done;
2402 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
2403 if (((uint8_t*)handoff < handoffStart) ||
2404 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2405 panic("handoff out of range");
2406 }
2407 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2408 uint8_t * data = &handoff->data[0];
2409 switch (handoff->type) {
2410 case kIOHibernateHandoffTypeEnd:
2411 done = true;
2412 break;
2413
2414 case kIOHibernateHandoffTypeGraphicsInfo:
2415 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2416 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2417 }
2418 break;
2419
2420 case kIOHibernateHandoffTypeCryptVars:
2421 if (cryptvars) {
2422 hibernate_cryptwakevars_t *
2423 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2424 if (handoff->bytecount == sizeof(*wakevars)) {
2425 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2426 } else {
2427 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2428 }
2429 }
2430 foundCryptData = true;
2431 bzero(data, handoff->bytecount);
2432 break;
2433
2434 case kIOHibernateHandoffTypeVolumeCryptKey:
2435 if (handoff->bytecount == vars->volumeCryptKeySize) {
2436 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2437 foundVolumeEncryptData = true;
2438 } else {
2439 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2440 }
2441 break;
2442
2443 #if defined(__i386__) || defined(__x86_64__)
2444 case kIOHibernateHandoffTypeMemoryMap:
2445
2446 clock_get_uptime(&allTime);
2447
2448 hibernate_newruntime_map(data, handoff->bytecount,
2449 gIOHibernateCurrentHeader->systemTableOffset);
2450
2451 clock_get_uptime(&endTime);
2452
2453 SUB_ABSOLUTETIME(&endTime, &allTime);
2454 absolutetime_to_nanoseconds(endTime, &nsec);
2455
2456 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2457
2458 break;
2459
2460 case kIOHibernateHandoffTypeDeviceTree:
2461 {
2462 // DTEntry chosen = NULL;
2463 // HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2464 }
2465 break;
2466 #endif /* defined(__i386__) || defined(__x86_64__) */
2467
2468 default:
2469 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2470 break;
2471 }
2472 }
2473
2474 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2475 panic("no volumeCryptKey");
2476 } else if (cryptvars && !foundCryptData) {
2477 panic("hibernate handoff");
2478 }
2479
2480 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2481 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2482 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2483
2484 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2485 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2486 * gIOHibernateGraphicsInfo->rowBytes);
2487 if (vars->videoMapSize > vars->videoAllocSize) {
2488 vars->videoMapSize = 0;
2489 } else {
2490 IOMapPages(kernel_map,
2491 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2492 vars->videoMapSize, kIOMapInhibitCache );
2493 }
2494 }
2495
2496 if (vars->videoMapSize) {
2497 ProgressUpdate(gIOHibernateGraphicsInfo,
2498 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2499 }
2500
2501 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2502 uint8_t * compressed = src + page_size;
2503 uint8_t * scratch = compressed + page_size;
2504 uint32_t decoOffset;
2505
2506 clock_get_uptime(&allTime);
2507 AbsoluteTime_to_scalar(&compTime) = 0;
2508 compBytes = 0;
2509
2510 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2511 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2512 clock_get_uptime(&startIOTime);
2513 endTime = startIOTime;
2514 SUB_ABSOLUTETIME(&endTime, &allTime);
2515 absolutetime_to_nanoseconds(endTime, &nsec);
2516 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2517
2518 if (vars->hwEncrypt) {
2519 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2520 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2521 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2522 if (kIOReturnSuccess != err) {
2523 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2524 }
2525 cryptvars = NULL;
2526 }
2527
2528 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2529
2530 // kick off the read ahead
2531 vars->fileVars->bufferHalf = 0;
2532 vars->fileVars->bufferLimit = 0;
2533 vars->fileVars->lastRead = 0;
2534 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2535 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2536 vars->fileVars->cryptBytes = 0;
2537 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2538
2539 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
2540 if (kIOReturnSuccess != err) {
2541 panic("Hibernate restore error %x", err);
2542 }
2543 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2544 // --
2545
2546 HIBLOG("hibernate_machine_init reading\n");
2547
2548
2549 uint32_t * header = (uint32_t *) src;
2550 sum = 0;
2551
2552 while (kIOReturnSuccess == err) {
2553 unsigned int count;
2554 unsigned int page;
2555 uint32_t tag;
2556 vm_offset_t compressedSize;
2557 ppnum_t ppnum;
2558
2559 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2560 if (kIOReturnSuccess != err) {
2561 panic("Hibernate restore error %x", err);
2562 }
2563
2564 ppnum = header[0];
2565 count = header[1];
2566
2567 // HIBPRINT("(%x, %x)\n", ppnum, count);
2568
2569 if (!count) {
2570 break;
2571 }
2572
2573 for (page = 0; page < count; page++) {
2574 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2575 if (kIOReturnSuccess != err) {
2576 panic("Hibernate restore error %x", err);
2577 }
2578
2579 compressedSize = kIOHibernateTagLength & tag;
2580 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2581 err = kIOReturnIPCError;
2582 panic("Hibernate restore error %x", err);
2583 }
2584
2585 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2586 if (kIOReturnSuccess != err) {
2587 panic("Hibernate restore error %x", err);
2588 }
2589
2590 if (compressedSize < page_size) {
2591 decoOffset = ((uint32_t) page_size);
2592 clock_get_uptime(&startTime);
2593
2594 if (compressedSize == 4) {
2595 int i;
2596 uint32_t *s, *d;
2597
2598 s = (uint32_t *)src;
2599 d = (uint32_t *)(uintptr_t)compressed;
2600
2601 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2602 *d++ = *s;
2603 }
2604 } else {
2605 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
2606 }
2607 clock_get_uptime(&endTime);
2608 ADD_ABSOLUTETIME(&compTime, &endTime);
2609 SUB_ABSOLUTETIME(&compTime, &startTime);
2610 compBytes += page_size;
2611 } else {
2612 decoOffset = 0;
2613 }
2614
2615 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
2616 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2617 if (err) {
2618 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2619 panic("Hibernate restore error %x", err);
2620 }
2621
2622
2623 ppnum++;
2624 pagesDone++;
2625 pagesRead++;
2626
2627 if (0 == (8191 & pagesDone)) {
2628 clock_get_uptime(&endTime);
2629 SUB_ABSOLUTETIME(&endTime, &allTime);
2630 absolutetime_to_nanoseconds(endTime, &nsec);
2631 progressStamp = nsec / 750000000ULL;
2632 if (progressStamp != lastProgressStamp) {
2633 lastProgressStamp = progressStamp;
2634 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2635 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2636 }
2637 }
2638 }
2639 }
2640 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2641 err = kIOReturnLockedRead;
2642 }
2643
2644 if (kIOReturnSuccess != err) {
2645 panic("Hibernate restore error %x", err);
2646 }
2647
2648
2649 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2650 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2651
2652 clock_get_uptime(&endIOTime);
2653
2654 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2655
2656 clock_get_uptime(&endTime);
2657
2658 IOService::getPMRootDomain()->pmStatsRecordEvent(
2659 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2660 IOService::getPMRootDomain()->pmStatsRecordEvent(
2661 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2662
2663 SUB_ABSOLUTETIME(&endTime, &allTime);
2664 absolutetime_to_nanoseconds(endTime, &nsec);
2665
2666 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2667 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2668
2669 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
2670 gIOHibernateStats->imagePages = pagesDone;
2671
2672 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2673 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2674 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2675
2676 absolutetime_to_nanoseconds(compTime, &nsec);
2677 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2678 compBytes,
2679 nsec / 1000000ULL,
2680 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2681
2682 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2683 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2684 vars->fileVars->cryptBytes,
2685 nsec / 1000000ULL,
2686 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2687
2688 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2689 }
2690
2691 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2692
2693 void
IOHibernateSetWakeCapabilities(uint32_t capability)2694 IOHibernateSetWakeCapabilities(uint32_t capability)
2695 {
2696 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2697 gIOHibernateStats->wakeCapability = capability;
2698
2699 if (kIOPMSystemCapabilityGraphics & capability) {
2700 vm_compressor_do_warmup();
2701 }
2702 }
2703 }
2704
2705 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2706
2707 void
IOHibernateSystemRestart(void)2708 IOHibernateSystemRestart(void)
2709 {
2710 static uint8_t noteStore[32] __attribute__((aligned(32)));
2711 IORegistryEntry * regEntry;
2712 const OSSymbol * sym;
2713 OSData * noteProp;
2714 OSData * data;
2715 uintptr_t * smcVars;
2716 uint8_t * smcBytes;
2717 size_t len;
2718 addr64_t element;
2719
2720 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2721 if (!data) {
2722 return;
2723 }
2724
2725 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2726 smcBytes = (typeof(smcBytes))smcVars[1];
2727 len = smcVars[0];
2728 if (len > sizeof(noteStore)) {
2729 len = sizeof(noteStore);
2730 }
2731 noteProp = OSData::withCapacity(3 * sizeof(element));
2732 if (!noteProp) {
2733 return;
2734 }
2735 element = len;
2736 noteProp->appendBytes(&element, sizeof(element));
2737 element = crc32(0, smcBytes, len);
2738 noteProp->appendBytes(&element, sizeof(element));
2739
2740 bcopy(smcBytes, noteStore, len);
2741 element = (addr64_t) ¬eStore[0];
2742 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2743 noteProp->appendBytes(&element, sizeof(element));
2744
2745 if (!gIOOptionsEntry) {
2746 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2747 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2748 if (regEntry && !gIOOptionsEntry) {
2749 regEntry->release();
2750 }
2751 }
2752
2753 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2754 if (gIOOptionsEntry && sym) {
2755 gIOOptionsEntry->setProperty(sym, noteProp);
2756 }
2757 if (noteProp) {
2758 noteProp->release();
2759 }
2760 if (sym) {
2761 sym->release();
2762 }
2763 }
2764