1 /*
2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 *
31 * Sleep:
32 *
33 * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
34 * (devices awake, normal execution context)
35 * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 * grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 * The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 * the disk, and other ioctls to get the transfer constraints
39 * If successful, the file is written to make sure its initially not bootable (in case of
40 * later failure) and nvram set to point to the first block of the file. (Has to be done
41 * here so blocking is possible in nvram support).
42 * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 * page out any pages it wants to (currently zero, but probably some percentage of memory).
44 * Its assumed just allocating pages will cause the VM system to naturally select the best
45 * pages for eviction. It also copies processor flags needed for the restore path and sets
46 * a flag in the boot processor proc info.
47 * gIOHibernateState = kIOHibernateStateHibernating.
48 * - Regular sleep progresses - some drivers may inspect the root domain property
49 * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 * as usual but leaves motherboard I/O on.
51 * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 * all ppc RC bits out of the hash table and caches into the mapping structures.
54 * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 * The image header and block list are written. The header includes the second file extent so
59 * only the header block is needed to read the file, regardless of filesystem.
60 * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
61 * (only) is used to decompress the image during wake/boot.
62 * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 * The bitmaps are written to the image.
64 * More areas are removed from the bitmaps (after they have been written to the image) - the
65 * segment "__HIB" pages and interrupt stack.
66 * Each wired page is compressed and written and then each non-wired page. Compression and
67 * disk writes are in parallel.
68 * The image header is written to the start of the file and the polling driver closed.
69 * The machine powers down (or sleeps).
70 *
71 * Boot/Wake:
72 *
73 * - BootX sees the boot-image nvram variable containing the device and block number of the image,
74 * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75 * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 * that is in the kernel's __HIB section.
78 * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 * only code & data in that section is safe to call since all the other wired pages are still
81 * compressed in the image.
82 * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 * location directly, and copies those that can't to interim free pages. When the image has been
85 * completed, the copies are uncompressed, overwriting the wired image pages.
86 * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 * is used to get pages into place for 64bit.
88 * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 * are removed from the software strutures, and the hash table is reinitialized.
91 * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 * for the remaining non wired pages.
95 * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 * is closed via bsd.
98 *
99 * Polled Mode I/O:
100 *
101 * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102 * registry, specifying an object of calls IOPolledInterface.
103 *
104 * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105 * partition) that the image is going to live, looking for polled interface properties. If it finds
106 * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107 * interfaces found are kept in an ordered list.
108 *
109 * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110 * few different contexts things happen in:
111 *
112 * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113 * up and running) and after wake - this is safe to allocate memory and do anything. The device
114 * ignores sleep requests from that point since its a waste of time if it goes to sleep and
115 * immediately wakes back up for the image write.
116 *
117 * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118 * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119 * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120 * used to flush and set the disk to sleep.
121 *
122 * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123 * immediately after sleep. These can't block or allocate memory. This is happening after the platform
124 * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125 * one thread running.
126 *
127 * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128 * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129 * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130 * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131 * that is called for the hardware to check for events, and complete the I/O via the callback.
132 * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133 * to restrict I/O ops.
134 */
135
136 #include <sys/systm.h>
137
138 #include <IOKit/IOWorkLoop.h>
139 #include <IOKit/IOCommandGate.h>
140 #include <IOKit/IOTimerEventSource.h>
141 #include <IOKit/IOPlatformExpert.h>
142 #include <IOKit/IOKitDebug.h>
143 #include <IOKit/IOTimeStamp.h>
144 #include <IOKit/pwr_mgt/RootDomain.h>
145 #include <IOKit/pwr_mgt/IOPMPrivate.h>
146 #include <IOKit/IOMessage.h>
147 #include <IOKit/IODeviceTreeSupport.h>
148 #include <IOKit/IOBSD.h>
149 #include <IOKit/IOKitKeysPrivate.h>
150 #include "RootDomainUserClient.h"
151 #include <IOKit/pwr_mgt/IOPowerConnection.h>
152 #include "IOPMPowerStateQueue.h"
153 #include <IOKit/IOBufferMemoryDescriptor.h>
154 #include <IOKit/AppleKeyStoreInterface.h>
155 #include <libkern/crypto/aes.h>
156
157 #include <sys/uio.h>
158 #include <sys/conf.h>
159 #include <sys/stat.h>
160 #include <sys/fcntl.h> // (FWRITE, ...)
161 #include <sys/sysctl.h>
162 #include <sys/kdebug.h>
163 #include <stdint.h>
164
165 #include <IOKit/IOHibernatePrivate.h>
166 #include <IOKit/IOPolledInterface.h>
167 #include <IOKit/IONVRAM.h>
168 #include "IOHibernateInternal.h"
169 #include <vm/vm_protos.h>
170 #include "IOKitKernelInternal.h"
171 #include <pexpert/device_tree.h>
172
173 #include <machine/pal_routines.h>
174 #include <machine/pal_hibernate.h>
175 #if defined(__i386__) || defined(__x86_64__)
176 #include <i386/tsc.h>
177 #include <i386/cpuid.h>
178 #include <vm/WKdm_new.h>
179 #elif defined(__arm64__)
180 #include <arm64/amcc_rorgn.h>
181 #endif /* defined(__i386__) || defined(__x86_64__) */
182 #include <san/kasan.h>
183
184
185 extern "C" addr64_t kvtophys(vm_offset_t va);
186 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
187
188 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
189
190 #define DISABLE_TRIM 0
191 #define TRIM_DELAY 25000
192
193 extern unsigned int save_kdebug_enable;
194 extern uint32_t gIOHibernateState;
195 uint32_t gIOHibernateMode;
196 static char gIOHibernateBootSignature[256 + 1];
197 static char gIOHibernateFilename[MAXPATHLEN + 1];
198 uint32_t gIOHibernateCount;
199
200 static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
201
202 static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
203 uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
204
205 enum {
206 HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
207 HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
208 };
209
210 #if defined(__arm64__)
211 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
212 #else
213 static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
214 #endif /* __arm64__ */
215 boolean_t gIOHibernateStandbyDisabled;
216
217 static IODTNVRAM * gIOOptionsEntry;
218 static IORegistryEntry * gIOChosenEntry;
219
220 static const OSSymbol * gIOHibernateBootImageKey;
221 static const OSSymbol * gIOHibernateBootSignatureKey;
222 static const OSSymbol * gIOBridgeBootSessionUUIDKey;
223
224 #if defined(__i386__) || defined(__x86_64__)
225
226 static const OSSymbol * gIOHibernateRTCVariablesKey;
227 static const OSSymbol * gIOHibernateBoot0082Key;
228 static const OSSymbol * gIOHibernateBootNextKey;
229 static OSData * gIOHibernateBoot0082Data;
230 static OSData * gIOHibernateBootNextData;
231 static OSObject * gIOHibernateBootNextSave;
232
233 #endif /* defined(__i386__) || defined(__x86_64__) */
234
235 static IOLock * gFSLock;
236 uint32_t gFSState;
237 static thread_call_t gIOHibernateTrimCalloutEntry;
238 static IOPolledFileIOVars gFileVars;
239 static IOHibernateVars gIOHibernateVars;
240 static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
241 static hibernate_graphics_t _hibernateGraphics;
242 static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
243 static hibernate_statistics_t _hibernateStats;
244 static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
245
246 enum{
247 kFSIdle = 0,
248 kFSOpening = 2,
249 kFSOpened = 3,
250 kFSTimedOut = 4,
251 kFSTrimDelay = 5
252 };
253
254 static IOReturn IOHibernateDone(IOHibernateVars * vars);
255 static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
256 static void IOSetBootImageNVRAM(OSData * data);
257 static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
258
259 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
260
261 enum { kDefaultIOSize = 128 * 1024 };
262 enum { kVideoMapSize = 80 * 1024 * 1024 };
263
264 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
265
266 // copy from phys addr to MD
267
268 static IOReturn
IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)269 IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
270 IOByteCount offset, addr64_t bytes, IOByteCount length)
271 {
272 addr64_t srcAddr = bytes;
273 IOByteCount remaining;
274
275 remaining = length = min(length, md->getLength() - offset);
276 while (remaining) { // (process another target segment?)
277 addr64_t dstAddr64;
278 IOByteCount dstLen;
279
280 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
281 if (!dstAddr64) {
282 break;
283 }
284
285 // Clip segment length to remaining
286 if (dstLen > remaining) {
287 dstLen = remaining;
288 }
289
290 #if 1
291 bcopy_phys(srcAddr, dstAddr64, dstLen);
292 #else
293 copypv(srcAddr, dstAddr64, dstLen,
294 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
295 #endif
296 srcAddr += dstLen;
297 offset += dstLen;
298 remaining -= dstLen;
299 }
300
301 assert(!remaining);
302
303 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
304 }
305
306 // copy from MD to phys addr
307
308 static IOReturn
IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,IOByteCount offset,addr64_t bytes,IOByteCount length)309 IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
310 IOByteCount offset, addr64_t bytes, IOByteCount length)
311 {
312 addr64_t dstAddr = bytes;
313 IOByteCount remaining;
314
315 remaining = length = min(length, md->getLength() - offset);
316 while (remaining) { // (process another target segment?)
317 addr64_t srcAddr64;
318 IOByteCount dstLen;
319
320 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
321 if (!srcAddr64) {
322 break;
323 }
324
325 // Clip segment length to remaining
326 if (dstLen > remaining) {
327 dstLen = remaining;
328 }
329
330 #if 1
331 bcopy_phys(srcAddr64, dstAddr, dstLen);
332 #else
333 copypv(srcAddr, dstAddr64, dstLen,
334 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
335 #endif
336 dstAddr += dstLen;
337 offset += dstLen;
338 remaining -= dstLen;
339 }
340
341 assert(!remaining);
342
343 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
344 }
345
346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348 void
hibernate_set_page_state(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired,vm_offset_t ppnum,vm_offset_t count,uint32_t kind)349 hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
350 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
351 {
352 count += ppnum;
353
354 if (count > UINT_MAX) {
355 panic("hibernate_set_page_state ppnum");
356 }
357
358 switch (kind) {
359 case kIOHibernatePageStateUnwiredSave:
360 // unwired save
361 for (; ppnum < count; ppnum++) {
362 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
363 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
364 }
365 break;
366 case kIOHibernatePageStateWiredSave:
367 // wired save
368 for (; ppnum < count; ppnum++) {
369 hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
370 hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
371 }
372 break;
373 case kIOHibernatePageStateFree:
374 // free page
375 for (; ppnum < count; ppnum++) {
376 hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
377 hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
378 }
379 break;
380 default:
381 panic("hibernate_set_page_state");
382 }
383 }
384
385 static void
hibernate_set_descriptor_page_state(IOHibernateVars * vars,IOMemoryDescriptor * descriptor,uint32_t kind,uint32_t * pageCount)386 hibernate_set_descriptor_page_state(IOHibernateVars *vars,
387 IOMemoryDescriptor *descriptor,
388 uint32_t kind,
389 uint32_t *pageCount)
390 {
391 IOItemCount count;
392 addr64_t phys64;
393 IOByteCount segLen;
394 if (descriptor) {
395 for (count = 0;
396 (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
397 count += segLen) {
398 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
399 atop_64(phys64), atop_32(segLen),
400 kind);
401 *pageCount -= atop_32(segLen);
402 }
403 }
404 }
405
406 static vm_offset_t
hibernate_page_list_iterate(hibernate_page_list_t * list,ppnum_t * pPage)407 hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
408 {
409 uint32_t page = ((typeof(page)) * pPage);
410 uint32_t count;
411 hibernate_bitmap_t * bitmap;
412
413 while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
414 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
415 if (!count) {
416 break;
417 }
418 page += count;
419 if (page <= bitmap->last_page) {
420 break;
421 }
422 }
423
424 *pPage = page;
425 if (bitmap) {
426 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
427 } else {
428 count = 0;
429 }
430
431 return count;
432 }
433
434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
435
436 IOReturn
IOHibernateSystemSleep(void)437 IOHibernateSystemSleep(void)
438 {
439 IOReturn err;
440 OSData * nvramData;
441 OSObject * obj;
442 OSString * str;
443 OSNumber * num;
444 bool dsSSD, vmflush, swapPinned;
445 IOHibernateVars * vars;
446 uint64_t setFileSize = 0;
447
448 gIOHibernateState = kIOHibernateStateInactive;
449
450 gIOHibernateDebugFlags = 0;
451 if (kIOLogHibernate & gIOKitDebug) {
452 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
453 }
454
455 if (IOService::getPMRootDomain()->getHibernateSettings(
456 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
457 if (kIOHibernateModeSleep & gIOHibernateMode) {
458 // default to discard clean for safe sleep
459 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
460 | kIOHibernateModeDiscardCleanActive);
461 }
462 }
463
464 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
465 if ((str = OSDynamicCast(OSString, obj))) {
466 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
467 sizeof(gIOHibernateFilename));
468 }
469 obj->release();
470 }
471
472 if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
473 return kIOReturnUnsupported;
474 }
475
476 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
477
478 vars = IOMallocType(IOHibernateVars);
479
480 IOLockLock(gFSLock);
481 if (!gIOHibernateTrimCalloutEntry) {
482 gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
483 }
484 IOHibernateSystemPostWakeTrim(NULL, NULL);
485 thread_call_cancel(gIOHibernateTrimCalloutEntry);
486 if (kFSIdle != gFSState) {
487 HIBLOG("hibernate file busy\n");
488 IOLockUnlock(gFSLock);
489 IOFreeType(vars, IOHibernateVars);
490 return kIOReturnBusy;
491 }
492 gFSState = kFSOpening;
493 IOLockUnlock(gFSLock);
494
495 swapPinned = false;
496 do{
497 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
498 HIBERNATION_SRC_BUFFER_SIZE, page_size);
499
500 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
501 ptoa_64(gIOHibernateHandoffPageCount), page_size);
502
503 if (!vars->srcBuffer || !vars->handoffBuffer) {
504 err = kIOReturnNoMemory;
505 break;
506 }
507
508 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
509 if ((num = OSDynamicCast(OSNumber, obj))) {
510 vars->fileMinSize = num->unsigned64BitValue();
511 }
512 obj->release();
513 }
514 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
515 if ((num = OSDynamicCast(OSNumber, obj))) {
516 vars->fileMaxSize = num->unsigned64BitValue();
517 }
518 obj->release();
519 }
520
521 boolean_t encryptedswap = true;
522 uint32_t pageCount;
523 AbsoluteTime startTime, endTime;
524 uint64_t nsec;
525
526 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
527 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
528 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
529
530 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
531 err = hibernate_alloc_page_lists(&vars->page_list,
532 &vars->page_list_wired,
533 &vars->page_list_pal);
534 if (KERN_SUCCESS != err) {
535 HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
536 break;
537 }
538
539 err = hibernate_pin_swap(TRUE);
540 if (KERN_SUCCESS != err) {
541 HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
542 break;
543 }
544 swapPinned = true;
545
546 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
547 hibernate_page_list_setall(vars->page_list,
548 vars->page_list_wired,
549 vars->page_list_pal,
550 true /* preflight */,
551 vmflush /* discard */,
552 &pageCount);
553 PE_Video consoleInfo;
554 bzero(&consoleInfo, sizeof(consoleInfo));
555 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
556
557 // estimate: 6% increase in pages compressed
558 // screen preview 2 images compressed 0%
559 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
560 + vars->page_list->list_size
561 + (consoleInfo.v_width * consoleInfo.v_height * 8);
562 enum { setFileRound = 1024 * 1024ULL };
563 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
564
565 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
566 pageCount, (100ULL * gIOHibernateCompression) >> 8,
567 setFileSize, vars->fileMinSize);
568
569 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
570 && (setFileSize < vars->fileMinSize)) {
571 setFileSize = vars->fileMinSize;
572 }
573 }
574
575 vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
576 err = IOPolledFileOpen(gIOHibernateFilename,
577 (kIOPolledFileCreate | kIOPolledFileHibernate),
578 setFileSize, 0,
579 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
580 &vars->fileVars, &nvramData,
581 &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
582
583 if (KERN_SUCCESS != err) {
584 IOLockLock(gFSLock);
585 if (kFSOpening != gFSState) {
586 err = kIOReturnTimeout;
587 }
588 IOLockUnlock(gFSLock);
589 }
590
591 if (KERN_SUCCESS != err) {
592 HIBLOG("IOPolledFileOpen(%x)\n", err);
593 OSSafeReleaseNULL(nvramData);
594 break;
595 }
596
597 // write extents for debug data usage in EFI
598 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
599
600 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
601 if (KERN_SUCCESS != err) {
602 OSSafeReleaseNULL(nvramData);
603 break;
604 }
605
606 clock_get_uptime(&startTime);
607 err = hibernate_setup(gIOHibernateCurrentHeader,
608 vmflush,
609 vars->page_list, vars->page_list_wired, vars->page_list_pal);
610 clock_get_uptime(&endTime);
611 SUB_ABSOLUTETIME(&endTime, &startTime);
612 absolutetime_to_nanoseconds(endTime, &nsec);
613
614 boolean_t haveSwapPin, hibFileSSD;
615 haveSwapPin = vm_swap_files_pinned();
616
617 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
618
619 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
620 err, nsec / 1000000ULL,
621 haveSwapPin, hibFileSSD);
622 if (KERN_SUCCESS != err) {
623 OSSafeReleaseNULL(nvramData);
624 break;
625 }
626
627 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
628
629 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
630 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
631
632 if (dsSSD) {
633 gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
634 } else {
635 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
636 }
637
638
639 #if defined(__i386__) || defined(__x86_64__)
640 if (vars->volumeCryptKeySize &&
641 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
642 uintptr_t smcVars[2];
643 smcVars[0] = vars->volumeCryptKeySize;
644 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
645
646 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
647 bzero(smcVars, sizeof(smcVars));
648 }
649 #endif
650
651
652 if (encryptedswap || vars->volumeCryptKeySize) {
653 gIOHibernateMode ^= kIOHibernateModeEncrypt;
654 }
655
656 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
657 vars->videoAllocSize = kVideoMapSize;
658 if (KERN_SUCCESS != kmem_alloc(kernel_map, &vars->videoMapping, vars->videoAllocSize,
659 (kma_flags_t)(KMA_PAGEABLE | KMA_DATA), VM_KERN_MEMORY_IOKIT)) {
660 vars->videoMapping = 0;
661 }
662 }
663
664 // generate crypt keys
665 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
666 vars->wiredCryptKey[i] = ((uint8_t) random());
667 }
668 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
669 vars->cryptKey[i] = ((uint8_t) random());
670 }
671
672 // set nvram
673
674 IOSetBootImageNVRAM(nvramData);
675 OSSafeReleaseNULL(nvramData);
676
677 #if defined(__i386__) || defined(__x86_64__)
678 {
679 struct AppleRTCHibernateVars {
680 uint8_t signature[4];
681 uint32_t revision;
682 uint8_t booterSignature[20];
683 uint8_t wiredCryptKey[16];
684 };
685 AppleRTCHibernateVars rtcVars;
686 OSData * data;
687
688 rtcVars.signature[0] = 'A';
689 rtcVars.signature[1] = 'A';
690 rtcVars.signature[2] = 'P';
691 rtcVars.signature[3] = 'L';
692 rtcVars.revision = 1;
693 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
694
695 if (gIOChosenEntry
696 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
697 && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
698 bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
699 } else if (gIOHibernateBootSignature[0]) {
700 char c;
701 uint8_t value = 0;
702 uint32_t in, out, digits;
703 for (in = out = digits = 0;
704 (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
705 in++) {
706 if ((c >= 'a') && (c <= 'f')) {
707 c -= 'a' - 10;
708 } else if ((c >= 'A') && (c <= 'F')) {
709 c -= 'A' - 10;
710 } else if ((c >= '0') && (c <= '9')) {
711 c -= '0';
712 } else {
713 if (c == '=') {
714 out = digits = value = 0;
715 }
716 continue;
717 }
718 value = ((uint8_t) ((value << 4) | c));
719 if (digits & 1) {
720 rtcVars.booterSignature[out++] = value;
721 if (out >= sizeof(rtcVars.booterSignature)) {
722 break;
723 }
724 }
725 digits++;
726 }
727 }
728 #if DEBUG || DEVELOPMENT
729 if (kIOLogHibernate & gIOKitDebug) {
730 IOKitKernelLogBuffer("H> rtc:",
731 &rtcVars, sizeof(rtcVars), &kprintf);
732 }
733 #endif /* DEBUG || DEVELOPMENT */
734
735 data = OSData::withValue(rtcVars);
736 if (data) {
737 if (gIOHibernateRTCVariablesKey) {
738 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
739 }
740 data->release();
741 }
742 if (gIOChosenEntry && gIOOptionsEntry) {
743 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
744 if (data) {
745 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
746 }
747 // set BootNext
748 if (!gIOHibernateBoot0082Data) {
749 OSData * fileData = NULL;
750 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
751 if (data && data->getLength() >= 4) {
752 fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
753 }
754 if (data && (data->getLength() <= UINT16_MAX)) {
755 // AppleNVRAM_EFI_LOAD_OPTION
756 struct {
757 uint32_t Attributes;
758 uint16_t FilePathLength;
759 uint16_t Desc;
760 } loadOptionHeader;
761 loadOptionHeader.Attributes = 1;
762 loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
763 loadOptionHeader.Desc = 0;
764 if (fileData) {
765 loadOptionHeader.FilePathLength -= 4;
766 loadOptionHeader.FilePathLength += fileData->getLength();
767 }
768 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
769 if (gIOHibernateBoot0082Data) {
770 gIOHibernateBoot0082Data->appendValue(loadOptionHeader);
771 if (fileData) {
772 gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
773 gIOHibernateBoot0082Data->appendBytes(fileData);
774 } else {
775 gIOHibernateBoot0082Data->appendBytes(data);
776 }
777 }
778 }
779 }
780 if (!gIOHibernateBootNextData) {
781 uint16_t bits = 0x0082;
782 gIOHibernateBootNextData = OSData::withValue(bits);
783 }
784
785 #if DEBUG || DEVELOPMENT
786 if (kIOLogHibernate & gIOKitDebug) {
787 IOKitKernelLogBuffer("H> bootnext:",
788 gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
789 }
790 #endif /* DEBUG || DEVELOPMENT */
791 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
792 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
793 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
794 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
795 }
796 // BootNext
797 }
798 }
799 #endif /* !i386 && !x86_64 */
800 }while (false);
801
802 if (swapPinned) {
803 hibernate_pin_swap(FALSE);
804 }
805
806 IOLockLock(gFSLock);
807 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
808 HIBLOG("hibernate file close due timeout\n");
809 err = kIOReturnTimeout;
810 }
811 if (kIOReturnSuccess == err) {
812 gFSState = kFSOpened;
813 gIOHibernateVars = *vars;
814 gFileVars = *vars->fileVars;
815 gFileVars.allocated = false;
816 gIOHibernateVars.fileVars = &gFileVars;
817 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
818 gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
819 gIOHibernateState = kIOHibernateStateHibernating;
820
821 #if DEBUG || DEVELOPMENT
822 if (kIOLogHibernate & gIOKitDebug) {
823 OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
824 if (data) {
825 uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
826 IOKitKernelLogBuffer("H> smc:",
827 (const void *)smcVars[1], smcVars[0], &kprintf);
828 }
829 }
830 #endif /* DEBUG || DEVELOPMENT */
831 } else {
832 IOPolledFileIOVars * fileVars = vars->fileVars;
833 IOHibernateDone(vars);
834 IOPolledFileClose(&fileVars,
835 #if DISABLE_TRIM
836 0, NULL, 0, 0, 0);
837 #else
838 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
839 #endif
840 gFSState = kFSIdle;
841 }
842 IOLockUnlock(gFSLock);
843
844 if (vars->fileVars) {
845 IOFreeType(vars->fileVars, IOPolledFileIOVars);
846 }
847 IOFreeType(vars, IOHibernateVars);
848
849 return err;
850 }
851
852 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
853
854 static void
IOSetBootImageNVRAM(OSData * data)855 IOSetBootImageNVRAM(OSData * data)
856 {
857 IORegistryEntry * regEntry;
858
859 if (!gIOOptionsEntry) {
860 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
861 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
862 if (regEntry && !gIOOptionsEntry) {
863 regEntry->release();
864 }
865 }
866 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
867 if (data) {
868 gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
869 #if DEBUG || DEVELOPMENT
870 if (kIOLogHibernate & gIOKitDebug) {
871 IOKitKernelLogBuffer("H> boot-image:",
872 data->getBytesNoCopy(), data->getLength(), &kprintf);
873 }
874 #endif /* DEBUG || DEVELOPMENT */
875 } else {
876 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
877 #if __x86_64__
878 gIOOptionsEntry->sync();
879 #else
880 if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
881 // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
882 gIOOptionsEntry->sync();
883 }
884 #endif
885 }
886 }
887 }
888
889 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
890 /*
891 * Writes header to disk with signature, block size and file extents data.
892 * If there are more than 2 extents, then they are written on second block.
893 */
894 static IOReturn
IOWriteExtentsToFile(IOPolledFileIOVars * vars,uint32_t signature)895 IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
896 {
897 IOHibernateImageHeader hdr;
898 IOItemCount count;
899 IOReturn err = kIOReturnSuccess;
900 int rc;
901 IOPolledFileExtent * fileExtents;
902
903 fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
904
905 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
906 count = vars->fileExtents->getLength();
907 if (count > sizeof(hdr.fileExtentMap)) {
908 hdr.fileExtentMapSize = count;
909 count = sizeof(hdr.fileExtentMap);
910 } else {
911 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
912 }
913
914 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
915
916 // copy file block extent list if larger than header
917 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
918 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
919 rc = kern_write_file(vars->fileRef, vars->blockSize,
920 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
921 count, IO_SKIP_ENCRYPTION);
922 if (rc != 0) {
923 HIBLOG("kern_write_file returned %d\n", rc);
924 err = kIOReturnIOError;
925 goto exit;
926 }
927 }
928 hdr.signature = signature;
929 hdr.deviceBlockSize = vars->blockSize;
930
931 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
932 if (rc != 0) {
933 HIBLOG("kern_write_file returned %d\n", rc);
934 err = kIOReturnIOError;
935 goto exit;
936 }
937
938 exit:
939 return err;
940 }
941
942 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
943
944 DECLARE_IOHIBERNATEPROGRESSALPHA
945
946 static void
ProgressInit(hibernate_graphics_t * display,uint8_t * screen,uint8_t * saveunder,uint32_t savelen)947 ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
948 {
949 uint32_t rowBytes, pixelShift;
950 uint32_t x, y;
951 int32_t blob;
952 uint32_t alpha, color, result;
953 uint8_t * out, in;
954 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
955
956 rowBytes = display->rowBytes;
957 pixelShift = display->depth >> 4;
958 if (pixelShift < 1) {
959 return;
960 }
961
962 screen += ((display->width
963 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
964 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
965
966 for (y = 0; y < kIOHibernateProgressHeight; y++) {
967 out = screen + y * rowBytes;
968 for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
969 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
970 for (x = 0; x < kIOHibernateProgressWidth; x++) {
971 alpha = gIOHibernateProgressAlpha[y][x];
972 result = color;
973 if (alpha) {
974 if (0xff != alpha) {
975 if (1 == pixelShift) {
976 in = *((uint16_t *)out) & 0x1f; // 16
977 in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
978 } else {
979 in = *((uint32_t *)out) & 0xff; // 32
980 }
981 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
982 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
983 }
984 if (1 == pixelShift) {
985 result >>= 3;
986 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
987 } else {
988 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
989 }
990 }
991 out += (1 << pixelShift);
992 }
993 out += (kIOHibernateProgressSpacing << pixelShift);
994 }
995 }
996 }
997
998
999 static void
ProgressUpdate(hibernate_graphics_t * display,uint8_t * screen,int32_t firstBlob,int32_t select)1000 ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1001 {
1002 uint32_t rowBytes, pixelShift;
1003 uint32_t x, y;
1004 int32_t blob, lastBlob;
1005 uint32_t alpha, in, color, result;
1006 uint8_t * out;
1007 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1008
1009 pixelShift = display->depth >> 4;
1010 if (pixelShift < 1) {
1011 return;
1012 }
1013
1014 rowBytes = display->rowBytes;
1015
1016 screen += ((display->width
1017 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1018 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1019
1020 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1021
1022 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1023
1024 for (y = 0; y < kIOHibernateProgressHeight; y++) {
1025 out = screen + y * rowBytes;
1026 for (blob = firstBlob; blob <= lastBlob; blob++) {
1027 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1028 for (x = 0; x < kIOHibernateProgressWidth; x++) {
1029 alpha = gIOHibernateProgressAlpha[y][x];
1030 result = color;
1031 if (alpha) {
1032 if (0xff != alpha) {
1033 in = display->progressSaveUnder[blob][saveindex[blob]++];
1034 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1035 }
1036 if (1 == pixelShift) {
1037 result >>= 3;
1038 *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
1039 } else {
1040 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1041 }
1042 }
1043 out += (1 << pixelShift);
1044 }
1045 out += (kIOHibernateProgressSpacing << pixelShift);
1046 }
1047 }
1048 }
1049
1050 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1051
1052 IOReturn
IOHibernateIOKitSleep(void)1053 IOHibernateIOKitSleep(void)
1054 {
1055 IOReturn ret = kIOReturnSuccess;
1056 IOLockLock(gFSLock);
1057 if (kFSOpening == gFSState) {
1058 gFSState = kFSTimedOut;
1059 HIBLOG("hibernate file open timed out\n");
1060 ret = kIOReturnTimeout;
1061 }
1062 IOLockUnlock(gFSLock);
1063 return ret;
1064 }
1065
1066 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1067
1068 IOReturn
IOHibernateSystemHasSlept(void)1069 IOHibernateSystemHasSlept(void)
1070 {
1071 IOReturn ret = kIOReturnSuccess;
1072 IOHibernateVars * vars = &gIOHibernateVars;
1073 OSObject * obj = NULL;
1074 OSData * data;
1075
1076 IOLockLock(gFSLock);
1077 if ((kFSOpened != gFSState) && gIOHibernateMode) {
1078 ret = kIOReturnTimeout;
1079 }
1080 IOLockUnlock(gFSLock);
1081 if (kIOReturnSuccess != ret) {
1082 return ret;
1083 }
1084
1085 if (gIOHibernateMode) {
1086 obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
1087 }
1088 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1089 if (obj && !vars->previewBuffer) {
1090 obj->release();
1091 }
1092 if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
1093 OSSafeReleaseNULL(vars->previewBuffer);
1094 }
1095
1096 vars->consoleMapping = NULL;
1097 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
1098 vars->previewBuffer->release();
1099 vars->previewBuffer = NULL;
1100 }
1101
1102 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1103 && vars->previewBuffer
1104 && (data = OSDynamicCast(OSData,
1105 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
1106 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
1107 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
1108
1109 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1110
1111 if (kIOHibernatePreviewUpdates & flags) {
1112 PE_Video consoleInfo;
1113 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1114
1115 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1116
1117 graphicsInfo->width = (uint32_t) consoleInfo.v_width;
1118 graphicsInfo->height = (uint32_t) consoleInfo.v_height;
1119 graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
1120 graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
1121 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1122
1123 HIBPRINT("video %p %d %d %d\n",
1124 vars->consoleMapping, graphicsInfo->depth,
1125 graphicsInfo->width, graphicsInfo->height);
1126 if (vars->consoleMapping) {
1127 ProgressInit(graphicsInfo, vars->consoleMapping,
1128 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1129 }
1130 }
1131 }
1132
1133 if (gIOOptionsEntry) {
1134 #if __x86_64__
1135 gIOOptionsEntry->sync();
1136 #else
1137 if (gIOHibernateMode) {
1138 gIOOptionsEntry->sync();
1139 }
1140 #endif
1141 }
1142
1143 return ret;
1144 }
1145
1146 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1147
1148 static const DeviceTreeNode *
MergeDeviceTree(const DeviceTreeNode * entry,IORegistryEntry * regEntry,OSSet * entriesToUpdate,vm_offset_t region_start,vm_size_t region_size)1149 MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
1150 {
1151 DeviceTreeNodeProperty * prop;
1152 const DeviceTreeNode * child;
1153 IORegistryEntry * childRegEntry;
1154 const char * nameProp;
1155 unsigned int propLen, idx;
1156
1157 bool updateEntry = true;
1158 if (!regEntry) {
1159 updateEntry = false;
1160 } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
1161 updateEntry = false;
1162 }
1163
1164 prop = (DeviceTreeNodeProperty *) (entry + 1);
1165 for (idx = 0; idx < entry->nProperties; idx++) {
1166 if (updateEntry && (0 != strcmp("name", prop->name))) {
1167 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1168 // HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1169 }
1170 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1171 }
1172
1173 if (entriesToUpdate) {
1174 entriesToUpdate->removeObject(regEntry);
1175 if (entriesToUpdate->getCount() == 0) {
1176 // we've updated all the entries we care about so we can stop
1177 return NULL;
1178 }
1179 }
1180
1181 child = (const DeviceTreeNode *) prop;
1182 for (idx = 0; idx < entry->nChildren; idx++) {
1183 if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
1184 region_start, region_size)) {
1185 panic("no name");
1186 }
1187 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1188 // HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1189 child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
1190 OSSafeReleaseNULL(childRegEntry);
1191 if (!child) {
1192 // the recursive call updated the last entry we cared about, so we can stop
1193 break;
1194 }
1195 }
1196 return child;
1197 }
1198
1199 IOReturn
IOHibernateSystemWake(void)1200 IOHibernateSystemWake(void)
1201 {
1202 if (kFSOpened == gFSState) {
1203 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
1204 IOHibernateDone(&gIOHibernateVars);
1205 } else {
1206 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1207 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1208 }
1209
1210 if (gIOOptionsEntry && gIOHibernateBootImageKey) {
1211 // if we got this far, clear boot-image
1212 // we don't need to sync immediately; the booter should have already removed this entry
1213 // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
1214 // a stale boot-image value
1215 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
1216 }
1217
1218 return kIOReturnSuccess;
1219 }
1220
1221 static IOReturn
IOHibernateDone(IOHibernateVars * vars)1222 IOHibernateDone(IOHibernateVars * vars)
1223 {
1224 IOReturn err;
1225 OSData * data;
1226
1227 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
1228
1229 if (vars->videoMapping) {
1230 if (vars->videoMapSize) {
1231 // remove mappings
1232 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1233 }
1234 if (vars->videoAllocSize) {
1235 // dealloc range
1236 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
1237 }
1238 }
1239
1240 if (vars->previewBuffer) {
1241 vars->previewBuffer->release();
1242 vars->previewBuffer = NULL;
1243 }
1244
1245 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1246 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1247 gIOHibernateCurrentHeader->options, 32);
1248 } else {
1249 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1250 }
1251
1252 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1253 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
1254 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1255 &gIOHibernateGraphicsInfo->gfxStatus,
1256 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1257 } else {
1258 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1259 }
1260
1261 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1262
1263 #if defined(__i386__) || defined(__x86_64__)
1264 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
1265 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
1266
1267 /*
1268 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1269 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1270 */
1271 if (gIOOptionsEntry) {
1272 if (gIOHibernateRTCVariablesKey) {
1273 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1274 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1275 }
1276 }
1277
1278 if (gIOHibernateBootNextKey) {
1279 if (gIOHibernateBootNextSave) {
1280 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1281 gIOHibernateBootNextSave->release();
1282 gIOHibernateBootNextSave = NULL;
1283 } else {
1284 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1285 }
1286 }
1287 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
1288 gIOOptionsEntry->sync();
1289 }
1290 }
1291 #endif
1292
1293 if (vars->srcBuffer) {
1294 vars->srcBuffer->release();
1295 }
1296
1297
1298 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
1299 if (vars->handoffBuffer) {
1300 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1301 IOHibernateHandoff * handoff;
1302 bool done = false;
1303 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1304 !done;
1305 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
1306 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1307 uint8_t * __unused data = &handoff->data[0];
1308 switch (handoff->type) {
1309 case kIOHibernateHandoffTypeEnd:
1310 done = true;
1311 break;
1312
1313 case kIOHibernateHandoffTypeDeviceTree:
1314 {
1315 #if defined(__i386__) || defined(__x86_64__)
1316 // On Intel, process the entirety of the passed in device tree
1317 OSSet * entriesToUpdate = NULL;
1318 #elif defined(__arm64__)
1319 // On ARM, only allow hibernation to update specific entries
1320 const char *mergePaths[] = {
1321 kIODeviceTreePlane ":/chosen/boot-object-manifests",
1322 kIODeviceTreePlane ":/chosen/secure-boot-hashes",
1323 };
1324 const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
1325 OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
1326 for (size_t i = 0; i < mergePathCount; i++) {
1327 IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
1328 if (!entry) {
1329 panic("failed to find %s in IORegistry", mergePaths[i]);
1330 }
1331 entriesToUpdate->setObject(entry);
1332 OSSafeReleaseNULL(entry);
1333 }
1334 #endif
1335 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
1336 (vm_offset_t)data, (vm_size_t)handoff->bytecount);
1337 OSSafeReleaseNULL(entriesToUpdate);
1338 break;
1339 }
1340
1341 case kIOHibernateHandoffTypeKeyStore:
1342 #if defined(__i386__) || defined(__x86_64__)
1343 {
1344 IOBufferMemoryDescriptor *
1345 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1346 if (md) {
1347 IOSetKeyStoreData(md);
1348 }
1349 }
1350 #endif
1351 break;
1352
1353 default:
1354 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1355 break;
1356 }
1357 }
1358 #if defined(__i386__) || defined(__x86_64__)
1359 if (vars->volumeCryptKeySize) {
1360 IOBufferMemoryDescriptor *
1361 bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
1362 vars->volumeCryptKeySize, kIODirectionOutIn);
1363 if (!bmd) {
1364 panic("IOBufferMemoryDescriptor");
1365 }
1366 IOSetAPFSKeyStoreData(bmd);
1367 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1368 }
1369 #endif
1370 }
1371 vars->handoffBuffer->release();
1372 }
1373
1374 if (gIOChosenEntry
1375 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1376 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1377 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
1378 sizeof(gIOHibernateBridgeBootSessionUUIDString));
1379 }
1380
1381 if (vars->hwEncrypt) {
1382 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
1383 HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
1384 }
1385
1386 bzero(vars, sizeof(*vars));
1387
1388 // gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1389 gIOHibernateCount++;
1390
1391 return kIOReturnSuccess;
1392 }
1393
1394 static void
IOHibernateSystemPostWakeTrim(void * p1,void * p2)1395 IOHibernateSystemPostWakeTrim(void * p1, void * p2)
1396 {
1397 // invalidate & close the image file
1398 if (p1) {
1399 IOLockLock(gFSLock);
1400 }
1401 if (kFSTrimDelay == gFSState) {
1402 IOPolledFileIOVars * vars = &gFileVars;
1403 IOPolledFileClose(&vars,
1404 #if DISABLE_TRIM
1405 0, NULL, 0, 0, 0);
1406 #else
1407 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1408 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
1409 #endif
1410 gFSState = kFSIdle;
1411 }
1412 if (p1) {
1413 IOLockUnlock(gFSLock);
1414 }
1415 }
1416
1417 IOReturn
IOHibernateSystemPostWake(bool now)1418 IOHibernateSystemPostWake(bool now)
1419 {
1420 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1421 IOSetBootImageNVRAM(NULL);
1422
1423 IOLockLock(gFSLock);
1424 if (kFSTrimDelay == gFSState) {
1425 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1426 IOHibernateSystemPostWakeTrim(NULL, NULL);
1427 } else if (kFSOpened != gFSState) {
1428 gFSState = kFSIdle;
1429 } else {
1430 gFSState = kFSTrimDelay;
1431 if (now) {
1432 thread_call_cancel(gIOHibernateTrimCalloutEntry);
1433 IOHibernateSystemPostWakeTrim(NULL, NULL);
1434 } else {
1435 AbsoluteTime deadline;
1436 clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
1437 thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
1438 }
1439 }
1440 IOLockUnlock(gFSLock);
1441
1442 return kIOReturnSuccess;
1443 }
1444
1445 uint32_t
IOHibernateWasScreenLocked(void)1446 IOHibernateWasScreenLocked(void)
1447 {
1448 uint32_t ret = 0;
1449 if (gIOChosenEntry) {
1450 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
1451 OSData *
1452 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1453 if (data) {
1454 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1455 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1456 }
1457 } else {
1458 gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1459 }
1460 }
1461
1462 return ret;
1463 }
1464
1465 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1466
1467 SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1468 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1469 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1470 SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1471 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1472 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1473 SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1474 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
1475 &gIOHibernateMode, 0, "");
1476 SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1477 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
1478 &_hibernateStats, hibernate_statistics_t, "");
1479 SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
1480 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1481 gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
1482 sysctl_handle_string, "A", "");
1483
1484 SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1485 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1486 &_hibernateStats.graphicsReadyTime, 0, "");
1487 SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1488 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1489 &_hibernateStats.wakeNotificationTime, 0, "");
1490 SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1491 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1492 &_hibernateStats.lockScreenReadyTime, 0, "");
1493 SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1494 CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1495 &_hibernateStats.hidReadyTime, 0, "");
1496
1497 SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
1498 CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1499 &gIOHibernateCount, 0, "");
1500
1501 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1502
1503 static int
1504 hibernate_set_preview SYSCTL_HANDLER_ARGS
1505 {
1506 #pragma unused(oidp, arg1, arg2)
1507
1508 if (!IOCurrentTaskHasEntitlement(kIOHibernateSetPreviewEntitlementKey)) {
1509 return EPERM;
1510 }
1511
1512 if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
1513 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
1514 return 0;
1515 }
1516
1517 size_t rounded_size = round_page(req->newlen);
1518 IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
1519 if (!md) {
1520 return ENOMEM;
1521 }
1522
1523 uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
1524 int error = SYSCTL_IN(req, bytes, req->newlen);
1525 if (error) {
1526 md->release();
1527 return error;
1528 }
1529
1530 IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
1531 md->release();
1532
1533 return 0;
1534 }
1535
1536 SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
1537 CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
1538 hibernate_set_preview, "S", "");
1539
1540 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1541
1542 void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)1543 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1544 {
1545 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1546 gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1547 gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
1548
1549 #if defined(__i386__) || defined(__x86_64__)
1550 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1551 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1552 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1553 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1554 #endif /* defined(__i386__) || defined(__x86_64__) */
1555
1556 OSData * data = OSData::withValueNoCopy(gIOHibernateState);
1557 if (data) {
1558 rootDomain->setProperty(kIOHibernateStateKey, data);
1559 data->release();
1560 }
1561
1562 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
1563 gIOHibernateMode = kIOHibernateModeOn;
1564 } else {
1565 gIOHibernateFilename[0] = 0;
1566 }
1567
1568 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1569
1570 if (gIOChosenEntry
1571 && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
1572 && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
1573 sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
1574 bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
1575 }
1576
1577 gFSLock = IOLockAlloc();
1578 gIOHibernateCount = 0;
1579 }
1580
1581 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1582
1583 static IOReturn
IOHibernatePolledFileWrite(IOHibernateVars * vars,const uint8_t * bytes,IOByteCount size,IOPolledFileCryptVars * cryptvars)1584 IOHibernatePolledFileWrite(IOHibernateVars * vars,
1585 const uint8_t * bytes, IOByteCount size,
1586 IOPolledFileCryptVars * cryptvars)
1587 {
1588 IOReturn err;
1589
1590
1591 err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
1592 if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
1593 err = kIOReturnAborted;
1594 }
1595
1596
1597 return err;
1598 }
1599
1600 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1601
1602 extern "C" uint32_t
hibernate_write_image(void)1603 hibernate_write_image(void)
1604 {
1605 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1606 IOHibernateVars * vars = &gIOHibernateVars;
1607 IOPolledFileExtent * fileExtents;
1608
1609 #if !defined(__arm64__)
1610 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
1611 #endif /* !defined(__arm64__) */
1612
1613 uint32_t pageCount, pagesDone;
1614 IOReturn err;
1615 ppnum_t ppnum, page;
1616 vm_offset_t count;
1617 uint8_t * src;
1618 uint8_t * data;
1619 uint8_t * compressed;
1620 uint8_t * scratch;
1621 IOByteCount pageCompressedSize;
1622 uint64_t compressedSize, uncompressedSize;
1623 uint64_t image1Size = 0;
1624 uint32_t bitmap_size;
1625 bool iterDone, pollerOpen, needEncrypt;
1626 int wkresult;
1627 uint32_t tag;
1628 uint32_t pageType;
1629 uint32_t pageAndCount[2];
1630 addr64_t phys64;
1631 IOByteCount segLen;
1632 uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
1633 uintptr_t hibernateBase;
1634 uintptr_t hibernateEnd;
1635
1636 AbsoluteTime startTime, endTime;
1637 AbsoluteTime allTime, compTime;
1638 uint64_t compBytes;
1639 uint64_t nsec;
1640 uint64_t lastProgressStamp = 0;
1641 uint64_t progressStamp;
1642 uint32_t blob, lastBlob = (uint32_t) -1L;
1643
1644 uint32_t wiredPagesEncrypted;
1645 uint32_t dirtyPagesEncrypted;
1646 uint32_t wiredPagesClear;
1647 uint32_t svPageCount;
1648 uint32_t zvPageCount;
1649
1650 IOPolledFileCryptVars _cryptvars;
1651 IOPolledFileCryptVars * cryptvars = NULL;
1652
1653 wiredPagesEncrypted = 0;
1654 dirtyPagesEncrypted = 0;
1655 wiredPagesClear = 0;
1656 svPageCount = 0;
1657 zvPageCount = 0;
1658
1659 if (!vars->fileVars
1660 || !vars->fileVars->pollers
1661 || !(kIOHibernateModeOn & gIOHibernateMode)) {
1662 return kIOHibernatePostWriteSleep;
1663 }
1664
1665
1666 #if !defined(__arm64__)
1667 if (kIOHibernateModeSleep & gIOHibernateMode) {
1668 kdebug_enable = save_kdebug_enable;
1669 }
1670 #endif /* !defined(__arm64__) */
1671
1672 pal_hib_write_hook();
1673
1674 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
1675 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
1676
1677 #if CRYPTO
1678 // encryption data. "iv" is the "initial vector".
1679 if (kIOHibernateModeEncrypt & gIOHibernateMode) {
1680 static const unsigned char first_iv[AES_BLOCK_SIZE]
1681 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1682 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1683
1684 cryptvars = &gIOHibernateCryptWakeContext;
1685 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1686 aes_encrypt_key(vars->cryptKey,
1687 kIOHibernateAESKeySize,
1688 &cryptvars->ctx.encrypt);
1689 aes_decrypt_key(vars->cryptKey,
1690 kIOHibernateAESKeySize,
1691 &cryptvars->ctx.decrypt);
1692
1693 cryptvars = &_cryptvars;
1694 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
1695 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
1696 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1697 }
1698 aes_encrypt_key(vars->wiredCryptKey,
1699 kIOHibernateAESKeySize,
1700 &cryptvars->ctx.encrypt);
1701
1702 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1703 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1704 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1705 }
1706 #endif /* CRYPTO */
1707
1708 hibernate_page_list_setall(vars->page_list,
1709 vars->page_list_wired,
1710 vars->page_list_pal,
1711 false /* !preflight */,
1712 /* discard_all */
1713 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1714 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
1715 &pageCount);
1716
1717 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1718
1719 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
1720
1721 #if 0
1722 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1723 for (page = 0; page < count; page++) {
1724 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1725 fileExtents[page].start, fileExtents[page].length,
1726 fileExtents[page].start + fileExtents[page].length);
1727 }
1728 #endif
1729
1730 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1731 AbsoluteTime_to_scalar(&compTime) = 0;
1732 compBytes = 0;
1733
1734 clock_get_uptime(&allTime);
1735 IOService::getPMRootDomain()->pmStatsRecordEvent(
1736 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
1737 do{
1738 compressedSize = 0;
1739 uncompressedSize = 0;
1740 svPageCount = 0;
1741 zvPageCount = 0;
1742
1743 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
1744
1745 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1746 ml_get_interrupts_enabled());
1747 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
1748 // abortable if not low battery
1749 !IOService::getPMRootDomain()->mustHibernate());
1750 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1751 pollerOpen = (kIOReturnSuccess == err);
1752 if (!pollerOpen) {
1753 break;
1754 }
1755
1756 if (vars->volumeCryptKeySize) {
1757 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
1758 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
1759 vars->hwEncrypt = (kIOReturnSuccess == err);
1760 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
1761 if (vars->hwEncrypt) {
1762 header->options |= kIOHibernateOptionHWEncrypt;
1763 }
1764 }
1765
1766 // copy file block extent list if larger than header
1767
1768 count = vars->fileVars->fileExtents->getLength();
1769 if (count > sizeof(header->fileExtentMap)) {
1770 count -= sizeof(header->fileExtentMap);
1771 err = IOHibernatePolledFileWrite(vars,
1772 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1773 if (kIOReturnSuccess != err) {
1774 break;
1775 }
1776 }
1777
1778 // copy out restore1 code
1779
1780 for (count = 0;
1781 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1782 count += segLen) {
1783 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
1784 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
1785 }
1786 }
1787
1788 hibernateBase = HIB_BASE; /* Defined in PAL headers */
1789 hibernateEnd = (segHIBB + segSizeHIB);
1790
1791 page = atop_32(kvtophys(hibernateBase));
1792 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1793 uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1794 uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
1795 if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
1796 panic("malformed kernel layout");
1797 }
1798 header->restore1CodePhysPage = (ppnum_t) page;
1799 header->restore1CodeVirt = hibernateBase;
1800 header->restore1PageCount = (uint32_t) count;
1801 header->restore1CodeOffset = (uint32_t) entrypoint;
1802 header->restore1StackOffset = (uint32_t) stack;
1803
1804 if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
1805 bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
1806 }
1807
1808 // sum __HIB seg, with zeros for the stack
1809 src = (uint8_t *) trunc_page(hibernateBase);
1810 for (page = 0; page < count; page++) {
1811 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
1812 restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
1813 } else {
1814 restore1Sum += 0x00000000;
1815 }
1816 src += page_size;
1817 }
1818 sum1 = restore1Sum;
1819
1820 // write the __HIB seg, with zeros for the stack
1821
1822 src = (uint8_t *) trunc_page(hibernateBase);
1823 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
1824 if (count) {
1825 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1826 if (kIOReturnSuccess != err) {
1827 break;
1828 }
1829 }
1830 err = IOHibernatePolledFileWrite(vars,
1831 (uint8_t *) NULL,
1832 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1833 cryptvars);
1834 if (kIOReturnSuccess != err) {
1835 break;
1836 }
1837 src = &gIOHibernateRestoreStackEnd[0];
1838 count = round_page(hibernateEnd) - ((uintptr_t) src);
1839 if (count) {
1840 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1841 if (kIOReturnSuccess != err) {
1842 break;
1843 }
1844 }
1845
1846 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1847 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1848 vars->fileVars->encryptEnd = UINT64_MAX;
1849 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1850 }
1851
1852 // write the preview buffer
1853
1854 if (vars->previewBuffer) {
1855 ppnum = 0;
1856 count = 0;
1857 do{
1858 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
1859 pageAndCount[0] = atop_64_ppnum(phys64);
1860 pageAndCount[1] = atop_64_ppnum(segLen);
1861 err = IOHibernatePolledFileWrite(vars,
1862 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1863 cryptvars);
1864 if (kIOReturnSuccess != err) {
1865 break;
1866 }
1867 count += segLen;
1868 ppnum += sizeof(pageAndCount);
1869 }while (phys64);
1870 if (kIOReturnSuccess != err) {
1871 break;
1872 }
1873
1874 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
1875
1876 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1877
1878 count = (uint32_t) vars->previewBuffer->getLength();
1879
1880 header->previewPageListSize = ((uint32_t) ppnum);
1881 header->previewSize = ((uint32_t) (count + ppnum));
1882
1883 for (page = 0; page < count; page += page_size) {
1884 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1885 sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
1886 }
1887 if (kIOReturnSuccess != err) {
1888 break;
1889 }
1890 err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
1891 if (kIOReturnSuccess != err) {
1892 break;
1893 }
1894 }
1895
1896 // mark areas for no save
1897 hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
1898 kIOHibernatePageStateFree, &pageCount);
1899 hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
1900 kIOHibernatePageStateFree, &pageCount);
1901
1902 // copy out bitmap of pages available for trashing during restore
1903
1904 bitmap_size = vars->page_list_wired->list_size;
1905 src = (uint8_t *) vars->page_list_wired;
1906 err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
1907 if (kIOReturnSuccess != err) {
1908 break;
1909 }
1910
1911 // mark more areas for no save, but these are not available
1912 // for trashing during restore
1913
1914 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
1915
1916 #if defined(__i386__) || defined(__x86_64__)
1917 // __HIB is explicitly saved above so we don't have to save it again
1918 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1919 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
1920 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1921 page, count,
1922 kIOHibernatePageStateFree);
1923 pageCount -= count;
1924 #elif defined(__arm64__)
1925 // the segments described in IOHibernateHibSegInfo are stored directly in the
1926 // hibernation file, so they don't need to be saved again
1927 extern unsigned long gPhysBase, gPhysSize;
1928 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
1929 page = segInfo->segments[i].physPage;
1930 count = segInfo->segments[i].pageCount;
1931 uint64_t physAddr = ptoa_64(page);
1932 uint64_t size = ptoa_64(count);
1933 if (size &&
1934 (physAddr >= gPhysBase) &&
1935 (physAddr + size <= gPhysBase + gPhysSize)) {
1936 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1937 page, count,
1938 kIOHibernatePageStateFree);
1939 pageCount -= count;
1940 }
1941 }
1942 #else
1943 #error unimplemented
1944 #endif
1945
1946 hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
1947 kIOHibernatePageStateFree, &pageCount);
1948 hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
1949 kIOHibernatePageStateFree, &pageCount);
1950
1951 #if KASAN
1952 vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
1953
1954 /* no need to save unused shadow pages */
1955 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1956 atop_64(shadow_pnext),
1957 shadow_pages_free,
1958 kIOHibernatePageStateFree);
1959 #endif
1960
1961 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1962 compressed = src + page_size;
1963 scratch = compressed + page_size;
1964
1965 pagesDone = 0;
1966 lastBlob = 0;
1967
1968 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1969 bitmap_size, header->previewSize,
1970 pageCount, vars->fileVars->position);
1971
1972
1973 enum
1974 // pageType
1975 {
1976 kWired = 0x02,
1977 kEncrypt = 0x01,
1978 kWiredEncrypt = kWired | kEncrypt,
1979 kWiredClear = kWired,
1980 kUnwiredEncrypt = kEncrypt
1981 };
1982
1983 #if defined(__i386__) || defined(__x86_64__)
1984 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1985 #else /* defined(__i386__) || defined(__x86_64__) */
1986 static const bool cpuAES = true;
1987 #endif /* defined(__i386__) || defined(__x86_64__) */
1988
1989 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
1990 if (kUnwiredEncrypt == pageType) {
1991 // start unwired image
1992 if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
1993 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1994 vars->fileVars->encryptEnd = UINT64_MAX;
1995 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1996 }
1997 bcopy(&cryptvars->aes_iv[0],
1998 &gIOHibernateCryptWakeContext.aes_iv[0],
1999 sizeof(cryptvars->aes_iv));
2000 cryptvars = &gIOHibernateCryptWakeContext;
2001 }
2002 for (iterDone = false, ppnum = 0; !iterDone;) {
2003 if (cpuAES && (pageType == kWiredClear)) {
2004 count = 0;
2005 } else {
2006 count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
2007 &ppnum);
2008 if (count > UINT_MAX) {
2009 count = UINT_MAX;
2010 }
2011 }
2012 // kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2013 iterDone = !count;
2014
2015 if (!cpuAES) {
2016 if (count && (kWired & pageType) && needEncrypt) {
2017 uint32_t checkIndex;
2018 for (checkIndex = 0;
2019 (checkIndex < count)
2020 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
2021 checkIndex++) {
2022 }
2023 if (!checkIndex) {
2024 ppnum++;
2025 continue;
2026 }
2027 count = checkIndex;
2028 }
2029 }
2030
2031 switch (pageType) {
2032 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2033 case kWiredClear: wiredPagesClear += count; break;
2034 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
2035 }
2036
2037 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
2038 } else {
2039 pageAndCount[0] = (uint32_t) ppnum;
2040 pageAndCount[1] = (uint32_t) count;
2041 err = IOHibernatePolledFileWrite(vars,
2042 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2043 cryptvars);
2044 if (kIOReturnSuccess != err) {
2045 break;
2046 }
2047 }
2048
2049 for (page = ppnum; page < (ppnum + count); page++) {
2050 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2051 if (err) {
2052 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2053 break;
2054 }
2055
2056 sum = hibernate_sum_page(src, (uint32_t) page);
2057 if (kWired & pageType) {
2058 sum1 += sum;
2059 } else {
2060 sum2 += sum;
2061 }
2062
2063 clock_get_uptime(&startTime);
2064 wkresult = WKdm_compress_new((const WK_word*) src,
2065 (WK_word*) compressed,
2066 (WK_word*) scratch,
2067 (uint32_t) (page_size - 4));
2068
2069 clock_get_uptime(&endTime);
2070 ADD_ABSOLUTETIME(&compTime, &endTime);
2071 SUB_ABSOLUTETIME(&compTime, &startTime);
2072
2073 compBytes += page_size;
2074 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2075
2076 if (pageCompressedSize == 0) {
2077 pageCompressedSize = 4;
2078 data = src;
2079
2080 if (*(uint32_t *)src) {
2081 svPageCount++;
2082 } else {
2083 zvPageCount++;
2084 }
2085 } else {
2086 if (pageCompressedSize != page_size) {
2087 data = compressed;
2088 } else {
2089 data = src;
2090 }
2091 }
2092
2093 assert(pageCompressedSize <= page_size);
2094 tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
2095 err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2096 if (kIOReturnSuccess != err) {
2097 break;
2098 }
2099
2100 err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2101 if (kIOReturnSuccess != err) {
2102 break;
2103 }
2104
2105 compressedSize += pageCompressedSize;
2106 uncompressedSize += page_size;
2107 pagesDone++;
2108
2109 if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
2110 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2111 if (blob != lastBlob) {
2112 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2113 lastBlob = blob;
2114 }
2115 }
2116 if (0 == (8191 & pagesDone)) {
2117 clock_get_uptime(&endTime);
2118 SUB_ABSOLUTETIME(&endTime, &allTime);
2119 absolutetime_to_nanoseconds(endTime, &nsec);
2120 progressStamp = nsec / 750000000ULL;
2121 if (progressStamp != lastProgressStamp) {
2122 lastProgressStamp = progressStamp;
2123 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2124 }
2125 }
2126 }
2127 if (kIOReturnSuccess != err) {
2128 break;
2129 }
2130 ppnum = page;
2131 }
2132
2133 if (kIOReturnSuccess != err) {
2134 break;
2135 }
2136
2137 if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
2138 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
2139 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2140 }
2141
2142 if (kWiredEncrypt != pageType) {
2143 // end of image1/2 - fill to next block
2144 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2145 if (kIOReturnSuccess != err) {
2146 break;
2147 }
2148 }
2149 if (kWiredClear == pageType) {
2150 // enlarge wired image for test
2151 // err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
2152
2153 // end wired image
2154 header->encryptStart = vars->fileVars->encryptStart;
2155 header->encryptEnd = vars->fileVars->encryptEnd;
2156 image1Size = vars->fileVars->position;
2157 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
2158 image1Size, header->encryptStart, header->encryptEnd);
2159 }
2160 }
2161 if (kIOReturnSuccess != err) {
2162 if (kIOReturnOverrun == err) {
2163 // update actual compression ratio on not enough space (for retry)
2164 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2165 }
2166
2167 // update partial amount written (for IOPolledFileClose cleanup/unmap)
2168 header->imageSize = vars->fileVars->position;
2169 break;
2170 }
2171
2172
2173 // Header:
2174
2175 header->imageSize = vars->fileVars->position;
2176 header->image1Size = image1Size;
2177 header->bitmapSize = bitmap_size;
2178 header->pageCount = pageCount;
2179
2180 header->restore1Sum = restore1Sum;
2181 header->image1Sum = sum1;
2182 header->image2Sum = sum2;
2183 header->sleepTime = gIOLastSleepTime.tv_sec;
2184
2185 header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
2186 #if defined(__arm64__)
2187 /*
2188 * We don't support retry on hibernation failure and so
2189 * we don't want to set this value to anything smaller
2190 * just because we may have been lucky this time around.
2191 * Though we'll let it go higher.
2192 */
2193 if (header->compression < HIB_COMPR_RATIO_ARM64) {
2194 header->compression = HIB_COMPR_RATIO_ARM64;
2195 }
2196 #endif /* __arm64__ */
2197
2198 gIOHibernateCompression = header->compression;
2199
2200 count = vars->fileVars->fileExtents->getLength();
2201 if (count > sizeof(header->fileExtentMap)) {
2202 header->fileExtentMapSize = ((uint32_t) count);
2203 count = sizeof(header->fileExtentMap);
2204 } else {
2205 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2206 }
2207 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
2208
2209 header->deviceBase = vars->fileVars->block0;
2210 header->deviceBlockSize = vars->fileVars->blockSize;
2211 header->lastHibAbsTime = mach_absolute_time();
2212 header->lastHibContTime = mach_continuous_time();
2213
2214
2215 IOPolledFileSeek(vars->fileVars, 0);
2216 err = IOHibernatePolledFileWrite(vars,
2217 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2218 cryptvars);
2219 if (kIOReturnSuccess != err) {
2220 break;
2221 }
2222
2223 err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
2224 }while (false);
2225
2226 clock_get_uptime(&endTime);
2227
2228 IOService::getPMRootDomain()->pmStatsRecordEvent(
2229 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2230
2231 SUB_ABSOLUTETIME(&endTime, &allTime);
2232 absolutetime_to_nanoseconds(endTime, &nsec);
2233 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
2234
2235 absolutetime_to_nanoseconds(compTime, &nsec);
2236 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2237 compBytes,
2238 nsec / 1000000ULL,
2239 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2240
2241 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2242 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2243 vars->fileVars->cryptBytes,
2244 nsec / 1000000ULL,
2245 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2246
2247 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
2248 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
2249 uncompressedSize, atop_32(uncompressedSize), compressedSize,
2250 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
2251
2252 HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
2253
2254 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2255 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
2256
2257 if (pollerOpen) {
2258 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
2259 }
2260
2261 if (vars->consoleMapping) {
2262 ProgressUpdate(gIOHibernateGraphicsInfo,
2263 vars->consoleMapping, 0, kIOHibernateProgressCount);
2264 }
2265
2266 HIBLOG("hibernate_write_image done(%x)\n", err);
2267
2268 // should we come back via regular wake, set the state in memory.
2269 gIOHibernateState = kIOHibernateStateInactive;
2270
2271 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
2272 wiredPagesClear, dirtyPagesEncrypted);
2273
2274 #if defined(__arm64__)
2275 if (kIOReturnSuccess == err) {
2276 return kIOHibernatePostWriteHalt;
2277 } else {
2278 // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
2279 // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
2280 vm_panic_hibernate_write_image_failed(err);
2281 return err; //not coming here post panic
2282 }
2283 #else
2284 if (kIOReturnSuccess == err) {
2285 if (kIOHibernateModeSleep & gIOHibernateMode) {
2286 return kIOHibernatePostWriteSleep;
2287 } else if (kIOHibernateModeRestart & gIOHibernateMode) {
2288 return kIOHibernatePostWriteRestart;
2289 } else {
2290 /* by default, power down */
2291 return kIOHibernatePostWriteHalt;
2292 }
2293 } else if (kIOReturnAborted == err) {
2294 return kIOHibernatePostWriteWake;
2295 } else {
2296 /* on error, sleep */
2297 return kIOHibernatePostWriteSleep;
2298 }
2299 #endif
2300 }
2301
2302 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2303
2304 extern "C" void
hibernate_machine_init(void)2305 hibernate_machine_init(void)
2306 {
2307 IOReturn err;
2308 uint32_t sum;
2309 uint32_t pagesDone;
2310 uint32_t pagesRead = 0;
2311 AbsoluteTime startTime, compTime;
2312 AbsoluteTime allTime, endTime;
2313 AbsoluteTime startIOTime, endIOTime;
2314 uint64_t nsec, nsecIO;
2315 uint64_t compBytes;
2316 uint64_t lastProgressStamp = 0;
2317 uint64_t progressStamp;
2318 IOPolledFileCryptVars * cryptvars = NULL;
2319
2320 IOHibernateVars * vars = &gIOHibernateVars;
2321 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
2322
2323 if (!vars->fileVars || !vars->fileVars->pollers) {
2324 return;
2325 }
2326
2327 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2328 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2329
2330 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
2331 HIBLOG("regular wake\n");
2332 return;
2333 }
2334
2335 HIBPRINT("diag %x %x %x %x\n",
2336 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2337 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2338
2339 #if defined(__i386__) || defined(__x86_64__)
2340 #define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
2341 #else /* defined(__i386__) || defined(__x86_64__) */
2342 #define t40ms(x) x
2343 #endif /* defined(__i386__) || defined(__x86_64__) */
2344 #define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2345 tStat(booterStart, booterStart);
2346 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
2347 tStat(booterDuration0, booterTime0);
2348 tStat(booterDuration1, booterTime1);
2349 tStat(booterDuration2, booterTime2);
2350 tStat(booterDuration, booterTime);
2351 tStat(booterConnectDisplayDuration, connectDisplayTime);
2352 tStat(booterSplashDuration, splashTime);
2353 tStat(trampolineDuration, trampolineTime);
2354
2355 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2356 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2357 gIOHibernateStats->image1Pages = pagesDone;
2358
2359 /* HIBERNATE_stats */
2360 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
2361 gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
2362 gIOHibernateStats->trampolineDuration);
2363
2364 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2365 gIOHibernateStats->booterStart,
2366 gIOHibernateStats->smcStart,
2367 gIOHibernateStats->booterDuration0,
2368 gIOHibernateStats->booterDuration1,
2369 gIOHibernateStats->booterDuration2,
2370 gIOHibernateStats->booterDuration,
2371 gIOHibernateStats->booterConnectDisplayDuration,
2372 gIOHibernateStats->booterSplashDuration,
2373 gIOHibernateStats->trampolineDuration);
2374
2375 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2376 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2377 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2378
2379 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2380 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
2381 hibernate_page_list_discard(vars->page_list);
2382 }
2383
2384 if (vars->hwEncrypt) {
2385 // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
2386 // decryption key via IOPolledFilePollersSetEncryptionKey
2387 cryptvars = NULL;
2388 } else {
2389 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
2390 }
2391
2392 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
2393 panic("handoff overflow");
2394 }
2395
2396 IOHibernateHandoff * handoff;
2397 bool done = false;
2398 bool foundCryptData = false;
2399 bool foundVolumeEncryptData = false;
2400 const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
2401 const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
2402
2403 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2404 !done;
2405 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
2406 if (((uint8_t*)handoff < handoffStart) ||
2407 (&handoff->data[handoff->bytecount] > handoffEnd)) {
2408 panic("handoff out of range");
2409 }
2410 // HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2411 uint8_t * data = &handoff->data[0];
2412 switch (handoff->type) {
2413 case kIOHibernateHandoffTypeEnd:
2414 done = true;
2415 break;
2416
2417 case kIOHibernateHandoffTypeGraphicsInfo:
2418 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
2419 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2420 }
2421 break;
2422
2423 case kIOHibernateHandoffTypeCryptVars:
2424 if (cryptvars) {
2425 hibernate_cryptwakevars_t *
2426 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2427 if (handoff->bytecount == sizeof(*wakevars)) {
2428 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2429 } else {
2430 panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
2431 }
2432 }
2433 foundCryptData = true;
2434 bzero(data, handoff->bytecount);
2435 break;
2436
2437 case kIOHibernateHandoffTypeVolumeCryptKey:
2438 if (handoff->bytecount == vars->volumeCryptKeySize) {
2439 bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2440 foundVolumeEncryptData = true;
2441 } else {
2442 panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
2443 }
2444 break;
2445
2446 #if defined(__i386__) || defined(__x86_64__)
2447 case kIOHibernateHandoffTypeMemoryMap:
2448
2449 clock_get_uptime(&allTime);
2450
2451 hibernate_newruntime_map(data, handoff->bytecount,
2452 gIOHibernateCurrentHeader->systemTableOffset);
2453
2454 clock_get_uptime(&endTime);
2455
2456 SUB_ABSOLUTETIME(&endTime, &allTime);
2457 absolutetime_to_nanoseconds(endTime, &nsec);
2458
2459 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2460
2461 break;
2462
2463 case kIOHibernateHandoffTypeDeviceTree:
2464 {
2465 // DTEntry chosen = NULL;
2466 // HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2467 }
2468 break;
2469 #endif /* defined(__i386__) || defined(__x86_64__) */
2470
2471 default:
2472 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2473 break;
2474 }
2475 }
2476
2477 if (vars->hwEncrypt && !foundVolumeEncryptData) {
2478 panic("no volumeCryptKey");
2479 } else if (cryptvars && !foundCryptData) {
2480 panic("hibernate handoff");
2481 }
2482
2483 HIBPRINT("video 0x%llx %d %d %d status %x\n",
2484 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2485 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2486
2487 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
2488 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2489 * gIOHibernateGraphicsInfo->rowBytes);
2490 if (vars->videoMapSize > vars->videoAllocSize) {
2491 vars->videoMapSize = 0;
2492 } else {
2493 IOMapPages(kernel_map,
2494 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2495 vars->videoMapSize, kIOMapInhibitCache );
2496 }
2497 }
2498
2499 if (vars->videoMapSize) {
2500 ProgressUpdate(gIOHibernateGraphicsInfo,
2501 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2502 }
2503
2504 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2505 uint8_t * compressed = src + page_size;
2506 uint8_t * scratch = compressed + page_size;
2507 uint32_t decoOffset;
2508
2509 clock_get_uptime(&allTime);
2510 AbsoluteTime_to_scalar(&compTime) = 0;
2511 compBytes = 0;
2512
2513 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2514 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
2515 clock_get_uptime(&startIOTime);
2516 endTime = startIOTime;
2517 SUB_ABSOLUTETIME(&endTime, &allTime);
2518 absolutetime_to_nanoseconds(endTime, &nsec);
2519 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
2520
2521 if (vars->hwEncrypt) {
2522 err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
2523 &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
2524 HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
2525 if (kIOReturnSuccess != err) {
2526 panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
2527 }
2528 cryptvars = NULL;
2529 }
2530
2531 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2532
2533 // kick off the read ahead
2534 vars->fileVars->bufferHalf = 0;
2535 vars->fileVars->bufferLimit = 0;
2536 vars->fileVars->lastRead = 0;
2537 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
2538 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2539 vars->fileVars->cryptBytes = 0;
2540 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
2541
2542 err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
2543 if (kIOReturnSuccess != err) {
2544 panic("Hibernate restore error %x", err);
2545 }
2546 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2547 // --
2548
2549 HIBLOG("hibernate_machine_init reading\n");
2550
2551
2552 uint32_t * header = (uint32_t *) src;
2553 sum = 0;
2554
2555 while (kIOReturnSuccess == err) {
2556 unsigned int count;
2557 unsigned int page;
2558 uint32_t tag;
2559 vm_offset_t compressedSize;
2560 ppnum_t ppnum;
2561
2562 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2563 if (kIOReturnSuccess != err) {
2564 panic("Hibernate restore error %x", err);
2565 }
2566
2567 ppnum = header[0];
2568 count = header[1];
2569
2570 // HIBPRINT("(%x, %x)\n", ppnum, count);
2571
2572 if (!count) {
2573 break;
2574 }
2575
2576 for (page = 0; page < count; page++) {
2577 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2578 if (kIOReturnSuccess != err) {
2579 panic("Hibernate restore error %x", err);
2580 }
2581
2582 compressedSize = kIOHibernateTagLength & tag;
2583 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
2584 err = kIOReturnIPCError;
2585 panic("Hibernate restore error %x", err);
2586 }
2587
2588 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2589 if (kIOReturnSuccess != err) {
2590 panic("Hibernate restore error %x", err);
2591 }
2592
2593 if (compressedSize < page_size) {
2594 decoOffset = ((uint32_t) page_size);
2595 clock_get_uptime(&startTime);
2596
2597 if (compressedSize == 4) {
2598 int i;
2599 uint32_t *s, *d;
2600
2601 s = (uint32_t *)src;
2602 d = (uint32_t *)(uintptr_t)compressed;
2603
2604 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
2605 *d++ = *s;
2606 }
2607 } else {
2608 pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
2609 }
2610 clock_get_uptime(&endTime);
2611 ADD_ABSOLUTETIME(&compTime, &endTime);
2612 SUB_ABSOLUTETIME(&compTime, &startTime);
2613 compBytes += page_size;
2614 } else {
2615 decoOffset = 0;
2616 }
2617
2618 sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
2619 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2620 if (err) {
2621 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2622 panic("Hibernate restore error %x", err);
2623 }
2624
2625
2626 ppnum++;
2627 pagesDone++;
2628 pagesRead++;
2629
2630 if (0 == (8191 & pagesDone)) {
2631 clock_get_uptime(&endTime);
2632 SUB_ABSOLUTETIME(&endTime, &allTime);
2633 absolutetime_to_nanoseconds(endTime, &nsec);
2634 progressStamp = nsec / 750000000ULL;
2635 if (progressStamp != lastProgressStamp) {
2636 lastProgressStamp = progressStamp;
2637 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2638 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2639 }
2640 }
2641 }
2642 }
2643 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
2644 err = kIOReturnLockedRead;
2645 }
2646
2647 if (kIOReturnSuccess != err) {
2648 panic("Hibernate restore error %x", err);
2649 }
2650
2651
2652 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2653 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
2654
2655 clock_get_uptime(&endIOTime);
2656
2657 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
2658
2659 clock_get_uptime(&endTime);
2660
2661 IOService::getPMRootDomain()->pmStatsRecordEvent(
2662 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2663 IOService::getPMRootDomain()->pmStatsRecordEvent(
2664 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2665
2666 SUB_ABSOLUTETIME(&endTime, &allTime);
2667 absolutetime_to_nanoseconds(endTime, &nsec);
2668
2669 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2670 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2671
2672 gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
2673 gIOHibernateStats->imagePages = pagesDone;
2674
2675 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2676 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2677 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
2678
2679 absolutetime_to_nanoseconds(compTime, &nsec);
2680 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2681 compBytes,
2682 nsec / 1000000ULL,
2683 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2684
2685 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2686 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2687 vars->fileVars->cryptBytes,
2688 nsec / 1000000ULL,
2689 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2690
2691 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
2692 }
2693
2694 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2695
2696 void
IOHibernateSetWakeCapabilities(uint32_t capability)2697 IOHibernateSetWakeCapabilities(uint32_t capability)
2698 {
2699 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
2700 gIOHibernateStats->wakeCapability = capability;
2701
2702 if (kIOPMSystemCapabilityGraphics & capability) {
2703 vm_compressor_do_warmup();
2704 }
2705 }
2706 }
2707
2708 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2709
2710 void
IOHibernateSystemRestart(void)2711 IOHibernateSystemRestart(void)
2712 {
2713 static uint8_t noteStore[32] __attribute__((aligned(32)));
2714 IORegistryEntry * regEntry;
2715 const OSSymbol * sym;
2716 OSData * noteProp;
2717 OSData * data;
2718 uintptr_t * smcVars;
2719 uint8_t * smcBytes;
2720 size_t len;
2721 addr64_t element;
2722
2723 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2724 if (!data) {
2725 return;
2726 }
2727
2728 smcVars = (typeof(smcVars))data->getBytesNoCopy();
2729 smcBytes = (typeof(smcBytes))smcVars[1];
2730 len = smcVars[0];
2731 if (len > sizeof(noteStore)) {
2732 len = sizeof(noteStore);
2733 }
2734 noteProp = OSData::withCapacity(3 * sizeof(element));
2735 if (!noteProp) {
2736 return;
2737 }
2738 element = len;
2739 noteProp->appendValue(element);
2740 element = crc32(0, smcBytes, len);
2741 noteProp->appendValue(element);
2742
2743 bcopy(smcBytes, noteStore, len);
2744 element = (addr64_t) ¬eStore[0];
2745 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2746 noteProp->appendValue(element);
2747
2748 if (!gIOOptionsEntry) {
2749 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2750 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2751 if (regEntry && !gIOOptionsEntry) {
2752 regEntry->release();
2753 }
2754 }
2755
2756 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2757 if (gIOOptionsEntry && sym) {
2758 gIOOptionsEntry->setProperty(sym, noteProp);
2759 }
2760 if (noteProp) {
2761 noteProp->release();
2762 }
2763 if (sym) {
2764 sym->release();
2765 }
2766 }
2767