1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*!
29 * ARM64-specific functions required to support hibernation exit.
30 */
31
32 #include <mach/mach_types.h>
33 #include <kern/misc_protos.h>
34 #include <IOKit/IOHibernatePrivate.h>
35 #include <machine/pal_hibernate.h>
36 #include <pexpert/arm/dockchannel.h>
37 #include <ptrauth.h>
38 #include <arm/cpu_data_internal.h>
39 #include <arm/cpu_internal.h>
40 #include <libkern/section_keywords.h>
41
42
43 pal_hib_tramp_result_t gHibTramp;
44 pal_hib_globals_t gHibernateGlobals MARK_AS_HIBERNATE_DATA_CONST_LATE;
45
46 // as a workaround for <rdar://problem/70121432> References between different compile units in xnu shouldn't go through GOT
47 // all of the extern symbols that we refer to in this file have to be declared with hidden visibility
48 extern IOHibernateImageHeader *gIOHibernateCurrentHeader __attribute__((visibility("hidden")));
49 extern const uint32_t ccsha256_initial_state[8] __attribute__((visibility("hidden")));
50 extern void AccelerateCrypto_SHA256_compress(ccdigest_state_t state, size_t numBlocks, const void *data) __attribute__((visibility("hidden")));
51 extern void ccdigest_final_64be(const struct ccdigest_info *di, ccdigest_ctx_t, unsigned char *digest) __attribute__((visibility("hidden")));
52 extern struct pmap_cpu_data_array_entry pmap_cpu_data_array[MAX_CPUS] __attribute__((visibility("hidden")));
53 extern bool hib_entry_pmap_lockdown __attribute__((visibility("hidden")));
54
55 uintptr_t
hibernate_restore_phys_page(uint64_t src,uint64_t dst,uint32_t len,__unused uint32_t procFlags)56 hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, __unused uint32_t procFlags)
57 {
58 void *d = (void*)pal_hib_map(DEST_COPY_AREA, dst);
59 __nosan_memcpy(d, (void*)src, len);
60 return (uintptr_t)d;
61 }
62
63 uintptr_t
pal_hib_map(pal_hib_map_type_t virt,uint64_t phys)64 pal_hib_map(pal_hib_map_type_t virt, uint64_t phys)
65 {
66 switch (virt) {
67 case DEST_COPY_AREA:
68 case COPY_PAGE_AREA:
69 case SCRATCH_AREA:
70 case WKDM_AREA:
71 return phys + gHibTramp.memSlide;
72 case BITMAP_AREA:
73 case IMAGE_AREA:
74 case IMAGE2_AREA:
75 return phys;
76 default:
77 HIB_ASSERT(0);
78 }
79 }
80
81 void
pal_hib_restore_pal_state(__unused uint32_t * arg)82 pal_hib_restore_pal_state(__unused uint32_t *arg)
83 {
84 }
85
86 void
pal_hib_resume_init(pal_hib_ctx_t * ctx,hibernate_page_list_t * map,uint32_t * nextFree)87 pal_hib_resume_init(pal_hib_ctx_t *ctx, hibernate_page_list_t *map, uint32_t *nextFree)
88 {
89 }
90
91 void
pal_hib_restored_page(pal_hib_ctx_t * ctx,pal_hib_restore_stage_t stage,ppnum_t ppnum)92 pal_hib_restored_page(pal_hib_ctx_t *ctx, pal_hib_restore_stage_t stage, ppnum_t ppnum)
93 {
94 }
95
96 void
pal_hib_patchup(pal_hib_ctx_t * ctx)97 pal_hib_patchup(pal_hib_ctx_t *ctx)
98 {
99
100 /* Reinit the ppl hib lock as it was saved to the hibernation image held. */
101 ppl_hib_lock_reinit();
102
103 // DRAM pages are captured from a PPL context, so here we restore all cpu_data structures to a non-PPL context
104 for (int i = 0; i < MAX_CPUS; i++) {
105 pmap_cpu_data_array[i].cpu_data.ppl_state = PPL_STATE_KERNEL;
106 pmap_cpu_data_array[i].cpu_data.ppl_kern_saved_sp = 0;
107 }
108
109 // cluster CTRR state needs to be reconfigured
110 init_ctrr_cluster_states();
111
112 // Calls into the pmap that could potentially modify pmap data structures
113 // during image copying were explicitly blocked on hibernation entry.
114 // Resetting this variable to false allows those calls to be made again.
115 hib_entry_pmap_lockdown = false;
116 }
117
118 void
pal_hib_decompress_page(void * src,void * dst,void * scratch,unsigned int compressedSize)119 pal_hib_decompress_page(void *src, void *dst, void *scratch, unsigned int compressedSize)
120 {
121 const void *wkdmSrc;
122 if (((uint64_t)src) & 63) {
123 // the wkdm instruction requires that our source buffer be aligned, so copy into an aligned buffer if necessary
124 __nosan_memcpy(scratch, src, compressedSize);
125 wkdmSrc = scratch;
126 } else {
127 wkdmSrc = src;
128 }
129 HIB_ASSERT((((uint64_t)wkdmSrc) & 63) == 0);
130 HIB_ASSERT((((uint64_t)dst) & PAGE_MASK) == 0);
131 struct {
132 uint32_t reserved:12;
133 uint32_t status:3;
134 uint32_t reserved2:17;
135 uint32_t popcnt:18;
136 uint32_t reserved3:14;
137 } result = { .status = ~0u };
138 __asm__ volatile ("wkdmd %0, %1" : "=r"(result): "r"(dst), "0"(wkdmSrc));
139 HIB_ASSERT(result.status == 0);
140 }
141
142 // proc_reg's ARM_TTE_TABLE_NS has both NSTABLE and NS set
143 #define ARM_LPAE_NSTABLE 0x8000000000000000ULL
144
145 #define TOP_LEVEL 1
146 #define LAST_TABLE_LEVEL 3
147 #define PAGE_GRANULE_SHIFT 14
148 #define PAGE_GRANULE_SIZE ((size_t)1<<PAGE_GRANULE_SHIFT)
149 #define PAGE_GRANULE_MASK (PAGE_GRANULE_SIZE-1)
150 #define LEVEL_SHIFT(level) (47 - (level * 11))
151
152 #define PTE_EMPTY(ent) ((ent) == 0)
153
154 typedef struct {
155 hibernate_page_list_t *bitmap;
156 uint32_t nextFree;
157 uint64_t page_table_base;
158 } map_ctx;
159
160 static void
hib_bzero(volatile void * s,size_t n)161 hib_bzero(volatile void *s, size_t n)
162 {
163 uintptr_t p = (uintptr_t)s;
164
165 // can't use __nosan_bzero while the MMU is off, so do it manually
166 while (n > sizeof(uint64_t)) {
167 *(volatile uint64_t *)p = 0;
168 p += sizeof(uint64_t);
169 n -= sizeof(uint64_t);
170 }
171 while (n > sizeof(uint32_t)) {
172 *(volatile uint32_t *)p = 0;
173 p += sizeof(uint32_t);
174 n -= sizeof(uint32_t);
175 }
176 while (n) {
177 *(volatile char *)p = 0;
178 p++;
179 n--;
180 }
181 }
182
183 static uint64_t
allocate_page(map_ctx * ctx)184 allocate_page(map_ctx *ctx)
185 {
186 // pages that were unnecessary for preservation when we entered hibernation are
187 // marked as free in ctx->bitmap, so they are available for scratch usage during
188 // resume; here, we "borrow" one of these free pages to use as part of our temporary
189 // page tables
190 ppnum_t ppnum = hibernate_page_list_grab(ctx->bitmap, &ctx->nextFree);
191 hibernate_page_bitset(ctx->bitmap, FALSE, ppnum);
192 uint64_t result = ptoa_64(ppnum);
193 hib_bzero((void *)result, PAGE_SIZE);
194 return result;
195 }
196
197 static void
create_map_entries(map_ctx * ctx,uint64_t vaddr,uint64_t paddr,uint64_t size,uint64_t map_flags)198 create_map_entries(map_ctx *ctx, uint64_t vaddr, uint64_t paddr, uint64_t size, uint64_t map_flags)
199 {
200 // if we've set gHibTramp.memSlide, we should already be running with the MMU on;
201 // in this case, we don't permit further modification to the page table
202 HIB_ASSERT(!gHibTramp.memSlide);
203
204 int level = TOP_LEVEL;
205 volatile uint64_t *table_base = (uint64_t *)ctx->page_table_base;
206 if (map_flags == 0) {
207 paddr = 0; // no physical address for none mappings
208 }
209
210 while (size) {
211 HIB_ASSERT(level >= 1);
212 HIB_ASSERT(level <= LAST_TABLE_LEVEL);
213
214 size_t level_shift = LEVEL_SHIFT(level);
215 size_t level_entries = PAGE_GRANULE_SIZE / sizeof(uint64_t);
216 size_t level_size = 1ull << level_shift;
217 size_t level_mask = level_size - 1;
218 size_t index = (vaddr >> level_shift) & (level_entries - 1);
219 // Can we make block entries here? Must be permitted at this
220 // level, have enough bytes remaining, and both virtual and
221 // physical addresses aligned to a block.
222 if ((level >= 2) &&
223 size >= level_size &&
224 ((vaddr | paddr) & level_mask) == 0) {
225 // Map contiguous blocks.
226 size_t num_entries = MIN(size / level_size, level_entries - index);
227 if (map_flags) {
228 uint64_t entry = map_flags | ((level < LAST_TABLE_LEVEL) ? ARM_TTE_TYPE_BLOCK : ARM_TTE_TYPE_L3BLOCK);
229 for (size_t i = 0; i < num_entries; i++) {
230 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
231 table_base[index + i] = entry | paddr;
232 paddr += level_size;
233 }
234 } else {
235 // make sure all the corresponding entries are empty
236 for (size_t i = 0; i < num_entries; i++) {
237 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
238 }
239 }
240 size_t mapped = num_entries * level_size;
241 size -= mapped;
242 if (size) {
243 // map the remaining at the top level
244 level = TOP_LEVEL;
245 table_base = (uint64_t *)ctx->page_table_base;
246 vaddr += mapped;
247 // paddr already incremented above if necessary
248 }
249 } else {
250 // Sub-divide into a next level table.
251 HIB_ASSERT(level < LAST_TABLE_LEVEL);
252 uint64_t entry = table_base[index];
253 HIB_ASSERT((entry & (ARM_TTE_VALID | ARM_TTE_TYPE_MASK)) != (ARM_TTE_VALID | ARM_TTE_TYPE_BLOCK)); // Breaking down blocks not implemented
254 uint64_t sub_base = entry & ARM_TTE_TABLE_MASK;
255 if (!sub_base) {
256 sub_base = allocate_page(ctx);
257 HIB_ASSERT((sub_base & PAGE_GRANULE_MASK) == 0);
258 table_base[index] = sub_base | ARM_LPAE_NSTABLE | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
259 }
260 // map into the sub table
261 level++;
262 table_base = (uint64_t *)sub_base;
263 }
264 }
265 }
266
267 static void
map_range_start_end(map_ctx * ctx,uint64_t start,uint64_t end,uint64_t slide,uint64_t flags)268 map_range_start_end(map_ctx *ctx, uint64_t start, uint64_t end, uint64_t slide, uint64_t flags)
269 {
270 HIB_ASSERT(end >= start);
271 create_map_entries(ctx, start + slide, start, end - start, flags);
272 }
273
274 #define MAP_FLAGS_COMMON (ARM_PTE_AF | ARM_PTE_NS | ARM_TTE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK))
275 #define MAP_DEVICE (ARM_PTE_AF | ARM_TTE_VALID | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_SH(SH_NONE) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE))
276 #define MAP_RO (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_AP(AP_RONA))
277 #define MAP_RW (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX)
278 #define MAP_RX (MAP_FLAGS_COMMON | ARM_PTE_AP(AP_RONA))
279
280 static void
map_register_page(map_ctx * ctx,vm_address_t regPage)281 map_register_page(map_ctx *ctx, vm_address_t regPage)
282 {
283 uint64_t regBase = trunc_page(regPage);
284 if (regBase) {
285 map_range_start_end(ctx, regBase, regBase + PAGE_SIZE, 0, MAP_DEVICE);
286 }
287 }
288
289 static void
290 iterate_bitmaps(const map_ctx *ctx, bool (^callback)(const hibernate_bitmap_t *bank_bitmap))
291 {
292 hibernate_bitmap_t *bank_bitmap = &ctx->bitmap->bank_bitmap[0];
293 for (uint32_t bank = 0; bank < ctx->bitmap->bank_count; bank++) {
294 if (!callback(bank_bitmap)) {
295 return;
296 }
297 bank_bitmap = (hibernate_bitmap_t*)&bank_bitmap->bitmap[bank_bitmap->bitmapwords];
298 }
299 }
300
301 // during hibernation resume, we can't use the original kernel page table (because we don't know what it was), so we instead
302 // create a temporary page table to use during hibernation resume; since the original kernel page table was part of DRAM,
303 // it will be restored by the time we're done with hibernation resume, at which point we can jump through the reset vector
304 // to reload the original page table
305 void
pal_hib_resume_tramp(uint32_t headerPpnum)306 pal_hib_resume_tramp(uint32_t headerPpnum)
307 {
308 uint64_t header_phys = ptoa_64(headerPpnum);
309 IOHibernateImageHeader *header = (IOHibernateImageHeader *)header_phys;
310 IOHibernateHibSegInfo *seg_info = &header->hibSegInfo;
311 uint64_t hib_text_start = ptoa_64(header->restore1CodePhysPage);
312
313 __block map_ctx ctx = {};
314 uint64_t map_phys = header_phys
315 + (offsetof(IOHibernateImageHeader, fileExtentMap)
316 + header->fileExtentMapSize
317 + ptoa_32(header->restore1PageCount)
318 + header->previewSize);
319 ctx.bitmap = (hibernate_page_list_t *)map_phys;
320
321 // find the bank describing xnu's map
322 __block uint64_t phys_start = 0, phys_end = 0;
323 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
324 if ((bank_bitmap->first_page <= header->restore1CodePhysPage) &&
325 (bank_bitmap->last_page >= header->restore1CodePhysPage)) {
326 phys_start = ptoa_64(bank_bitmap->first_page);
327 phys_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
328 return false;
329 }
330 return true;
331 });
332
333 HIB_ASSERT(phys_start != 0);
334 HIB_ASSERT(phys_end != 0);
335
336 hib_bzero(&gHibTramp, sizeof(gHibTramp));
337
338 // During hibernation resume, we create temporary mappings that do not collide with where any of the kernel mappings were originally.
339 // Technically, non-collision isn't a requirement, but doing this means that if some code accidentally jumps to a VA in the original
340 // kernel map, it won't be present in our temporary map and we'll get an exception when jumping to an unmapped address.
341 // The base address of our temporary mappings is adjusted by a random amount as a "poor-man's ASLR". We don’t have a good source of random
342 // numbers in this context, so we just use some of the bits from one of imageHeaderHMMAC, which should be random enough.
343 uint16_t rand = (uint16_t)(((header->imageHeaderHMAC[0]) << 8) | header->imageHeaderHMAC[1]);
344 uint64_t mem_slide = gHibernateGlobals.kernelSlide - (phys_end - phys_start) * 4 - rand * 256 * PAGE_SIZE;
345
346 // make sure we don't clobber any of the pages we need for restore
347 hibernate_reserve_restore_pages(header_phys, header, ctx.bitmap);
348
349 // init nextFree
350 hibernate_page_list_grab(ctx.bitmap, &ctx.nextFree);
351
352 // map ttbr1 pages
353 ctx.page_table_base = allocate_page(&ctx);
354 gHibTramp.ttbr1 = ctx.page_table_base;
355
356 uint64_t first_seg_start = 0, last_seg_end = 0, hib_text_end = 0;
357 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
358 uint64_t size = ptoa_64(seg_info->segments[i].pageCount);
359 if (size) {
360 uint64_t seg_start = ptoa_64(seg_info->segments[i].physPage);
361 uint64_t seg_end = seg_start + size;
362 uint32_t protection = seg_info->segments[i].protection;
363 if (protection != VM_PROT_NONE) {
364 // make sure the segment is in bounds
365 HIB_ASSERT(seg_start >= phys_start);
366 HIB_ASSERT(seg_end <= phys_end);
367
368 if (!first_seg_start) {
369 first_seg_start = seg_start;
370 }
371 if (last_seg_end) {
372 // map the "hole" as RW
373 map_range_start_end(&ctx, last_seg_end, seg_start, mem_slide, MAP_RW);
374 }
375 // map the segments described in machine_header at their original locations
376 bool executable = (protection & VM_PROT_EXECUTE);
377 bool writeable = (protection & VM_PROT_WRITE);
378 uint64_t map_flags = executable ? MAP_RX : writeable ? MAP_RW : MAP_RO;
379 map_range_start_end(&ctx, seg_start, seg_end, gHibernateGlobals.kernelSlide, map_flags);
380 last_seg_end = seg_end;
381 }
382 if (seg_info->segments[i].physPage == header->restore1CodePhysPage) {
383 // this is the hibtext segment, so remember where it ends
384 hib_text_end = seg_end;
385 }
386 }
387 }
388 // map the rest of kernel memory (the pages that come before and after our segments) as RW
389 map_range_start_end(&ctx, phys_start, first_seg_start, mem_slide, MAP_RW);
390 map_range_start_end(&ctx, last_seg_end, phys_end, mem_slide, MAP_RW);
391
392 // map all of the remaining banks that we didn't already deal with
393 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
394 uint64_t bank_start = ptoa_64(bank_bitmap->first_page);
395 uint64_t bank_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
396 if (bank_start == phys_start) {
397 // skip this bank since we already covered it above
398 } else {
399 // map the bank RW
400 map_range_start_end(&ctx, bank_start, bank_end, mem_slide, MAP_RW);
401 }
402 return true;
403 });
404
405 // map ttbr0 pages
406 ctx.page_table_base = allocate_page(&ctx);
407 gHibTramp.ttbr0 = ctx.page_table_base;
408
409 // map hib text P=V so that we can still execute at its physical address
410 map_range_start_end(&ctx, hib_text_start, hib_text_end, 0, MAP_RX);
411
412 // map the hib image P=V, RW
413 uint64_t image_start = trunc_page(header_phys);
414 uint64_t image_end = round_page(header_phys + header->image1Size);
415 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RW);
416
417 // map the handoff pages P=V, RO
418 image_start = ptoa_64(header->handoffPages);
419 image_end = image_start + ptoa_64(header->handoffPageCount);
420 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RO);
421
422 // map some device register pages
423 if (gHibernateGlobals.dockChannelRegPhysBase) {
424 #define dockchannel_uart_base gHibernateGlobals.dockChannelRegPhysBase
425 vm_address_t dockChannelRegPhysBase = trunc_page(&rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL));
426 map_register_page(&ctx, dockChannelRegPhysBase);
427 }
428 map_register_page(&ctx, gHibernateGlobals.hibUartRegPhysBase);
429 map_register_page(&ctx, gHibernateGlobals.hmacRegBase);
430
431 gHibTramp.memSlide = mem_slide;
432 }
433