1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*!
29 * ARM64-specific functions required to support hibernation exit. Most of this
30 * logic won't get used on SPTM-based systems (the SPTM takes over the
31 * responsibilities usually handled by HIBTEXT and restoring pages).
32 */
33
34 #include <mach/mach_types.h>
35 #include <kern/misc_protos.h>
36 #include <IOKit/IOHibernatePrivate.h>
37 #include <machine/pal_hibernate.h>
38 #include <pexpert/arm/dockchannel.h>
39 #include <ptrauth.h>
40 #include <arm/cpu_data_internal.h>
41 #include <arm/cpu_internal.h>
42 #include <libkern/section_keywords.h>
43
44
45 pal_hib_tramp_result_t gHibTramp;
46 pal_hib_globals_t gHibernateGlobals MARK_AS_HIBERNATE_DATA_CONST_LATE;
47
48 // as a workaround for <rdar://problem/70121432> References between different compile units in xnu shouldn't go through GOT
49 // all of the extern symbols that we refer to in this file have to be declared with hidden visibility
50 extern IOHibernateImageHeader *gIOHibernateCurrentHeader __attribute__((visibility("hidden")));
51 extern const uint32_t ccsha256_initial_state[8] __attribute__((visibility("hidden")));
52 extern void AccelerateCrypto_SHA256_compress(ccdigest_state_t state, size_t numBlocks, const void *data) __attribute__((visibility("hidden")));
53 extern void ccdigest_final_64be(const struct ccdigest_info *di, ccdigest_ctx_t, unsigned char *digest) __attribute__((visibility("hidden")));
54 extern struct pmap_cpu_data_array_entry pmap_cpu_data_array[MAX_CPUS] __attribute__((visibility("hidden")));
55 extern bool hib_entry_pmap_lockdown __attribute__((visibility("hidden")));
56
57 uintptr_t
hibernate_restore_phys_page(uint64_t src,uint64_t dst,uint32_t len,__unused uint32_t procFlags)58 hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, __unused uint32_t procFlags)
59 {
60 void *d = (void*)pal_hib_map(DEST_COPY_AREA, dst);
61 __nosan_memcpy(d, (void*)src, len);
62 return (uintptr_t)d;
63 }
64
65 uintptr_t
pal_hib_map(pal_hib_map_type_t virt,uint64_t phys)66 pal_hib_map(pal_hib_map_type_t virt, uint64_t phys)
67 {
68 switch (virt) {
69 case DEST_COPY_AREA:
70 case COPY_PAGE_AREA:
71 case SCRATCH_AREA:
72 case WKDM_AREA:
73 return phys + gHibTramp.memSlide;
74 case BITMAP_AREA:
75 case IMAGE_AREA:
76 case IMAGE2_AREA:
77 return phys;
78 default:
79 HIB_ASSERT(0);
80 }
81 }
82
83 void
pal_hib_restore_pal_state(__unused uint32_t * arg)84 pal_hib_restore_pal_state(__unused uint32_t *arg)
85 {
86 }
87
88 void
pal_hib_resume_init(pal_hib_ctx_t * ctx,hibernate_page_list_t * map,uint32_t * nextFree)89 pal_hib_resume_init(pal_hib_ctx_t *ctx, hibernate_page_list_t *map, uint32_t *nextFree)
90 {
91 #pragma unused(ctx, map, nextFree)
92
93 }
94
95 void
pal_hib_restored_page(pal_hib_ctx_t * ctx,pal_hib_restore_stage_t stage,ppnum_t ppnum)96 pal_hib_restored_page(pal_hib_ctx_t *ctx, pal_hib_restore_stage_t stage, ppnum_t ppnum)
97 {
98 #pragma unused(ctx, stage, ppnum)
99
100 }
101
102 void
pal_hib_patchup(pal_hib_ctx_t * ctx)103 pal_hib_patchup(pal_hib_ctx_t *ctx)
104 {
105 #pragma unused(ctx)
106
107
108 // cluster CTRR state needs to be reconfigured
109 init_ctrr_cluster_states();
110 }
111
112 void
pal_hib_decompress_page(void * src,void * dst,void * scratch,unsigned int compressedSize)113 pal_hib_decompress_page(void *src, void *dst, void *scratch, unsigned int compressedSize)
114 {
115 const void *wkdmSrc;
116 if (((uint64_t)src) & 63) {
117 // the wkdm instruction requires that our source buffer be aligned, so copy into an aligned buffer if necessary
118 __nosan_memcpy(scratch, src, compressedSize);
119 wkdmSrc = scratch;
120 } else {
121 wkdmSrc = src;
122 }
123 HIB_ASSERT((((uint64_t)wkdmSrc) & 63) == 0);
124 HIB_ASSERT((((uint64_t)dst) & PAGE_MASK) == 0);
125 struct {
126 uint32_t reserved:12;
127 uint32_t status:3;
128 uint32_t reserved2:17;
129 uint32_t popcnt:18;
130 uint32_t reserved3:14;
131 } result = { .status = ~0u };
132 __asm__ volatile ("wkdmd %0, %1" : "=r"(result): "r"(dst), "0"(wkdmSrc));
133
134 #if defined APPLEH16 || defined APPLEACC8
135 __builtin_arm_dmb(DMB_ISH);
136 #endif /* defined APPLEH16 || defined APPLEACC8 */
137
138 HIB_ASSERT(result.status == 0);
139 }
140
141 // proc_reg's ARM_TTE_TABLE_NS has both NSTABLE and NS set
142 #define ARM_LPAE_NSTABLE 0x8000000000000000ULL
143
144 #define TOP_LEVEL 1
145 #define LAST_TABLE_LEVEL 3
146 #define PAGE_GRANULE_SHIFT 14
147 #define PAGE_GRANULE_SIZE ((size_t)1<<PAGE_GRANULE_SHIFT)
148 #define PAGE_GRANULE_MASK (PAGE_GRANULE_SIZE-1)
149 #define LEVEL_SHIFT(level) (47 - (level * 11))
150
151 #define PTE_EMPTY(ent) ((ent) == 0)
152
153 typedef struct {
154 hibernate_page_list_t *bitmap;
155 uint32_t nextFree;
156 uint64_t page_table_base;
157 } map_ctx;
158
159 static void
hib_bzero(volatile void * s,size_t n)160 hib_bzero(volatile void *s, size_t n)
161 {
162 uintptr_t p = (uintptr_t)s;
163
164 // can't use __nosan_bzero while the MMU is off, so do it manually
165 while (n > sizeof(uint64_t)) {
166 *(volatile uint64_t *)p = 0;
167 p += sizeof(uint64_t);
168 n -= sizeof(uint64_t);
169 }
170 while (n > sizeof(uint32_t)) {
171 *(volatile uint32_t *)p = 0;
172 p += sizeof(uint32_t);
173 n -= sizeof(uint32_t);
174 }
175 while (n) {
176 *(volatile char *)p = 0;
177 p++;
178 n--;
179 }
180 }
181
182 static uint64_t
allocate_page(map_ctx * ctx)183 allocate_page(map_ctx *ctx)
184 {
185 // pages that were unnecessary for preservation when we entered hibernation are
186 // marked as free in ctx->bitmap, so they are available for scratch usage during
187 // resume; here, we "borrow" one of these free pages to use as part of our temporary
188 // page tables
189 ppnum_t ppnum = hibernate_page_list_grab(ctx->bitmap, &ctx->nextFree);
190 hibernate_page_bitset(ctx->bitmap, FALSE, ppnum);
191 uint64_t result = ptoa_64(ppnum);
192 hib_bzero((void *)result, PAGE_SIZE);
193 return result;
194 }
195
196 static void
create_map_entries(map_ctx * ctx,uint64_t vaddr,uint64_t paddr,uint64_t size,uint64_t map_flags)197 create_map_entries(map_ctx *ctx, uint64_t vaddr, uint64_t paddr, uint64_t size, uint64_t map_flags)
198 {
199 // if we've set gHibTramp.memSlide, we should already be running with the MMU on;
200 // in this case, we don't permit further modification to the page table
201 HIB_ASSERT(!gHibTramp.memSlide);
202
203 int level = TOP_LEVEL;
204 volatile uint64_t *table_base = (uint64_t *)ctx->page_table_base;
205 if (map_flags == 0) {
206 paddr = 0; // no physical address for none mappings
207 }
208
209 while (size) {
210 HIB_ASSERT(level >= 1);
211 HIB_ASSERT(level <= LAST_TABLE_LEVEL);
212
213 size_t level_shift = LEVEL_SHIFT(level);
214 size_t level_entries = PAGE_GRANULE_SIZE / sizeof(uint64_t);
215 size_t level_size = 1ull << level_shift;
216 size_t level_mask = level_size - 1;
217 size_t index = (vaddr >> level_shift) & (level_entries - 1);
218 // Can we make block entries here? Must be permitted at this
219 // level, have enough bytes remaining, and both virtual and
220 // physical addresses aligned to a block.
221 if ((level >= 2) &&
222 size >= level_size &&
223 ((vaddr | paddr) & level_mask) == 0) {
224 // Map contiguous blocks.
225 size_t num_entries = MIN(size / level_size, level_entries - index);
226 if (map_flags) {
227 uint64_t entry = map_flags | ((level < LAST_TABLE_LEVEL) ? ARM_TTE_TYPE_BLOCK : ARM_TTE_TYPE_L3BLOCK);
228 for (size_t i = 0; i < num_entries; i++) {
229 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
230 table_base[index + i] = entry | paddr;
231 paddr += level_size;
232 }
233 } else {
234 // make sure all the corresponding entries are empty
235 for (size_t i = 0; i < num_entries; i++) {
236 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
237 }
238 }
239 size_t mapped = num_entries * level_size;
240 size -= mapped;
241 if (size) {
242 // map the remaining at the top level
243 level = TOP_LEVEL;
244 table_base = (uint64_t *)ctx->page_table_base;
245 vaddr += mapped;
246 // paddr already incremented above if necessary
247 }
248 } else {
249 // Sub-divide into a next level table.
250 HIB_ASSERT(level < LAST_TABLE_LEVEL);
251 uint64_t entry = table_base[index];
252 HIB_ASSERT((entry & (ARM_TTE_VALID | ARM_TTE_TYPE_MASK)) != (ARM_TTE_VALID | ARM_TTE_TYPE_BLOCK)); // Breaking down blocks not implemented
253 uint64_t sub_base = entry & ARM_TTE_TABLE_MASK;
254 if (!sub_base) {
255 sub_base = allocate_page(ctx);
256 HIB_ASSERT((sub_base & PAGE_GRANULE_MASK) == 0);
257 table_base[index] = sub_base | ARM_LPAE_NSTABLE | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
258 }
259 // map into the sub table
260 level++;
261 table_base = (uint64_t *)sub_base;
262 }
263 }
264 }
265
266 static void
map_range_start_end(map_ctx * ctx,uint64_t start,uint64_t end,uint64_t slide,uint64_t flags)267 map_range_start_end(map_ctx *ctx, uint64_t start, uint64_t end, uint64_t slide, uint64_t flags)
268 {
269 HIB_ASSERT(end >= start);
270 create_map_entries(ctx, start + slide, start, end - start, flags);
271 }
272
273 #define MAP_FLAGS_COMMON (ARM_PTE_AF | ARM_PTE_NS | ARM_TTE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK))
274 #define MAP_DEVICE (ARM_PTE_AF | ARM_TTE_VALID | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_SH(SH_NONE) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE))
275 #define MAP_RO (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_AP(AP_RONA))
276 #define MAP_RW (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX)
277 #define MAP_RX (MAP_FLAGS_COMMON | ARM_PTE_AP(AP_RONA))
278
279 static void
map_register_page(map_ctx * ctx,vm_address_t regPage)280 map_register_page(map_ctx *ctx, vm_address_t regPage)
281 {
282 uint64_t regBase = trunc_page(regPage);
283 if (regBase) {
284 map_range_start_end(ctx, regBase, regBase + PAGE_SIZE, 0, MAP_DEVICE);
285 }
286 }
287
288 static void
289 iterate_bitmaps(const map_ctx *ctx, bool (^callback)(const hibernate_bitmap_t *bank_bitmap))
290 {
291 hibernate_bitmap_t *bank_bitmap = &ctx->bitmap->bank_bitmap[0];
292 for (uint32_t bank = 0; bank < ctx->bitmap->bank_count; bank++) {
293 if (!callback(bank_bitmap)) {
294 return;
295 }
296 bank_bitmap = (hibernate_bitmap_t*)&bank_bitmap->bitmap[bank_bitmap->bitmapwords];
297 }
298 }
299
300 // during hibernation resume, we can't use the original kernel page table (because we don't know what it was), so we instead
301 // create a temporary page table to use during hibernation resume; since the original kernel page table was part of DRAM,
302 // it will be restored by the time we're done with hibernation resume, at which point we can jump through the reset vector
303 // to reload the original page table
304 void
pal_hib_resume_tramp(uint32_t headerPpnum)305 pal_hib_resume_tramp(uint32_t headerPpnum)
306 {
307 uint64_t header_phys = ptoa_64(headerPpnum);
308 IOHibernateImageHeader *header = (IOHibernateImageHeader *)header_phys;
309 IOHibernateHibSegInfo *seg_info = &header->hibSegInfo;
310 uint64_t hib_text_start = ptoa_64(header->restore1CodePhysPage);
311
312 __block map_ctx ctx = {};
313 uint64_t map_phys = header_phys
314 + (offsetof(IOHibernateImageHeader, fileExtentMap)
315 + header->fileExtentMapSize
316 + ptoa_32(header->restore1PageCount)
317 + header->previewSize);
318 ctx.bitmap = (hibernate_page_list_t *)map_phys;
319
320 // find the bank describing xnu's map
321 __block uint64_t phys_start = 0, phys_end = 0;
322 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
323 if ((bank_bitmap->first_page <= header->restore1CodePhysPage) &&
324 (bank_bitmap->last_page >= header->restore1CodePhysPage)) {
325 phys_start = ptoa_64(bank_bitmap->first_page);
326 phys_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
327 return false;
328 }
329 return true;
330 });
331
332 HIB_ASSERT(phys_start != 0);
333 HIB_ASSERT(phys_end != 0);
334
335 hib_bzero(&gHibTramp, sizeof(gHibTramp));
336
337 // During hibernation resume, we create temporary mappings that do not collide with where any of the kernel mappings were originally.
338 // Technically, non-collision isn't a requirement, but doing this means that if some code accidentally jumps to a VA in the original
339 // kernel map, it won't be present in our temporary map and we'll get an exception when jumping to an unmapped address.
340 // The base address of our temporary mappings is adjusted by a random amount as a "poor-man's ASLR". We don’t have a good source of random
341 // numbers in this context, so we just use some of the bits from one of imageHeaderHMMAC, which should be random enough.
342 uint16_t rand = (uint16_t)(((header->imageHeaderHMAC[0]) << 8) | header->imageHeaderHMAC[1]);
343 uint64_t mem_slide = gHibernateGlobals.kernelSlide - (phys_end - phys_start) * 4 - rand * 256 * PAGE_SIZE;
344
345 // make sure we don't clobber any of the pages we need for restore
346 hibernate_reserve_restore_pages(header_phys, header, ctx.bitmap);
347
348 // init nextFree
349 hibernate_page_list_grab(ctx.bitmap, &ctx.nextFree);
350
351 // map ttbr1 pages
352 ctx.page_table_base = allocate_page(&ctx);
353 gHibTramp.ttbr1 = ctx.page_table_base;
354
355 uint64_t first_seg_start = 0, last_seg_end = 0, hib_text_end = 0;
356 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
357 uint64_t size = ptoa_64(seg_info->segments[i].pageCount);
358 if (size) {
359 uint64_t seg_start = ptoa_64(seg_info->segments[i].physPage);
360 uint64_t seg_end = seg_start + size;
361 uint32_t protection = seg_info->segments[i].protection;
362 if (protection != VM_PROT_NONE) {
363 // make sure the segment is in bounds
364 HIB_ASSERT(seg_start >= phys_start);
365 HIB_ASSERT(seg_end <= phys_end);
366
367 if (!first_seg_start) {
368 first_seg_start = seg_start;
369 }
370 if (last_seg_end) {
371 // map the "hole" as RW
372 map_range_start_end(&ctx, last_seg_end, seg_start, mem_slide, MAP_RW);
373 }
374 // map the segments described in machine_header at their original locations
375 bool executable = (protection & VM_PROT_EXECUTE);
376 bool writeable = (protection & VM_PROT_WRITE);
377 uint64_t map_flags = executable ? MAP_RX : writeable ? MAP_RW : MAP_RO;
378 map_range_start_end(&ctx, seg_start, seg_end, gHibernateGlobals.kernelSlide, map_flags);
379 last_seg_end = seg_end;
380 }
381 if (seg_info->segments[i].physPage == header->restore1CodePhysPage) {
382 // this is the hibtext segment, so remember where it ends
383 hib_text_end = seg_end;
384 }
385 }
386 }
387 // map the rest of kernel memory (the pages that come before and after our segments) as RW
388 map_range_start_end(&ctx, phys_start, first_seg_start, mem_slide, MAP_RW);
389 map_range_start_end(&ctx, last_seg_end, phys_end, mem_slide, MAP_RW);
390
391 // map all of the remaining banks that we didn't already deal with
392 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
393 uint64_t bank_start = ptoa_64(bank_bitmap->first_page);
394 uint64_t bank_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
395 if (bank_start == phys_start) {
396 // skip this bank since we already covered it above
397 } else {
398 // map the bank RW
399 map_range_start_end(&ctx, bank_start, bank_end, mem_slide, MAP_RW);
400 }
401 return true;
402 });
403
404 // map ttbr0 pages
405 ctx.page_table_base = allocate_page(&ctx);
406 gHibTramp.ttbr0 = ctx.page_table_base;
407
408 // map hib text P=V so that we can still execute at its physical address
409 map_range_start_end(&ctx, hib_text_start, hib_text_end, 0, MAP_RX);
410
411 // map the hib image P=V, RW
412 uint64_t image_start = trunc_page(header_phys);
413 uint64_t image_end = round_page(header_phys + header->image1Size);
414 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RW);
415
416 // map the handoff pages P=V, RO
417 image_start = ptoa_64(header->handoffPages);
418 image_end = image_start + ptoa_64(header->handoffPageCount);
419 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RO);
420
421 // map some device register pages
422 if (gHibernateGlobals.dockChannelRegPhysBase) {
423 #define dockchannel_uart_base gHibernateGlobals.dockChannelRegPhysBase
424 vm_address_t dockChannelRegPhysBase =
425 (vm_address_t)rDOCKCHANNELS_DEV_WSTAT(dockchannel_uart_base, DOCKCHANNEL_UART_CHANNEL);
426 map_register_page(&ctx, dockChannelRegPhysBase);
427 }
428 map_register_page(&ctx, gHibernateGlobals.hibUartRegPhysBase);
429 map_register_page(&ctx, gHibernateGlobals.hmacRegBase);
430
431 gHibTramp.memSlide = mem_slide;
432 }
433