1 /*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*!
29 * ARM64-specific functions required to support hibernation exit. Most of this
30 * logic won't get used on SPTM-based systems (the SPTM takes over the
31 * responsibilities usually handled by HIBTEXT and restoring pages).
32 */
33
34 #include <mach/mach_types.h>
35 #include <kern/misc_protos.h>
36 #include <IOKit/IOHibernatePrivate.h>
37 #include <machine/pal_hibernate.h>
38 #include <pexpert/arm/dockchannel.h>
39 #include <ptrauth.h>
40 #include <arm/cpu_data_internal.h>
41 #include <arm/cpu_internal.h>
42 #include <libkern/section_keywords.h>
43
44
45 pal_hib_tramp_result_t gHibTramp;
46 pal_hib_globals_t gHibernateGlobals MARK_AS_HIBERNATE_DATA_CONST_LATE;
47
48 // as a workaround for <rdar://problem/70121432> References between different compile units in xnu shouldn't go through GOT
49 // all of the extern symbols that we refer to in this file have to be declared with hidden visibility
50 extern IOHibernateImageHeader *gIOHibernateCurrentHeader __attribute__((visibility("hidden")));
51 extern const uint32_t ccsha256_initial_state[8] __attribute__((visibility("hidden")));
52 extern void AccelerateCrypto_SHA256_compress(ccdigest_state_t state, size_t numBlocks, const void *data) __attribute__((visibility("hidden")));
53 extern void ccdigest_final_64be(const struct ccdigest_info *di, ccdigest_ctx_t, unsigned char *digest) __attribute__((visibility("hidden")));
54 extern struct pmap_cpu_data_array_entry pmap_cpu_data_array[MAX_CPUS] __attribute__((visibility("hidden")));
55 extern bool hib_entry_pmap_lockdown __attribute__((visibility("hidden")));
56
57 uintptr_t
hibernate_restore_phys_page(uint64_t src,uint64_t dst,uint32_t len,__unused uint32_t procFlags)58 hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, __unused uint32_t procFlags)
59 {
60 void *d = (void*)pal_hib_map(DEST_COPY_AREA, dst);
61 __nosan_memcpy(d, (void*)src, len);
62 return (uintptr_t)d;
63 }
64
65 uintptr_t
pal_hib_map(pal_hib_map_type_t virt,uint64_t phys)66 pal_hib_map(pal_hib_map_type_t virt, uint64_t phys)
67 {
68 switch (virt) {
69 case DEST_COPY_AREA:
70 case COPY_PAGE_AREA:
71 case SCRATCH_AREA:
72 case WKDM_AREA:
73 return phys + gHibTramp.memSlide;
74 case BITMAP_AREA:
75 case IMAGE_AREA:
76 case IMAGE2_AREA:
77 return phys;
78 default:
79 HIB_ASSERT(0);
80 }
81 }
82
83 void
pal_hib_restore_pal_state(__unused uint32_t * arg)84 pal_hib_restore_pal_state(__unused uint32_t *arg)
85 {
86 }
87
88 void
pal_hib_resume_init(pal_hib_ctx_t * ctx,hibernate_page_list_t * map,uint32_t * nextFree)89 pal_hib_resume_init(pal_hib_ctx_t *ctx, hibernate_page_list_t *map, uint32_t *nextFree)
90 {
91 #pragma unused(ctx, map, nextFree)
92
93 }
94
95 void
pal_hib_restored_page(pal_hib_ctx_t * ctx,pal_hib_restore_stage_t stage,ppnum_t ppnum)96 pal_hib_restored_page(pal_hib_ctx_t *ctx, pal_hib_restore_stage_t stage, ppnum_t ppnum)
97 {
98 #pragma unused(ctx, stage, ppnum)
99
100 }
101
102 void
pal_hib_patchup(pal_hib_ctx_t * ctx)103 pal_hib_patchup(pal_hib_ctx_t *ctx)
104 {
105 #pragma unused(ctx)
106
107
108 // cluster CTRR state needs to be reconfigured
109 init_ctrr_cluster_states();
110 }
111
112 void
pal_hib_decompress_page(void * src,void * dst,void * scratch,unsigned int compressedSize)113 pal_hib_decompress_page(void *src, void *dst, void *scratch, unsigned int compressedSize)
114 {
115 const void *wkdmSrc;
116 if (((uint64_t)src) & 63) {
117 // the wkdm instruction requires that our source buffer be aligned, so copy into an aligned buffer if necessary
118 __nosan_memcpy(scratch, src, compressedSize);
119 wkdmSrc = scratch;
120 } else {
121 wkdmSrc = src;
122 }
123 HIB_ASSERT((((uint64_t)wkdmSrc) & 63) == 0);
124 HIB_ASSERT((((uint64_t)dst) & PAGE_MASK) == 0);
125 struct {
126 uint32_t reserved:12;
127 uint32_t status:3;
128 uint32_t reserved2:17;
129 uint32_t popcnt:18;
130 uint32_t reserved3:14;
131 } result = { .status = ~0u };
132 __asm__ volatile ("wkdmd %0, %1" : "=r"(result): "r"(dst), "0"(wkdmSrc));
133 HIB_ASSERT(result.status == 0);
134 }
135
136 // proc_reg's ARM_TTE_TABLE_NS has both NSTABLE and NS set
137 #define ARM_LPAE_NSTABLE 0x8000000000000000ULL
138
139 #define TOP_LEVEL 1
140 #define LAST_TABLE_LEVEL 3
141 #define PAGE_GRANULE_SHIFT 14
142 #define PAGE_GRANULE_SIZE ((size_t)1<<PAGE_GRANULE_SHIFT)
143 #define PAGE_GRANULE_MASK (PAGE_GRANULE_SIZE-1)
144 #define LEVEL_SHIFT(level) (47 - (level * 11))
145
146 #define PTE_EMPTY(ent) ((ent) == 0)
147
148 typedef struct {
149 hibernate_page_list_t *bitmap;
150 uint32_t nextFree;
151 uint64_t page_table_base;
152 } map_ctx;
153
154 static void
hib_bzero(volatile void * s,size_t n)155 hib_bzero(volatile void *s, size_t n)
156 {
157 uintptr_t p = (uintptr_t)s;
158
159 // can't use __nosan_bzero while the MMU is off, so do it manually
160 while (n > sizeof(uint64_t)) {
161 *(volatile uint64_t *)p = 0;
162 p += sizeof(uint64_t);
163 n -= sizeof(uint64_t);
164 }
165 while (n > sizeof(uint32_t)) {
166 *(volatile uint32_t *)p = 0;
167 p += sizeof(uint32_t);
168 n -= sizeof(uint32_t);
169 }
170 while (n) {
171 *(volatile char *)p = 0;
172 p++;
173 n--;
174 }
175 }
176
177 static uint64_t
allocate_page(map_ctx * ctx)178 allocate_page(map_ctx *ctx)
179 {
180 // pages that were unnecessary for preservation when we entered hibernation are
181 // marked as free in ctx->bitmap, so they are available for scratch usage during
182 // resume; here, we "borrow" one of these free pages to use as part of our temporary
183 // page tables
184 ppnum_t ppnum = hibernate_page_list_grab(ctx->bitmap, &ctx->nextFree);
185 hibernate_page_bitset(ctx->bitmap, FALSE, ppnum);
186 uint64_t result = ptoa_64(ppnum);
187 hib_bzero((void *)result, PAGE_SIZE);
188 return result;
189 }
190
191 static void
create_map_entries(map_ctx * ctx,uint64_t vaddr,uint64_t paddr,uint64_t size,uint64_t map_flags)192 create_map_entries(map_ctx *ctx, uint64_t vaddr, uint64_t paddr, uint64_t size, uint64_t map_flags)
193 {
194 // if we've set gHibTramp.memSlide, we should already be running with the MMU on;
195 // in this case, we don't permit further modification to the page table
196 HIB_ASSERT(!gHibTramp.memSlide);
197
198 int level = TOP_LEVEL;
199 volatile uint64_t *table_base = (uint64_t *)ctx->page_table_base;
200 if (map_flags == 0) {
201 paddr = 0; // no physical address for none mappings
202 }
203
204 while (size) {
205 HIB_ASSERT(level >= 1);
206 HIB_ASSERT(level <= LAST_TABLE_LEVEL);
207
208 size_t level_shift = LEVEL_SHIFT(level);
209 size_t level_entries = PAGE_GRANULE_SIZE / sizeof(uint64_t);
210 size_t level_size = 1ull << level_shift;
211 size_t level_mask = level_size - 1;
212 size_t index = (vaddr >> level_shift) & (level_entries - 1);
213 // Can we make block entries here? Must be permitted at this
214 // level, have enough bytes remaining, and both virtual and
215 // physical addresses aligned to a block.
216 if ((level >= 2) &&
217 size >= level_size &&
218 ((vaddr | paddr) & level_mask) == 0) {
219 // Map contiguous blocks.
220 size_t num_entries = MIN(size / level_size, level_entries - index);
221 if (map_flags) {
222 uint64_t entry = map_flags | ((level < LAST_TABLE_LEVEL) ? ARM_TTE_TYPE_BLOCK : ARM_TTE_TYPE_L3BLOCK);
223 for (size_t i = 0; i < num_entries; i++) {
224 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
225 table_base[index + i] = entry | paddr;
226 paddr += level_size;
227 }
228 } else {
229 // make sure all the corresponding entries are empty
230 for (size_t i = 0; i < num_entries; i++) {
231 HIB_ASSERT(PTE_EMPTY(table_base[index + i]));
232 }
233 }
234 size_t mapped = num_entries * level_size;
235 size -= mapped;
236 if (size) {
237 // map the remaining at the top level
238 level = TOP_LEVEL;
239 table_base = (uint64_t *)ctx->page_table_base;
240 vaddr += mapped;
241 // paddr already incremented above if necessary
242 }
243 } else {
244 // Sub-divide into a next level table.
245 HIB_ASSERT(level < LAST_TABLE_LEVEL);
246 uint64_t entry = table_base[index];
247 HIB_ASSERT((entry & (ARM_TTE_VALID | ARM_TTE_TYPE_MASK)) != (ARM_TTE_VALID | ARM_TTE_TYPE_BLOCK)); // Breaking down blocks not implemented
248 uint64_t sub_base = entry & ARM_TTE_TABLE_MASK;
249 if (!sub_base) {
250 sub_base = allocate_page(ctx);
251 HIB_ASSERT((sub_base & PAGE_GRANULE_MASK) == 0);
252 table_base[index] = sub_base | ARM_LPAE_NSTABLE | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
253 }
254 // map into the sub table
255 level++;
256 table_base = (uint64_t *)sub_base;
257 }
258 }
259 }
260
261 static void
map_range_start_end(map_ctx * ctx,uint64_t start,uint64_t end,uint64_t slide,uint64_t flags)262 map_range_start_end(map_ctx *ctx, uint64_t start, uint64_t end, uint64_t slide, uint64_t flags)
263 {
264 HIB_ASSERT(end >= start);
265 create_map_entries(ctx, start + slide, start, end - start, flags);
266 }
267
268 #define MAP_FLAGS_COMMON (ARM_PTE_AF | ARM_PTE_NS | ARM_TTE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK))
269 #define MAP_DEVICE (ARM_PTE_AF | ARM_TTE_VALID | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_SH(SH_NONE) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE))
270 #define MAP_RO (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX | ARM_PTE_AP(AP_RONA))
271 #define MAP_RW (MAP_FLAGS_COMMON | ARM_PTE_PNX | ARM_PTE_NX)
272 #define MAP_RX (MAP_FLAGS_COMMON | ARM_PTE_AP(AP_RONA))
273
274 static void
map_register_page(map_ctx * ctx,vm_address_t regPage)275 map_register_page(map_ctx *ctx, vm_address_t regPage)
276 {
277 uint64_t regBase = trunc_page(regPage);
278 if (regBase) {
279 map_range_start_end(ctx, regBase, regBase + PAGE_SIZE, 0, MAP_DEVICE);
280 }
281 }
282
283 static void
284 iterate_bitmaps(const map_ctx *ctx, bool (^callback)(const hibernate_bitmap_t *bank_bitmap))
285 {
286 hibernate_bitmap_t *bank_bitmap = &ctx->bitmap->bank_bitmap[0];
287 for (uint32_t bank = 0; bank < ctx->bitmap->bank_count; bank++) {
288 if (!callback(bank_bitmap)) {
289 return;
290 }
291 bank_bitmap = (hibernate_bitmap_t*)&bank_bitmap->bitmap[bank_bitmap->bitmapwords];
292 }
293 }
294
295 // during hibernation resume, we can't use the original kernel page table (because we don't know what it was), so we instead
296 // create a temporary page table to use during hibernation resume; since the original kernel page table was part of DRAM,
297 // it will be restored by the time we're done with hibernation resume, at which point we can jump through the reset vector
298 // to reload the original page table
299 void
pal_hib_resume_tramp(uint32_t headerPpnum)300 pal_hib_resume_tramp(uint32_t headerPpnum)
301 {
302 uint64_t header_phys = ptoa_64(headerPpnum);
303 IOHibernateImageHeader *header = (IOHibernateImageHeader *)header_phys;
304 IOHibernateHibSegInfo *seg_info = &header->hibSegInfo;
305 uint64_t hib_text_start = ptoa_64(header->restore1CodePhysPage);
306
307 __block map_ctx ctx = {};
308 uint64_t map_phys = header_phys
309 + (offsetof(IOHibernateImageHeader, fileExtentMap)
310 + header->fileExtentMapSize
311 + ptoa_32(header->restore1PageCount)
312 + header->previewSize);
313 ctx.bitmap = (hibernate_page_list_t *)map_phys;
314
315 // find the bank describing xnu's map
316 __block uint64_t phys_start = 0, phys_end = 0;
317 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
318 if ((bank_bitmap->first_page <= header->restore1CodePhysPage) &&
319 (bank_bitmap->last_page >= header->restore1CodePhysPage)) {
320 phys_start = ptoa_64(bank_bitmap->first_page);
321 phys_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
322 return false;
323 }
324 return true;
325 });
326
327 HIB_ASSERT(phys_start != 0);
328 HIB_ASSERT(phys_end != 0);
329
330 hib_bzero(&gHibTramp, sizeof(gHibTramp));
331
332 // During hibernation resume, we create temporary mappings that do not collide with where any of the kernel mappings were originally.
333 // Technically, non-collision isn't a requirement, but doing this means that if some code accidentally jumps to a VA in the original
334 // kernel map, it won't be present in our temporary map and we'll get an exception when jumping to an unmapped address.
335 // The base address of our temporary mappings is adjusted by a random amount as a "poor-man's ASLR". We don’t have a good source of random
336 // numbers in this context, so we just use some of the bits from one of imageHeaderHMMAC, which should be random enough.
337 uint16_t rand = (uint16_t)(((header->imageHeaderHMAC[0]) << 8) | header->imageHeaderHMAC[1]);
338 uint64_t mem_slide = gHibernateGlobals.kernelSlide - (phys_end - phys_start) * 4 - rand * 256 * PAGE_SIZE;
339
340 // make sure we don't clobber any of the pages we need for restore
341 hibernate_reserve_restore_pages(header_phys, header, ctx.bitmap);
342
343 // init nextFree
344 hibernate_page_list_grab(ctx.bitmap, &ctx.nextFree);
345
346 // map ttbr1 pages
347 ctx.page_table_base = allocate_page(&ctx);
348 gHibTramp.ttbr1 = ctx.page_table_base;
349
350 uint64_t first_seg_start = 0, last_seg_end = 0, hib_text_end = 0;
351 for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
352 uint64_t size = ptoa_64(seg_info->segments[i].pageCount);
353 if (size) {
354 uint64_t seg_start = ptoa_64(seg_info->segments[i].physPage);
355 uint64_t seg_end = seg_start + size;
356 uint32_t protection = seg_info->segments[i].protection;
357 if (protection != VM_PROT_NONE) {
358 // make sure the segment is in bounds
359 HIB_ASSERT(seg_start >= phys_start);
360 HIB_ASSERT(seg_end <= phys_end);
361
362 if (!first_seg_start) {
363 first_seg_start = seg_start;
364 }
365 if (last_seg_end) {
366 // map the "hole" as RW
367 map_range_start_end(&ctx, last_seg_end, seg_start, mem_slide, MAP_RW);
368 }
369 // map the segments described in machine_header at their original locations
370 bool executable = (protection & VM_PROT_EXECUTE);
371 bool writeable = (protection & VM_PROT_WRITE);
372 uint64_t map_flags = executable ? MAP_RX : writeable ? MAP_RW : MAP_RO;
373 map_range_start_end(&ctx, seg_start, seg_end, gHibernateGlobals.kernelSlide, map_flags);
374 last_seg_end = seg_end;
375 }
376 if (seg_info->segments[i].physPage == header->restore1CodePhysPage) {
377 // this is the hibtext segment, so remember where it ends
378 hib_text_end = seg_end;
379 }
380 }
381 }
382 // map the rest of kernel memory (the pages that come before and after our segments) as RW
383 map_range_start_end(&ctx, phys_start, first_seg_start, mem_slide, MAP_RW);
384 map_range_start_end(&ctx, last_seg_end, phys_end, mem_slide, MAP_RW);
385
386 // map all of the remaining banks that we didn't already deal with
387 iterate_bitmaps(&ctx, ^bool (const hibernate_bitmap_t *bank_bitmap) {
388 uint64_t bank_start = ptoa_64(bank_bitmap->first_page);
389 uint64_t bank_end = ptoa_64(bank_bitmap->last_page) + PAGE_SIZE;
390 if (bank_start == phys_start) {
391 // skip this bank since we already covered it above
392 } else {
393 // map the bank RW
394 map_range_start_end(&ctx, bank_start, bank_end, mem_slide, MAP_RW);
395 }
396 return true;
397 });
398
399 // map ttbr0 pages
400 ctx.page_table_base = allocate_page(&ctx);
401 gHibTramp.ttbr0 = ctx.page_table_base;
402
403 // map hib text P=V so that we can still execute at its physical address
404 map_range_start_end(&ctx, hib_text_start, hib_text_end, 0, MAP_RX);
405
406 // map the hib image P=V, RW
407 uint64_t image_start = trunc_page(header_phys);
408 uint64_t image_end = round_page(header_phys + header->image1Size);
409 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RW);
410
411 // map the handoff pages P=V, RO
412 image_start = ptoa_64(header->handoffPages);
413 image_end = image_start + ptoa_64(header->handoffPageCount);
414 map_range_start_end(&ctx, image_start, image_end, 0, MAP_RO);
415
416 // map some device register pages
417 if (gHibernateGlobals.dockChannelRegPhysBase) {
418 #define dockchannel_uart_base gHibernateGlobals.dockChannelRegPhysBase
419 vm_address_t dockChannelRegPhysBase =
420 (vm_address_t)rDOCKCHANNELS_DEV_WSTAT(dockchannel_uart_base, DOCKCHANNEL_UART_CHANNEL);
421 map_register_page(&ctx, dockChannelRegPhysBase);
422 }
423 map_register_page(&ctx, gHibernateGlobals.hibUartRegPhysBase);
424 map_register_page(&ctx, gHibernateGlobals.hmacRegBase);
425
426 gHibTramp.memSlide = mem_slide;
427 }
428