1 /*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41 #include <mach/mach_vm.h>
42
43 #include <kern/host.h>
44 #include <kern/kalloc.h>
45 #include <kern/thread.h>
46 #include <kern/ipc_kobject.h>
47
48 #include <ipc/ipc_port.h>
49 #include <ipc/ipc_space.h>
50
51 #include <vm/memory_object.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_fault.h>
54 #include <vm/vm_map.h>
55 #include <vm/vm_pageout.h>
56 #include <vm/vm_protos.h>
57 #include <vm/vm_dyld_pager.h>
58
59 #include <sys/kdebug_triage.h>
60 #include <mach-o/fixup-chains.h>
61 #if defined(HAS_APPLE_PAC)
62 #include <ptrauth.h>
63 #include <arm/misc_protos.h>
64 #endif /* defined(HAS_APPLE_PAC) */
65
66 /*
67 * DYLD page in linking pager.
68 *
69 * This external memory manager (EMM) applies dyld fixup to data
70 * pages, allowing the modified page to appear "clean".
71 *
72 * The modified pages will never be dirtied, so the memory manager doesn't
73 * need to handle page-out requests (from memory_object_data_return()). The
74 * pages are mapped copy-on-write, so that the originals stay clean.
75 */
76
77 /* forward declarations */
78 typedef struct dyld_pager *dyld_pager_t;
79 static void dyld_pager_reference(memory_object_t mem_obj);
80 static void dyld_pager_deallocate(memory_object_t mem_obj);
81 static void dyld_pager_deallocate_internal(dyld_pager_t pager, bool locked);
82 static kern_return_t dyld_pager_init(memory_object_t mem_obj,
83 memory_object_control_t control,
84 memory_object_cluster_size_t pg_size);
85 static kern_return_t dyld_pager_terminate(memory_object_t mem_obj);
86 static void dyld_pager_terminate_internal(dyld_pager_t pager);
87 static kern_return_t dyld_pager_data_request(memory_object_t mem_obj,
88 memory_object_offset_t offset,
89 memory_object_cluster_size_t length,
90 vm_prot_t protection_required,
91 memory_object_fault_info_t fault_info);
92 static kern_return_t dyld_pager_data_return(memory_object_t mem_obj,
93 memory_object_offset_t offset,
94 memory_object_cluster_size_t data_cnt,
95 memory_object_offset_t *resid_offset,
96 int *io_error,
97 boolean_t dirty,
98 boolean_t kernel_copy,
99 int upl_flags);
100 static kern_return_t dyld_pager_data_initialize(memory_object_t mem_obj,
101 memory_object_offset_t offset,
102 memory_object_cluster_size_t data_cnt);
103 static kern_return_t dyld_pager_map(memory_object_t mem_obj,
104 vm_prot_t prot);
105 static kern_return_t dyld_pager_last_unmap(memory_object_t mem_obj);
106 static boolean_t dyld_pager_backing_object(
107 memory_object_t mem_obj,
108 memory_object_offset_t mem_obj_offset,
109 vm_object_t *backing_object,
110 vm_object_offset_t *backing_offset);
111 static dyld_pager_t dyld_pager_lookup(memory_object_t mem_obj);
112
113 /*
114 * Vector of VM operations for this EMM.
115 * These routines are invoked by VM via the memory_object_*() interfaces.
116 */
117 const struct memory_object_pager_ops dyld_pager_ops = {
118 .memory_object_reference = dyld_pager_reference,
119 .memory_object_deallocate = dyld_pager_deallocate,
120 .memory_object_init = dyld_pager_init,
121 .memory_object_terminate = dyld_pager_terminate,
122 .memory_object_data_request = dyld_pager_data_request,
123 .memory_object_data_return = dyld_pager_data_return,
124 .memory_object_data_initialize = dyld_pager_data_initialize,
125 .memory_object_map = dyld_pager_map,
126 .memory_object_last_unmap = dyld_pager_last_unmap,
127 .memory_object_backing_object = dyld_pager_backing_object,
128 .memory_object_pager_name = "dyld"
129 };
130
131 /*
132 * The "dyld_pager" structure. We create one of these for each use of
133 * map_with_linking_np() that dyld uses.
134 */
135 struct dyld_pager {
136 struct memory_object dyld_header; /* mandatory generic header */
137
138 #if MEMORY_OBJECT_HAS_REFCOUNT
139 #define dyld_ref_count dyld_header.mo_ref
140 #else
141 os_ref_atomic_t dyld_ref_count; /* active uses */
142 #endif
143 queue_chain_t dyld_pager_queue; /* next & prev pagers */
144 bool dyld_is_mapped; /* has active mappings */
145 bool dyld_is_ready; /* is this pager ready? */
146 vm_object_t dyld_backing_object; /* VM object for shared cache */
147 void *dyld_link_info;
148 uint32_t dyld_link_info_size;
149 uint32_t dyld_num_range;
150 memory_object_offset_t dyld_file_offset[MWL_MAX_REGION_COUNT];
151 mach_vm_address_t dyld_address[MWL_MAX_REGION_COUNT];
152 mach_vm_size_t dyld_size[MWL_MAX_REGION_COUNT];
153 #if defined(HAS_APPLE_PAC)
154 uint64_t dyld_a_key;
155 #endif /* defined(HAS_APPLE_PAC) */
156 };
157
158 queue_head_t dyld_pager_queue = QUEUE_HEAD_INITIALIZER(dyld_pager_queue);
159
160 /*
161 * "dyld_pager_lock" for counters, ref counting, etc.
162 */
163 LCK_GRP_DECLARE(dyld_pager_lck_grp, "dyld_pager");
164 LCK_MTX_DECLARE(dyld_pager_lock, &dyld_pager_lck_grp);
165
166 /*
167 * Statistics & counters.
168 */
169 uint32_t dyld_pager_count = 0;
170 uint32_t dyld_pager_count_max = 0;
171
172 /*
173 * dyld_pager_dequeue()
174 *
175 * Removes a pager from the list of pagers.
176 *
177 * The caller must hold "dyld_pager".
178 */
179 static void
dyld_pager_dequeue(__unused dyld_pager_t pager)180 dyld_pager_dequeue(
181 __unused dyld_pager_t pager)
182 {
183 queue_remove(&dyld_pager_queue,
184 pager,
185 dyld_pager_t,
186 dyld_pager_queue);
187 pager->dyld_pager_queue.next = NULL;
188 pager->dyld_pager_queue.prev = NULL;
189 dyld_pager_count--;
190 }
191
192 /*
193 * dyld_pager_init()
194 *
195 * Initialize the memory object and makes it ready to be used and mapped.
196 */
197 static kern_return_t
dyld_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)198 dyld_pager_init(
199 memory_object_t mem_obj,
200 memory_object_control_t control,
201 __unused
202 memory_object_cluster_size_t pg_size)
203 {
204 dyld_pager_t pager;
205 kern_return_t kr;
206 memory_object_attr_info_data_t attributes;
207
208 if (control == MEMORY_OBJECT_CONTROL_NULL) {
209 printf("%s(): control NULL\n", __func__);
210 return KERN_INVALID_ARGUMENT;
211 }
212
213 pager = dyld_pager_lookup(mem_obj);
214
215 memory_object_control_reference(control);
216
217 pager->dyld_header.mo_control = control;
218
219 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
220 attributes.cluster_size = (1 << (PAGE_SHIFT));
221 attributes.may_cache_object = FALSE;
222 attributes.temporary = TRUE;
223
224 kr = memory_object_change_attributes(
225 control,
226 MEMORY_OBJECT_ATTRIBUTE_INFO,
227 (memory_object_info_t) &attributes,
228 MEMORY_OBJECT_ATTR_INFO_COUNT);
229 if (kr != KERN_SUCCESS) {
230 panic("dyld_pager_init: " "memory_object_change_attributes() failed");
231 }
232
233 return KERN_SUCCESS;
234 }
235
236 /*
237 * dyld_data_return()
238 *
239 * A page-out request from VM -- should never happen so panic.
240 */
241 static kern_return_t
dyld_pager_data_return(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt,__unused memory_object_offset_t * resid_offset,__unused int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,__unused int upl_flags)242 dyld_pager_data_return(
243 __unused memory_object_t mem_obj,
244 __unused memory_object_offset_t offset,
245 __unused memory_object_cluster_size_t data_cnt,
246 __unused memory_object_offset_t *resid_offset,
247 __unused int *io_error,
248 __unused boolean_t dirty,
249 __unused boolean_t kernel_copy,
250 __unused int upl_flags)
251 {
252 panic("dyld_pager_data_return: should never happen!");
253 return KERN_FAILURE;
254 }
255
256 static kern_return_t
dyld_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)257 dyld_pager_data_initialize(
258 __unused memory_object_t mem_obj,
259 __unused memory_object_offset_t offset,
260 __unused memory_object_cluster_size_t data_cnt)
261 {
262 panic("dyld_pager_data_initialize: should never happen");
263 return KERN_FAILURE;
264 }
265
266
267 /*
268 * Apply fixups to a page used by a 64 bit process.
269 */
270 static kern_return_t
fixupPage64(vm_offset_t contents,vm_offset_t end_contents,void * link_info,struct dyld_chained_starts_in_segment * segInfo,uint32_t pageIndex,bool offsetBased)271 fixupPage64(
272 vm_offset_t contents,
273 vm_offset_t end_contents,
274 void *link_info,
275 struct dyld_chained_starts_in_segment *segInfo,
276 uint32_t pageIndex,
277 bool offsetBased)
278 {
279 struct mwl_info_hdr *hdr = (struct mwl_info_hdr *)link_info;
280 uint64_t *bindsArray = (uint64_t *)((uintptr_t)hdr + hdr->mwli_binds_offset);
281 uint16_t firstStartOffset = segInfo->page_start[pageIndex];
282
283 /*
284 * Done if no fixups on the page
285 */
286 if (firstStartOffset == DYLD_CHAINED_PTR_START_NONE) {
287 return KERN_SUCCESS;
288 }
289
290 /*
291 * walk the chain
292 */
293 uint64_t *chain = (uint64_t *)(contents + firstStartOffset);
294 uint64_t targetAdjust = (offsetBased ? hdr->mwli_image_address : hdr->mwli_slide);
295 uint64_t delta = 0;
296 do {
297 if ((uintptr_t)chain < contents || (uintptr_t)chain + sizeof(*chain) > end_contents) {
298 printf("%s(): chain 0x%llx out of range 0x%llx..0x%llx", __func__,
299 (long long)chain, (long long)contents, (long long)end_contents);
300 return KERN_FAILURE;
301 }
302 uint64_t value = *chain;
303 bool isBind = (value & 0x8000000000000000ULL);
304 delta = (value >> 51) & 0xFFF;
305 if (isBind) {
306 uint32_t bindOrdinal = value & 0x00FFFFFF;
307 if (bindOrdinal >= hdr->mwli_binds_count) {
308 printf("%s out of range bind ordinal %u (max %u)\n", __func__,
309 bindOrdinal, hdr->mwli_binds_count);
310 return KERN_FAILURE;
311 }
312 uint32_t addend = (value >> 24) & 0xFF;
313 *chain = bindsArray[bindOrdinal] + addend;
314 } else {
315 /* is rebase */
316 uint64_t target = value & 0xFFFFFFFFFULL;
317 uint64_t high8 = (value >> 36) & 0xFF;
318 *chain = target + targetAdjust + (high8 << 56);
319 }
320 if (delta * 4 >= PAGE_SIZE) {
321 printf("%s(): delta offset > page size %lld\n", __func__, delta * 4);
322 return KERN_FAILURE;
323 }
324 chain = (uint64_t *)((uintptr_t)chain + (delta * 4)); // 4-byte stride
325 } while (delta != 0);
326 return KERN_SUCCESS;
327 }
328
329
330 /*
331 * Apply fixups within a page used by a 32 bit process.
332 */
333 static kern_return_t
fixupChain32(uint32_t * chain,vm_offset_t contents,vm_offset_t end_contents,void * link_info,struct dyld_chained_starts_in_segment * segInfo,uint32_t * bindsArray)334 fixupChain32(
335 uint32_t *chain,
336 vm_offset_t contents,
337 vm_offset_t end_contents,
338 void *link_info,
339 struct dyld_chained_starts_in_segment *segInfo,
340 uint32_t *bindsArray)
341 {
342 struct mwl_info_hdr *hdr = (struct mwl_info_hdr *)link_info;
343 uint32_t delta = 0;
344
345 do {
346 if ((uintptr_t)chain < contents || (uintptr_t)chain + sizeof(*chain) > end_contents) {
347 printf("%s(): chain 0x%llx out of range 0x%llx..0x%llx", __func__,
348 (long long)chain, (long long)contents, (long long)end_contents);
349 return KERN_FAILURE;
350 }
351 uint32_t value = *chain;
352 delta = (value >> 26) & 0x1F;
353 if (value & 0x80000000) {
354 // is bind
355 uint32_t bindOrdinal = value & 0x000FFFFF;
356 if (bindOrdinal >= hdr->mwli_binds_count) {
357 printf("%s(): out of range bind ordinal %u (max %u)",
358 __func__, bindOrdinal, hdr->mwli_binds_count);
359 return KERN_FAILURE;
360 }
361 uint32_t addend = (value >> 20) & 0x3F;
362 *chain = bindsArray[bindOrdinal] + addend;
363 } else {
364 // is rebase
365 uint32_t target = value & 0x03FFFFFF;
366 if (target > segInfo->max_valid_pointer) {
367 // handle non-pointers in chain
368 uint32_t bias = (0x04000000 + segInfo->max_valid_pointer) / 2;
369 *chain = target - bias;
370 } else {
371 *chain = target + (uint32_t)hdr->mwli_slide;
372 }
373 }
374 chain += delta;
375 } while (delta != 0);
376 return KERN_SUCCESS;
377 }
378
379
380 /*
381 * Apply fixups to a page used by a 32 bit process.
382 */
383 static kern_return_t
fixupPage32(vm_offset_t contents,vm_offset_t end_contents,void * link_info,uint32_t link_info_size,struct dyld_chained_starts_in_segment * segInfo,uint32_t pageIndex)384 fixupPage32(
385 vm_offset_t contents,
386 vm_offset_t end_contents,
387 void *link_info,
388 uint32_t link_info_size,
389 struct dyld_chained_starts_in_segment *segInfo,
390 uint32_t pageIndex)
391 {
392 struct mwl_info_hdr *hdr = (struct mwl_info_hdr *)link_info;
393 uint32_t *bindsArray = (uint32_t *)((uintptr_t)hdr + hdr->mwli_binds_offset);
394 uint16_t startOffset = segInfo->page_start[pageIndex];
395
396 /*
397 * done if no fixups
398 */
399 if (startOffset == DYLD_CHAINED_PTR_START_NONE) {
400 return KERN_SUCCESS;
401 }
402
403 if (startOffset & DYLD_CHAINED_PTR_START_MULTI) {
404 // some fixups in the page are too far apart, so page has multiple starts
405 uint32_t overflowIndex = startOffset & ~DYLD_CHAINED_PTR_START_MULTI;
406 bool chainEnd = false;
407 while (!chainEnd) {
408 /*
409 * range check against link_info, note +1 to include data we'll dereference
410 */
411 if ((uintptr_t)&segInfo->page_start[overflowIndex + 1] > (uintptr_t)link_info + link_info_size) {
412 printf("%s(): out of range segInfo->page_start[overflowIndex]", __func__);
413 return KERN_FAILURE;
414 }
415 chainEnd = (segInfo->page_start[overflowIndex] & DYLD_CHAINED_PTR_START_LAST);
416 startOffset = (segInfo->page_start[overflowIndex] & ~DYLD_CHAINED_PTR_START_LAST);
417 uint32_t *chain = (uint32_t *)(contents + startOffset);
418 fixupChain32(chain, contents, end_contents, link_info, segInfo, bindsArray);
419 ++overflowIndex;
420 }
421 } else {
422 uint32_t *chain = (uint32_t *)(contents + startOffset);
423 fixupChain32(chain, contents, end_contents, link_info, segInfo, bindsArray);
424 }
425 return KERN_SUCCESS;
426 }
427
428 #if defined(HAS_APPLE_PAC)
429 /*
430 * Sign a pointer needed for fixups.
431 */
432 static kern_return_t
signPointer(uint64_t unsignedAddr,void * loc,bool addrDiv,uint16_t diversity,ptrauth_key key,dyld_pager_t pager,uint64_t * signedAddr)433 signPointer(
434 uint64_t unsignedAddr,
435 void *loc,
436 bool addrDiv,
437 uint16_t diversity,
438 ptrauth_key key,
439 dyld_pager_t pager,
440 uint64_t *signedAddr)
441 {
442 // don't sign NULL
443 if (unsignedAddr == 0) {
444 *signedAddr = 0;
445 return KERN_SUCCESS;
446 }
447
448 uint64_t extendedDiscriminator = diversity;
449 if (addrDiv) {
450 extendedDiscriminator = __builtin_ptrauth_blend_discriminator(loc, extendedDiscriminator);
451 }
452
453 switch (key) {
454 case ptrauth_key_asia:
455 case ptrauth_key_asda:
456 if (pager->dyld_a_key == 0 || arm_user_jop_disabled()) {
457 *signedAddr = unsignedAddr;
458 } else {
459 *signedAddr = (uintptr_t)pmap_sign_user_ptr((void *)unsignedAddr, key, extendedDiscriminator, pager->dyld_a_key);
460 }
461 break;
462
463 default:
464 printf("%s(): Invalid ptr auth key %d\n", __func__, key);
465 return KERN_FAILURE;
466 }
467 return KERN_SUCCESS;
468 }
469
470 /*
471 * Apply fixups to a page used by a 64 bit process using pointer authentication.
472 */
473 static kern_return_t
fixupPageAuth64(uint64_t userVA,vm_offset_t contents,vm_offset_t end_contents,dyld_pager_t pager,struct dyld_chained_starts_in_segment * segInfo,uint32_t pageIndex,bool offsetBased)474 fixupPageAuth64(
475 uint64_t userVA,
476 vm_offset_t contents,
477 vm_offset_t end_contents,
478 dyld_pager_t pager,
479 struct dyld_chained_starts_in_segment *segInfo,
480 uint32_t pageIndex,
481 bool offsetBased)
482 {
483 void *link_info = pager->dyld_link_info;
484 uint32_t link_info_size = pager->dyld_link_info_size;
485 struct mwl_info_hdr *hdr = (struct mwl_info_hdr *)link_info;
486 uint64_t *bindsArray = (uint64_t*)((uintptr_t)link_info + hdr->mwli_binds_offset);
487
488 /*
489 * range check against link_info, note +1 to include data we'll dereference
490 */
491 if ((uintptr_t)&segInfo->page_start[pageIndex + 1] > (uintptr_t)link_info + link_info_size) {
492 printf("%s(): out of range segInfo->page_start[pageIndex]", __func__);
493 return KERN_FAILURE;
494 }
495 uint16_t firstStartOffset = segInfo->page_start[pageIndex];
496
497 /*
498 * All done if no fixups on the page
499 */
500 if (firstStartOffset == DYLD_CHAINED_PTR_START_NONE) {
501 return KERN_SUCCESS;
502 }
503
504 /*
505 * Walk the chain of offsets to fix up
506 */
507 uint64_t *chain = (uint64_t *)(contents + firstStartOffset);
508 uint64_t targetAdjust = (offsetBased ? hdr->mwli_image_address : hdr->mwli_slide);
509 uint64_t delta = 0;
510 do {
511 if ((uintptr_t)chain < contents || (uintptr_t)chain + sizeof(*chain) > end_contents) {
512 printf("%s(): chain 0x%llx out of range 0x%llx..0x%llx", __func__,
513 (long long)chain, (long long)contents, (long long)end_contents);
514 return KERN_FAILURE;
515 }
516 uint64_t value = *chain;
517 delta = (value >> 51) & 0x7FF;
518 bool isAuth = (value & 0x8000000000000000ULL);
519 bool isBind = (value & 0x4000000000000000ULL);
520 if (isAuth) {
521 ptrauth_key key = (ptrauth_key)((value >> 49) & 0x3);
522 bool addrDiv = ((value & (1ULL << 48)) != 0);
523 uint16_t diversity = (uint16_t)((value >> 32) & 0xFFFF);
524 uintptr_t uVA = userVA + ((uintptr_t)chain - contents);
525 if (isBind) {
526 uint32_t bindOrdinal = value & 0x00FFFFFF;
527 if (bindOrdinal >= hdr->mwli_binds_count) {
528 printf("%s(): out of range bind ordinal %u (max %u)",
529 __func__, bindOrdinal, hdr->mwli_binds_count);
530 return KERN_FAILURE;
531 }
532 if (signPointer(bindsArray[bindOrdinal], (void *)uVA, addrDiv, diversity, key, pager, chain) != KERN_SUCCESS) {
533 return KERN_FAILURE;
534 }
535 } else {
536 /* note: in auth rebases only have 32-bits, so target is always offset - never vmaddr */
537 uint64_t target = (value & 0xFFFFFFFF) + hdr->mwli_image_address;
538 if (signPointer(target, (void *)uVA, addrDiv, diversity, key, pager, chain) != KERN_SUCCESS) {
539 return KERN_FAILURE;
540 }
541 }
542 } else {
543 if (isBind) {
544 uint32_t bindOrdinal = value & 0x00FFFFFF;
545 if (bindOrdinal >= hdr->mwli_binds_count) {
546 printf("%s(): out of range bind ordinal %u (max %u)",
547 __func__, bindOrdinal, hdr->mwli_binds_count);
548 return KERN_FAILURE;
549 } else {
550 uint64_t addend19 = (value >> 32) & 0x0007FFFF;
551 if (addend19 & 0x40000) {
552 addend19 |= 0xFFFFFFFFFFFC0000ULL;
553 }
554 *chain = bindsArray[bindOrdinal] + addend19;
555 }
556 } else {
557 uint64_t target = (value & 0x7FFFFFFFFFFULL);
558 uint64_t high8 = (value << 13) & 0xFF00000000000000ULL;
559 *chain = target + targetAdjust + high8;
560 }
561 }
562 chain += delta;
563 } while (delta != 0);
564 return KERN_SUCCESS;
565 }
566 #endif /* defined(HAS_APPLE_PAC) */
567
568
569 /*
570 * Handle dyld fixups for a page.
571 */
572 static kern_return_t
fixup_page(vm_offset_t contents,vm_offset_t end_contents,uint64_t userVA,dyld_pager_t pager)573 fixup_page(
574 vm_offset_t contents,
575 vm_offset_t end_contents,
576 uint64_t userVA,
577 dyld_pager_t pager)
578 {
579 void *link_info = pager->dyld_link_info;
580 uint32_t link_info_size = pager->dyld_link_info_size;
581 struct mwl_info_hdr *hdr = (struct mwl_info_hdr *)link_info;
582 struct dyld_chained_starts_in_segment *segInfo = NULL;
583 uint32_t pageIndex = 0;
584 uint32_t segIndex;
585 struct dyld_chained_starts_in_image *startsInfo;
586 struct dyld_chained_starts_in_segment *seg;
587 uint64_t segStartAddress;
588 uint64_t segEndAddress;
589
590 /*
591 * Note this is a linear search done for every page we have to fix up.
592 * However, it should be quick as there should only be 2 or 4 segments:
593 * - data
594 * - data const
595 * - data auth (for arm64e)
596 * - data const auth (for arm64e)
597 */
598 startsInfo = (struct dyld_chained_starts_in_image *)((uintptr_t)hdr + hdr->mwli_chains_offset);
599 for (segIndex = 0; segIndex < startsInfo->seg_count; ++segIndex) {
600 seg = (struct dyld_chained_starts_in_segment *)
601 ((uintptr_t)startsInfo + startsInfo->seg_info_offset[segIndex]);
602
603 /*
604 * ensure we don't go out of bounds of the link_info
605 */
606 if ((uintptr_t)seg + sizeof(*seg) > (uintptr_t)link_info + link_info_size) {
607 printf("%s(): seg_info out of bounds\n", __func__);
608 return KERN_FAILURE;
609 }
610
611 segStartAddress = hdr->mwli_image_address + seg->segment_offset;
612 segEndAddress = segStartAddress + seg->page_count * seg->page_size;
613 if (segStartAddress <= userVA && userVA < segEndAddress) {
614 segInfo = seg;
615 pageIndex = (uint32_t)(userVA - segStartAddress) / PAGE_SIZE;
616
617 /* ensure seg->size fits in link_info_size */
618 if ((uintptr_t)seg + seg->size > (uintptr_t)link_info + link_info_size) {
619 printf("%s(): seg->size out of bounds\n", __func__);
620 return KERN_FAILURE;
621 }
622 if (seg->size < sizeof(struct dyld_chained_starts_in_segment)) {
623 printf("%s(): seg->size too small\n", __func__);
624 return KERN_FAILURE;
625 }
626 /* ensure page_count and pageIndex are valid too */
627 if ((uintptr_t)&seg->page_start[seg->page_count] > (uintptr_t)link_info + link_info_size) {
628 printf("%s(): seg->page_count out of bounds\n", __func__);
629 return KERN_FAILURE;
630 }
631 if (pageIndex >= seg->page_count) {
632 printf("%s(): seg->page_count too small\n", __func__);
633 return KERN_FAILURE;
634 }
635
636 break;
637 }
638 }
639
640 /*
641 * Question for Nick.. or can we make this OK and just return KERN_SUCCESS, nothing to do?
642 */
643 if (segInfo == NULL) {
644 printf("%s(): No segment for user VA 0x%llx\n", __func__, (long long)userVA);
645 return KERN_FAILURE;
646 }
647
648 /*
649 * Route to the appropriate fixup routine
650 */
651 switch (hdr->mwli_pointer_format) {
652 #if defined(HAS_APPLE_PAC)
653 case DYLD_CHAINED_PTR_ARM64E:
654 fixupPageAuth64(userVA, contents, end_contents, pager, segInfo, pageIndex, false);
655 break;
656 case DYLD_CHAINED_PTR_ARM64E_USERLAND:
657 case DYLD_CHAINED_PTR_ARM64E_USERLAND24:
658 fixupPageAuth64(userVA, contents, end_contents, pager, segInfo, pageIndex, true);
659 break;
660 #endif /* defined(HAS_APPLE_PAC) */
661 case DYLD_CHAINED_PTR_64:
662 fixupPage64(contents, end_contents, link_info, segInfo, pageIndex, false);
663 break;
664 case DYLD_CHAINED_PTR_64_OFFSET:
665 fixupPage64(contents, end_contents, link_info, segInfo, pageIndex, true);
666 break;
667 case DYLD_CHAINED_PTR_32:
668 fixupPage32(contents, end_contents, link_info, link_info_size, segInfo, pageIndex);
669 break;
670 default:
671 printf("%s(): unknown pointer_format %d\n", __func__, hdr->mwli_pointer_format);
672 return KERN_FAILURE;
673 }
674 return KERN_SUCCESS;
675 }
676
677 /*
678 * dyld_pager_data_request()
679 *
680 * Handles page-in requests from VM.
681 */
682 static kern_return_t
dyld_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t length,__unused vm_prot_t protection_required,memory_object_fault_info_t mo_fault_info)683 dyld_pager_data_request(
684 memory_object_t mem_obj,
685 memory_object_offset_t offset,
686 memory_object_cluster_size_t length,
687 __unused vm_prot_t protection_required,
688 memory_object_fault_info_t mo_fault_info)
689 {
690 dyld_pager_t pager;
691 memory_object_control_t mo_control;
692 upl_t upl = NULL;
693 int upl_flags;
694 upl_size_t upl_size;
695 upl_page_info_t *upl_pl = NULL;
696 unsigned int pl_count;
697 vm_object_t src_top_object = VM_OBJECT_NULL;
698 vm_object_t src_page_object = VM_OBJECT_NULL;
699 vm_object_t dst_object;
700 kern_return_t kr;
701 kern_return_t retval = KERN_SUCCESS;
702 vm_offset_t src_vaddr;
703 vm_offset_t dst_vaddr;
704 vm_offset_t cur_offset;
705 kern_return_t error_code;
706 vm_prot_t prot;
707 vm_page_t src_page, top_page;
708 int interruptible;
709 struct vm_object_fault_info fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info);
710 struct mwl_info_hdr *hdr;
711 uint32_t r;
712 uint64_t userVA;
713
714 fault_info.stealth = TRUE;
715 fault_info.io_sync = FALSE;
716 fault_info.mark_zf_absent = FALSE;
717 fault_info.batch_pmap_op = FALSE;
718 interruptible = fault_info.interruptible;
719
720 pager = dyld_pager_lookup(mem_obj);
721 assert(pager->dyld_is_ready);
722 assert(os_ref_get_count_raw(&pager->dyld_ref_count) > 1); /* pager is alive */
723 assert(pager->dyld_is_mapped); /* pager is mapped */
724 hdr = (struct mwl_info_hdr *)pager->dyld_link_info;
725
726 /*
727 * Gather in a UPL all the VM pages requested by VM.
728 */
729 mo_control = pager->dyld_header.mo_control;
730
731 upl_size = length;
732 upl_flags =
733 UPL_RET_ONLY_ABSENT |
734 UPL_SET_LITE |
735 UPL_NO_SYNC |
736 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */
737 UPL_SET_INTERNAL;
738 pl_count = 0;
739 kr = memory_object_upl_request(mo_control,
740 offset, upl_size,
741 &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
742 if (kr != KERN_SUCCESS) {
743 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_DYLD_PAGER, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_DYLD_PAGER_NO_UPL), 0 /* arg */);
744 retval = kr;
745 goto done;
746 }
747 dst_object = memory_object_control_to_vm_object(mo_control);
748 assert(dst_object != VM_OBJECT_NULL);
749
750 /*
751 * We'll map the original data in the kernel address space from the
752 * backing VM object, itself backed by the executable/library file via
753 * the vnode pager.
754 */
755 src_top_object = pager->dyld_backing_object;
756 assert(src_top_object != VM_OBJECT_NULL);
757 vm_object_reference(src_top_object); /* keep the source object alive */
758
759 /*
760 * Fill in the contents of the pages requested by VM.
761 */
762 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
763 pl_count = length / PAGE_SIZE;
764 for (cur_offset = 0;
765 retval == KERN_SUCCESS && cur_offset < length;
766 cur_offset += PAGE_SIZE) {
767 ppnum_t dst_pnum;
768
769 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
770 /* this page is not in the UPL: skip it */
771 continue;
772 }
773
774 /*
775 * Map the source page in the kernel's virtual address space.
776 * We already hold a reference on the src_top_object.
777 */
778 retry_src_fault:
779 vm_object_lock(src_top_object);
780 vm_object_paging_begin(src_top_object);
781 error_code = 0;
782 prot = VM_PROT_READ;
783 src_page = VM_PAGE_NULL;
784 kr = vm_fault_page(src_top_object,
785 offset + cur_offset,
786 VM_PROT_READ,
787 FALSE,
788 FALSE, /* src_page not looked up */
789 &prot,
790 &src_page,
791 &top_page,
792 NULL,
793 &error_code,
794 FALSE,
795 &fault_info);
796 switch (kr) {
797 case VM_FAULT_SUCCESS:
798 break;
799 case VM_FAULT_RETRY:
800 goto retry_src_fault;
801 case VM_FAULT_MEMORY_SHORTAGE:
802 if (vm_page_wait(interruptible)) {
803 goto retry_src_fault;
804 }
805 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_DYLD_PAGER, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_DYLD_PAGER_MEMORY_SHORTAGE), 0 /* arg */);
806 OS_FALLTHROUGH;
807 case VM_FAULT_INTERRUPTED:
808 retval = MACH_SEND_INTERRUPTED;
809 goto done;
810 case VM_FAULT_SUCCESS_NO_VM_PAGE:
811 /* success but no VM page: fail */
812 vm_object_paging_end(src_top_object);
813 vm_object_unlock(src_top_object);
814 OS_FALLTHROUGH;
815 case VM_FAULT_MEMORY_ERROR:
816 /* the page is not there ! */
817 if (error_code) {
818 retval = error_code;
819 } else {
820 retval = KERN_MEMORY_ERROR;
821 }
822 goto done;
823 default:
824 panic("dyld_pager_data_request: vm_fault_page() unexpected error 0x%x\n", kr);
825 }
826 assert(src_page != VM_PAGE_NULL);
827 assert(src_page->vmp_busy);
828
829 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
830 vm_page_lockspin_queues();
831 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
832 vm_page_speculate(src_page, FALSE);
833 }
834 vm_page_unlock_queues();
835 }
836
837 /*
838 * Establish pointers to the source and destination physical pages.
839 */
840 dst_pnum = (ppnum_t)upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
841 assert(dst_pnum != 0);
842
843 src_vaddr = (vm_map_offset_t)phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) << PAGE_SHIFT);
844 dst_vaddr = (vm_map_offset_t)phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
845 src_page_object = VM_PAGE_OBJECT(src_page);
846
847 /*
848 * Validate the original page...
849 */
850 if (src_page_object->code_signed) {
851 vm_page_validate_cs_mapped(src_page, PAGE_SIZE, 0, (const void *)src_vaddr);
852 }
853
854 /*
855 * ... and transfer the results to the destination page.
856 */
857 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, src_page->vmp_cs_validated);
858 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, src_page->vmp_cs_tainted);
859 UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE, src_page->vmp_cs_nx);
860
861 /*
862 * The page provider might access a mapped file, so let's
863 * release the object lock for the source page to avoid a
864 * potential deadlock.
865 * The source page is kept busy and we have a
866 * "paging_in_progress" reference on its object, so it's safe
867 * to unlock the object here.
868 */
869 assert(src_page->vmp_busy);
870 assert(src_page_object->paging_in_progress > 0);
871 vm_object_unlock(src_page_object);
872
873 /*
874 * Process the original contents of the source page
875 * into the destination page.
876 */
877 bcopy((const char *)src_vaddr, (char *)dst_vaddr, PAGE_SIZE);
878
879 /*
880 * Figure out what the original user virtual address was, based on the offset.
881 */
882 userVA = 0;
883 for (r = 0; r < pager->dyld_num_range; ++r) {
884 vm_offset_t o = offset + cur_offset;
885 if (pager->dyld_file_offset[r] <= o &&
886 o < pager->dyld_file_offset[r] + pager->dyld_size[r]) {
887 userVA = pager->dyld_address[r] + (o - pager->dyld_file_offset[r]);
888 break;
889 }
890 }
891
892 /*
893 * If we have a valid range fixup the page.
894 */
895 if (r == pager->dyld_num_range) {
896 printf("%s(): Range not found for offset 0x%llx\n", __func__, (long long)cur_offset);
897 retval = KERN_FAILURE;
898 } else if (fixup_page(dst_vaddr, dst_vaddr + PAGE_SIZE, userVA, pager) != KERN_SUCCESS) {
899 /* printf was done under fixup_page() */
900 retval = KERN_FAILURE;
901 }
902 if (retval != KERN_SUCCESS) {
903 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_DYLD_PAGER, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_DYLD_PAGER_SLIDE_ERROR), 0 /* arg */);
904 }
905
906 assert(VM_PAGE_OBJECT(src_page) == src_page_object);
907 assert(src_page->vmp_busy);
908 assert(src_page_object->paging_in_progress > 0);
909 vm_object_lock(src_page_object);
910
911 /*
912 * Cleanup the result of vm_fault_page() of the source page.
913 */
914 PAGE_WAKEUP_DONE(src_page);
915 src_page = VM_PAGE_NULL;
916 vm_object_paging_end(src_page_object);
917 vm_object_unlock(src_page_object);
918
919 if (top_page != VM_PAGE_NULL) {
920 assert(VM_PAGE_OBJECT(top_page) == src_top_object);
921 vm_object_lock(src_top_object);
922 VM_PAGE_FREE(top_page);
923 vm_object_paging_end(src_top_object);
924 vm_object_unlock(src_top_object);
925 }
926 }
927
928 done:
929 if (upl != NULL) {
930 /* clean up the UPL */
931
932 /*
933 * The pages are currently dirty because we've just been
934 * writing on them, but as far as we're concerned, they're
935 * clean since they contain their "original" contents as
936 * provided by us, the pager.
937 * Tell the UPL to mark them "clean".
938 */
939 upl_clear_dirty(upl, TRUE);
940
941 /* abort or commit the UPL */
942 if (retval != KERN_SUCCESS) {
943 upl_abort(upl, 0);
944 } else {
945 boolean_t empty;
946 assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size),
947 "upl %p offset 0x%llx size 0x%x\n",
948 upl, upl->u_offset, upl->u_size);
949 upl_commit_range(upl, 0, upl->u_size,
950 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
951 upl_pl, pl_count, &empty);
952 }
953
954 /* and deallocate the UPL */
955 upl_deallocate(upl);
956 upl = NULL;
957 }
958 if (src_top_object != VM_OBJECT_NULL) {
959 vm_object_deallocate(src_top_object);
960 }
961 return retval;
962 }
963
964 /*
965 * dyld_pager_reference()
966 *
967 * Get a reference on this memory object.
968 * For external usage only. Assumes that the initial reference count is not 0,
969 * i.e one should not "revive" a dead pager this way.
970 */
971 static void
dyld_pager_reference(memory_object_t mem_obj)972 dyld_pager_reference(
973 memory_object_t mem_obj)
974 {
975 dyld_pager_t pager;
976
977 pager = dyld_pager_lookup(mem_obj);
978
979 lck_mtx_lock(&dyld_pager_lock);
980 os_ref_retain_locked_raw(&pager->dyld_ref_count, NULL);
981 lck_mtx_unlock(&dyld_pager_lock);
982 }
983
984
985
986 /*
987 * dyld_pager_terminate_internal:
988 *
989 * Trigger the asynchronous termination of the memory object associated
990 * with this pager.
991 * When the memory object is terminated, there will be one more call
992 * to memory_object_deallocate() (i.e. dyld_pager_deallocate())
993 * to finish the clean up.
994 *
995 * "dyld_pager_lock" should not be held by the caller.
996 */
997 static void
dyld_pager_terminate_internal(dyld_pager_t pager)998 dyld_pager_terminate_internal(
999 dyld_pager_t pager)
1000 {
1001 assert(pager->dyld_is_ready);
1002 assert(!pager->dyld_is_mapped);
1003 assert(os_ref_get_count_raw(&pager->dyld_ref_count) == 1);
1004
1005 if (pager->dyld_backing_object != VM_OBJECT_NULL) {
1006 vm_object_deallocate(pager->dyld_backing_object);
1007 pager->dyld_backing_object = VM_OBJECT_NULL;
1008 }
1009 /* trigger the destruction of the memory object */
1010 memory_object_destroy(pager->dyld_header.mo_control, 0);
1011 }
1012
1013 /*
1014 * dyld_pager_deallocate_internal()
1015 *
1016 * Release a reference on this pager and free it when the last reference goes away.
1017 * Can be called with dyld_pager_lock held or not, but always returns
1018 * with it unlocked.
1019 */
1020 static void
dyld_pager_deallocate_internal(dyld_pager_t pager,bool locked)1021 dyld_pager_deallocate_internal(
1022 dyld_pager_t pager,
1023 bool locked)
1024 {
1025 os_ref_count_t ref_count;
1026
1027 if (!locked) {
1028 lck_mtx_lock(&dyld_pager_lock);
1029 }
1030
1031 /* drop a reference on this pager */
1032 ref_count = os_ref_release_locked_raw(&pager->dyld_ref_count, NULL);
1033
1034 if (ref_count == 1) {
1035 /*
1036 * Only this reference is left, which means that
1037 * no one is really holding on to this pager anymore.
1038 * Terminate it.
1039 */
1040 dyld_pager_dequeue(pager);
1041 /* the pager is all ours: no need for the lock now */
1042 lck_mtx_unlock(&dyld_pager_lock);
1043 dyld_pager_terminate_internal(pager);
1044 } else if (ref_count == 0) {
1045 /*
1046 * Dropped all references; the memory object has
1047 * been terminated. Do some final cleanup and release the
1048 * pager structure.
1049 */
1050 lck_mtx_unlock(&dyld_pager_lock);
1051
1052 kfree_data(pager->dyld_link_info, pager->dyld_link_info_size);
1053 pager->dyld_link_info = NULL;
1054
1055 if (pager->dyld_header.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
1056 memory_object_control_deallocate(pager->dyld_header.mo_control);
1057 pager->dyld_header.mo_control = MEMORY_OBJECT_CONTROL_NULL;
1058 }
1059 kfree_type(struct dyld_pager, pager);
1060 pager = NULL;
1061 } else {
1062 /* there are still plenty of references: keep going... */
1063 lck_mtx_unlock(&dyld_pager_lock);
1064 }
1065
1066 /* caution: lock is not held on return... */
1067 }
1068
1069 /*
1070 * dyld_pager_deallocate()
1071 *
1072 * Release a reference on this pager and free it when the last
1073 * reference goes away.
1074 */
1075 static void
dyld_pager_deallocate(memory_object_t mem_obj)1076 dyld_pager_deallocate(
1077 memory_object_t mem_obj)
1078 {
1079 dyld_pager_t pager;
1080
1081 pager = dyld_pager_lookup(mem_obj);
1082 dyld_pager_deallocate_internal(pager, FALSE);
1083 }
1084
1085 /*
1086 *
1087 */
1088 static kern_return_t
dyld_pager_terminate(__unused memory_object_t mem_obj)1089 dyld_pager_terminate(
1090 #if !DEBUG
1091 __unused
1092 #endif
1093 memory_object_t mem_obj)
1094 {
1095 return KERN_SUCCESS;
1096 }
1097
1098 /*
1099 * dyld_pager_map()
1100 *
1101 * This allows VM to let us, the EMM, know that this memory object
1102 * is currently mapped one or more times. This is called by VM each time
1103 * the memory object gets mapped, but we only take one extra reference the
1104 * first time it is called.
1105 */
1106 static kern_return_t
dyld_pager_map(memory_object_t mem_obj,__unused vm_prot_t prot)1107 dyld_pager_map(
1108 memory_object_t mem_obj,
1109 __unused vm_prot_t prot)
1110 {
1111 dyld_pager_t pager;
1112
1113 pager = dyld_pager_lookup(mem_obj);
1114
1115 lck_mtx_lock(&dyld_pager_lock);
1116 assert(pager->dyld_is_ready);
1117 assert(os_ref_get_count_raw(&pager->dyld_ref_count) > 0); /* pager is alive */
1118 if (!pager->dyld_is_mapped) {
1119 pager->dyld_is_mapped = TRUE;
1120 os_ref_retain_locked_raw(&pager->dyld_ref_count, NULL);
1121 }
1122 lck_mtx_unlock(&dyld_pager_lock);
1123
1124 return KERN_SUCCESS;
1125 }
1126
1127 /*
1128 * dyld_pager_last_unmap()
1129 *
1130 * This is called by VM when this memory object is no longer mapped anywhere.
1131 */
1132 static kern_return_t
dyld_pager_last_unmap(memory_object_t mem_obj)1133 dyld_pager_last_unmap(
1134 memory_object_t mem_obj)
1135 {
1136 dyld_pager_t pager;
1137
1138 pager = dyld_pager_lookup(mem_obj);
1139
1140 lck_mtx_lock(&dyld_pager_lock);
1141 if (pager->dyld_is_mapped) {
1142 /*
1143 * All the mappings are gone, so let go of the one extra
1144 * reference that represents all the mappings of this pager.
1145 */
1146 pager->dyld_is_mapped = FALSE;
1147 dyld_pager_deallocate_internal(pager, TRUE);
1148 /* caution: deallocate_internal() released the lock ! */
1149 } else {
1150 lck_mtx_unlock(&dyld_pager_lock);
1151 }
1152
1153 return KERN_SUCCESS;
1154 }
1155
1156 static boolean_t
dyld_pager_backing_object(memory_object_t mem_obj,memory_object_offset_t offset,vm_object_t * backing_object,vm_object_offset_t * backing_offset)1157 dyld_pager_backing_object(
1158 memory_object_t mem_obj,
1159 memory_object_offset_t offset,
1160 vm_object_t *backing_object,
1161 vm_object_offset_t *backing_offset)
1162 {
1163 dyld_pager_t pager;
1164
1165 pager = dyld_pager_lookup(mem_obj);
1166
1167 *backing_object = pager->dyld_backing_object;
1168 *backing_offset = offset;
1169
1170 return TRUE;
1171 }
1172
1173
1174 /*
1175 * Convert from memory_object to dyld_pager.
1176 */
1177 static dyld_pager_t
dyld_pager_lookup(memory_object_t mem_obj)1178 dyld_pager_lookup(
1179 memory_object_t mem_obj)
1180 {
1181 dyld_pager_t pager;
1182
1183 assert(mem_obj->mo_pager_ops == &dyld_pager_ops);
1184 pager = (dyld_pager_t)(uintptr_t) mem_obj;
1185 assert(os_ref_get_count_raw(&pager->dyld_ref_count) > 0);
1186 return pager;
1187 }
1188
1189 /*
1190 * Create and return a pager for the given object with the
1191 * given slide information.
1192 */
1193 static dyld_pager_t
dyld_pager_create(__unused task_t task,vm_object_t backing_object,struct mwl_region * regions,uint32_t region_cnt,void * link_info,uint32_t link_info_size)1194 dyld_pager_create(
1195 #if !defined(HAS_APPLE_PAC)
1196 __unused
1197 #endif /* defined(HAS_APPLE_PAC) */
1198 task_t task,
1199 vm_object_t backing_object,
1200 struct mwl_region *regions,
1201 uint32_t region_cnt,
1202 void *link_info,
1203 uint32_t link_info_size)
1204 {
1205 dyld_pager_t pager;
1206 memory_object_control_t control;
1207 kern_return_t kr;
1208
1209 pager = kalloc_type(struct dyld_pager, Z_WAITOK);
1210 if (pager == NULL) {
1211 return NULL;
1212 }
1213
1214 /*
1215 * The vm_map call takes both named entry ports and raw memory
1216 * objects in the same parameter. We need to make sure that
1217 * vm_map does not see this object as a named entry port. So,
1218 * we reserve the first word in the object for a fake ip_kotype
1219 * setting - that will tell vm_map to use it as a memory object.
1220 */
1221 pager->dyld_header.mo_ikot = IKOT_MEMORY_OBJECT;
1222 pager->dyld_header.mo_pager_ops = &dyld_pager_ops;
1223 pager->dyld_header.mo_control = MEMORY_OBJECT_CONTROL_NULL;
1224
1225 pager->dyld_is_ready = FALSE;/* not ready until it has a "name" */
1226 /* existence reference for the caller */
1227 os_ref_init_count_raw(&pager->dyld_ref_count, NULL, 1);
1228 pager->dyld_is_mapped = FALSE;
1229 pager->dyld_backing_object = backing_object;
1230 pager->dyld_link_info = link_info;
1231 pager->dyld_link_info_size = link_info_size;
1232 #if defined(HAS_APPLE_PAC)
1233 pager->dyld_a_key = (task->map && task->map->pmap && !task->map->pmap->disable_jop) ? task->jop_pid : 0;
1234 #endif /* defined(HAS_APPLE_PAC) */
1235
1236 /*
1237 * Record the regions so the pager can find the offset from an address.
1238 */
1239 pager->dyld_num_range = region_cnt;
1240 for (uint32_t r = 0; r < region_cnt; ++r) {
1241 pager->dyld_file_offset[r] = regions[r].mwlr_file_offset;
1242 pager->dyld_address[r] = regions[r].mwlr_address;
1243 pager->dyld_size[r] = regions[r].mwlr_size;
1244 }
1245
1246 vm_object_reference(backing_object);
1247
1248 lck_mtx_lock(&dyld_pager_lock);
1249 queue_enter_first(&dyld_pager_queue,
1250 pager,
1251 dyld_pager_t,
1252 dyld_pager_queue);
1253 dyld_pager_count++;
1254 if (dyld_pager_count > dyld_pager_count_max) {
1255 dyld_pager_count_max = dyld_pager_count;
1256 }
1257 lck_mtx_unlock(&dyld_pager_lock);
1258
1259 kr = memory_object_create_named((memory_object_t) pager, 0, &control);
1260 assert(kr == KERN_SUCCESS);
1261
1262 memory_object_mark_trusted(control);
1263
1264 lck_mtx_lock(&dyld_pager_lock);
1265 /* the new pager is now ready to be used */
1266 pager->dyld_is_ready = TRUE;
1267 lck_mtx_unlock(&dyld_pager_lock);
1268
1269 /* wakeup anyone waiting for this pager to be ready */
1270 thread_wakeup(&pager->dyld_is_ready);
1271
1272 return pager;
1273 }
1274
1275 /*
1276 * dyld_pager_setup()
1277 *
1278 * Provide the caller with a memory object backed by the provided
1279 * "backing_object" VM object.
1280 */
1281 static memory_object_t
dyld_pager_setup(task_t task,vm_object_t backing_object,struct mwl_region * regions,uint32_t region_cnt,void * link_info,uint32_t link_info_size)1282 dyld_pager_setup(
1283 task_t task,
1284 vm_object_t backing_object,
1285 struct mwl_region *regions,
1286 uint32_t region_cnt,
1287 void *link_info,
1288 uint32_t link_info_size)
1289 {
1290 dyld_pager_t pager;
1291
1292 /* create new pager */
1293 pager = dyld_pager_create(task, backing_object, regions, region_cnt, link_info, link_info_size);
1294 if (pager == NULL) {
1295 /* could not create a new pager */
1296 return MEMORY_OBJECT_NULL;
1297 }
1298
1299 lck_mtx_lock(&dyld_pager_lock);
1300 while (!pager->dyld_is_ready) {
1301 lck_mtx_sleep(&dyld_pager_lock,
1302 LCK_SLEEP_DEFAULT,
1303 &pager->dyld_is_ready,
1304 THREAD_UNINT);
1305 }
1306 lck_mtx_unlock(&dyld_pager_lock);
1307
1308 return (memory_object_t) pager;
1309 }
1310
1311 /*
1312 * Set up regions which use a special pager to apply dyld fixups.
1313 *
1314 * The arguments to this function are mostly just used as input.
1315 * Except for the link_info! That is saved off in the pager that
1316 * gets created, so shouldn't be free'd by the caller, if KERN_SUCCES.
1317 */
1318 kern_return_t
vm_map_with_linking(task_t task,struct mwl_region * regions,uint32_t region_cnt,void * link_info,uint32_t link_info_size,memory_object_control_t file_control)1319 vm_map_with_linking(
1320 task_t task,
1321 struct mwl_region *regions,
1322 uint32_t region_cnt,
1323 void *link_info,
1324 uint32_t link_info_size,
1325 memory_object_control_t file_control)
1326 {
1327 vm_map_t map = task->map;
1328 vm_object_t object = VM_OBJECT_NULL;
1329 memory_object_t pager = MEMORY_OBJECT_NULL;
1330 uint32_t r;
1331 vm_map_address_t map_addr;
1332 kern_return_t kr = KERN_SUCCESS;
1333
1334 object = memory_object_control_to_vm_object(file_control);
1335 if (object == VM_OBJECT_NULL || object->internal) {
1336 printf("%s no object for file_control\n", __func__);
1337 object = VM_OBJECT_NULL;
1338 kr = KERN_INVALID_ADDRESS;
1339 goto done;
1340 }
1341
1342 /* create a pager */
1343 pager = dyld_pager_setup(task, object, regions, region_cnt, link_info, link_info_size);
1344 if (pager == MEMORY_OBJECT_NULL) {
1345 kr = KERN_RESOURCE_SHORTAGE;
1346 goto done;
1347 }
1348
1349 for (r = 0; r < region_cnt; ++r) {
1350 vm_map_kernel_flags_t vmk_flags = {
1351 .vmf_fixed = true,
1352 .vmf_overwrite = true,
1353 .vmkf_overwrite_immutable = true,
1354 };
1355 struct mwl_region *rp = ®ions[r];
1356
1357 /* map that pager over the portion of the mapping that needs sliding */
1358 map_addr = (vm_map_address_t)rp->mwlr_address;
1359 kr = vm_map_enter_mem_object(map,
1360 &map_addr,
1361 rp->mwlr_size,
1362 (mach_vm_offset_t) 0,
1363 vmk_flags,
1364 (ipc_port_t)(uintptr_t)pager,
1365 rp->mwlr_file_offset,
1366 TRUE, /* copy == TRUE, as this is MAP_PRIVATE so COW may happen */
1367 rp->mwlr_protections,
1368 rp->mwlr_protections,
1369 VM_INHERIT_DEFAULT);
1370 if (kr != KERN_SUCCESS) {
1371 /* no need to clean up earlier regions, this will be process fatal */
1372 goto done;
1373 }
1374 }
1375
1376 /* success! */
1377 kr = KERN_SUCCESS;
1378
1379 done:
1380
1381 if (pager != MEMORY_OBJECT_NULL) {
1382 /*
1383 * Release the pager reference obtained by dyld_pager_setup().
1384 * The mapping, if it succeeded, is now holding a reference on the memory object.
1385 */
1386 memory_object_deallocate(pager);
1387 pager = MEMORY_OBJECT_NULL;
1388 }
1389 return kr;
1390 }
1391
1392 static uint64_t
dyld_pager_purge(dyld_pager_t pager)1393 dyld_pager_purge(
1394 dyld_pager_t pager)
1395 {
1396 uint64_t pages_purged;
1397 vm_object_t object;
1398
1399 pages_purged = 0;
1400 object = memory_object_to_vm_object((memory_object_t) pager);
1401 assert(object != VM_OBJECT_NULL);
1402 vm_object_lock(object);
1403 pages_purged = object->resident_page_count;
1404 vm_object_reap_pages(object, REAP_DATA_FLUSH);
1405 pages_purged -= object->resident_page_count;
1406 // printf(" %s:%d pager %p object %p purged %llu left %d\n", __FUNCTION__, __LINE__, pager, object, pages_purged, object->resident_page_count);
1407 vm_object_unlock(object);
1408 return pages_purged;
1409 }
1410
1411 uint64_t
dyld_pager_purge_all(void)1412 dyld_pager_purge_all(void)
1413 {
1414 uint64_t pages_purged;
1415 dyld_pager_t pager;
1416
1417 pages_purged = 0;
1418 lck_mtx_lock(&dyld_pager_lock);
1419 queue_iterate(&dyld_pager_queue, pager, dyld_pager_t, dyld_pager_queue) {
1420 pages_purged += dyld_pager_purge(pager);
1421 }
1422 lck_mtx_unlock(&dyld_pager_lock);
1423 #if DEVELOPMENT || DEBUG
1424 printf(" %s:%d pages purged: %llu\n", __FUNCTION__, __LINE__, pages_purged);
1425 #endif /* DEVELOPMENT || DEBUG */
1426 return pages_purged;
1427 }
1428