1 /*
2 * Copyright (c) 2018-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41 #include <mach/mach_vm.h>
42
43 #include <kern/host.h>
44 #include <kern/kalloc.h>
45 #include <kern/queue.h>
46 #include <kern/thread.h>
47 #include <kern/ipc_kobject.h>
48
49 #include <ipc/ipc_port.h>
50 #include <ipc/ipc_space.h>
51
52 #include <vm/memory_object.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_fault.h>
55 #include <vm/vm_map.h>
56 #include <vm/vm_pageout.h>
57 #include <vm/vm_protos.h>
58 #include <vm/vm_shared_region.h>
59
60 #include <sys/kdebug_triage.h>
61
62 #if __has_feature(ptrauth_calls)
63 #include <ptrauth.h>
64 extern boolean_t diversify_user_jop;
65 #endif /* __has_feature(ptrauth_calls) */
66
67 /*
68 * SHARED REGION MEMORY PAGER
69 *
70 * This external memory manager (EMM) handles mappings of a dyld shared cache
71 * in shared regions, applying any necessary modifications (sliding,
72 * pointer signing, ...).
73 *
74 * It mostly handles page-in requests (from memory_object_data_request()) by
75 * getting the original data from its backing VM object, itself backed by
76 * the dyld shared cache file, modifying it if needed and providing it to VM.
77 *
78 * The modified pages will never be dirtied, so the memory manager doesn't
79 * need to handle page-out requests (from memory_object_data_return()). The
80 * pages need to be mapped copy-on-write, so that the originals stay clean.
81 *
82 * We don't expect to have to handle a large number of shared cache files,
83 * so the data structures are very simple (simple linked list) for now.
84 */
85
86 /* forward declarations */
87 void shared_region_pager_reference(memory_object_t mem_obj);
88 void shared_region_pager_deallocate(memory_object_t mem_obj);
89 kern_return_t shared_region_pager_init(memory_object_t mem_obj,
90 memory_object_control_t control,
91 memory_object_cluster_size_t pg_size);
92 kern_return_t shared_region_pager_terminate(memory_object_t mem_obj);
93 kern_return_t shared_region_pager_data_request(memory_object_t mem_obj,
94 memory_object_offset_t offset,
95 memory_object_cluster_size_t length,
96 vm_prot_t protection_required,
97 memory_object_fault_info_t fault_info);
98 kern_return_t shared_region_pager_data_return(memory_object_t mem_obj,
99 memory_object_offset_t offset,
100 memory_object_cluster_size_t data_cnt,
101 memory_object_offset_t *resid_offset,
102 int *io_error,
103 boolean_t dirty,
104 boolean_t kernel_copy,
105 int upl_flags);
106 kern_return_t shared_region_pager_data_initialize(memory_object_t mem_obj,
107 memory_object_offset_t offset,
108 memory_object_cluster_size_t data_cnt);
109 kern_return_t shared_region_pager_map(memory_object_t mem_obj,
110 vm_prot_t prot);
111 kern_return_t shared_region_pager_last_unmap(memory_object_t mem_obj);
112 boolean_t shared_region_pager_backing_object(
113 memory_object_t mem_obj,
114 memory_object_offset_t mem_obj_offset,
115 vm_object_t *backing_object,
116 vm_object_offset_t *backing_offset);
117
118 /*
119 * Vector of VM operations for this EMM.
120 * These routines are invoked by VM via the memory_object_*() interfaces.
121 */
122 const struct memory_object_pager_ops shared_region_pager_ops = {
123 .memory_object_reference = shared_region_pager_reference,
124 .memory_object_deallocate = shared_region_pager_deallocate,
125 .memory_object_init = shared_region_pager_init,
126 .memory_object_terminate = shared_region_pager_terminate,
127 .memory_object_data_request = shared_region_pager_data_request,
128 .memory_object_data_return = shared_region_pager_data_return,
129 .memory_object_data_initialize = shared_region_pager_data_initialize,
130 .memory_object_map = shared_region_pager_map,
131 .memory_object_last_unmap = shared_region_pager_last_unmap,
132 .memory_object_backing_object = shared_region_pager_backing_object,
133 .memory_object_pager_name = "shared_region"
134 };
135
136 #if __has_feature(ptrauth_calls)
137 /*
138 * Track mappings between shared_region_id and the key used to sign
139 * authenticated pointers.
140 */
141 typedef struct shared_region_jop_key_map {
142 queue_chain_t srk_queue;
143 char *srk_shared_region_id;
144 uint64_t srk_jop_key;
145 os_refcnt_t srk_ref_count; /* count of tasks active with this shared_region_id */
146 } *shared_region_jop_key_map_t;
147
148 os_refgrp_decl(static, srk_refgrp, "shared region key ref cnts", NULL);
149
150 /*
151 * The list is protected by the "shared_region_key_map" lock.
152 */
153 int shared_region_key_count = 0; /* number of active shared_region_id keys */
154 queue_head_t shared_region_jop_key_queue = QUEUE_HEAD_INITIALIZER(shared_region_jop_key_queue);
155 LCK_GRP_DECLARE(shared_region_jop_key_lck_grp, "shared_region_jop_key");
156 LCK_MTX_DECLARE(shared_region_jop_key_lock, &shared_region_jop_key_lck_grp);
157
158 /*
159 * Find the pointer signing key for the give shared_region_id.
160 */
161 uint64_t
shared_region_find_key(char * shared_region_id)162 shared_region_find_key(char *shared_region_id)
163 {
164 shared_region_jop_key_map_t region;
165 uint64_t key;
166
167 lck_mtx_lock(&shared_region_jop_key_lock);
168 queue_iterate(&shared_region_jop_key_queue, region, shared_region_jop_key_map_t, srk_queue) {
169 if (strcmp(region->srk_shared_region_id, shared_region_id) == 0) {
170 goto found;
171 }
172 }
173 panic("shared_region_find_key() no key for region '%s'", shared_region_id);
174
175 found:
176 key = region->srk_jop_key;
177 lck_mtx_unlock(&shared_region_jop_key_lock);
178 return key;
179 }
180
181 /*
182 * Return a authentication key to use for the given shared_region_id.
183 * If inherit is TRUE, then the key must match inherited_key.
184 * Creates an additional reference when successful.
185 */
186 void
shared_region_key_alloc(char * shared_region_id,bool inherit,uint64_t inherited_key)187 shared_region_key_alloc(char *shared_region_id, bool inherit, uint64_t inherited_key)
188 {
189 shared_region_jop_key_map_t region;
190 shared_region_jop_key_map_t new = NULL;
191
192 assert(shared_region_id != NULL);
193 again:
194 lck_mtx_lock(&shared_region_jop_key_lock);
195 queue_iterate(&shared_region_jop_key_queue, region, shared_region_jop_key_map_t, srk_queue) {
196 if (strcmp(region->srk_shared_region_id, shared_region_id) == 0) {
197 os_ref_retain_locked(®ion->srk_ref_count);
198 goto done;
199 }
200 }
201
202 /*
203 * ID was not found, if first time, allocate a new one and redo the lookup.
204 */
205 if (new == NULL) {
206 lck_mtx_unlock(&shared_region_jop_key_lock);
207 new = kalloc_type(struct shared_region_jop_key_map, Z_WAITOK);
208 uint_t len = strlen(shared_region_id) + 1;
209 new->srk_shared_region_id = kalloc_data(len, Z_WAITOK);
210 strlcpy(new->srk_shared_region_id, shared_region_id, len);
211 os_ref_init(&new->srk_ref_count, &srk_refgrp);
212
213 if (diversify_user_jop && inherit) {
214 new->srk_jop_key = inherited_key;
215 } else if (diversify_user_jop && strlen(shared_region_id) > 0) {
216 new->srk_jop_key = generate_jop_key();
217 } else {
218 new->srk_jop_key = ml_default_jop_pid();
219 }
220
221 goto again;
222 }
223
224 /*
225 * Use the newly allocated entry
226 */
227 ++shared_region_key_count;
228 queue_enter_first(&shared_region_jop_key_queue, new, shared_region_jop_key_map_t, srk_queue);
229 region = new;
230 new = NULL;
231
232 done:
233 if (inherit && inherited_key != region->srk_jop_key) {
234 panic("shared_region_key_alloc() inherited key mismatch");
235 }
236 lck_mtx_unlock(&shared_region_jop_key_lock);
237
238 /*
239 * free any unused new entry
240 */
241 if (new != NULL) {
242 kfree_data(new->srk_shared_region_id,
243 strlen(new->srk_shared_region_id) + 1);
244 kfree_type(struct shared_region_jop_key_map, new);
245 }
246 }
247
248 /*
249 * Mark the end of using a shared_region_id's key
250 */
251 extern void
shared_region_key_dealloc(char * shared_region_id)252 shared_region_key_dealloc(char *shared_region_id)
253 {
254 shared_region_jop_key_map_t region;
255
256 assert(shared_region_id != NULL);
257 lck_mtx_lock(&shared_region_jop_key_lock);
258 queue_iterate(&shared_region_jop_key_queue, region, shared_region_jop_key_map_t, srk_queue) {
259 if (strcmp(region->srk_shared_region_id, shared_region_id) == 0) {
260 goto done;
261 }
262 }
263 panic("shared_region_key_dealloc() Shared region ID '%s' not found", shared_region_id);
264
265 done:
266 if (os_ref_release_locked(®ion->srk_ref_count) == 0) {
267 queue_remove(&shared_region_jop_key_queue, region, shared_region_jop_key_map_t, srk_queue);
268 --shared_region_key_count;
269 } else {
270 region = NULL;
271 }
272 lck_mtx_unlock(&shared_region_jop_key_lock);
273
274 if (region != NULL) {
275 kfree_data(region->srk_shared_region_id,
276 strlen(region->srk_shared_region_id) + 1);
277 kfree_type(struct shared_region_jop_key_map, region);
278 }
279 }
280 #endif /* __has_feature(ptrauth_calls) */
281
282 /*
283 * The "shared_region_pager" describes a memory object backed by
284 * the "shared_region" EMM.
285 */
286 typedef struct shared_region_pager {
287 struct memory_object srp_header; /* mandatory generic header */
288
289 /* pager-specific data */
290 queue_chain_t srp_queue; /* next & prev pagers */
291 #if MEMORY_OBJECT_HAS_REFCOUNT
292 #define srp_ref_count srp_header.mo_ref
293 #else
294 os_ref_atomic_t srp_ref_count; /* active uses */
295 #endif
296 bool srp_is_mapped; /* has active mappings */
297 bool srp_is_ready; /* is this pager ready? */
298 vm_object_t srp_backing_object; /* VM object for shared cache */
299 vm_object_offset_t srp_backing_offset;
300 vm_shared_region_slide_info_t srp_slide_info;
301 #if __has_feature(ptrauth_calls)
302 uint64_t srp_jop_key; /* zero if used for arm64 */
303 #endif /* __has_feature(ptrauth_calls) */
304 } *shared_region_pager_t;
305 #define SHARED_REGION_PAGER_NULL ((shared_region_pager_t) NULL)
306
307 /*
308 * List of memory objects managed by this EMM.
309 * The list is protected by the "shared_region_pager_lock" lock.
310 */
311 int shared_region_pager_count = 0; /* number of pagers */
312 int shared_region_pager_count_mapped = 0; /* number of unmapped pagers */
313 queue_head_t shared_region_pager_queue = QUEUE_HEAD_INITIALIZER(shared_region_pager_queue);
314 LCK_GRP_DECLARE(shared_region_pager_lck_grp, "shared_region_pager");
315 LCK_MTX_DECLARE(shared_region_pager_lock, &shared_region_pager_lck_grp);
316
317 /*
318 * Maximum number of unmapped pagers we're willing to keep around.
319 */
320 int shared_region_pager_cache_limit = 0;
321
322 /*
323 * Statistics & counters.
324 */
325 int shared_region_pager_count_max = 0;
326 int shared_region_pager_count_unmapped_max = 0;
327 int shared_region_pager_num_trim_max = 0;
328 int shared_region_pager_num_trim_total = 0;
329
330 uint64_t shared_region_pager_copied = 0;
331 uint64_t shared_region_pager_slid = 0;
332 uint64_t shared_region_pager_slid_error = 0;
333 uint64_t shared_region_pager_reclaimed = 0;
334
335 /* internal prototypes */
336 shared_region_pager_t shared_region_pager_lookup(memory_object_t mem_obj);
337 void shared_region_pager_dequeue(shared_region_pager_t pager);
338 void shared_region_pager_deallocate_internal(shared_region_pager_t pager,
339 boolean_t locked);
340 void shared_region_pager_terminate_internal(shared_region_pager_t pager);
341 void shared_region_pager_trim(void);
342
343
344 #if DEBUG
345 int shared_region_pagerdebug = 0;
346 #define PAGER_ALL 0xffffffff
347 #define PAGER_INIT 0x00000001
348 #define PAGER_PAGEIN 0x00000002
349
350 #define PAGER_DEBUG(LEVEL, A) \
351 MACRO_BEGIN \
352 if ((shared_region_pagerdebug & (LEVEL)) == (LEVEL)) { \
353 printf A; \
354 } \
355 MACRO_END
356 #else
357 #define PAGER_DEBUG(LEVEL, A)
358 #endif
359
360 /*
361 * shared_region_pager_init()
362 *
363 * Initialize the memory object and makes it ready to be used and mapped.
364 */
365 kern_return_t
shared_region_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)366 shared_region_pager_init(
367 memory_object_t mem_obj,
368 memory_object_control_t control,
369 #if !DEBUG
370 __unused
371 #endif
372 memory_object_cluster_size_t pg_size)
373 {
374 shared_region_pager_t pager;
375 kern_return_t kr;
376 memory_object_attr_info_data_t attributes;
377
378 PAGER_DEBUG(PAGER_ALL,
379 ("shared_region_pager_init: %p, %p, %x\n",
380 mem_obj, control, pg_size));
381
382 if (control == MEMORY_OBJECT_CONTROL_NULL) {
383 return KERN_INVALID_ARGUMENT;
384 }
385
386 pager = shared_region_pager_lookup(mem_obj);
387
388 memory_object_control_reference(control);
389
390 pager->srp_header.mo_control = control;
391
392 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
393 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
394 attributes.cluster_size = (1 << (PAGE_SHIFT));
395 attributes.may_cache_object = FALSE;
396 attributes.temporary = TRUE;
397
398 kr = memory_object_change_attributes(
399 control,
400 MEMORY_OBJECT_ATTRIBUTE_INFO,
401 (memory_object_info_t) &attributes,
402 MEMORY_OBJECT_ATTR_INFO_COUNT);
403 if (kr != KERN_SUCCESS) {
404 panic("shared_region_pager_init: "
405 "memory_object_change_attributes() failed");
406 }
407
408 #if CONFIG_SECLUDED_MEMORY
409 if (secluded_for_filecache) {
410 #if 00
411 /*
412 * XXX FBDP do we want this in the secluded pool?
413 * Ideally, we'd want the shared region used by Camera to
414 * NOT be in the secluded pool, but all other shared regions
415 * in the secluded pool...
416 */
417 memory_object_mark_eligible_for_secluded(control, TRUE);
418 #endif /* 00 */
419 }
420 #endif /* CONFIG_SECLUDED_MEMORY */
421
422 return KERN_SUCCESS;
423 }
424
425 /*
426 * shared_region_data_return()
427 *
428 * Handles page-out requests from VM. This should never happen since
429 * the pages provided by this EMM are not supposed to be dirty or dirtied
430 * and VM should simply discard the contents and reclaim the pages if it
431 * needs to.
432 */
433 kern_return_t
shared_region_pager_data_return(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt,__unused memory_object_offset_t * resid_offset,__unused int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,__unused int upl_flags)434 shared_region_pager_data_return(
435 __unused memory_object_t mem_obj,
436 __unused memory_object_offset_t offset,
437 __unused memory_object_cluster_size_t data_cnt,
438 __unused memory_object_offset_t *resid_offset,
439 __unused int *io_error,
440 __unused boolean_t dirty,
441 __unused boolean_t kernel_copy,
442 __unused int upl_flags)
443 {
444 panic("shared_region_pager_data_return: should never get called");
445 return KERN_FAILURE;
446 }
447
448 kern_return_t
shared_region_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)449 shared_region_pager_data_initialize(
450 __unused memory_object_t mem_obj,
451 __unused memory_object_offset_t offset,
452 __unused memory_object_cluster_size_t data_cnt)
453 {
454 panic("shared_region_pager_data_initialize: should never get called");
455 return KERN_FAILURE;
456 }
457
458 /*
459 * shared_region_pager_data_request()
460 *
461 * Handles page-in requests from VM.
462 */
463 int shared_region_pager_data_request_debug = 0;
464 kern_return_t
shared_region_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t length,__unused vm_prot_t protection_required,memory_object_fault_info_t mo_fault_info)465 shared_region_pager_data_request(
466 memory_object_t mem_obj,
467 memory_object_offset_t offset,
468 memory_object_cluster_size_t length,
469 #if !DEBUG
470 __unused
471 #endif
472 vm_prot_t protection_required,
473 memory_object_fault_info_t mo_fault_info)
474 {
475 shared_region_pager_t pager;
476 memory_object_control_t mo_control;
477 upl_t upl;
478 int upl_flags;
479 upl_size_t upl_size;
480 upl_page_info_t *upl_pl;
481 unsigned int pl_count;
482 vm_object_t src_top_object, src_page_object, dst_object;
483 kern_return_t kr, retval;
484 vm_offset_t src_vaddr, dst_vaddr;
485 vm_offset_t cur_offset;
486 vm_offset_t offset_in_page;
487 kern_return_t error_code;
488 vm_prot_t prot;
489 vm_page_t src_page, top_page;
490 int interruptible;
491 struct vm_object_fault_info fault_info;
492 mach_vm_offset_t slide_start_address;
493
494 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
495
496 retval = KERN_SUCCESS;
497 src_top_object = VM_OBJECT_NULL;
498 src_page_object = VM_OBJECT_NULL;
499 upl = NULL;
500 upl_pl = NULL;
501 fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info);
502 fault_info.stealth = TRUE;
503 fault_info.io_sync = FALSE;
504 fault_info.mark_zf_absent = FALSE;
505 fault_info.batch_pmap_op = FALSE;
506 interruptible = fault_info.interruptible;
507
508 pager = shared_region_pager_lookup(mem_obj);
509 assert(pager->srp_is_ready);
510 assert(os_ref_get_count_raw(&pager->srp_ref_count) > 1); /* pager is alive */
511 assert(pager->srp_is_mapped); /* pager is mapped */
512
513 PAGER_DEBUG(PAGER_PAGEIN, ("shared_region_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
514
515 /*
516 * Gather in a UPL all the VM pages requested by VM.
517 */
518 mo_control = pager->srp_header.mo_control;
519
520 upl_size = length;
521 upl_flags =
522 UPL_RET_ONLY_ABSENT |
523 UPL_SET_LITE |
524 UPL_NO_SYNC |
525 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */
526 UPL_SET_INTERNAL;
527 pl_count = 0;
528 kr = memory_object_upl_request(mo_control,
529 offset, upl_size,
530 &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY);
531 if (kr != KERN_SUCCESS) {
532 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_SHARED_REGION, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_SHARED_REGION_NO_UPL), 0 /* arg */);
533 retval = kr;
534 goto done;
535 }
536 dst_object = memory_object_control_to_vm_object(mo_control);
537 assert(dst_object != VM_OBJECT_NULL);
538
539 /*
540 * We'll map the original data in the kernel address space from the
541 * backing VM object (itself backed by the shared cache file via
542 * the vnode pager).
543 */
544 src_top_object = pager->srp_backing_object;
545 assert(src_top_object != VM_OBJECT_NULL);
546 vm_object_reference(src_top_object); /* keep the source object alive */
547
548 slide_start_address = pager->srp_slide_info->si_slid_address;
549
550 fault_info.lo_offset += pager->srp_backing_offset;
551 fault_info.hi_offset += pager->srp_backing_offset;
552
553 /*
554 * Fill in the contents of the pages requested by VM.
555 */
556 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
557 pl_count = length / PAGE_SIZE;
558 for (cur_offset = 0;
559 retval == KERN_SUCCESS && cur_offset < length;
560 cur_offset += PAGE_SIZE) {
561 ppnum_t dst_pnum;
562
563 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
564 /* this page is not in the UPL: skip it */
565 continue;
566 }
567
568 /*
569 * Map the source (dyld shared cache) page in the kernel's
570 * virtual address space.
571 * We already hold a reference on the src_top_object.
572 */
573 retry_src_fault:
574 vm_object_lock(src_top_object);
575 vm_object_paging_begin(src_top_object);
576 error_code = 0;
577 prot = VM_PROT_READ;
578 src_page = VM_PAGE_NULL;
579 kr = vm_fault_page(src_top_object,
580 pager->srp_backing_offset + offset + cur_offset,
581 VM_PROT_READ,
582 FALSE,
583 FALSE, /* src_page not looked up */
584 &prot,
585 &src_page,
586 &top_page,
587 NULL,
588 &error_code,
589 FALSE,
590 &fault_info);
591 switch (kr) {
592 case VM_FAULT_SUCCESS:
593 break;
594 case VM_FAULT_RETRY:
595 goto retry_src_fault;
596 case VM_FAULT_MEMORY_SHORTAGE:
597 if (vm_page_wait(interruptible)) {
598 goto retry_src_fault;
599 }
600 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_SHARED_REGION, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_SHARED_REGION_PAGER_MEMORY_SHORTAGE), 0 /* arg */);
601 OS_FALLTHROUGH;
602 case VM_FAULT_INTERRUPTED:
603 retval = MACH_SEND_INTERRUPTED;
604 goto done;
605 case VM_FAULT_SUCCESS_NO_VM_PAGE:
606 /* success but no VM page: fail */
607 vm_object_paging_end(src_top_object);
608 vm_object_unlock(src_top_object);
609 OS_FALLTHROUGH;
610 case VM_FAULT_MEMORY_ERROR:
611 /* the page is not there ! */
612 if (error_code) {
613 retval = error_code;
614 } else {
615 retval = KERN_MEMORY_ERROR;
616 }
617 goto done;
618 default:
619 panic("shared_region_pager_data_request: "
620 "vm_fault_page() unexpected error 0x%x\n",
621 kr);
622 }
623 assert(src_page != VM_PAGE_NULL);
624 assert(src_page->vmp_busy);
625
626 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
627 vm_page_lockspin_queues();
628 if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
629 vm_page_speculate(src_page, FALSE);
630 }
631 vm_page_unlock_queues();
632 }
633
634 /*
635 * Establish pointers to the source
636 * and destination physical pages.
637 */
638 dst_pnum = (ppnum_t)
639 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
640 assert(dst_pnum != 0);
641
642 src_vaddr = (vm_map_offset_t)
643 phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page)
644 << PAGE_SHIFT);
645 dst_vaddr = (vm_map_offset_t)
646 phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT);
647 src_page_object = VM_PAGE_OBJECT(src_page);
648
649 /*
650 * Validate the original page...
651 */
652 if (src_page_object->code_signed) {
653 vm_page_validate_cs_mapped(
654 src_page, PAGE_SIZE, 0,
655 (const void *) src_vaddr);
656 }
657 /*
658 * ... and transfer the results to the destination page.
659 */
660 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
661 src_page->vmp_cs_validated);
662 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
663 src_page->vmp_cs_tainted);
664 UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
665 src_page->vmp_cs_nx);
666
667 /*
668 * The page provider might access a mapped file, so let's
669 * release the object lock for the source page to avoid a
670 * potential deadlock.
671 * The source page is kept busy and we have a
672 * "paging_in_progress" reference on its object, so it's safe
673 * to unlock the object here.
674 */
675 assert(src_page->vmp_busy);
676 assert(src_page_object->paging_in_progress > 0);
677 vm_object_unlock(src_page_object);
678
679 /*
680 * Process the original contents of the source page
681 * into the destination page.
682 */
683 for (offset_in_page = 0;
684 offset_in_page < PAGE_SIZE;
685 offset_in_page += PAGE_SIZE_FOR_SR_SLIDE) {
686 vm_object_offset_t chunk_offset;
687 vm_object_offset_t offset_in_backing_object;
688 vm_object_offset_t offset_in_sliding_range;
689
690 chunk_offset = offset + cur_offset + offset_in_page;
691
692 bcopy((const char *)(src_vaddr +
693 offset_in_page),
694 (char *)(dst_vaddr + offset_in_page),
695 PAGE_SIZE_FOR_SR_SLIDE);
696
697 offset_in_backing_object = (chunk_offset +
698 pager->srp_backing_offset);
699 if ((offset_in_backing_object < pager->srp_slide_info->si_start) ||
700 (offset_in_backing_object >= pager->srp_slide_info->si_end)) {
701 /* chunk is outside of sliding range: done */
702 shared_region_pager_copied++;
703 continue;
704 }
705
706 offset_in_sliding_range = offset_in_backing_object - pager->srp_slide_info->si_start;
707 kr = vm_shared_region_slide_page(pager->srp_slide_info,
708 dst_vaddr + offset_in_page,
709 (mach_vm_offset_t) (offset_in_sliding_range + slide_start_address),
710 (uint32_t) (offset_in_sliding_range / PAGE_SIZE_FOR_SR_SLIDE),
711 #if __has_feature(ptrauth_calls)
712 pager->srp_slide_info->si_ptrauth ? pager->srp_jop_key : 0
713 #else /* __has_feature(ptrauth_calls) */
714 0
715 #endif /* __has_feature(ptrauth_calls) */
716 );
717 if (shared_region_pager_data_request_debug) {
718 printf("shared_region_data_request"
719 "(%p,0x%llx+0x%llx+0x%04llx): 0x%llx "
720 "in sliding range [0x%llx:0x%llx]: "
721 "SLIDE offset 0x%llx="
722 "(0x%llx+0x%llx+0x%llx+0x%04llx)"
723 "[0x%016llx 0x%016llx] "
724 "code_signed=%d "
725 "cs_validated=%d "
726 "cs_tainted=%d "
727 "cs_nx=%d "
728 "kr=0x%x\n",
729 pager,
730 offset,
731 (uint64_t) cur_offset,
732 (uint64_t) offset_in_page,
733 chunk_offset,
734 pager->srp_slide_info->si_start,
735 pager->srp_slide_info->si_end,
736 (pager->srp_backing_offset +
737 offset +
738 cur_offset +
739 offset_in_page),
740 pager->srp_backing_offset,
741 offset,
742 (uint64_t) cur_offset,
743 (uint64_t) offset_in_page,
744 *(uint64_t *)(dst_vaddr + offset_in_page),
745 *(uint64_t *)(dst_vaddr + offset_in_page + 8),
746 src_page_object->code_signed,
747 src_page->vmp_cs_validated,
748 src_page->vmp_cs_tainted,
749 src_page->vmp_cs_nx,
750 kr);
751 }
752 if (kr != KERN_SUCCESS) {
753 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_SHARED_REGION, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_SHARED_REGION_SLIDE_ERROR), 0 /* arg */);
754 shared_region_pager_slid_error++;
755 retval = KERN_MEMORY_ERROR;
756 break;
757 }
758 shared_region_pager_slid++;
759 }
760
761 assert(VM_PAGE_OBJECT(src_page) == src_page_object);
762 assert(src_page->vmp_busy);
763 assert(src_page_object->paging_in_progress > 0);
764 vm_object_lock(src_page_object);
765
766 /*
767 * Cleanup the result of vm_fault_page() of the source page.
768 */
769 PAGE_WAKEUP_DONE(src_page);
770 src_page = VM_PAGE_NULL;
771 vm_object_paging_end(src_page_object);
772 vm_object_unlock(src_page_object);
773
774 if (top_page != VM_PAGE_NULL) {
775 assert(VM_PAGE_OBJECT(top_page) == src_top_object);
776 vm_object_lock(src_top_object);
777 VM_PAGE_FREE(top_page);
778 vm_object_paging_end(src_top_object);
779 vm_object_unlock(src_top_object);
780 }
781 }
782
783 done:
784 if (upl != NULL) {
785 /* clean up the UPL */
786
787 /*
788 * The pages are currently dirty because we've just been
789 * writing on them, but as far as we're concerned, they're
790 * clean since they contain their "original" contents as
791 * provided by us, the pager.
792 * Tell the UPL to mark them "clean".
793 */
794 upl_clear_dirty(upl, TRUE);
795
796 /* abort or commit the UPL */
797 if (retval != KERN_SUCCESS) {
798 upl_abort(upl, 0);
799 } else {
800 boolean_t empty;
801 assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size),
802 "upl %p offset 0x%llx size 0x%x\n",
803 upl, upl->u_offset, upl->u_size);
804 upl_commit_range(upl, 0, upl->u_size,
805 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
806 upl_pl, pl_count, &empty);
807 }
808
809 /* and deallocate the UPL */
810 upl_deallocate(upl);
811 upl = NULL;
812 }
813 if (src_top_object != VM_OBJECT_NULL) {
814 vm_object_deallocate(src_top_object);
815 }
816 return retval;
817 }
818
819 /*
820 * shared_region_pager_reference()
821 *
822 * Get a reference on this memory object.
823 * For external usage only. Assumes that the initial reference count is not 0,
824 * i.e one should not "revive" a dead pager this way.
825 */
826 void
shared_region_pager_reference(memory_object_t mem_obj)827 shared_region_pager_reference(
828 memory_object_t mem_obj)
829 {
830 shared_region_pager_t pager;
831
832 pager = shared_region_pager_lookup(mem_obj);
833
834 lck_mtx_lock(&shared_region_pager_lock);
835 os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
836 lck_mtx_unlock(&shared_region_pager_lock);
837 }
838
839
840 /*
841 * shared_region_pager_dequeue:
842 *
843 * Removes a pager from the list of pagers.
844 *
845 * The caller must hold "shared_region_pager_lock".
846 */
847 void
shared_region_pager_dequeue(shared_region_pager_t pager)848 shared_region_pager_dequeue(
849 shared_region_pager_t pager)
850 {
851 assert(!pager->srp_is_mapped);
852
853 queue_remove(&shared_region_pager_queue,
854 pager,
855 shared_region_pager_t,
856 srp_queue);
857 pager->srp_queue.next = NULL;
858 pager->srp_queue.prev = NULL;
859
860 shared_region_pager_count--;
861 }
862
863 /*
864 * shared_region_pager_terminate_internal:
865 *
866 * Trigger the asynchronous termination of the memory object associated
867 * with this pager.
868 * When the memory object is terminated, there will be one more call
869 * to memory_object_deallocate() (i.e. shared_region_pager_deallocate())
870 * to finish the clean up.
871 *
872 * "shared_region_pager_lock" should not be held by the caller.
873 * We don't need the lock because the pager has already been removed from
874 * the pagers' list and is now ours exclusively.
875 */
876 void
shared_region_pager_terminate_internal(shared_region_pager_t pager)877 shared_region_pager_terminate_internal(
878 shared_region_pager_t pager)
879 {
880 assert(pager->srp_is_ready);
881 assert(!pager->srp_is_mapped);
882 assert(os_ref_get_count_raw(&pager->srp_ref_count) == 1);
883
884 if (pager->srp_backing_object != VM_OBJECT_NULL) {
885 vm_object_deallocate(pager->srp_backing_object);
886 pager->srp_backing_object = VM_OBJECT_NULL;
887 }
888 /* trigger the destruction of the memory object */
889 memory_object_destroy(pager->srp_header.mo_control, 0);
890 }
891
892 /*
893 * shared_region_pager_deallocate_internal()
894 *
895 * Release a reference on this pager and free it when the last reference goes away.
896 * Can be called with shared_region_pager_lock held or not, but always returns
897 * with it unlocked.
898 */
899 void
shared_region_pager_deallocate_internal(shared_region_pager_t pager,boolean_t locked)900 shared_region_pager_deallocate_internal(
901 shared_region_pager_t pager,
902 boolean_t locked)
903 {
904 boolean_t needs_trimming;
905 int count_unmapped;
906 os_ref_count_t ref_count;
907
908 if (!locked) {
909 lck_mtx_lock(&shared_region_pager_lock);
910 }
911
912 /* if we have too many unmapped pagers, trim some */
913 count_unmapped = shared_region_pager_count - shared_region_pager_count_mapped;
914 needs_trimming = (count_unmapped > shared_region_pager_cache_limit);
915
916 /* drop a reference on this pager */
917 ref_count = os_ref_release_locked_raw(&pager->srp_ref_count, NULL);
918
919 if (ref_count == 1) {
920 /*
921 * Only the "named" reference is left, which means that
922 * no one is really holding on to this pager anymore.
923 * Terminate it.
924 */
925 shared_region_pager_dequeue(pager);
926 /* the pager is all ours: no need for the lock now */
927 lck_mtx_unlock(&shared_region_pager_lock);
928 shared_region_pager_terminate_internal(pager);
929 } else if (ref_count == 0) {
930 /*
931 * Dropped the existence reference; the memory object has
932 * been terminated. Do some final cleanup and release the
933 * pager structure.
934 */
935 lck_mtx_unlock(&shared_region_pager_lock);
936
937 vm_shared_region_slide_info_t si = pager->srp_slide_info;
938 #if __has_feature(ptrauth_calls)
939 /*
940 * The slide_info for auth sections lives in the shared region.
941 * Just deallocate() on the shared region and clear the field.
942 */
943 if (si != NULL) {
944 if (si->si_shared_region != NULL) {
945 assert(si->si_ptrauth);
946 vm_shared_region_deallocate(si->si_shared_region);
947 pager->srp_slide_info = NULL;
948 si = NULL;
949 }
950 }
951 #endif /* __has_feature(ptrauth_calls) */
952 if (si != NULL) {
953 vm_object_deallocate(si->si_slide_object);
954 /* free the slide_info_entry */
955 kfree_data(si->si_slide_info_entry,
956 si->si_slide_info_size);
957 kfree_type(struct vm_shared_region_slide_info, si);
958 pager->srp_slide_info = NULL;
959 }
960
961 if (pager->srp_header.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
962 memory_object_control_deallocate(pager->srp_header.mo_control);
963 pager->srp_header.mo_control = MEMORY_OBJECT_CONTROL_NULL;
964 }
965 kfree_type(struct shared_region_pager, pager);
966 pager = SHARED_REGION_PAGER_NULL;
967 } else {
968 /* there are still plenty of references: keep going... */
969 lck_mtx_unlock(&shared_region_pager_lock);
970 }
971
972 if (needs_trimming) {
973 shared_region_pager_trim();
974 }
975 /* caution: lock is not held on return... */
976 }
977
978 /*
979 * shared_region_pager_deallocate()
980 *
981 * Release a reference on this pager and free it when the last
982 * reference goes away.
983 */
984 void
shared_region_pager_deallocate(memory_object_t mem_obj)985 shared_region_pager_deallocate(
986 memory_object_t mem_obj)
987 {
988 shared_region_pager_t pager;
989
990 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_deallocate: %p\n", mem_obj));
991 pager = shared_region_pager_lookup(mem_obj);
992 shared_region_pager_deallocate_internal(pager, FALSE);
993 }
994
995 /*
996 *
997 */
998 kern_return_t
shared_region_pager_terminate(__unused memory_object_t mem_obj)999 shared_region_pager_terminate(
1000 #if !DEBUG
1001 __unused
1002 #endif
1003 memory_object_t mem_obj)
1004 {
1005 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_terminate: %p\n", mem_obj));
1006
1007 return KERN_SUCCESS;
1008 }
1009
1010 /*
1011 * shared_region_pager_map()
1012 *
1013 * This allows VM to let us, the EMM, know that this memory object
1014 * is currently mapped one or more times. This is called by VM each time
1015 * the memory object gets mapped, but we only take one extra reference the
1016 * first time it is called.
1017 */
1018 kern_return_t
shared_region_pager_map(memory_object_t mem_obj,__unused vm_prot_t prot)1019 shared_region_pager_map(
1020 memory_object_t mem_obj,
1021 __unused vm_prot_t prot)
1022 {
1023 shared_region_pager_t pager;
1024
1025 PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_map: %p\n", mem_obj));
1026
1027 pager = shared_region_pager_lookup(mem_obj);
1028
1029 lck_mtx_lock(&shared_region_pager_lock);
1030 assert(pager->srp_is_ready);
1031 assert(os_ref_get_count_raw(&pager->srp_ref_count) > 0); /* pager is alive */
1032 if (!pager->srp_is_mapped) {
1033 pager->srp_is_mapped = TRUE;
1034 os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
1035 shared_region_pager_count_mapped++;
1036 }
1037 lck_mtx_unlock(&shared_region_pager_lock);
1038
1039 return KERN_SUCCESS;
1040 }
1041
1042 /*
1043 * shared_region_pager_last_unmap()
1044 *
1045 * This is called by VM when this memory object is no longer mapped anywhere.
1046 */
1047 kern_return_t
shared_region_pager_last_unmap(memory_object_t mem_obj)1048 shared_region_pager_last_unmap(
1049 memory_object_t mem_obj)
1050 {
1051 shared_region_pager_t pager;
1052 int count_unmapped;
1053
1054 PAGER_DEBUG(PAGER_ALL,
1055 ("shared_region_pager_last_unmap: %p\n", mem_obj));
1056
1057 pager = shared_region_pager_lookup(mem_obj);
1058
1059 lck_mtx_lock(&shared_region_pager_lock);
1060 if (pager->srp_is_mapped) {
1061 /*
1062 * All the mappings are gone, so let go of the one extra
1063 * reference that represents all the mappings of this pager.
1064 */
1065 shared_region_pager_count_mapped--;
1066 count_unmapped = (shared_region_pager_count - shared_region_pager_count_mapped);
1067 if (count_unmapped > shared_region_pager_count_unmapped_max) {
1068 shared_region_pager_count_unmapped_max = count_unmapped;
1069 }
1070 pager->srp_is_mapped = FALSE;
1071 shared_region_pager_deallocate_internal(pager, TRUE);
1072 /* caution: deallocate_internal() released the lock ! */
1073 } else {
1074 lck_mtx_unlock(&shared_region_pager_lock);
1075 }
1076
1077 return KERN_SUCCESS;
1078 }
1079
1080 boolean_t
shared_region_pager_backing_object(memory_object_t mem_obj,memory_object_offset_t offset,vm_object_t * backing_object,vm_object_offset_t * backing_offset)1081 shared_region_pager_backing_object(
1082 memory_object_t mem_obj,
1083 memory_object_offset_t offset,
1084 vm_object_t *backing_object,
1085 vm_object_offset_t *backing_offset)
1086 {
1087 shared_region_pager_t pager;
1088
1089 PAGER_DEBUG(PAGER_ALL,
1090 ("shared_region_pager_backing_object: %p\n", mem_obj));
1091
1092 pager = shared_region_pager_lookup(mem_obj);
1093
1094 *backing_object = pager->srp_backing_object;
1095 *backing_offset = pager->srp_backing_offset + offset;
1096
1097 return TRUE;
1098 }
1099
1100
1101 /*
1102 *
1103 */
1104 shared_region_pager_t
shared_region_pager_lookup(memory_object_t mem_obj)1105 shared_region_pager_lookup(
1106 memory_object_t mem_obj)
1107 {
1108 shared_region_pager_t pager;
1109
1110 assert(mem_obj->mo_pager_ops == &shared_region_pager_ops);
1111 pager = (shared_region_pager_t)(uintptr_t) mem_obj;
1112 assert(os_ref_get_count_raw(&pager->srp_ref_count) > 0);
1113 return pager;
1114 }
1115
1116 /*
1117 * Create and return a pager for the given object with the
1118 * given slide information.
1119 */
1120 static shared_region_pager_t
shared_region_pager_create(vm_object_t backing_object,vm_object_offset_t backing_offset,struct vm_shared_region_slide_info * slide_info,__unused uint64_t jop_key)1121 shared_region_pager_create(
1122 vm_object_t backing_object,
1123 vm_object_offset_t backing_offset,
1124 struct vm_shared_region_slide_info *slide_info,
1125 #if !__has_feature(ptrauth_calls)
1126 __unused
1127 #endif /* !__has_feature(ptrauth_calls) */
1128 uint64_t jop_key)
1129 {
1130 shared_region_pager_t pager;
1131 memory_object_control_t control;
1132 kern_return_t kr;
1133 vm_object_t object;
1134
1135 pager = kalloc_type(struct shared_region_pager, Z_WAITOK);
1136 if (pager == SHARED_REGION_PAGER_NULL) {
1137 return SHARED_REGION_PAGER_NULL;
1138 }
1139
1140 /*
1141 * The vm_map call takes both named entry ports and raw memory
1142 * objects in the same parameter. We need to make sure that
1143 * vm_map does not see this object as a named entry port. So,
1144 * we reserve the first word in the object for a fake ip_kotype
1145 * setting - that will tell vm_map to use it as a memory object.
1146 */
1147 pager->srp_header.mo_ikot = IKOT_MEMORY_OBJECT;
1148 pager->srp_header.mo_pager_ops = &shared_region_pager_ops;
1149 pager->srp_header.mo_control = MEMORY_OBJECT_CONTROL_NULL;
1150
1151 pager->srp_is_ready = FALSE;/* not ready until it has a "name" */
1152 /* existence reference (for the cache) + 1 for the caller */
1153 os_ref_init_count_raw(&pager->srp_ref_count, NULL, 2);
1154 pager->srp_is_mapped = FALSE;
1155 pager->srp_backing_object = backing_object;
1156 pager->srp_backing_offset = backing_offset;
1157 pager->srp_slide_info = slide_info;
1158 #if __has_feature(ptrauth_calls)
1159 pager->srp_jop_key = jop_key;
1160 /*
1161 * If we're getting slide_info from the shared_region,
1162 * take a reference, so it can't disappear from under us.
1163 */
1164 if (slide_info->si_shared_region) {
1165 assert(slide_info->si_ptrauth);
1166 vm_shared_region_reference(slide_info->si_shared_region);
1167 }
1168 #endif /* __has_feature(ptrauth_calls) */
1169
1170 vm_object_reference(backing_object);
1171
1172 lck_mtx_lock(&shared_region_pager_lock);
1173 /* enter new pager at the head of our list of pagers */
1174 queue_enter_first(&shared_region_pager_queue,
1175 pager,
1176 shared_region_pager_t,
1177 srp_queue);
1178 shared_region_pager_count++;
1179 if (shared_region_pager_count > shared_region_pager_count_max) {
1180 shared_region_pager_count_max = shared_region_pager_count;
1181 }
1182 lck_mtx_unlock(&shared_region_pager_lock);
1183
1184 kr = memory_object_create_named((memory_object_t) pager,
1185 0,
1186 &control);
1187 assert(kr == KERN_SUCCESS);
1188
1189 memory_object_mark_trusted(control);
1190
1191 lck_mtx_lock(&shared_region_pager_lock);
1192 /* the new pager is now ready to be used */
1193 pager->srp_is_ready = TRUE;
1194 object = memory_object_to_vm_object((memory_object_t) pager);
1195 assert(object);
1196 /*
1197 * No one knows about this object and so we get away without the object lock.
1198 * This object is _eventually_ backed by the dyld shared cache and so we want
1199 * to benefit from the lock priority boosting.
1200 */
1201 object->object_is_shared_cache = TRUE;
1202 lck_mtx_unlock(&shared_region_pager_lock);
1203
1204 /* wakeup anyone waiting for this pager to be ready */
1205 thread_wakeup(&pager->srp_is_ready);
1206
1207 return pager;
1208 }
1209
1210 /*
1211 * shared_region_pager_setup()
1212 *
1213 * Provide the caller with a memory object backed by the provided
1214 * "backing_object" VM object.
1215 */
1216 memory_object_t
shared_region_pager_setup(vm_object_t backing_object,vm_object_offset_t backing_offset,struct vm_shared_region_slide_info * slide_info,uint64_t jop_key)1217 shared_region_pager_setup(
1218 vm_object_t backing_object,
1219 vm_object_offset_t backing_offset,
1220 struct vm_shared_region_slide_info *slide_info,
1221 uint64_t jop_key)
1222 {
1223 shared_region_pager_t pager;
1224
1225 /* create new pager */
1226 pager = shared_region_pager_create(backing_object,
1227 backing_offset, slide_info, jop_key);
1228 if (pager == SHARED_REGION_PAGER_NULL) {
1229 /* could not create a new pager */
1230 return MEMORY_OBJECT_NULL;
1231 }
1232
1233 lck_mtx_lock(&shared_region_pager_lock);
1234 while (!pager->srp_is_ready) {
1235 lck_mtx_sleep(&shared_region_pager_lock,
1236 LCK_SLEEP_DEFAULT,
1237 &pager->srp_is_ready,
1238 THREAD_UNINT);
1239 }
1240 lck_mtx_unlock(&shared_region_pager_lock);
1241
1242 return (memory_object_t) pager;
1243 }
1244
1245 #if __has_feature(ptrauth_calls)
1246 /*
1247 * shared_region_pager_match()
1248 *
1249 * Provide the caller with a memory object backed by the provided
1250 * "backing_object" VM object.
1251 */
1252 memory_object_t
shared_region_pager_match(vm_object_t backing_object,vm_object_offset_t backing_offset,vm_shared_region_slide_info_t slide_info,uint64_t jop_key)1253 shared_region_pager_match(
1254 vm_object_t backing_object,
1255 vm_object_offset_t backing_offset,
1256 vm_shared_region_slide_info_t slide_info,
1257 uint64_t jop_key)
1258 {
1259 shared_region_pager_t pager;
1260 vm_shared_region_slide_info_t si;
1261
1262 lck_mtx_lock(&shared_region_pager_lock);
1263 queue_iterate(&shared_region_pager_queue, pager, shared_region_pager_t, srp_queue) {
1264 if (pager->srp_backing_object != backing_object->vo_copy) {
1265 continue;
1266 }
1267 if (pager->srp_backing_offset != backing_offset) {
1268 continue;
1269 }
1270 si = pager->srp_slide_info;
1271
1272 /* If there's no AUTH section then it can't match (slide_info is always !NULL) */
1273 if (!si->si_ptrauth) {
1274 continue;
1275 }
1276 if (pager->srp_jop_key != jop_key) {
1277 continue;
1278 }
1279 if (si->si_slide != slide_info->si_slide) {
1280 continue;
1281 }
1282 if (si->si_start != slide_info->si_start) {
1283 continue;
1284 }
1285 if (si->si_end != slide_info->si_end) {
1286 continue;
1287 }
1288 if (si->si_slide_object != slide_info->si_slide_object) {
1289 continue;
1290 }
1291 if (si->si_slide_info_size != slide_info->si_slide_info_size) {
1292 continue;
1293 }
1294 if (memcmp(si->si_slide_info_entry, slide_info->si_slide_info_entry, si->si_slide_info_size) != 0) {
1295 continue;
1296 }
1297 /* the caller expects a reference on this */
1298 os_ref_retain_locked_raw(&pager->srp_ref_count, NULL);
1299 lck_mtx_unlock(&shared_region_pager_lock);
1300 return (memory_object_t)pager;
1301 }
1302
1303 /*
1304 * We didn't find a pre-existing pager, so create one.
1305 *
1306 * Note slight race condition here since we drop the lock. This could lead to more than one
1307 * thread calling setup with the same arguments here. That shouldn't break anything, just
1308 * waste a little memory.
1309 */
1310 lck_mtx_unlock(&shared_region_pager_lock);
1311 return shared_region_pager_setup(backing_object->vo_copy, backing_offset, slide_info, jop_key);
1312 }
1313
1314 void
shared_region_pager_match_task_key(memory_object_t memobj,__unused task_t task)1315 shared_region_pager_match_task_key(memory_object_t memobj, __unused task_t task)
1316 {
1317 __unused shared_region_pager_t pager = (shared_region_pager_t)memobj;
1318
1319 assert(pager->srp_jop_key == task->jop_pid);
1320 }
1321 #endif /* __has_feature(ptrauth_calls) */
1322
1323 void
shared_region_pager_trim(void)1324 shared_region_pager_trim(void)
1325 {
1326 shared_region_pager_t pager, prev_pager;
1327 queue_head_t trim_queue;
1328 int num_trim;
1329 int count_unmapped;
1330
1331 lck_mtx_lock(&shared_region_pager_lock);
1332
1333 /*
1334 * We have too many pagers, try and trim some unused ones,
1335 * starting with the oldest pager at the end of the queue.
1336 */
1337 queue_init(&trim_queue);
1338 num_trim = 0;
1339
1340 for (pager = (shared_region_pager_t)queue_last(&shared_region_pager_queue);
1341 !queue_end(&shared_region_pager_queue, (queue_entry_t) pager);
1342 pager = prev_pager) {
1343 /* get prev elt before we dequeue */
1344 prev_pager = (shared_region_pager_t)queue_prev(&pager->srp_queue);
1345
1346 if (os_ref_get_count_raw(&pager->srp_ref_count) == 2 &&
1347 pager->srp_is_ready &&
1348 !pager->srp_is_mapped) {
1349 /* this pager can be trimmed */
1350 num_trim++;
1351 /* remove this pager from the main list ... */
1352 shared_region_pager_dequeue(pager);
1353 /* ... and add it to our trim queue */
1354 queue_enter_first(&trim_queue,
1355 pager,
1356 shared_region_pager_t,
1357 srp_queue);
1358
1359 /* do we have enough pagers to trim? */
1360 count_unmapped = (shared_region_pager_count - shared_region_pager_count_mapped);
1361 if (count_unmapped <= shared_region_pager_cache_limit) {
1362 break;
1363 }
1364 }
1365 }
1366 if (num_trim > shared_region_pager_num_trim_max) {
1367 shared_region_pager_num_trim_max = num_trim;
1368 }
1369 shared_region_pager_num_trim_total += num_trim;
1370
1371 lck_mtx_unlock(&shared_region_pager_lock);
1372
1373 /* terminate the trimmed pagers */
1374 while (!queue_empty(&trim_queue)) {
1375 queue_remove_first(&trim_queue,
1376 pager,
1377 shared_region_pager_t,
1378 srp_queue);
1379 pager->srp_queue.next = NULL;
1380 pager->srp_queue.prev = NULL;
1381 assert(os_ref_get_count_raw(&pager->srp_ref_count) == 2);
1382 /*
1383 * We can't call deallocate_internal() because the pager
1384 * has already been dequeued, but we still need to remove
1385 * a reference.
1386 */
1387 (void)os_ref_release_locked_raw(&pager->srp_ref_count, NULL);
1388 shared_region_pager_terminate_internal(pager);
1389 }
1390 }
1391
1392 static uint64_t
shared_region_pager_purge(shared_region_pager_t pager)1393 shared_region_pager_purge(
1394 shared_region_pager_t pager)
1395 {
1396 uint64_t pages_purged;
1397 vm_object_t object;
1398
1399 pages_purged = 0;
1400 object = memory_object_to_vm_object((memory_object_t) pager);
1401 assert(object != VM_OBJECT_NULL);
1402 vm_object_lock(object);
1403 pages_purged = object->resident_page_count;
1404 vm_object_reap_pages(object, REAP_DATA_FLUSH);
1405 pages_purged -= object->resident_page_count;
1406 // printf(" %s:%d pager %p object %p purged %llu left %d\n", __FUNCTION__, __LINE__, pager, object, pages_purged, object->resident_page_count);
1407 vm_object_unlock(object);
1408 return pages_purged;
1409 }
1410
1411 uint64_t
shared_region_pager_purge_all(void)1412 shared_region_pager_purge_all(void)
1413 {
1414 uint64_t pages_purged;
1415 shared_region_pager_t pager;
1416
1417 pages_purged = 0;
1418 lck_mtx_lock(&shared_region_pager_lock);
1419 queue_iterate(&shared_region_pager_queue, pager, shared_region_pager_t, srp_queue) {
1420 pages_purged += shared_region_pager_purge(pager);
1421 }
1422 lck_mtx_unlock(&shared_region_pager_lock);
1423 #if DEVELOPMENT || DEBUG
1424 printf(" %s:%d pages purged: %llu\n", __FUNCTION__, __LINE__, pages_purged);
1425 #endif /* DEVELOPMENT || DEBUG */
1426 return pages_purged;
1427 }
1428