1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41
42 #include <kern/assert.h>
43 #include <kern/host.h>
44 #include <kern/ledger.h>
45 #include <kern/thread.h>
46 #include <kern/ipc_kobject.h>
47 #include <os/refcnt.h>
48
49 #include <vm/vm_map_internal.h>
50 #include <vm/vm_pageout_internal.h>
51 #include <vm/memory_object_internal.h>
52 #include <vm/vm_pageout.h>
53 #include <vm/vm_protos_internal.h>
54 #include <vm/vm_purgeable_internal.h>
55 #include <vm/vm_ubc.h>
56 #include <vm/vm_page_internal.h>
57 #include <vm/vm_object_internal.h>
58
59 #include <sys/kdebug_triage.h>
60
61 /* BSD VM COMPONENT INTERFACES */
62 int
get_map_nentries(vm_map_t map)63 get_map_nentries(
64 vm_map_t map)
65 {
66 return map->hdr.nentries;
67 }
68
69 /*
70 * BSD VNODE PAGER
71 */
72
73 const struct memory_object_pager_ops vnode_pager_ops = {
74 .memory_object_reference = vnode_pager_reference,
75 .memory_object_deallocate = vnode_pager_deallocate,
76 .memory_object_init = vnode_pager_init,
77 .memory_object_terminate = vnode_pager_terminate,
78 .memory_object_data_request = vnode_pager_data_request,
79 .memory_object_data_return = vnode_pager_data_return,
80 .memory_object_data_initialize = vnode_pager_data_initialize,
81 .memory_object_map = vnode_pager_map,
82 .memory_object_last_unmap = vnode_pager_last_unmap,
83 .memory_object_backing_object = NULL,
84 .memory_object_pager_name = "vnode pager"
85 };
86
87 typedef struct vnode_pager {
88 /* mandatory generic header */
89 struct memory_object vn_pgr_hdr;
90
91 /* pager-specific */
92 #if MEMORY_OBJECT_HAS_REFCOUNT
93 #define vn_pgr_hdr_ref vn_pgr_hdr.mo_ref
94 #else
95 os_ref_atomic_t vn_pgr_hdr_ref;
96 #endif
97 struct vnode *vnode_handle; /* vnode handle */
98 } *vnode_pager_t;
99
100
101 kern_return_t
102 vnode_pager_cluster_read( /* forward */
103 vnode_pager_t,
104 vm_object_offset_t,
105 vm_object_offset_t,
106 uint32_t,
107 vm_size_t);
108
109 void
110 vnode_pager_cluster_write( /* forward */
111 vnode_pager_t,
112 vm_object_offset_t,
113 vm_size_t,
114 vm_object_offset_t *,
115 int *,
116 int);
117
118
119 vnode_pager_t
120 vnode_object_create( /* forward */
121 struct vnode *);
122
123 vnode_pager_t
124 vnode_pager_lookup( /* forward */
125 memory_object_t);
126
127 struct vnode *
128 vnode_pager_lookup_vnode( /* forward */
129 memory_object_t);
130
131 ZONE_DEFINE_TYPE(vnode_pager_zone, "vnode pager structures",
132 struct vnode_pager, ZC_NOENCRYPT);
133
134 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
135
136 /* TODO: Should be set dynamically by vnode_pager_init() */
137 #define CLUSTER_SHIFT 1
138
139
140 #if DEBUG
141 int pagerdebug = 0;
142
143 #define PAGER_ALL 0xffffffff
144 #define PAGER_INIT 0x00000001
145 #define PAGER_PAGEIN 0x00000002
146
147 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
148 #else
149 #define PAGER_DEBUG(LEVEL, A)
150 #endif
151
152 extern int proc_resetpcontrol(int);
153
154
155 extern int uiomove64(addr64_t, int, void *);
156 #define MAX_RUN 32
157
158 int
memory_object_control_uiomove(memory_object_control_t control,memory_object_offset_t offset,void * uio,int start_offset,int io_requested,int mark_dirty,int take_reference)159 memory_object_control_uiomove(
160 memory_object_control_t control,
161 memory_object_offset_t offset,
162 void * uio,
163 int start_offset,
164 int io_requested,
165 int mark_dirty,
166 int take_reference)
167 {
168 vm_object_t object;
169 vm_page_t dst_page;
170 int xsize;
171 int retval = 0;
172 int cur_run;
173 int cur_needed;
174 int i;
175 int orig_offset;
176 vm_page_t page_run[MAX_RUN];
177 int dirty_count; /* keeps track of number of pages dirtied as part of this uiomove */
178
179 object = memory_object_control_to_vm_object(control);
180 if (object == VM_OBJECT_NULL) {
181 return 0;
182 }
183 assert(!object->internal);
184
185 vm_object_lock(object);
186
187 if (mark_dirty && object->vo_copy != VM_OBJECT_NULL) {
188 /*
189 * We can't modify the pages without honoring
190 * copy-on-write obligations first, so fall off
191 * this optimized path and fall back to the regular
192 * path.
193 */
194 vm_object_unlock(object);
195 return 0;
196 }
197 orig_offset = start_offset;
198
199 dirty_count = 0;
200 while (io_requested && retval == 0) {
201 cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
202
203 if (cur_needed > MAX_RUN) {
204 cur_needed = MAX_RUN;
205 }
206
207 for (cur_run = 0; cur_run < cur_needed;) {
208 if (mark_dirty && object->vo_copy != VM_OBJECT_NULL) {
209 /*
210 * We checked that this file-backed object did not have
211 * a copy object when we entered this routine but it now has
212 * one, so we can't stay on this optimized path.
213 * We can finish processing the pages we have already grabbed
214 * because they were made "busy" before the copy object was
215 * created so they can't have been seen through that copy
216 * object yet.
217 */
218 break;
219 }
220
221 if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) {
222 break;
223 }
224
225 if (__improbable(dst_page->vmp_error)) {
226 retval = EIO;
227 break;
228 }
229 if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
230 /*
231 * someone else is playing with the page... if we've
232 * already collected pages into this run, go ahead
233 * and process now, we can't block on this
234 * page while holding other pages in the BUSY state
235 * otherwise we will wait
236 */
237 if (cur_run) {
238 break;
239 }
240 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_EXCLUSIVE);
241 continue;
242 }
243 if (dst_page->vmp_laundry) {
244 vm_pageout_steal_laundry(dst_page, FALSE);
245 }
246 if (__improbable(dst_page->vmp_absent)) {
247 printf("absent page %p (obj %p offset 0x%llx) -> EIO",
248 dst_page, object, offset);
249 retval = EIO;
250 break;
251 }
252
253 if (mark_dirty) {
254 #if CONFIG_SPTM
255 if (__improbable(PMAP_PAGE_IS_USER_EXECUTABLE(dst_page))) {
256 /*
257 * This is analogous to the PMAP_OPTIONS_RETYPE disconnect we perform
258 * in vm_object_upl_request() when setting up a UPL to overwrite the
259 * destination pages, which is the UPL-based analogue of this path.
260 * See the comment there for the gory details, but it essentially boils
261 * down to the same situation of being asked to overwrite page contents
262 * that were already marked executable from some prior use of the vnode
263 * associated with this VM object.
264 */
265 pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(dst_page), PMAP_OPTIONS_RETYPE, NULL);
266 }
267 #endif /* CONFIG_SPTM */
268 if (dst_page->vmp_dirty == FALSE) {
269 dirty_count++;
270 }
271 SET_PAGE_DIRTY(dst_page, FALSE);
272 if (dst_page->vmp_cs_validated &&
273 !dst_page->vmp_cs_tainted) {
274 /*
275 * CODE SIGNING:
276 * We're modifying a code-signed
277 * page: force revalidate
278 */
279 dst_page->vmp_cs_validated = VMP_CS_ALL_FALSE;
280
281 VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
282
283 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
284 }
285 }
286 dst_page->vmp_busy = TRUE;
287
288 page_run[cur_run++] = dst_page;
289
290 offset += PAGE_SIZE_64;
291 }
292 if (cur_run == 0) {
293 /*
294 * we hit a 'hole' in the cache or
295 * a page we don't want to try to handle,
296 * so bail at this point
297 * we'll unlock the object below
298 */
299 break;
300 }
301 vm_object_unlock(object);
302
303 for (i = 0; i < cur_run; i++) {
304 dst_page = page_run[i];
305
306 if ((xsize = PAGE_SIZE - start_offset) > io_requested) {
307 xsize = io_requested;
308 }
309
310 /* Such phyiscal pages should never be restricted pages */
311 if (vm_page_is_restricted(dst_page)) {
312 panic("%s: cannot uiomove64 into restricted page", __func__);
313 }
314
315 if ((retval = uiomove64((addr64_t)(((addr64_t)(VM_PAGE_GET_PHYS_PAGE(dst_page)) << PAGE_SHIFT) + start_offset), xsize, uio))) {
316 break;
317 }
318
319 io_requested -= xsize;
320 start_offset = 0;
321 }
322 vm_object_lock(object);
323
324 /*
325 * if we have more than 1 page to work on
326 * in the current run, or the original request
327 * started at offset 0 of the page, or we're
328 * processing multiple batches, we will move
329 * the pages to the tail of the inactive queue
330 * to implement an LRU for read/write accesses
331 *
332 * the check for orig_offset == 0 is there to
333 * mitigate the cost of small (< page_size) requests
334 * to the same page (this way we only move it once)
335 */
336 if (take_reference && (cur_run > 1 || orig_offset == 0)) {
337 vm_page_lockspin_queues();
338
339 for (i = 0; i < cur_run; i++) {
340 vm_page_lru(page_run[i]);
341 }
342
343 vm_page_unlock_queues();
344 }
345 for (i = 0; i < cur_run; i++) {
346 dst_page = page_run[i];
347
348 /*
349 * someone is explicitly referencing this page...
350 * update clustered and speculative state
351 *
352 */
353 if (dst_page->vmp_clustered) {
354 VM_PAGE_CONSUME_CLUSTERED(dst_page);
355 }
356
357 vm_page_wakeup_done(object, dst_page);
358 }
359 orig_offset = 0;
360 }
361 vm_object_unlock(object);
362 return retval;
363 }
364
365
366 bool
memory_object_is_vnode_pager(memory_object_t mem_obj)367 memory_object_is_vnode_pager(
368 memory_object_t mem_obj)
369 {
370 if (mem_obj != NULL &&
371 mem_obj->mo_pager_ops == &vnode_pager_ops) {
372 return true;
373 }
374 return false;
375 }
376
377 /*
378 *
379 */
380 memory_object_t
vnode_pager_setup(struct vnode * vp,__unused memory_object_t pager)381 vnode_pager_setup(
382 struct vnode *vp,
383 __unused memory_object_t pager)
384 {
385 vnode_pager_t vnode_object;
386
387 vnode_object = vnode_object_create(vp);
388 if (vnode_object == VNODE_PAGER_NULL) {
389 panic("vnode_pager_setup: vnode_object_create() failed");
390 }
391 return (memory_object_t)vnode_object;
392 }
393
394 /*
395 *
396 */
397 kern_return_t
vnode_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)398 vnode_pager_init(memory_object_t mem_obj,
399 memory_object_control_t control,
400 #if !DEBUG
401 __unused
402 #endif
403 memory_object_cluster_size_t pg_size)
404 {
405 vnode_pager_t vnode_object;
406 kern_return_t kr;
407 memory_object_attr_info_data_t attributes;
408
409
410 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %lx\n", mem_obj, control, (unsigned long)pg_size));
411
412 if (control == MEMORY_OBJECT_CONTROL_NULL) {
413 return KERN_INVALID_ARGUMENT;
414 }
415
416 vnode_object = vnode_pager_lookup(mem_obj);
417
418 memory_object_control_reference(control);
419
420 vnode_object->vn_pgr_hdr.mo_control = control;
421
422 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
423 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
424 attributes.cluster_size = (1 << (PAGE_SHIFT));
425 attributes.may_cache_object = TRUE;
426 attributes.temporary = TRUE;
427
428 kr = memory_object_change_attributes(
429 control,
430 MEMORY_OBJECT_ATTRIBUTE_INFO,
431 (memory_object_info_t) &attributes,
432 MEMORY_OBJECT_ATTR_INFO_COUNT);
433 if (kr != KERN_SUCCESS) {
434 panic("vnode_pager_init: memory_object_change_attributes() failed");
435 }
436
437 return KERN_SUCCESS;
438 }
439
440 /*
441 *
442 */
443 kern_return_t
vnode_pager_data_return(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t data_cnt,memory_object_offset_t * resid_offset,int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,int upl_flags)444 vnode_pager_data_return(
445 memory_object_t mem_obj,
446 memory_object_offset_t offset,
447 memory_object_cluster_size_t data_cnt,
448 memory_object_offset_t *resid_offset,
449 int *io_error,
450 __unused boolean_t dirty,
451 __unused boolean_t kernel_copy,
452 int upl_flags)
453 {
454 vnode_pager_t vnode_object;
455
456 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
457
458 vnode_object = vnode_pager_lookup(mem_obj);
459
460 vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags);
461
462 return KERN_SUCCESS;
463 }
464
465 kern_return_t
vnode_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)466 vnode_pager_data_initialize(
467 __unused memory_object_t mem_obj,
468 __unused memory_object_offset_t offset,
469 __unused memory_object_cluster_size_t data_cnt)
470 {
471 panic("vnode_pager_data_initialize");
472 return KERN_FAILURE;
473 }
474
475 void
vnode_pager_dirtied(memory_object_t mem_obj,vm_object_offset_t s_offset,vm_object_offset_t e_offset)476 vnode_pager_dirtied(
477 memory_object_t mem_obj,
478 vm_object_offset_t s_offset,
479 vm_object_offset_t e_offset)
480 {
481 vnode_pager_t vnode_object;
482
483 if (mem_obj && mem_obj->mo_pager_ops == &vnode_pager_ops) {
484 vnode_object = vnode_pager_lookup(mem_obj);
485 vnode_pager_was_dirtied(vnode_object->vnode_handle, s_offset, e_offset);
486 }
487 }
488
489 kern_return_t
vnode_pager_get_isinuse(memory_object_t mem_obj,uint32_t * isinuse)490 vnode_pager_get_isinuse(
491 memory_object_t mem_obj,
492 uint32_t *isinuse)
493 {
494 vnode_pager_t vnode_object;
495
496 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
497 *isinuse = 1;
498 return KERN_INVALID_ARGUMENT;
499 }
500
501 vnode_object = vnode_pager_lookup(mem_obj);
502
503 *isinuse = vnode_pager_isinuse(vnode_object->vnode_handle);
504 return KERN_SUCCESS;
505 }
506
507 kern_return_t
vnode_pager_get_throttle_io_limit(memory_object_t mem_obj,uint32_t * limit)508 vnode_pager_get_throttle_io_limit(
509 memory_object_t mem_obj,
510 uint32_t *limit)
511 {
512 vnode_pager_t vnode_object;
513
514 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
515 return KERN_INVALID_ARGUMENT;
516 }
517
518 vnode_object = vnode_pager_lookup(mem_obj);
519
520 (void)vnode_pager_return_throttle_io_limit(vnode_object->vnode_handle, limit);
521 return KERN_SUCCESS;
522 }
523
524 kern_return_t
vnode_pager_get_isSSD(memory_object_t mem_obj,boolean_t * isSSD)525 vnode_pager_get_isSSD(
526 memory_object_t mem_obj,
527 boolean_t *isSSD)
528 {
529 vnode_pager_t vnode_object;
530
531 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
532 return KERN_INVALID_ARGUMENT;
533 }
534
535 vnode_object = vnode_pager_lookup(mem_obj);
536
537 *isSSD = vnode_pager_isSSD(vnode_object->vnode_handle);
538 return KERN_SUCCESS;
539 }
540
541 #if FBDP_DEBUG_OBJECT_NO_PAGER
542 kern_return_t
vnode_pager_get_forced_unmount(memory_object_t mem_obj,bool * forced_unmount)543 vnode_pager_get_forced_unmount(
544 memory_object_t mem_obj,
545 bool *forced_unmount)
546 {
547 vnode_pager_t vnode_object;
548
549 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
550 return KERN_INVALID_ARGUMENT;
551 }
552
553 vnode_object = vnode_pager_lookup(mem_obj);
554
555 *forced_unmount = vnode_pager_forced_unmount(vnode_object->vnode_handle);
556 return KERN_SUCCESS;
557 }
558 #endif /* FBDP_DEBUG_OBJECT_NO_PAGER */
559
560 kern_return_t
vnode_pager_get_object_size(memory_object_t mem_obj,memory_object_offset_t * length)561 vnode_pager_get_object_size(
562 memory_object_t mem_obj,
563 memory_object_offset_t *length)
564 {
565 vnode_pager_t vnode_object;
566
567 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
568 *length = 0;
569 return KERN_INVALID_ARGUMENT;
570 }
571
572 vnode_object = vnode_pager_lookup(mem_obj);
573
574 *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
575 return KERN_SUCCESS;
576 }
577
578 kern_return_t
vnode_pager_get_object_name(memory_object_t mem_obj,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)579 vnode_pager_get_object_name(
580 memory_object_t mem_obj,
581 char *pathname,
582 vm_size_t pathname_len,
583 char *filename,
584 vm_size_t filename_len,
585 boolean_t *truncated_path_p)
586 {
587 vnode_pager_t vnode_object;
588
589 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
590 return KERN_INVALID_ARGUMENT;
591 }
592
593 vnode_object = vnode_pager_lookup(mem_obj);
594
595 return vnode_pager_get_name(vnode_object->vnode_handle,
596 pathname,
597 pathname_len,
598 filename,
599 filename_len,
600 truncated_path_p);
601 }
602
603 kern_return_t
vnode_pager_get_object_mtime(memory_object_t mem_obj,struct timespec * mtime,struct timespec * cs_mtime)604 vnode_pager_get_object_mtime(
605 memory_object_t mem_obj,
606 struct timespec *mtime,
607 struct timespec *cs_mtime)
608 {
609 vnode_pager_t vnode_object;
610
611 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
612 return KERN_INVALID_ARGUMENT;
613 }
614
615 vnode_object = vnode_pager_lookup(mem_obj);
616
617 return vnode_pager_get_mtime(vnode_object->vnode_handle,
618 mtime,
619 cs_mtime);
620 }
621
622 #if CHECK_CS_VALIDATION_BITMAP
623 kern_return_t
vnode_pager_cs_check_validation_bitmap(memory_object_t mem_obj,memory_object_offset_t offset,int optype)624 vnode_pager_cs_check_validation_bitmap(
625 memory_object_t mem_obj,
626 memory_object_offset_t offset,
627 int optype )
628 {
629 vnode_pager_t vnode_object;
630
631 if (mem_obj == MEMORY_OBJECT_NULL ||
632 mem_obj->mo_pager_ops != &vnode_pager_ops) {
633 return KERN_INVALID_ARGUMENT;
634 }
635
636 vnode_object = vnode_pager_lookup(mem_obj);
637 return ubc_cs_check_validation_bitmap( vnode_object->vnode_handle, offset, optype );
638 }
639 #endif /* CHECK_CS_VALIDATION_BITMAP */
640
641 /*
642 *
643 */
644 kern_return_t
vnode_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,__unused memory_object_cluster_size_t length,__unused vm_prot_t desired_access,memory_object_fault_info_t fault_info)645 vnode_pager_data_request(
646 memory_object_t mem_obj,
647 memory_object_offset_t offset,
648 __unused memory_object_cluster_size_t length,
649 __unused vm_prot_t desired_access,
650 memory_object_fault_info_t fault_info)
651 {
652 vnode_pager_t vnode_object;
653 memory_object_offset_t base_offset;
654 vm_size_t size;
655 uint32_t io_streaming = 0;
656
657 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
658
659 vnode_object = vnode_pager_lookup(mem_obj);
660
661 size = MAX_UPL_TRANSFER_BYTES;
662 base_offset = offset;
663
664 if (memory_object_cluster_size(vnode_object->vn_pgr_hdr.mo_control,
665 &base_offset, &size, &io_streaming,
666 fault_info) != KERN_SUCCESS) {
667 size = PAGE_SIZE;
668 }
669
670 assert(offset >= base_offset &&
671 offset < base_offset + size);
672
673 return vnode_pager_cluster_read(vnode_object, base_offset, offset, io_streaming, size);
674 }
675
676 /*
677 *
678 */
679 void
vnode_pager_reference(memory_object_t mem_obj)680 vnode_pager_reference(
681 memory_object_t mem_obj)
682 {
683 vnode_pager_t vnode_object;
684
685 vnode_object = vnode_pager_lookup(mem_obj);
686 os_ref_retain_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
687 }
688
689 /*
690 *
691 */
692 void
vnode_pager_deallocate(memory_object_t mem_obj)693 vnode_pager_deallocate(
694 memory_object_t mem_obj)
695 {
696 vnode_pager_t vnode_object;
697
698 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %p\n", mem_obj));
699
700 vnode_object = vnode_pager_lookup(mem_obj);
701
702 if (os_ref_release_raw(&vnode_object->vn_pgr_hdr_ref, NULL) == 0) {
703 if (vnode_object->vnode_handle != NULL) {
704 vnode_pager_vrele(vnode_object->vnode_handle);
705 }
706 zfree(vnode_pager_zone, vnode_object);
707 }
708 }
709
710 /*
711 *
712 */
713 kern_return_t
vnode_pager_terminate(__unused memory_object_t mem_obj)714 vnode_pager_terminate(
715 #if !DEBUG
716 __unused
717 #endif
718 memory_object_t mem_obj)
719 {
720 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %p\n", mem_obj));
721
722 return KERN_SUCCESS;
723 }
724
725 /*
726 *
727 */
728 kern_return_t
vnode_pager_map(memory_object_t mem_obj,vm_prot_t prot)729 vnode_pager_map(
730 memory_object_t mem_obj,
731 vm_prot_t prot)
732 {
733 vnode_pager_t vnode_object;
734 int ret;
735 kern_return_t kr;
736
737 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_map: %p %x\n", mem_obj, prot));
738
739 vnode_object = vnode_pager_lookup(mem_obj);
740
741 ret = ubc_map(vnode_object->vnode_handle, prot);
742
743 if (ret != 0) {
744 kr = KERN_FAILURE;
745 } else {
746 kr = KERN_SUCCESS;
747 }
748
749 return kr;
750 }
751
752 kern_return_t
vnode_pager_last_unmap(memory_object_t mem_obj)753 vnode_pager_last_unmap(
754 memory_object_t mem_obj)
755 {
756 vnode_pager_t vnode_object;
757
758 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_last_unmap: %p\n", mem_obj));
759
760 vnode_object = vnode_pager_lookup(mem_obj);
761
762 ubc_unmap(vnode_object->vnode_handle);
763 return KERN_SUCCESS;
764 }
765
766
767
768 /*
769 *
770 */
771 void
vnode_pager_cluster_write(vnode_pager_t vnode_object,vm_object_offset_t offset,vm_size_t cnt,vm_object_offset_t * resid_offset,int * io_error,int upl_flags)772 vnode_pager_cluster_write(
773 vnode_pager_t vnode_object,
774 vm_object_offset_t offset,
775 vm_size_t cnt,
776 vm_object_offset_t * resid_offset,
777 int * io_error,
778 int upl_flags)
779 {
780 vm_size_t size;
781 int errno;
782
783 if (upl_flags & UPL_MSYNC) {
784 upl_flags |= UPL_VNODE_PAGER;
785
786 if ((upl_flags & UPL_IOSYNC) && io_error) {
787 upl_flags |= UPL_KEEPCACHED;
788 }
789
790 while (cnt) {
791 size = (cnt < MAX_UPL_TRANSFER_BYTES) ? cnt : MAX_UPL_TRANSFER_BYTES; /* effective max */
792
793 assert((upl_size_t) size == size);
794 vnode_pageout(vnode_object->vnode_handle,
795 NULL, (upl_offset_t)0, offset, (upl_size_t)size, upl_flags, &errno);
796
797 if ((upl_flags & UPL_KEEPCACHED)) {
798 if ((*io_error = errno)) {
799 break;
800 }
801 }
802 cnt -= size;
803 offset += size;
804 }
805 if (resid_offset) {
806 *resid_offset = offset;
807 }
808 } else {
809 vm_object_offset_t vnode_size;
810 vm_object_offset_t base_offset;
811
812 /*
813 * this is the pageout path
814 */
815 vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle);
816
817 if (vnode_size > (offset + PAGE_SIZE)) {
818 /*
819 * preset the maximum size of the cluster
820 * and put us on a nice cluster boundary...
821 * and then clip the size to insure we
822 * don't request past the end of the underlying file
823 */
824 size = MAX_UPL_TRANSFER_BYTES;
825 base_offset = offset & ~((signed)(size - 1));
826
827 if ((base_offset + size) > vnode_size) {
828 size = round_page(((vm_size_t)(vnode_size - base_offset)));
829 }
830 } else {
831 /*
832 * we've been requested to page out a page beyond the current
833 * end of the 'file'... don't try to cluster in this case...
834 * we still need to send this page through because it might
835 * be marked precious and the underlying filesystem may need
836 * to do something with it (besides page it out)...
837 */
838 base_offset = offset;
839 size = PAGE_SIZE;
840 }
841 assert((upl_size_t) size == size);
842 vnode_pageout(vnode_object->vnode_handle,
843 NULL, (upl_offset_t)(offset - base_offset), base_offset, (upl_size_t) size,
844 (upl_flags & UPL_IOSYNC) | UPL_VNODE_PAGER, NULL);
845 }
846 }
847
848
849 /*
850 *
851 */
852 kern_return_t
vnode_pager_cluster_read(vnode_pager_t vnode_object,vm_object_offset_t base_offset,vm_object_offset_t offset,uint32_t io_streaming,vm_size_t cnt)853 vnode_pager_cluster_read(
854 vnode_pager_t vnode_object,
855 vm_object_offset_t base_offset,
856 vm_object_offset_t offset,
857 uint32_t io_streaming,
858 vm_size_t cnt)
859 {
860 int local_error = 0;
861 int kret;
862 int flags = 0;
863
864 assert(!(cnt & PAGE_MASK));
865
866 if (io_streaming) {
867 flags |= UPL_IOSTREAMING;
868 }
869
870 assert((upl_size_t) cnt == cnt);
871 kret = vnode_pagein(vnode_object->vnode_handle,
872 (upl_t) NULL,
873 (upl_offset_t) (offset - base_offset),
874 base_offset,
875 (upl_size_t) cnt,
876 flags,
877 &local_error);
878 /*
879 * if(kret == PAGER_ABSENT) {
880 * Need to work out the defs here, 1 corresponds to PAGER_ABSENT
881 * defined in bsd/vm/vm_pager_xnu.h However, we should not be including
882 * that file here it is a layering violation.
883 */
884 if (kret == 1) {
885 int uplflags;
886 upl_t upl = NULL;
887 unsigned int count = 0;
888 kern_return_t kr;
889
890 uplflags = (UPL_NO_SYNC |
891 UPL_CLEAN_IN_PLACE |
892 UPL_SET_INTERNAL);
893 count = 0;
894 assert((upl_size_t) cnt == cnt);
895 kr = memory_object_upl_request(vnode_object->vn_pgr_hdr.mo_control,
896 base_offset, (upl_size_t) cnt,
897 &upl, NULL, &count, uplflags, VM_KERN_MEMORY_NONE);
898 if (kr == KERN_SUCCESS) {
899 upl_abort(upl, 0);
900 upl_deallocate(upl);
901 } else {
902 /*
903 * We couldn't gather the page list, probably
904 * because the memory object doesn't have a link
905 * to a VM object anymore (forced unmount, for
906 * example). Just return an error to the vm_fault()
907 * path and let it handle it.
908 */
909 }
910
911 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGER_CLREAD_NO_UPL), 0 /* arg */);
912 return KERN_FAILURE;
913 }
914
915 return KERN_SUCCESS;
916 }
917
918 /*
919 *
920 */
921 vnode_pager_t
vnode_object_create(struct vnode * vp)922 vnode_object_create(
923 struct vnode *vp)
924 {
925 vnode_pager_t vnode_object;
926
927 vnode_object = zalloc_flags(vnode_pager_zone, Z_WAITOK | Z_NOFAIL);
928
929 /*
930 * The vm_map call takes both named entry ports and raw memory
931 * objects in the same parameter. We need to make sure that
932 * vm_map does not see this object as a named entry port. So,
933 * we reserve the first word in the object for a fake object type
934 * setting - that will tell vm_map to use it as a memory object.
935 */
936 vnode_object->vn_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
937 vnode_object->vn_pgr_hdr.mo_pager_ops = &vnode_pager_ops;
938 vnode_object->vn_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
939
940 os_ref_init_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
941 vnode_object->vnode_handle = vp;
942
943 return vnode_object;
944 }
945
946 /*
947 *
948 */
949 vnode_pager_t
vnode_pager_lookup(memory_object_t name)950 vnode_pager_lookup(
951 memory_object_t name)
952 {
953 vnode_pager_t vnode_object;
954
955 vnode_object = (vnode_pager_t)name;
956 assert(vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops);
957 return vnode_object;
958 }
959
960
961 struct vnode *
vnode_pager_lookup_vnode(memory_object_t name)962 vnode_pager_lookup_vnode(
963 memory_object_t name)
964 {
965 vnode_pager_t vnode_object;
966 vnode_object = (vnode_pager_t)name;
967 if (vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops) {
968 return vnode_object->vnode_handle;
969 } else {
970 return NULL;
971 }
972 }
973
974 /*********************** proc_info implementation *************/
975
976 #include <sys/bsdtask_info.h>
977
978 static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uintptr_t * vnodeaddr, uint32_t * vid, bool *is_map_shared);
979
980 int
fill_procregioninfo(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)981 fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
982 {
983 vm_map_t map;
984 vm_map_offset_t address = (vm_map_offset_t)arg;
985 vm_map_entry_t tmp_entry;
986 vm_map_entry_t entry;
987 vm_map_offset_t start;
988 vm_region_extended_info_data_t extended;
989 vm_region_top_info_data_t top;
990 boolean_t do_region_footprint;
991 int effective_page_shift, effective_page_size;
992
993 vmlp_api_start(FILL_PROCREGIONINFO);
994
995 task_lock(task);
996 map = task->map;
997 if (map == VM_MAP_NULL) {
998 task_unlock(task);
999 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1000 return 0;
1001 }
1002
1003 effective_page_shift = vm_self_region_page_shift(map);
1004 effective_page_size = (1 << effective_page_shift);
1005
1006 vm_map_reference(map);
1007 task_unlock(task);
1008
1009 do_region_footprint = task_self_region_footprint();
1010
1011 vm_map_lock_read(map);
1012
1013 start = address;
1014
1015 if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
1016 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1017 if (do_region_footprint &&
1018 address == tmp_entry->vme_end) {
1019 ledger_amount_t ledger_resident;
1020 ledger_amount_t ledger_compressed;
1021
1022 /*
1023 * This request is right after the last valid
1024 * memory region; instead of reporting the
1025 * end of the address space, report a fake
1026 * memory region to account for non-volatile
1027 * purgeable and/or ledger-tagged memory
1028 * owned by this task.
1029 */
1030 task_ledgers_footprint(task->ledger,
1031 &ledger_resident,
1032 &ledger_compressed);
1033 if (ledger_resident + ledger_compressed == 0) {
1034 /* nothing to report */
1035 vm_map_unlock_read(map);
1036 vm_map_deallocate(map);
1037 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1038 return 0;
1039 }
1040
1041 /* provide fake region for purgeable */
1042 pinfo->pri_offset = address;
1043 pinfo->pri_protection = VM_PROT_DEFAULT;
1044 pinfo->pri_max_protection = VM_PROT_DEFAULT;
1045 pinfo->pri_inheritance = VM_INHERIT_NONE;
1046 pinfo->pri_behavior = VM_BEHAVIOR_DEFAULT;
1047 pinfo->pri_user_wired_count = 0;
1048 pinfo->pri_user_tag = -1;
1049 pinfo->pri_pages_resident =
1050 (uint32_t) (ledger_resident / effective_page_size);
1051 pinfo->pri_pages_shared_now_private = 0;
1052 pinfo->pri_pages_swapped_out =
1053 (uint32_t) (ledger_compressed / effective_page_size);
1054 pinfo->pri_pages_dirtied =
1055 (uint32_t) (ledger_resident / effective_page_size);
1056 pinfo->pri_ref_count = 1;
1057 pinfo->pri_shadow_depth = 0;
1058 pinfo->pri_share_mode = SM_PRIVATE;
1059 pinfo->pri_private_pages_resident =
1060 (uint32_t) (ledger_resident / effective_page_size);
1061 pinfo->pri_shared_pages_resident = 0;
1062 pinfo->pri_obj_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile);
1063 pinfo->pri_address = address;
1064 pinfo->pri_size =
1065 (uint64_t) (ledger_resident + ledger_compressed);
1066 pinfo->pri_depth = 0;
1067
1068 vm_map_unlock_read(map);
1069 vm_map_deallocate(map);
1070 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1071 return 1;
1072 }
1073 vm_map_unlock_read(map);
1074 vm_map_deallocate(map);
1075 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1076 return 0;
1077 }
1078 } else {
1079 entry = tmp_entry;
1080 }
1081
1082 start = entry->vme_start;
1083 vmlp_range_event_entry(map, entry);
1084
1085 pinfo->pri_offset = VME_OFFSET(entry);
1086 pinfo->pri_protection = entry->protection;
1087 pinfo->pri_max_protection = entry->max_protection;
1088 pinfo->pri_inheritance = entry->inheritance;
1089 pinfo->pri_behavior = entry->behavior;
1090 pinfo->pri_user_wired_count = entry->user_wired_count;
1091 pinfo->pri_user_tag = VME_ALIAS(entry);
1092
1093 if (entry->is_sub_map) {
1094 pinfo->pri_flags |= PROC_REGION_SUBMAP;
1095 } else {
1096 if (entry->is_shared) {
1097 pinfo->pri_flags |= PROC_REGION_SHARED;
1098 }
1099 }
1100
1101
1102 extended.protection = entry->protection;
1103 extended.user_tag = VME_ALIAS(entry);
1104 extended.pages_resident = 0;
1105 extended.pages_swapped_out = 0;
1106 extended.pages_shared_now_private = 0;
1107 extended.pages_dirtied = 0;
1108 extended.external_pager = 0;
1109 extended.shadow_depth = 0;
1110
1111 vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, &extended, TRUE, VM_REGION_EXTENDED_INFO_COUNT);
1112
1113 top.private_pages_resident = 0;
1114 top.shared_pages_resident = 0;
1115 vm_map_region_top_walk(entry, &top);
1116
1117
1118 pinfo->pri_pages_resident = extended.pages_resident;
1119 pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private;
1120 pinfo->pri_pages_swapped_out = extended.pages_swapped_out;
1121 pinfo->pri_pages_dirtied = extended.pages_dirtied;
1122 pinfo->pri_ref_count = extended.ref_count;
1123 pinfo->pri_shadow_depth = extended.shadow_depth;
1124 pinfo->pri_share_mode = extended.share_mode;
1125
1126 pinfo->pri_private_pages_resident = top.private_pages_resident;
1127 pinfo->pri_shared_pages_resident = top.shared_pages_resident;
1128 pinfo->pri_obj_id = top.obj_id;
1129
1130 pinfo->pri_address = (uint64_t)start;
1131 pinfo->pri_size = (uint64_t)(entry->vme_end - start);
1132 pinfo->pri_depth = 0;
1133
1134 if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) {
1135 *vnodeaddr = (uintptr_t)0;
1136
1137 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid, NULL) == 0) {
1138 vm_map_unlock_read(map);
1139 vm_map_deallocate(map);
1140 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1141 return 1;
1142 }
1143 }
1144
1145 vm_map_unlock_read(map);
1146 vm_map_deallocate(map);
1147 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1148 return 1;
1149 }
1150
1151 int
fill_procregioninfo_onlymappedvnodes(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)1152 fill_procregioninfo_onlymappedvnodes(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
1153 {
1154 vm_map_t map;
1155 vm_map_offset_t address = (vm_map_offset_t)arg;
1156 vm_map_entry_t tmp_entry;
1157 vm_map_entry_t entry;
1158
1159 vmlp_api_start(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES);
1160
1161 task_lock(task);
1162 map = task->map;
1163 if (map == VM_MAP_NULL) {
1164 task_unlock(task);
1165 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1166 return 0;
1167 }
1168 vm_map_reference(map);
1169 task_unlock(task);
1170
1171 vm_map_lock_read(map);
1172
1173 if (!vm_map_lookup_entry(map, address, &tmp_entry)) {
1174 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1175 vm_map_unlock_read(map);
1176 vm_map_deallocate(map);
1177 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1178 return 0;
1179 }
1180 } else {
1181 entry = tmp_entry;
1182 }
1183
1184 while (entry != vm_map_to_entry(map)) {
1185 vmlp_range_event_entry(map, entry);
1186 *vnodeaddr = 0;
1187 *vid = 0;
1188
1189 if (entry->is_sub_map == 0) {
1190 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid, NULL)) {
1191 pinfo->pri_offset = VME_OFFSET(entry);
1192 pinfo->pri_protection = entry->protection;
1193 pinfo->pri_max_protection = entry->max_protection;
1194 pinfo->pri_inheritance = entry->inheritance;
1195 pinfo->pri_behavior = entry->behavior;
1196 pinfo->pri_user_wired_count = entry->user_wired_count;
1197 pinfo->pri_user_tag = VME_ALIAS(entry);
1198
1199 if (entry->is_shared) {
1200 pinfo->pri_flags |= PROC_REGION_SHARED;
1201 }
1202
1203 pinfo->pri_pages_resident = 0;
1204 pinfo->pri_pages_shared_now_private = 0;
1205 pinfo->pri_pages_swapped_out = 0;
1206 pinfo->pri_pages_dirtied = 0;
1207 pinfo->pri_ref_count = 0;
1208 pinfo->pri_shadow_depth = 0;
1209 pinfo->pri_share_mode = 0;
1210
1211 pinfo->pri_private_pages_resident = 0;
1212 pinfo->pri_shared_pages_resident = 0;
1213 pinfo->pri_obj_id = 0;
1214
1215 pinfo->pri_address = (uint64_t)entry->vme_start;
1216 pinfo->pri_size = (uint64_t)(entry->vme_end - entry->vme_start);
1217 pinfo->pri_depth = 0;
1218
1219 vm_map_unlock_read(map);
1220 vm_map_deallocate(map);
1221 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 1);
1222 return 1;
1223 }
1224 }
1225
1226 /* Keep searching for a vnode-backed mapping */
1227 entry = entry->vme_next;
1228 }
1229
1230 vm_map_unlock_read(map);
1231 vm_map_deallocate(map);
1232 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1233 return 0;
1234 }
1235
1236 extern int vnode_get(struct vnode *vp);
1237 int
task_find_region_details(task_t task,vm_map_offset_t offset,find_region_details_options_t options,uintptr_t * vp_p,uint32_t * vid_p,bool * is_map_shared_p,uint64_t * start_p,uint64_t * len_p)1238 task_find_region_details(
1239 task_t task,
1240 vm_map_offset_t offset,
1241 find_region_details_options_t options,
1242 uintptr_t *vp_p,
1243 uint32_t *vid_p,
1244 bool *is_map_shared_p,
1245 uint64_t *start_p,
1246 uint64_t *len_p)
1247 {
1248 vm_map_t map;
1249 vm_map_entry_t entry;
1250 int rc;
1251
1252 vmlp_api_start(TASK_FIND_REGION_DETAILS);
1253
1254 rc = 0;
1255 *vp_p = 0;
1256 *vid_p = 0;
1257 *is_map_shared_p = false;
1258 *start_p = 0;
1259 *len_p = 0;
1260 if (options & ~FIND_REGION_DETAILS_OPTIONS_ALL) {
1261 vmlp_api_end(TASK_FIND_REGION_DETAILS, 0);
1262 return 0;
1263 }
1264
1265 task_lock(task);
1266 map = task->map;
1267 if (map == VM_MAP_NULL) {
1268 task_unlock(task);
1269 vmlp_api_end(TASK_FIND_REGION_DETAILS, 0);
1270 return 0;
1271 }
1272 vm_map_reference(map);
1273 task_unlock(task);
1274
1275 vm_map_lock_read(map);
1276 if (!vm_map_lookup_entry(map, offset, &entry)) {
1277 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1278 /* no mapping at this offset */
1279 goto ret;
1280 }
1281 /* check next entry */
1282 entry = entry->vme_next;
1283 if (entry == vm_map_to_entry(map)) {
1284 /* no next entry */
1285 goto ret;
1286 }
1287 }
1288
1289 for (;
1290 entry != vm_map_to_entry(map);
1291 entry = entry->vme_next) {
1292 vmlp_range_event_entry(map, entry);
1293
1294 if (entry->is_sub_map) {
1295 /* fallthru to check next entry */
1296 } else if (fill_vnodeinfoforaddr(entry, vp_p, vid_p, is_map_shared_p)) {
1297 if ((options & FIND_REGION_DETAILS_GET_VNODE) &&
1298 vnode_get((struct vnode *)*vp_p)) {
1299 /* tried but could not get an iocount */
1300 *vp_p = 0;
1301 *vid_p = 0;
1302 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1303 /* done */
1304 break;
1305 }
1306 /* check next entry */
1307 continue;
1308 }
1309 *start_p = entry->vme_start;
1310 *len_p = entry->vme_end - entry->vme_start;
1311 rc = 1; /* success */
1312 break;
1313 }
1314 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1315 /* no file mapping at this offset: done */
1316 break;
1317 }
1318 /* check next entry */
1319 }
1320
1321 ret:
1322 vm_map_unlock_read(map);
1323 vm_map_deallocate(map);
1324 vmlp_api_end(TASK_FIND_REGION_DETAILS, rc);
1325 return rc;
1326 }
1327
1328 static int
fill_vnodeinfoforaddr(vm_map_entry_t entry,uintptr_t * vnodeaddr,uint32_t * vid,bool * is_map_shared)1329 fill_vnodeinfoforaddr(
1330 vm_map_entry_t entry,
1331 uintptr_t * vnodeaddr,
1332 uint32_t * vid,
1333 bool *is_map_shared)
1334 {
1335 vm_object_t top_object, object;
1336 memory_object_t memory_object;
1337 memory_object_pager_ops_t pager_ops;
1338 kern_return_t kr;
1339 int shadow_depth;
1340
1341
1342 if (entry->is_sub_map) {
1343 return 0;
1344 } else {
1345 /*
1346 * The last object in the shadow chain has the
1347 * relevant pager information.
1348 */
1349 top_object = VME_OBJECT(entry);
1350 if (top_object == VM_OBJECT_NULL) {
1351 object = VM_OBJECT_NULL;
1352 shadow_depth = 0;
1353 } else {
1354 vm_object_lock(top_object);
1355 for (object = top_object, shadow_depth = 0;
1356 object->shadow != VM_OBJECT_NULL;
1357 object = object->shadow, shadow_depth++) {
1358 vm_object_lock(object->shadow);
1359 vm_object_unlock(object);
1360 }
1361 }
1362 }
1363
1364 if (object == VM_OBJECT_NULL) {
1365 return 0;
1366 } else if (object->internal) {
1367 vm_object_unlock(object);
1368 return 0;
1369 } else if (!object->pager_ready ||
1370 object->terminating ||
1371 !object->alive ||
1372 object->pager == NULL) {
1373 vm_object_unlock(object);
1374 return 0;
1375 } else {
1376 memory_object = object->pager;
1377 pager_ops = memory_object->mo_pager_ops;
1378 if (pager_ops == &vnode_pager_ops) {
1379 kr = vnode_pager_get_object_vnode(
1380 memory_object,
1381 vnodeaddr, vid);
1382 if (kr != KERN_SUCCESS) {
1383 vm_object_unlock(object);
1384 return 0;
1385 }
1386 } else {
1387 vm_object_unlock(object);
1388 return 0;
1389 }
1390 }
1391 if (is_map_shared) {
1392 *is_map_shared = (shadow_depth == 0);
1393 }
1394 vm_object_unlock(object);
1395 return 1;
1396 }
1397
1398 kern_return_t
vnode_pager_get_object_vnode(memory_object_t mem_obj,uintptr_t * vnodeaddr,uint32_t * vid)1399 vnode_pager_get_object_vnode(
1400 memory_object_t mem_obj,
1401 uintptr_t * vnodeaddr,
1402 uint32_t * vid)
1403 {
1404 vnode_pager_t vnode_object;
1405
1406 vnode_object = vnode_pager_lookup(mem_obj);
1407 if (vnode_object->vnode_handle) {
1408 *vnodeaddr = (uintptr_t)vnode_object->vnode_handle;
1409 *vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle);
1410
1411 return KERN_SUCCESS;
1412 }
1413
1414 return KERN_FAILURE;
1415 }
1416
1417 #if CONFIG_IOSCHED
1418 kern_return_t
vnode_pager_get_object_devvp(memory_object_t mem_obj,uintptr_t * devvp)1419 vnode_pager_get_object_devvp(
1420 memory_object_t mem_obj,
1421 uintptr_t *devvp)
1422 {
1423 struct vnode *vp;
1424 uint32_t vid;
1425
1426 if (vnode_pager_get_object_vnode(mem_obj, (uintptr_t *)&vp, (uint32_t *)&vid) != KERN_SUCCESS) {
1427 return KERN_FAILURE;
1428 }
1429 *devvp = (uintptr_t)vnode_mountdevvp(vp);
1430 if (*devvp) {
1431 return KERN_SUCCESS;
1432 }
1433 return KERN_FAILURE;
1434 }
1435 #endif
1436
1437 /*
1438 * Find the underlying vnode object for the given vm_map_entry. If found, return with the
1439 * object locked, otherwise return NULL with nothing locked.
1440 */
1441
1442 vm_object_t
find_vnode_object(vm_map_entry_t entry)1443 find_vnode_object(
1444 vm_map_entry_t entry
1445 )
1446 {
1447 vm_object_t top_object, object;
1448 memory_object_t memory_object;
1449 memory_object_pager_ops_t pager_ops;
1450
1451 if (!entry->is_sub_map) {
1452 /*
1453 * The last object in the shadow chain has the
1454 * relevant pager information.
1455 */
1456
1457 top_object = VME_OBJECT(entry);
1458
1459 if (top_object) {
1460 vm_object_lock(top_object);
1461
1462 for (object = top_object; object->shadow != VM_OBJECT_NULL; object = object->shadow) {
1463 vm_object_lock(object->shadow);
1464 vm_object_unlock(object);
1465 }
1466
1467 if (object &&
1468 !object->internal &&
1469 object->pager_ready &&
1470 !object->terminating &&
1471 object->alive &&
1472 object->pager != NULL) {
1473 memory_object = object->pager;
1474 pager_ops = memory_object->mo_pager_ops;
1475
1476 /*
1477 * If this object points to the vnode_pager_ops, then we found what we're
1478 * looking for. Otherwise, this vm_map_entry doesn't have an underlying
1479 * vnode and so we fall through to the bottom and return NULL.
1480 */
1481
1482 if (pager_ops == &vnode_pager_ops) {
1483 return object; /* we return with the object locked */
1484 }
1485 }
1486
1487 vm_object_unlock(object);
1488 }
1489 }
1490
1491 return VM_OBJECT_NULL;
1492 }
1493