1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41
42 #include <kern/assert.h>
43 #include <kern/host.h>
44 #include <kern/ledger.h>
45 #include <kern/thread.h>
46 #include <kern/ipc_kobject.h>
47 #include <os/refcnt.h>
48
49 #include <vm/vm_map_internal.h>
50 #include <vm/vm_pageout_internal.h>
51 #include <vm/memory_object_internal.h>
52 #include <vm/vm_pageout.h>
53 #include <vm/vm_protos_internal.h>
54 #include <vm/vm_purgeable_internal.h>
55 #include <vm/vm_ubc.h>
56 #include <vm/vm_page_internal.h>
57 #include <vm/vm_object_internal.h>
58
59 #include <sys/kdebug_triage.h>
60
61 /* BSD VM COMPONENT INTERFACES */
62 int
get_map_nentries(vm_map_t map)63 get_map_nentries(
64 vm_map_t map)
65 {
66 return map->hdr.nentries;
67 }
68
69 /*
70 * BSD VNODE PAGER
71 */
72
73 const struct memory_object_pager_ops vnode_pager_ops = {
74 .memory_object_reference = vnode_pager_reference,
75 .memory_object_deallocate = vnode_pager_deallocate,
76 .memory_object_init = vnode_pager_init,
77 .memory_object_terminate = vnode_pager_terminate,
78 .memory_object_data_request = vnode_pager_data_request,
79 .memory_object_data_return = vnode_pager_data_return,
80 .memory_object_data_initialize = vnode_pager_data_initialize,
81 .memory_object_map = vnode_pager_map,
82 .memory_object_last_unmap = vnode_pager_last_unmap,
83 .memory_object_backing_object = NULL,
84 .memory_object_pager_name = "vnode pager"
85 };
86
87 typedef struct vnode_pager {
88 /* mandatory generic header */
89 struct memory_object vn_pgr_hdr;
90
91 /* pager-specific */
92 #if MEMORY_OBJECT_HAS_REFCOUNT
93 #define vn_pgr_hdr_ref vn_pgr_hdr.mo_ref
94 #else
95 os_ref_atomic_t vn_pgr_hdr_ref;
96 #endif
97 struct vnode *vnode_handle; /* vnode handle */
98 } *vnode_pager_t;
99
100
101 kern_return_t
102 vnode_pager_cluster_read( /* forward */
103 vnode_pager_t,
104 vm_object_offset_t,
105 vm_object_offset_t,
106 uint32_t,
107 vm_size_t);
108
109 void
110 vnode_pager_cluster_write( /* forward */
111 vnode_pager_t,
112 vm_object_offset_t,
113 vm_size_t,
114 vm_object_offset_t *,
115 int *,
116 int);
117
118
119 vnode_pager_t
120 vnode_object_create( /* forward */
121 struct vnode *);
122
123 vnode_pager_t
124 vnode_pager_lookup( /* forward */
125 memory_object_t);
126
127 struct vnode *
128 vnode_pager_lookup_vnode( /* forward */
129 memory_object_t);
130
131 ZONE_DEFINE_TYPE(vnode_pager_zone, "vnode pager structures",
132 struct vnode_pager, ZC_NOENCRYPT);
133
134 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
135
136 /* TODO: Should be set dynamically by vnode_pager_init() */
137 #define CLUSTER_SHIFT 1
138
139
140 #if DEBUG
141 int pagerdebug = 0;
142
143 #define PAGER_ALL 0xffffffff
144 #define PAGER_INIT 0x00000001
145 #define PAGER_PAGEIN 0x00000002
146
147 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
148 #else
149 #define PAGER_DEBUG(LEVEL, A)
150 #endif
151
152 extern int proc_resetpcontrol(int);
153
154
155 extern int uiomove64(addr64_t, int, void *);
156 #define MAX_RUN 32
157
158 int
memory_object_control_uiomove(memory_object_control_t control,memory_object_offset_t offset,void * uio,int start_offset,int io_requested,int mark_dirty,int take_reference)159 memory_object_control_uiomove(
160 memory_object_control_t control,
161 memory_object_offset_t offset,
162 void * uio,
163 int start_offset,
164 int io_requested,
165 int mark_dirty,
166 int take_reference)
167 {
168 vm_object_t object;
169 vm_page_t dst_page;
170 int xsize;
171 int retval = 0;
172 int cur_run;
173 int cur_needed;
174 int i;
175 int orig_offset;
176 vm_page_t page_run[MAX_RUN];
177 int dirty_count; /* keeps track of number of pages dirtied as part of this uiomove */
178
179 object = memory_object_control_to_vm_object(control);
180 if (object == VM_OBJECT_NULL) {
181 return 0;
182 }
183 assert(!object->internal);
184
185 vm_object_lock(object);
186
187 if (mark_dirty && object->vo_copy != VM_OBJECT_NULL) {
188 /*
189 * We can't modify the pages without honoring
190 * copy-on-write obligations first, so fall off
191 * this optimized path and fall back to the regular
192 * path.
193 */
194 vm_object_unlock(object);
195 return 0;
196 }
197 orig_offset = start_offset;
198
199 dirty_count = 0;
200 while (io_requested && retval == 0) {
201 cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
202
203 if (cur_needed > MAX_RUN) {
204 cur_needed = MAX_RUN;
205 }
206
207 for (cur_run = 0; cur_run < cur_needed;) {
208 if (mark_dirty && object->vo_copy != VM_OBJECT_NULL) {
209 /*
210 * We checked that this file-backed object did not have
211 * a copy object when we entered this routine but it now has
212 * one, so we can't stay on this optimized path.
213 * We can finish processing the pages we have already grabbed
214 * because they were made "busy" before the copy object was
215 * created so they can't have been seen through that copy
216 * object yet.
217 */
218 break;
219 }
220
221 if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) {
222 break;
223 }
224
225 if (__improbable(dst_page->vmp_error)) {
226 retval = EIO;
227 break;
228 }
229 if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
230 /*
231 * someone else is playing with the page... if we've
232 * already collected pages into this run, go ahead
233 * and process now, we can't block on this
234 * page while holding other pages in the BUSY state
235 * otherwise we will wait
236 */
237 if (cur_run) {
238 break;
239 }
240 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_EXCLUSIVE);
241 continue;
242 }
243 if (dst_page->vmp_laundry) {
244 vm_pageout_steal_laundry(dst_page, FALSE);
245 }
246 if (__improbable(dst_page->vmp_absent)) {
247 printf("absent page %p (obj %p offset 0x%llx) -> EIO",
248 dst_page, object, offset);
249 retval = EIO;
250 break;
251 }
252
253 if (mark_dirty) {
254 #if CONFIG_SPTM
255 if (__improbable(PMAP_PAGE_IS_USER_EXECUTABLE(dst_page))) {
256 /*
257 * This is analogous to the PMAP_OPTIONS_RETYPE disconnect we perform
258 * in vm_object_upl_request() when setting up a UPL to overwrite the
259 * destination pages, which is the UPL-based analogue of this path.
260 * See the comment there for the gory details, but it essentially boils
261 * down to the same situation of being asked to overwrite page contents
262 * that were already marked executable from some prior use of the vnode
263 * associated with this VM object.
264 */
265 pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(dst_page), PMAP_OPTIONS_RETYPE, NULL);
266 }
267 #endif /* CONFIG_SPTM */
268 if (dst_page->vmp_dirty == FALSE) {
269 dirty_count++;
270 }
271 SET_PAGE_DIRTY(dst_page, FALSE);
272 if (dst_page->vmp_cs_validated &&
273 !dst_page->vmp_cs_tainted) {
274 /*
275 * CODE SIGNING:
276 * We're modifying a code-signed
277 * page: force revalidate
278 */
279 dst_page->vmp_cs_validated = VMP_CS_ALL_FALSE;
280
281 VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
282
283 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
284 }
285 }
286 dst_page->vmp_busy = TRUE;
287
288 page_run[cur_run++] = dst_page;
289
290 offset += PAGE_SIZE_64;
291 }
292 if (cur_run == 0) {
293 /*
294 * we hit a 'hole' in the cache or
295 * a page we don't want to try to handle,
296 * so bail at this point
297 * we'll unlock the object below
298 */
299 break;
300 }
301 vm_object_unlock(object);
302
303 for (i = 0; i < cur_run; i++) {
304 dst_page = page_run[i];
305
306 if ((xsize = PAGE_SIZE - start_offset) > io_requested) {
307 xsize = io_requested;
308 }
309
310 /* Such phyiscal pages should never be restricted pages */
311 if (vm_page_is_restricted(dst_page)) {
312 panic("%s: cannot uiomove64 into restricted page", __func__);
313 }
314
315 if ((retval = uiomove64((addr64_t)(((addr64_t)(VM_PAGE_GET_PHYS_PAGE(dst_page)) << PAGE_SHIFT) + start_offset), xsize, uio))) {
316 break;
317 }
318
319 io_requested -= xsize;
320 start_offset = 0;
321 }
322 vm_object_lock(object);
323
324 /*
325 * if we have more than 1 page to work on
326 * in the current run, or the original request
327 * started at offset 0 of the page, or we're
328 * processing multiple batches, we will move
329 * the pages to the tail of the inactive queue
330 * to implement an LRU for read/write accesses
331 *
332 * the check for orig_offset == 0 is there to
333 * mitigate the cost of small (< page_size) requests
334 * to the same page (this way we only move it once)
335 */
336 if (take_reference && (cur_run > 1 || orig_offset == 0)) {
337 vm_page_lockspin_queues();
338
339 for (i = 0; i < cur_run; i++) {
340 vm_page_lru(page_run[i]);
341 }
342
343 vm_page_unlock_queues();
344 }
345 for (i = 0; i < cur_run; i++) {
346 dst_page = page_run[i];
347
348 /*
349 * someone is explicitly referencing this page...
350 * update clustered and speculative state
351 *
352 */
353 if (dst_page->vmp_clustered) {
354 VM_PAGE_CONSUME_CLUSTERED(dst_page);
355 }
356
357 vm_page_wakeup_done(object, dst_page);
358 }
359 orig_offset = 0;
360 }
361 vm_object_unlock(object);
362 return retval;
363 }
364
365
366 bool
memory_object_is_vnode_pager(memory_object_t mem_obj)367 memory_object_is_vnode_pager(
368 memory_object_t mem_obj)
369 {
370 if (mem_obj != NULL &&
371 mem_obj->mo_pager_ops == &vnode_pager_ops) {
372 return true;
373 }
374 return false;
375 }
376
377 /*
378 *
379 */
380 memory_object_t
vnode_pager_setup(struct vnode * vp,__unused memory_object_t pager)381 vnode_pager_setup(
382 struct vnode *vp,
383 __unused memory_object_t pager)
384 {
385 vnode_pager_t vnode_object;
386
387 vnode_object = vnode_object_create(vp);
388 if (vnode_object == VNODE_PAGER_NULL) {
389 panic("vnode_pager_setup: vnode_object_create() failed");
390 }
391 return (memory_object_t)vnode_object;
392 }
393
394 /*
395 *
396 */
397 kern_return_t
vnode_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)398 vnode_pager_init(memory_object_t mem_obj,
399 memory_object_control_t control,
400 #if !DEBUG
401 __unused
402 #endif
403 memory_object_cluster_size_t pg_size)
404 {
405 vnode_pager_t vnode_object;
406 kern_return_t kr;
407 memory_object_attr_info_data_t attributes;
408
409
410 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %lx\n", mem_obj, control, (unsigned long)pg_size));
411
412 if (control == MEMORY_OBJECT_CONTROL_NULL) {
413 return KERN_INVALID_ARGUMENT;
414 }
415
416 vnode_object = vnode_pager_lookup(mem_obj);
417
418 memory_object_control_reference(control);
419
420 vnode_object->vn_pgr_hdr.mo_control = control;
421
422 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
423 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
424 attributes.cluster_size = (1 << (PAGE_SHIFT));
425 attributes.may_cache_object = TRUE;
426 attributes.temporary = TRUE;
427
428 kr = memory_object_change_attributes(
429 control,
430 MEMORY_OBJECT_ATTRIBUTE_INFO,
431 (memory_object_info_t) &attributes,
432 MEMORY_OBJECT_ATTR_INFO_COUNT);
433 if (kr != KERN_SUCCESS) {
434 panic("vnode_pager_init: memory_object_change_attributes() failed");
435 }
436
437 return KERN_SUCCESS;
438 }
439
440 /*
441 *
442 */
443 kern_return_t
vnode_pager_data_return(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t data_cnt,memory_object_offset_t * resid_offset,int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,int upl_flags)444 vnode_pager_data_return(
445 memory_object_t mem_obj,
446 memory_object_offset_t offset,
447 memory_object_cluster_size_t data_cnt,
448 memory_object_offset_t *resid_offset,
449 int *io_error,
450 __unused boolean_t dirty,
451 __unused boolean_t kernel_copy,
452 int upl_flags)
453 {
454 vnode_pager_t vnode_object;
455
456 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
457
458 vnode_object = vnode_pager_lookup(mem_obj);
459
460 vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags);
461
462 return KERN_SUCCESS;
463 }
464
465 kern_return_t
vnode_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)466 vnode_pager_data_initialize(
467 __unused memory_object_t mem_obj,
468 __unused memory_object_offset_t offset,
469 __unused memory_object_cluster_size_t data_cnt)
470 {
471 panic("vnode_pager_data_initialize");
472 return KERN_FAILURE;
473 }
474
475 void
vnode_pager_dirtied(memory_object_t mem_obj,vm_object_offset_t s_offset,vm_object_offset_t e_offset)476 vnode_pager_dirtied(
477 memory_object_t mem_obj,
478 vm_object_offset_t s_offset,
479 vm_object_offset_t e_offset)
480 {
481 vnode_pager_t vnode_object;
482
483 if (mem_obj && mem_obj->mo_pager_ops == &vnode_pager_ops) {
484 vnode_object = vnode_pager_lookup(mem_obj);
485 vnode_pager_was_dirtied(vnode_object->vnode_handle, s_offset, e_offset);
486 }
487 }
488
489 kern_return_t
vnode_pager_get_isinuse(memory_object_t mem_obj,uint32_t * isinuse)490 vnode_pager_get_isinuse(
491 memory_object_t mem_obj,
492 uint32_t *isinuse)
493 {
494 vnode_pager_t vnode_object;
495
496 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
497 *isinuse = 1;
498 return KERN_INVALID_ARGUMENT;
499 }
500
501 vnode_object = vnode_pager_lookup(mem_obj);
502
503 *isinuse = vnode_pager_isinuse(vnode_object->vnode_handle);
504 return KERN_SUCCESS;
505 }
506
507 kern_return_t
vnode_pager_get_throttle_io_limit(memory_object_t mem_obj,uint32_t * limit)508 vnode_pager_get_throttle_io_limit(
509 memory_object_t mem_obj,
510 uint32_t *limit)
511 {
512 vnode_pager_t vnode_object;
513
514 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
515 return KERN_INVALID_ARGUMENT;
516 }
517
518 vnode_object = vnode_pager_lookup(mem_obj);
519
520 (void)vnode_pager_return_throttle_io_limit(vnode_object->vnode_handle, limit);
521 return KERN_SUCCESS;
522 }
523
524 kern_return_t
vnode_pager_get_isSSD(memory_object_t mem_obj,boolean_t * isSSD)525 vnode_pager_get_isSSD(
526 memory_object_t mem_obj,
527 boolean_t *isSSD)
528 {
529 vnode_pager_t vnode_object;
530
531 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
532 return KERN_INVALID_ARGUMENT;
533 }
534
535 vnode_object = vnode_pager_lookup(mem_obj);
536
537 *isSSD = vnode_pager_isSSD(vnode_object->vnode_handle);
538 return KERN_SUCCESS;
539 }
540
541 #if FBDP_DEBUG_OBJECT_NO_PAGER
542 kern_return_t
vnode_pager_get_forced_unmount(memory_object_t mem_obj,bool * forced_unmount)543 vnode_pager_get_forced_unmount(
544 memory_object_t mem_obj,
545 bool *forced_unmount)
546 {
547 vnode_pager_t vnode_object;
548
549 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
550 return KERN_INVALID_ARGUMENT;
551 }
552
553 vnode_object = vnode_pager_lookup(mem_obj);
554
555 *forced_unmount = vnode_pager_forced_unmount(vnode_object->vnode_handle);
556 return KERN_SUCCESS;
557 }
558 #endif /* FBDP_DEBUG_OBJECT_NO_PAGER */
559
560 kern_return_t
vnode_pager_get_object_size(memory_object_t mem_obj,memory_object_offset_t * length)561 vnode_pager_get_object_size(
562 memory_object_t mem_obj,
563 memory_object_offset_t *length)
564 {
565 vnode_pager_t vnode_object;
566
567 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
568 *length = 0;
569 return KERN_INVALID_ARGUMENT;
570 }
571
572 vnode_object = vnode_pager_lookup(mem_obj);
573
574 *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
575 return KERN_SUCCESS;
576 }
577
578 kern_return_t
vnode_pager_get_object_name(memory_object_t mem_obj,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)579 vnode_pager_get_object_name(
580 memory_object_t mem_obj,
581 char *pathname,
582 vm_size_t pathname_len,
583 char *filename,
584 vm_size_t filename_len,
585 boolean_t *truncated_path_p)
586 {
587 vnode_pager_t vnode_object;
588
589 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
590 return KERN_INVALID_ARGUMENT;
591 }
592
593 vnode_object = vnode_pager_lookup(mem_obj);
594
595 return vnode_pager_get_name(vnode_object->vnode_handle,
596 pathname,
597 pathname_len,
598 filename,
599 filename_len,
600 truncated_path_p);
601 }
602
603 kern_return_t
vnode_pager_get_object_mtime(memory_object_t mem_obj,struct timespec * mtime,struct timespec * cs_mtime)604 vnode_pager_get_object_mtime(
605 memory_object_t mem_obj,
606 struct timespec *mtime,
607 struct timespec *cs_mtime)
608 {
609 vnode_pager_t vnode_object;
610
611 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
612 return KERN_INVALID_ARGUMENT;
613 }
614
615 vnode_object = vnode_pager_lookup(mem_obj);
616
617 return vnode_pager_get_mtime(vnode_object->vnode_handle,
618 mtime,
619 cs_mtime);
620 }
621
622 #if CHECK_CS_VALIDATION_BITMAP
623 kern_return_t
vnode_pager_cs_check_validation_bitmap(memory_object_t mem_obj,memory_object_offset_t offset,int optype)624 vnode_pager_cs_check_validation_bitmap(
625 memory_object_t mem_obj,
626 memory_object_offset_t offset,
627 int optype )
628 {
629 vnode_pager_t vnode_object;
630
631 if (mem_obj == MEMORY_OBJECT_NULL ||
632 mem_obj->mo_pager_ops != &vnode_pager_ops) {
633 return KERN_INVALID_ARGUMENT;
634 }
635
636 vnode_object = vnode_pager_lookup(mem_obj);
637 return ubc_cs_check_validation_bitmap( vnode_object->vnode_handle, offset, optype );
638 }
639 #endif /* CHECK_CS_VALIDATION_BITMAP */
640
641 /*
642 *
643 */
644 kern_return_t
vnode_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,__unused memory_object_cluster_size_t length,__unused vm_prot_t desired_access,memory_object_fault_info_t fault_info)645 vnode_pager_data_request(
646 memory_object_t mem_obj,
647 memory_object_offset_t offset,
648 __unused memory_object_cluster_size_t length,
649 __unused vm_prot_t desired_access,
650 memory_object_fault_info_t fault_info)
651 {
652 vnode_pager_t vnode_object;
653 memory_object_offset_t base_offset;
654 vm_size_t size;
655 uint32_t io_streaming = 0;
656
657 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
658
659 vnode_object = vnode_pager_lookup(mem_obj);
660
661 size = MAX_UPL_TRANSFER_BYTES;
662 base_offset = offset;
663
664 if (memory_object_cluster_size(vnode_object->vn_pgr_hdr.mo_control,
665 &base_offset, &size, &io_streaming,
666 fault_info) != KERN_SUCCESS) {
667 size = PAGE_SIZE;
668 }
669
670 assert(offset >= base_offset &&
671 offset < base_offset + size);
672
673 return vnode_pager_cluster_read(vnode_object, base_offset, offset, io_streaming, size);
674 }
675
676 /*
677 *
678 */
679 void
vnode_pager_reference(memory_object_t mem_obj)680 vnode_pager_reference(
681 memory_object_t mem_obj)
682 {
683 vnode_pager_t vnode_object;
684
685 vnode_object = vnode_pager_lookup(mem_obj);
686 os_ref_retain_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
687 }
688
689 /*
690 *
691 */
692 void
vnode_pager_deallocate(memory_object_t mem_obj)693 vnode_pager_deallocate(
694 memory_object_t mem_obj)
695 {
696 vnode_pager_t vnode_object;
697
698 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %p\n", mem_obj));
699
700 vnode_object = vnode_pager_lookup(mem_obj);
701
702 if (os_ref_release_raw(&vnode_object->vn_pgr_hdr_ref, NULL) == 0) {
703 if (vnode_object->vnode_handle != NULL) {
704 vnode_pager_vrele(vnode_object->vnode_handle);
705 }
706 zfree(vnode_pager_zone, vnode_object);
707 }
708 }
709
710 /*
711 *
712 */
713 kern_return_t
vnode_pager_terminate(__unused memory_object_t mem_obj)714 vnode_pager_terminate(
715 #if !DEBUG
716 __unused
717 #endif
718 memory_object_t mem_obj)
719 {
720 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %p\n", mem_obj));
721
722 return KERN_SUCCESS;
723 }
724
725 /*
726 *
727 */
728 kern_return_t
vnode_pager_map(memory_object_t mem_obj,vm_prot_t prot)729 vnode_pager_map(
730 memory_object_t mem_obj,
731 vm_prot_t prot)
732 {
733 vnode_pager_t vnode_object;
734 int ret;
735 kern_return_t kr;
736
737 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_map: %p %x\n", mem_obj, prot));
738
739 vnode_object = vnode_pager_lookup(mem_obj);
740
741 ret = ubc_map(vnode_object->vnode_handle, prot);
742
743 if (ret != 0) {
744 kr = KERN_FAILURE;
745 } else {
746 kr = KERN_SUCCESS;
747 }
748
749 return kr;
750 }
751
752 kern_return_t
vnode_pager_last_unmap(memory_object_t mem_obj)753 vnode_pager_last_unmap(
754 memory_object_t mem_obj)
755 {
756 vnode_pager_t vnode_object;
757
758 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_last_unmap: %p\n", mem_obj));
759
760 vnode_object = vnode_pager_lookup(mem_obj);
761
762 ubc_unmap(vnode_object->vnode_handle);
763 return KERN_SUCCESS;
764 }
765
766
767
768 /*
769 *
770 */
771 void
vnode_pager_cluster_write(vnode_pager_t vnode_object,vm_object_offset_t offset,vm_size_t cnt,vm_object_offset_t * resid_offset,int * io_error,int upl_flags)772 vnode_pager_cluster_write(
773 vnode_pager_t vnode_object,
774 vm_object_offset_t offset,
775 vm_size_t cnt,
776 vm_object_offset_t * resid_offset,
777 int * io_error,
778 int upl_flags)
779 {
780 vm_size_t size;
781 int errno;
782
783 if (upl_flags & UPL_MSYNC) {
784 upl_flags |= UPL_VNODE_PAGER;
785
786 if ((upl_flags & UPL_IOSYNC) && io_error) {
787 upl_flags |= UPL_KEEPCACHED;
788 }
789
790 while (cnt) {
791 size = (cnt < MAX_UPL_TRANSFER_BYTES) ? cnt : MAX_UPL_TRANSFER_BYTES; /* effective max */
792
793 assert((upl_size_t) size == size);
794 vnode_pageout(vnode_object->vnode_handle,
795 NULL, (upl_offset_t)0, offset, (upl_size_t)size, upl_flags, &errno);
796
797 if ((upl_flags & UPL_KEEPCACHED)) {
798 if ((*io_error = errno)) {
799 break;
800 }
801 }
802 cnt -= size;
803 offset += size;
804 }
805 if (resid_offset) {
806 *resid_offset = offset;
807 }
808 } else {
809 vm_object_offset_t vnode_size;
810 vm_object_offset_t base_offset;
811
812 /*
813 * this is the pageout path
814 */
815 vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle);
816
817 if (vnode_size > (offset + PAGE_SIZE)) {
818 /*
819 * preset the maximum size of the cluster
820 * and put us on a nice cluster boundary...
821 * and then clip the size to insure we
822 * don't request past the end of the underlying file
823 */
824 size = MAX_UPL_TRANSFER_BYTES;
825 base_offset = offset & ~((signed)(size - 1));
826
827 if ((base_offset + size) > vnode_size) {
828 size = round_page(((vm_size_t)(vnode_size - base_offset)));
829 }
830 } else {
831 /*
832 * we've been requested to page out a page beyond the current
833 * end of the 'file'... don't try to cluster in this case...
834 * we still need to send this page through because it might
835 * be marked precious and the underlying filesystem may need
836 * to do something with it (besides page it out)...
837 */
838 base_offset = offset;
839 size = PAGE_SIZE;
840 }
841 assert((upl_size_t) size == size);
842 vnode_pageout(vnode_object->vnode_handle,
843 NULL, (upl_offset_t)(offset - base_offset), base_offset, (upl_size_t) size,
844 (upl_flags & UPL_IOSYNC) | UPL_VNODE_PAGER, NULL);
845 }
846 }
847
848
849 /*
850 *
851 */
852 kern_return_t
vnode_pager_cluster_read(vnode_pager_t vnode_object,vm_object_offset_t base_offset,vm_object_offset_t offset,uint32_t io_streaming,vm_size_t cnt)853 vnode_pager_cluster_read(
854 vnode_pager_t vnode_object,
855 vm_object_offset_t base_offset,
856 vm_object_offset_t offset,
857 uint32_t io_streaming,
858 vm_size_t cnt)
859 {
860 int local_error = 0;
861 int kret;
862 int flags = 0;
863
864 assert(!(cnt & PAGE_MASK));
865
866 if (io_streaming) {
867 flags |= UPL_IOSTREAMING;
868 }
869
870 assert((upl_size_t) cnt == cnt);
871 kret = vnode_pagein(vnode_object->vnode_handle,
872 (upl_t) NULL,
873 (upl_offset_t) (offset - base_offset),
874 base_offset,
875 (upl_size_t) cnt,
876 flags,
877 &local_error);
878 /*
879 * if(kret == PAGER_ABSENT) {
880 * Need to work out the defs here, 1 corresponds to PAGER_ABSENT
881 * defined in bsd/vm/vm_pager_xnu.h However, we should not be including
882 * that file here it is a layering violation.
883 */
884 if (kret == 1) {
885 int uplflags;
886 upl_t upl = NULL;
887 unsigned int count = 0;
888 kern_return_t kr;
889
890 uplflags = (UPL_NO_SYNC |
891 UPL_CLEAN_IN_PLACE |
892 UPL_SET_INTERNAL);
893 count = 0;
894 assert((upl_size_t) cnt == cnt);
895 kr = memory_object_upl_request(vnode_object->vn_pgr_hdr.mo_control,
896 base_offset, (upl_size_t) cnt,
897 &upl, NULL, &count, uplflags, VM_KERN_MEMORY_NONE);
898 if (kr == KERN_SUCCESS) {
899 upl_abort(upl, 0);
900 upl_deallocate(upl);
901 } else {
902 /*
903 * We couldn't gather the page list, probably
904 * because the memory object doesn't have a link
905 * to a VM object anymore (forced unmount, for
906 * example). Just return an error to the vm_fault()
907 * path and let it handle it.
908 */
909 }
910
911 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGER_CLREAD_NO_UPL), 0 /* arg */);
912 return KERN_FAILURE;
913 }
914
915 return KERN_SUCCESS;
916 }
917
918 /*
919 *
920 */
921 vnode_pager_t
vnode_object_create(struct vnode * vp)922 vnode_object_create(
923 struct vnode *vp)
924 {
925 vnode_pager_t vnode_object;
926
927 vnode_object = zalloc_flags(vnode_pager_zone, Z_WAITOK | Z_NOFAIL);
928
929 /*
930 * The vm_map call takes both named entry ports and raw memory
931 * objects in the same parameter. We need to make sure that
932 * vm_map does not see this object as a named entry port. So,
933 * we reserve the first word in the object for a fake object type
934 * setting - that will tell vm_map to use it as a memory object.
935 */
936 vnode_object->vn_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
937 vnode_object->vn_pgr_hdr.mo_pager_ops = &vnode_pager_ops;
938 vnode_object->vn_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
939 vnode_object->vn_pgr_hdr.mo_last_unmap_ctid = 0;
940
941 os_ref_init_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
942 vnode_object->vnode_handle = vp;
943
944 return vnode_object;
945 }
946
947 /*
948 *
949 */
950 vnode_pager_t
vnode_pager_lookup(memory_object_t name)951 vnode_pager_lookup(
952 memory_object_t name)
953 {
954 vnode_pager_t vnode_object;
955
956 vnode_object = (vnode_pager_t)name;
957 assert(vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops);
958 return vnode_object;
959 }
960
961
962 struct vnode *
vnode_pager_lookup_vnode(memory_object_t name)963 vnode_pager_lookup_vnode(
964 memory_object_t name)
965 {
966 vnode_pager_t vnode_object;
967 vnode_object = (vnode_pager_t)name;
968 if (vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops) {
969 return vnode_object->vnode_handle;
970 } else {
971 return NULL;
972 }
973 }
974
975 /*********************** proc_info implementation *************/
976
977 #include <sys/bsdtask_info.h>
978
979 static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uintptr_t * vnodeaddr, uint32_t * vid, bool *is_map_shared);
980
981 int
fill_procregioninfo(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)982 fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
983 {
984 vm_map_t map;
985 vm_map_offset_t address = (vm_map_offset_t)arg;
986 vm_map_entry_t tmp_entry;
987 vm_map_entry_t entry;
988 vm_map_offset_t start;
989 vm_region_extended_info_data_t extended;
990 vm_region_top_info_data_t top;
991 boolean_t do_region_footprint;
992 int effective_page_shift, effective_page_size;
993
994 vmlp_api_start(FILL_PROCREGIONINFO);
995
996 task_lock(task);
997 map = task->map;
998 if (map == VM_MAP_NULL) {
999 task_unlock(task);
1000 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1001 return 0;
1002 }
1003
1004 effective_page_shift = vm_self_region_page_shift(map);
1005 effective_page_size = (1 << effective_page_shift);
1006
1007 vm_map_reference(map);
1008 task_unlock(task);
1009
1010 do_region_footprint = task_self_region_footprint();
1011
1012 vm_map_lock_read(map);
1013
1014 start = address;
1015
1016 if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
1017 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1018 if (do_region_footprint &&
1019 address == tmp_entry->vme_end) {
1020 ledger_amount_t ledger_resident;
1021 ledger_amount_t ledger_compressed;
1022
1023 /*
1024 * This request is right after the last valid
1025 * memory region; instead of reporting the
1026 * end of the address space, report a fake
1027 * memory region to account for non-volatile
1028 * purgeable and/or ledger-tagged memory
1029 * owned by this task.
1030 */
1031 task_ledgers_footprint(task->ledger,
1032 &ledger_resident,
1033 &ledger_compressed);
1034 if (ledger_resident + ledger_compressed == 0) {
1035 /* nothing to report */
1036 vm_map_unlock_read(map);
1037 vm_map_deallocate(map);
1038 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1039 return 0;
1040 }
1041
1042 /* provide fake region for purgeable */
1043 pinfo->pri_offset = address;
1044 pinfo->pri_protection = VM_PROT_DEFAULT;
1045 pinfo->pri_max_protection = VM_PROT_DEFAULT;
1046 pinfo->pri_inheritance = VM_INHERIT_NONE;
1047 pinfo->pri_behavior = VM_BEHAVIOR_DEFAULT;
1048 pinfo->pri_user_wired_count = 0;
1049 pinfo->pri_user_tag = -1;
1050 pinfo->pri_pages_resident =
1051 (uint32_t) (ledger_resident / effective_page_size);
1052 pinfo->pri_pages_shared_now_private = 0;
1053 pinfo->pri_pages_swapped_out =
1054 (uint32_t) (ledger_compressed / effective_page_size);
1055 pinfo->pri_pages_dirtied =
1056 (uint32_t) (ledger_resident / effective_page_size);
1057 pinfo->pri_ref_count = 1;
1058 pinfo->pri_shadow_depth = 0;
1059 pinfo->pri_share_mode = SM_PRIVATE;
1060 pinfo->pri_private_pages_resident =
1061 (uint32_t) (ledger_resident / effective_page_size);
1062 pinfo->pri_shared_pages_resident = 0;
1063 pinfo->pri_obj_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile);
1064 pinfo->pri_address = address;
1065 pinfo->pri_size =
1066 (uint64_t) (ledger_resident + ledger_compressed);
1067 pinfo->pri_depth = 0;
1068
1069 vm_map_unlock_read(map);
1070 vm_map_deallocate(map);
1071 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1072 return 1;
1073 }
1074 vm_map_unlock_read(map);
1075 vm_map_deallocate(map);
1076 vmlp_api_end(FILL_PROCREGIONINFO, 0);
1077 return 0;
1078 }
1079 } else {
1080 entry = tmp_entry;
1081 }
1082
1083 start = entry->vme_start;
1084 vmlp_range_event_entry(map, entry);
1085
1086 pinfo->pri_offset = VME_OFFSET(entry);
1087 pinfo->pri_protection = entry->protection;
1088 pinfo->pri_max_protection = entry->max_protection;
1089 pinfo->pri_inheritance = entry->inheritance;
1090 pinfo->pri_behavior = entry->behavior;
1091 pinfo->pri_user_wired_count = entry->user_wired_count;
1092 pinfo->pri_user_tag = VME_ALIAS(entry);
1093
1094 if (entry->is_sub_map) {
1095 pinfo->pri_flags |= PROC_REGION_SUBMAP;
1096 } else {
1097 if (entry->is_shared) {
1098 pinfo->pri_flags |= PROC_REGION_SHARED;
1099 }
1100 }
1101
1102
1103 extended.protection = entry->protection;
1104 extended.user_tag = VME_ALIAS(entry);
1105 extended.pages_resident = 0;
1106 extended.pages_swapped_out = 0;
1107 extended.pages_shared_now_private = 0;
1108 extended.pages_dirtied = 0;
1109 extended.external_pager = 0;
1110 extended.shadow_depth = 0;
1111
1112 vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, &extended, TRUE, VM_REGION_EXTENDED_INFO_COUNT);
1113
1114 top.private_pages_resident = 0;
1115 top.shared_pages_resident = 0;
1116 vm_map_region_top_walk(entry, &top);
1117
1118
1119 pinfo->pri_pages_resident = extended.pages_resident;
1120 pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private;
1121 pinfo->pri_pages_swapped_out = extended.pages_swapped_out;
1122 pinfo->pri_pages_dirtied = extended.pages_dirtied;
1123 pinfo->pri_ref_count = extended.ref_count;
1124 pinfo->pri_shadow_depth = extended.shadow_depth;
1125 pinfo->pri_share_mode = extended.share_mode;
1126
1127 pinfo->pri_private_pages_resident = top.private_pages_resident;
1128 pinfo->pri_shared_pages_resident = top.shared_pages_resident;
1129 pinfo->pri_obj_id = top.obj_id;
1130
1131 pinfo->pri_address = (uint64_t)start;
1132 pinfo->pri_size = (uint64_t)(entry->vme_end - start);
1133 pinfo->pri_depth = 0;
1134
1135 if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) {
1136 *vnodeaddr = (uintptr_t)0;
1137
1138 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid, NULL) == 0) {
1139 vm_map_unlock_read(map);
1140 vm_map_deallocate(map);
1141 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1142 return 1;
1143 }
1144 }
1145
1146 vm_map_unlock_read(map);
1147 vm_map_deallocate(map);
1148 vmlp_api_end(FILL_PROCREGIONINFO, 1);
1149 return 1;
1150 }
1151
1152 int
fill_procregioninfo_onlymappedvnodes(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)1153 fill_procregioninfo_onlymappedvnodes(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
1154 {
1155 vm_map_t map;
1156 vm_map_offset_t address = (vm_map_offset_t)arg;
1157 vm_map_entry_t tmp_entry;
1158 vm_map_entry_t entry;
1159
1160 vmlp_api_start(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES);
1161
1162 task_lock(task);
1163 map = task->map;
1164 if (map == VM_MAP_NULL) {
1165 task_unlock(task);
1166 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1167 return 0;
1168 }
1169 vm_map_reference(map);
1170 task_unlock(task);
1171
1172 vm_map_lock_read(map);
1173
1174 if (!vm_map_lookup_entry(map, address, &tmp_entry)) {
1175 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1176 vm_map_unlock_read(map);
1177 vm_map_deallocate(map);
1178 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1179 return 0;
1180 }
1181 } else {
1182 entry = tmp_entry;
1183 }
1184
1185 while (entry != vm_map_to_entry(map)) {
1186 vmlp_range_event_entry(map, entry);
1187 *vnodeaddr = 0;
1188 *vid = 0;
1189
1190 if (entry->is_sub_map == 0) {
1191 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid, NULL)) {
1192 pinfo->pri_offset = VME_OFFSET(entry);
1193 pinfo->pri_protection = entry->protection;
1194 pinfo->pri_max_protection = entry->max_protection;
1195 pinfo->pri_inheritance = entry->inheritance;
1196 pinfo->pri_behavior = entry->behavior;
1197 pinfo->pri_user_wired_count = entry->user_wired_count;
1198 pinfo->pri_user_tag = VME_ALIAS(entry);
1199
1200 if (entry->is_shared) {
1201 pinfo->pri_flags |= PROC_REGION_SHARED;
1202 }
1203
1204 pinfo->pri_pages_resident = 0;
1205 pinfo->pri_pages_shared_now_private = 0;
1206 pinfo->pri_pages_swapped_out = 0;
1207 pinfo->pri_pages_dirtied = 0;
1208 pinfo->pri_ref_count = 0;
1209 pinfo->pri_shadow_depth = 0;
1210 pinfo->pri_share_mode = 0;
1211
1212 pinfo->pri_private_pages_resident = 0;
1213 pinfo->pri_shared_pages_resident = 0;
1214 pinfo->pri_obj_id = 0;
1215
1216 pinfo->pri_address = (uint64_t)entry->vme_start;
1217 pinfo->pri_size = (uint64_t)(entry->vme_end - entry->vme_start);
1218 pinfo->pri_depth = 0;
1219
1220 vm_map_unlock_read(map);
1221 vm_map_deallocate(map);
1222 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 1);
1223 return 1;
1224 }
1225 }
1226
1227 /* Keep searching for a vnode-backed mapping */
1228 entry = entry->vme_next;
1229 }
1230
1231 vm_map_unlock_read(map);
1232 vm_map_deallocate(map);
1233 vmlp_api_end(FILL_PROCREGIONINFO_ONLYMAPPEDVNODES, 0);
1234 return 0;
1235 }
1236
1237 extern int vnode_get(struct vnode *vp);
1238 int
task_find_region_details(task_t task,vm_map_offset_t offset,find_region_details_options_t options,uintptr_t * vp_p,uint32_t * vid_p,bool * is_map_shared_p,uint64_t * start_p,uint64_t * len_p)1239 task_find_region_details(
1240 task_t task,
1241 vm_map_offset_t offset,
1242 find_region_details_options_t options,
1243 uintptr_t *vp_p,
1244 uint32_t *vid_p,
1245 bool *is_map_shared_p,
1246 uint64_t *start_p,
1247 uint64_t *len_p)
1248 {
1249 vm_map_t map;
1250 vm_map_entry_t entry;
1251 int rc;
1252
1253 vmlp_api_start(TASK_FIND_REGION_DETAILS);
1254
1255 rc = 0;
1256 *vp_p = 0;
1257 *vid_p = 0;
1258 *is_map_shared_p = false;
1259 *start_p = 0;
1260 *len_p = 0;
1261 if (options & ~FIND_REGION_DETAILS_OPTIONS_ALL) {
1262 vmlp_api_end(TASK_FIND_REGION_DETAILS, 0);
1263 return 0;
1264 }
1265
1266 task_lock(task);
1267 map = task->map;
1268 if (map == VM_MAP_NULL) {
1269 task_unlock(task);
1270 vmlp_api_end(TASK_FIND_REGION_DETAILS, 0);
1271 return 0;
1272 }
1273 vm_map_reference(map);
1274 task_unlock(task);
1275
1276 vm_map_lock_read(map);
1277 if (!vm_map_lookup_entry(map, offset, &entry)) {
1278 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1279 /* no mapping at this offset */
1280 goto ret;
1281 }
1282 /* check next entry */
1283 entry = entry->vme_next;
1284 if (entry == vm_map_to_entry(map)) {
1285 /* no next entry */
1286 goto ret;
1287 }
1288 }
1289
1290 for (;
1291 entry != vm_map_to_entry(map);
1292 entry = entry->vme_next) {
1293 vmlp_range_event_entry(map, entry);
1294
1295 if (entry->is_sub_map) {
1296 /* fallthru to check next entry */
1297 } else if (fill_vnodeinfoforaddr(entry, vp_p, vid_p, is_map_shared_p)) {
1298 if ((options & FIND_REGION_DETAILS_GET_VNODE) &&
1299 vnode_get((struct vnode *)*vp_p)) {
1300 /* tried but could not get an iocount */
1301 *vp_p = 0;
1302 *vid_p = 0;
1303 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1304 /* done */
1305 break;
1306 }
1307 /* check next entry */
1308 continue;
1309 }
1310 *start_p = entry->vme_start;
1311 *len_p = entry->vme_end - entry->vme_start;
1312 rc = 1; /* success */
1313 break;
1314 }
1315 if (options & FIND_REGION_DETAILS_AT_OFFSET) {
1316 /* no file mapping at this offset: done */
1317 break;
1318 }
1319 /* check next entry */
1320 }
1321
1322 ret:
1323 vm_map_unlock_read(map);
1324 vm_map_deallocate(map);
1325 vmlp_api_end(TASK_FIND_REGION_DETAILS, rc);
1326 return rc;
1327 }
1328
1329 static int
fill_vnodeinfoforaddr(vm_map_entry_t entry,uintptr_t * vnodeaddr,uint32_t * vid,bool * is_map_shared)1330 fill_vnodeinfoforaddr(
1331 vm_map_entry_t entry,
1332 uintptr_t * vnodeaddr,
1333 uint32_t * vid,
1334 bool *is_map_shared)
1335 {
1336 vm_object_t top_object, object;
1337 memory_object_t memory_object;
1338 memory_object_pager_ops_t pager_ops;
1339 kern_return_t kr;
1340 int shadow_depth;
1341
1342
1343 if (entry->is_sub_map) {
1344 return 0;
1345 } else {
1346 /*
1347 * The last object in the shadow chain has the
1348 * relevant pager information.
1349 */
1350 top_object = VME_OBJECT(entry);
1351 if (top_object == VM_OBJECT_NULL) {
1352 object = VM_OBJECT_NULL;
1353 shadow_depth = 0;
1354 } else {
1355 vm_object_lock(top_object);
1356 for (object = top_object, shadow_depth = 0;
1357 object->shadow != VM_OBJECT_NULL;
1358 object = object->shadow, shadow_depth++) {
1359 vm_object_lock(object->shadow);
1360 vm_object_unlock(object);
1361 }
1362 }
1363 }
1364
1365 if (object == VM_OBJECT_NULL) {
1366 return 0;
1367 } else if (object->internal) {
1368 vm_object_unlock(object);
1369 return 0;
1370 } else if (!object->pager_ready ||
1371 object->terminating ||
1372 !object->alive ||
1373 object->pager == NULL) {
1374 vm_object_unlock(object);
1375 return 0;
1376 } else {
1377 memory_object = object->pager;
1378 pager_ops = memory_object->mo_pager_ops;
1379 if (pager_ops == &vnode_pager_ops) {
1380 kr = vnode_pager_get_object_vnode(
1381 memory_object,
1382 vnodeaddr, vid);
1383 if (kr != KERN_SUCCESS) {
1384 vm_object_unlock(object);
1385 return 0;
1386 }
1387 } else {
1388 vm_object_unlock(object);
1389 return 0;
1390 }
1391 }
1392 if (is_map_shared) {
1393 *is_map_shared = (shadow_depth == 0);
1394 }
1395 vm_object_unlock(object);
1396 return 1;
1397 }
1398
1399 kern_return_t
vnode_pager_get_object_vnode(memory_object_t mem_obj,uintptr_t * vnodeaddr,uint32_t * vid)1400 vnode_pager_get_object_vnode(
1401 memory_object_t mem_obj,
1402 uintptr_t * vnodeaddr,
1403 uint32_t * vid)
1404 {
1405 vnode_pager_t vnode_object;
1406
1407 vnode_object = vnode_pager_lookup(mem_obj);
1408 if (vnode_object->vnode_handle) {
1409 *vnodeaddr = (uintptr_t)vnode_object->vnode_handle;
1410 *vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle);
1411
1412 return KERN_SUCCESS;
1413 }
1414
1415 return KERN_FAILURE;
1416 }
1417
1418 #if CONFIG_IOSCHED
1419 kern_return_t
vnode_pager_get_object_devvp(memory_object_t mem_obj,uintptr_t * devvp)1420 vnode_pager_get_object_devvp(
1421 memory_object_t mem_obj,
1422 uintptr_t *devvp)
1423 {
1424 struct vnode *vp;
1425 uint32_t vid;
1426
1427 if (vnode_pager_get_object_vnode(mem_obj, (uintptr_t *)&vp, (uint32_t *)&vid) != KERN_SUCCESS) {
1428 return KERN_FAILURE;
1429 }
1430 *devvp = (uintptr_t)vnode_mountdevvp(vp);
1431 if (*devvp) {
1432 return KERN_SUCCESS;
1433 }
1434 return KERN_FAILURE;
1435 }
1436 #endif
1437
1438 /*
1439 * Find the underlying vnode object for the given vm_map_entry. If found, return with the
1440 * object locked, otherwise return NULL with nothing locked.
1441 */
1442
1443 vm_object_t
find_vnode_object(vm_map_entry_t entry)1444 find_vnode_object(
1445 vm_map_entry_t entry
1446 )
1447 {
1448 vm_object_t top_object, object;
1449 memory_object_t memory_object;
1450 memory_object_pager_ops_t pager_ops;
1451
1452 if (!entry->is_sub_map) {
1453 /*
1454 * The last object in the shadow chain has the
1455 * relevant pager information.
1456 */
1457
1458 top_object = VME_OBJECT(entry);
1459
1460 if (top_object) {
1461 vm_object_lock(top_object);
1462
1463 for (object = top_object; object->shadow != VM_OBJECT_NULL; object = object->shadow) {
1464 vm_object_lock(object->shadow);
1465 vm_object_unlock(object);
1466 }
1467
1468 if (object &&
1469 !object->internal &&
1470 object->pager_ready &&
1471 !object->terminating &&
1472 object->alive &&
1473 object->pager != NULL) {
1474 memory_object = object->pager;
1475 pager_ops = memory_object->mo_pager_ops;
1476
1477 /*
1478 * If this object points to the vnode_pager_ops, then we found what we're
1479 * looking for. Otherwise, this vm_map_entry doesn't have an underlying
1480 * vnode and so we fall through to the bottom and return NULL.
1481 */
1482
1483 if (pager_ops == &vnode_pager_ops) {
1484 return object; /* we return with the object locked */
1485 }
1486 }
1487
1488 vm_object_unlock(object);
1489 }
1490 }
1491
1492 return VM_OBJECT_NULL;
1493 }
1494