1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41
42 #include <kern/assert.h>
43 #include <kern/host.h>
44 #include <kern/ledger.h>
45 #include <kern/thread.h>
46 #include <kern/ipc_kobject.h>
47 #include <os/refcnt.h>
48
49 #include <ipc/ipc_port.h>
50 #include <ipc/ipc_space.h>
51
52 #include <vm/vm_map.h>
53 #include <vm/vm_pageout.h>
54 #include <vm/memory_object.h>
55 #include <vm/vm_pageout.h>
56 #include <vm/vm_protos.h>
57 #include <vm/vm_purgeable_internal.h>
58
59 #include <sys/kdebug_triage.h>
60
61 /* BSD VM COMPONENT INTERFACES */
62 int
63 get_map_nentries(
64 vm_map_t);
65
66 int
get_map_nentries(vm_map_t map)67 get_map_nentries(
68 vm_map_t map)
69 {
70 return map->hdr.nentries;
71 }
72
73 /*
74 * BSD VNODE PAGER
75 */
76
77 const struct memory_object_pager_ops vnode_pager_ops = {
78 .memory_object_reference = vnode_pager_reference,
79 .memory_object_deallocate = vnode_pager_deallocate,
80 .memory_object_init = vnode_pager_init,
81 .memory_object_terminate = vnode_pager_terminate,
82 .memory_object_data_request = vnode_pager_data_request,
83 .memory_object_data_return = vnode_pager_data_return,
84 .memory_object_data_initialize = vnode_pager_data_initialize,
85 .memory_object_map = vnode_pager_map,
86 .memory_object_last_unmap = vnode_pager_last_unmap,
87 .memory_object_backing_object = NULL,
88 .memory_object_pager_name = "vnode pager"
89 };
90
91 typedef struct vnode_pager {
92 /* mandatory generic header */
93 struct memory_object vn_pgr_hdr;
94
95 /* pager-specific */
96 #if MEMORY_OBJECT_HAS_REFCOUNT
97 #define vn_pgr_hdr_ref vn_pgr_hdr.mo_ref
98 #else
99 os_ref_atomic_t vn_pgr_hdr_ref;
100 #endif
101 struct vnode *vnode_handle; /* vnode handle */
102 } *vnode_pager_t;
103
104
105 kern_return_t
106 vnode_pager_cluster_read( /* forward */
107 vnode_pager_t,
108 vm_object_offset_t,
109 vm_object_offset_t,
110 uint32_t,
111 vm_size_t);
112
113 void
114 vnode_pager_cluster_write( /* forward */
115 vnode_pager_t,
116 vm_object_offset_t,
117 vm_size_t,
118 vm_object_offset_t *,
119 int *,
120 int);
121
122
123 vnode_pager_t
124 vnode_object_create( /* forward */
125 struct vnode *);
126
127 vnode_pager_t
128 vnode_pager_lookup( /* forward */
129 memory_object_t);
130
131 struct vnode *
132 vnode_pager_lookup_vnode( /* forward */
133 memory_object_t);
134
135 ZONE_DEFINE_TYPE(vnode_pager_zone, "vnode pager structures",
136 struct vnode_pager, ZC_NOENCRYPT);
137
138 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
139
140 /* TODO: Should be set dynamically by vnode_pager_init() */
141 #define CLUSTER_SHIFT 1
142
143
144 #if DEBUG
145 int pagerdebug = 0;
146
147 #define PAGER_ALL 0xffffffff
148 #define PAGER_INIT 0x00000001
149 #define PAGER_PAGEIN 0x00000002
150
151 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
152 #else
153 #define PAGER_DEBUG(LEVEL, A)
154 #endif
155
156 extern int proc_resetpcontrol(int);
157
158
159 extern int uiomove64(addr64_t, int, void *);
160 #define MAX_RUN 32
161
162 int
memory_object_control_uiomove(memory_object_control_t control,memory_object_offset_t offset,void * uio,int start_offset,int io_requested,int mark_dirty,int take_reference)163 memory_object_control_uiomove(
164 memory_object_control_t control,
165 memory_object_offset_t offset,
166 void * uio,
167 int start_offset,
168 int io_requested,
169 int mark_dirty,
170 int take_reference)
171 {
172 vm_object_t object;
173 vm_page_t dst_page;
174 int xsize;
175 int retval = 0;
176 int cur_run;
177 int cur_needed;
178 int i;
179 int orig_offset;
180 vm_page_t page_run[MAX_RUN];
181 int dirty_count; /* keeps track of number of pages dirtied as part of this uiomove */
182
183 object = memory_object_control_to_vm_object(control);
184 if (object == VM_OBJECT_NULL) {
185 return 0;
186 }
187 assert(!object->internal);
188
189 vm_object_lock(object);
190
191 if (mark_dirty && object->vo_copy != VM_OBJECT_NULL) {
192 /*
193 * We can't modify the pages without honoring
194 * copy-on-write obligations first, so fall off
195 * this optimized path and fall back to the regular
196 * path.
197 */
198 vm_object_unlock(object);
199 return 0;
200 }
201 orig_offset = start_offset;
202
203 dirty_count = 0;
204 while (io_requested && retval == 0) {
205 cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
206
207 if (cur_needed > MAX_RUN) {
208 cur_needed = MAX_RUN;
209 }
210
211 for (cur_run = 0; cur_run < cur_needed;) {
212 if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) {
213 break;
214 }
215
216
217 if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
218 /*
219 * someone else is playing with the page... if we've
220 * already collected pages into this run, go ahead
221 * and process now, we can't block on this
222 * page while holding other pages in the BUSY state
223 * otherwise we will wait
224 */
225 if (cur_run) {
226 break;
227 }
228 PAGE_SLEEP(object, dst_page, THREAD_UNINT);
229 continue;
230 }
231 if (dst_page->vmp_laundry) {
232 vm_pageout_steal_laundry(dst_page, FALSE);
233 }
234
235 if (mark_dirty) {
236 if (dst_page->vmp_dirty == FALSE) {
237 dirty_count++;
238 }
239 SET_PAGE_DIRTY(dst_page, FALSE);
240 if (dst_page->vmp_cs_validated &&
241 !dst_page->vmp_cs_tainted) {
242 /*
243 * CODE SIGNING:
244 * We're modifying a code-signed
245 * page: force revalidate
246 */
247 dst_page->vmp_cs_validated = VMP_CS_ALL_FALSE;
248
249 VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
250
251 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
252 }
253 }
254 dst_page->vmp_busy = TRUE;
255
256 page_run[cur_run++] = dst_page;
257
258 offset += PAGE_SIZE_64;
259 }
260 if (cur_run == 0) {
261 /*
262 * we hit a 'hole' in the cache or
263 * a page we don't want to try to handle,
264 * so bail at this point
265 * we'll unlock the object below
266 */
267 break;
268 }
269 vm_object_unlock(object);
270
271 for (i = 0; i < cur_run; i++) {
272 dst_page = page_run[i];
273
274 if ((xsize = PAGE_SIZE - start_offset) > io_requested) {
275 xsize = io_requested;
276 }
277
278 if ((retval = uiomove64((addr64_t)(((addr64_t)(VM_PAGE_GET_PHYS_PAGE(dst_page)) << PAGE_SHIFT) + start_offset), xsize, uio))) {
279 break;
280 }
281
282 io_requested -= xsize;
283 start_offset = 0;
284 }
285 vm_object_lock(object);
286
287 /*
288 * if we have more than 1 page to work on
289 * in the current run, or the original request
290 * started at offset 0 of the page, or we're
291 * processing multiple batches, we will move
292 * the pages to the tail of the inactive queue
293 * to implement an LRU for read/write accesses
294 *
295 * the check for orig_offset == 0 is there to
296 * mitigate the cost of small (< page_size) requests
297 * to the same page (this way we only move it once)
298 */
299 if (take_reference && (cur_run > 1 || orig_offset == 0)) {
300 vm_page_lockspin_queues();
301
302 for (i = 0; i < cur_run; i++) {
303 vm_page_lru(page_run[i]);
304 }
305
306 vm_page_unlock_queues();
307 }
308 for (i = 0; i < cur_run; i++) {
309 dst_page = page_run[i];
310
311 /*
312 * someone is explicitly referencing this page...
313 * update clustered and speculative state
314 *
315 */
316 if (dst_page->vmp_clustered) {
317 VM_PAGE_CONSUME_CLUSTERED(dst_page);
318 }
319
320 PAGE_WAKEUP_DONE(dst_page);
321 }
322 orig_offset = 0;
323 }
324 if (object->pager) {
325 task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_DEFERRED, vnode_pager_lookup_vnode(object->pager));
326 }
327 vm_object_unlock(object);
328 return retval;
329 }
330
331
332 /*
333 *
334 */
335 memory_object_t
vnode_pager_setup(struct vnode * vp,__unused memory_object_t pager)336 vnode_pager_setup(
337 struct vnode *vp,
338 __unused memory_object_t pager)
339 {
340 vnode_pager_t vnode_object;
341
342 vnode_object = vnode_object_create(vp);
343 if (vnode_object == VNODE_PAGER_NULL) {
344 panic("vnode_pager_setup: vnode_object_create() failed");
345 }
346 return (memory_object_t)vnode_object;
347 }
348
349 /*
350 *
351 */
352 kern_return_t
vnode_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)353 vnode_pager_init(memory_object_t mem_obj,
354 memory_object_control_t control,
355 #if !DEBUG
356 __unused
357 #endif
358 memory_object_cluster_size_t pg_size)
359 {
360 vnode_pager_t vnode_object;
361 kern_return_t kr;
362 memory_object_attr_info_data_t attributes;
363
364
365 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %lx\n", mem_obj, control, (unsigned long)pg_size));
366
367 if (control == MEMORY_OBJECT_CONTROL_NULL) {
368 return KERN_INVALID_ARGUMENT;
369 }
370
371 vnode_object = vnode_pager_lookup(mem_obj);
372
373 memory_object_control_reference(control);
374
375 vnode_object->vn_pgr_hdr.mo_control = control;
376
377 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
378 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
379 attributes.cluster_size = (1 << (PAGE_SHIFT));
380 attributes.may_cache_object = TRUE;
381 attributes.temporary = TRUE;
382
383 kr = memory_object_change_attributes(
384 control,
385 MEMORY_OBJECT_ATTRIBUTE_INFO,
386 (memory_object_info_t) &attributes,
387 MEMORY_OBJECT_ATTR_INFO_COUNT);
388 if (kr != KERN_SUCCESS) {
389 panic("vnode_pager_init: memory_object_change_attributes() failed");
390 }
391
392 return KERN_SUCCESS;
393 }
394
395 /*
396 *
397 */
398 kern_return_t
vnode_pager_data_return(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t data_cnt,memory_object_offset_t * resid_offset,int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,int upl_flags)399 vnode_pager_data_return(
400 memory_object_t mem_obj,
401 memory_object_offset_t offset,
402 memory_object_cluster_size_t data_cnt,
403 memory_object_offset_t *resid_offset,
404 int *io_error,
405 __unused boolean_t dirty,
406 __unused boolean_t kernel_copy,
407 int upl_flags)
408 {
409 vnode_pager_t vnode_object;
410
411 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
412
413 vnode_object = vnode_pager_lookup(mem_obj);
414
415 vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags);
416
417 return KERN_SUCCESS;
418 }
419
420 kern_return_t
vnode_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)421 vnode_pager_data_initialize(
422 __unused memory_object_t mem_obj,
423 __unused memory_object_offset_t offset,
424 __unused memory_object_cluster_size_t data_cnt)
425 {
426 panic("vnode_pager_data_initialize");
427 return KERN_FAILURE;
428 }
429
430 void
vnode_pager_dirtied(memory_object_t mem_obj,vm_object_offset_t s_offset,vm_object_offset_t e_offset)431 vnode_pager_dirtied(
432 memory_object_t mem_obj,
433 vm_object_offset_t s_offset,
434 vm_object_offset_t e_offset)
435 {
436 vnode_pager_t vnode_object;
437
438 if (mem_obj && mem_obj->mo_pager_ops == &vnode_pager_ops) {
439 vnode_object = vnode_pager_lookup(mem_obj);
440 vnode_pager_was_dirtied(vnode_object->vnode_handle, s_offset, e_offset);
441 }
442 }
443
444 kern_return_t
vnode_pager_get_isinuse(memory_object_t mem_obj,uint32_t * isinuse)445 vnode_pager_get_isinuse(
446 memory_object_t mem_obj,
447 uint32_t *isinuse)
448 {
449 vnode_pager_t vnode_object;
450
451 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
452 *isinuse = 1;
453 return KERN_INVALID_ARGUMENT;
454 }
455
456 vnode_object = vnode_pager_lookup(mem_obj);
457
458 *isinuse = vnode_pager_isinuse(vnode_object->vnode_handle);
459 return KERN_SUCCESS;
460 }
461
462 kern_return_t
vnode_pager_get_throttle_io_limit(memory_object_t mem_obj,uint32_t * limit)463 vnode_pager_get_throttle_io_limit(
464 memory_object_t mem_obj,
465 uint32_t *limit)
466 {
467 vnode_pager_t vnode_object;
468
469 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
470 return KERN_INVALID_ARGUMENT;
471 }
472
473 vnode_object = vnode_pager_lookup(mem_obj);
474
475 (void)vnode_pager_return_throttle_io_limit(vnode_object->vnode_handle, limit);
476 return KERN_SUCCESS;
477 }
478
479 kern_return_t
vnode_pager_get_isSSD(memory_object_t mem_obj,boolean_t * isSSD)480 vnode_pager_get_isSSD(
481 memory_object_t mem_obj,
482 boolean_t *isSSD)
483 {
484 vnode_pager_t vnode_object;
485
486 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
487 return KERN_INVALID_ARGUMENT;
488 }
489
490 vnode_object = vnode_pager_lookup(mem_obj);
491
492 *isSSD = vnode_pager_isSSD(vnode_object->vnode_handle);
493 return KERN_SUCCESS;
494 }
495
496 #if FBDP_DEBUG_OBJECT_NO_PAGER
497 kern_return_t
vnode_pager_get_forced_unmount(memory_object_t mem_obj,bool * forced_unmount)498 vnode_pager_get_forced_unmount(
499 memory_object_t mem_obj,
500 bool *forced_unmount)
501 {
502 vnode_pager_t vnode_object;
503
504 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
505 return KERN_INVALID_ARGUMENT;
506 }
507
508 vnode_object = vnode_pager_lookup(mem_obj);
509
510 *forced_unmount = vnode_pager_forced_unmount(vnode_object->vnode_handle);
511 return KERN_SUCCESS;
512 }
513 #endif /* FBDP_DEBUG_OBJECT_NO_PAGER */
514
515 kern_return_t
vnode_pager_get_object_size(memory_object_t mem_obj,memory_object_offset_t * length)516 vnode_pager_get_object_size(
517 memory_object_t mem_obj,
518 memory_object_offset_t *length)
519 {
520 vnode_pager_t vnode_object;
521
522 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
523 *length = 0;
524 return KERN_INVALID_ARGUMENT;
525 }
526
527 vnode_object = vnode_pager_lookup(mem_obj);
528
529 *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
530 return KERN_SUCCESS;
531 }
532
533 kern_return_t
vnode_pager_get_object_name(memory_object_t mem_obj,char * pathname,vm_size_t pathname_len,char * filename,vm_size_t filename_len,boolean_t * truncated_path_p)534 vnode_pager_get_object_name(
535 memory_object_t mem_obj,
536 char *pathname,
537 vm_size_t pathname_len,
538 char *filename,
539 vm_size_t filename_len,
540 boolean_t *truncated_path_p)
541 {
542 vnode_pager_t vnode_object;
543
544 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
545 return KERN_INVALID_ARGUMENT;
546 }
547
548 vnode_object = vnode_pager_lookup(mem_obj);
549
550 return vnode_pager_get_name(vnode_object->vnode_handle,
551 pathname,
552 pathname_len,
553 filename,
554 filename_len,
555 truncated_path_p);
556 }
557
558 kern_return_t
vnode_pager_get_object_mtime(memory_object_t mem_obj,struct timespec * mtime,struct timespec * cs_mtime)559 vnode_pager_get_object_mtime(
560 memory_object_t mem_obj,
561 struct timespec *mtime,
562 struct timespec *cs_mtime)
563 {
564 vnode_pager_t vnode_object;
565
566 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
567 return KERN_INVALID_ARGUMENT;
568 }
569
570 vnode_object = vnode_pager_lookup(mem_obj);
571
572 return vnode_pager_get_mtime(vnode_object->vnode_handle,
573 mtime,
574 cs_mtime);
575 }
576
577 #if CHECK_CS_VALIDATION_BITMAP
578 kern_return_t
vnode_pager_cs_check_validation_bitmap(memory_object_t mem_obj,memory_object_offset_t offset,int optype)579 vnode_pager_cs_check_validation_bitmap(
580 memory_object_t mem_obj,
581 memory_object_offset_t offset,
582 int optype )
583 {
584 vnode_pager_t vnode_object;
585
586 if (mem_obj == MEMORY_OBJECT_NULL ||
587 mem_obj->mo_pager_ops != &vnode_pager_ops) {
588 return KERN_INVALID_ARGUMENT;
589 }
590
591 vnode_object = vnode_pager_lookup(mem_obj);
592 return ubc_cs_check_validation_bitmap( vnode_object->vnode_handle, offset, optype );
593 }
594 #endif /* CHECK_CS_VALIDATION_BITMAP */
595
596 /*
597 *
598 */
599 kern_return_t
vnode_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,__unused memory_object_cluster_size_t length,__unused vm_prot_t desired_access,memory_object_fault_info_t fault_info)600 vnode_pager_data_request(
601 memory_object_t mem_obj,
602 memory_object_offset_t offset,
603 __unused memory_object_cluster_size_t length,
604 __unused vm_prot_t desired_access,
605 memory_object_fault_info_t fault_info)
606 {
607 vnode_pager_t vnode_object;
608 memory_object_offset_t base_offset;
609 vm_size_t size;
610 uint32_t io_streaming = 0;
611
612 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
613
614 vnode_object = vnode_pager_lookup(mem_obj);
615
616 size = MAX_UPL_TRANSFER_BYTES;
617 base_offset = offset;
618
619 if (memory_object_cluster_size(vnode_object->vn_pgr_hdr.mo_control,
620 &base_offset, &size, &io_streaming,
621 fault_info) != KERN_SUCCESS) {
622 size = PAGE_SIZE;
623 }
624
625 assert(offset >= base_offset &&
626 offset < base_offset + size);
627
628 return vnode_pager_cluster_read(vnode_object, base_offset, offset, io_streaming, size);
629 }
630
631 /*
632 *
633 */
634 void
vnode_pager_reference(memory_object_t mem_obj)635 vnode_pager_reference(
636 memory_object_t mem_obj)
637 {
638 vnode_pager_t vnode_object;
639
640 vnode_object = vnode_pager_lookup(mem_obj);
641 os_ref_retain_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
642 }
643
644 /*
645 *
646 */
647 void
vnode_pager_deallocate(memory_object_t mem_obj)648 vnode_pager_deallocate(
649 memory_object_t mem_obj)
650 {
651 vnode_pager_t vnode_object;
652
653 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %p\n", mem_obj));
654
655 vnode_object = vnode_pager_lookup(mem_obj);
656
657 if (os_ref_release_raw(&vnode_object->vn_pgr_hdr_ref, NULL) == 0) {
658 if (vnode_object->vnode_handle != NULL) {
659 vnode_pager_vrele(vnode_object->vnode_handle);
660 }
661 zfree(vnode_pager_zone, vnode_object);
662 }
663 }
664
665 /*
666 *
667 */
668 kern_return_t
vnode_pager_terminate(__unused memory_object_t mem_obj)669 vnode_pager_terminate(
670 #if !DEBUG
671 __unused
672 #endif
673 memory_object_t mem_obj)
674 {
675 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %p\n", mem_obj));
676
677 return KERN_SUCCESS;
678 }
679
680 /*
681 *
682 */
683 kern_return_t
vnode_pager_map(memory_object_t mem_obj,vm_prot_t prot)684 vnode_pager_map(
685 memory_object_t mem_obj,
686 vm_prot_t prot)
687 {
688 vnode_pager_t vnode_object;
689 int ret;
690 kern_return_t kr;
691
692 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_map: %p %x\n", mem_obj, prot));
693
694 vnode_object = vnode_pager_lookup(mem_obj);
695
696 ret = ubc_map(vnode_object->vnode_handle, prot);
697
698 if (ret != 0) {
699 kr = KERN_FAILURE;
700 } else {
701 kr = KERN_SUCCESS;
702 }
703
704 return kr;
705 }
706
707 kern_return_t
vnode_pager_last_unmap(memory_object_t mem_obj)708 vnode_pager_last_unmap(
709 memory_object_t mem_obj)
710 {
711 vnode_pager_t vnode_object;
712
713 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_last_unmap: %p\n", mem_obj));
714
715 vnode_object = vnode_pager_lookup(mem_obj);
716
717 ubc_unmap(vnode_object->vnode_handle);
718 return KERN_SUCCESS;
719 }
720
721
722
723 /*
724 *
725 */
726 void
vnode_pager_cluster_write(vnode_pager_t vnode_object,vm_object_offset_t offset,vm_size_t cnt,vm_object_offset_t * resid_offset,int * io_error,int upl_flags)727 vnode_pager_cluster_write(
728 vnode_pager_t vnode_object,
729 vm_object_offset_t offset,
730 vm_size_t cnt,
731 vm_object_offset_t * resid_offset,
732 int * io_error,
733 int upl_flags)
734 {
735 vm_size_t size;
736 int errno;
737
738 if (upl_flags & UPL_MSYNC) {
739 upl_flags |= UPL_VNODE_PAGER;
740
741 if ((upl_flags & UPL_IOSYNC) && io_error) {
742 upl_flags |= UPL_KEEPCACHED;
743 }
744
745 while (cnt) {
746 size = (cnt < MAX_UPL_TRANSFER_BYTES) ? cnt : MAX_UPL_TRANSFER_BYTES; /* effective max */
747
748 assert((upl_size_t) size == size);
749 vnode_pageout(vnode_object->vnode_handle,
750 NULL, (upl_offset_t)0, offset, (upl_size_t)size, upl_flags, &errno);
751
752 if ((upl_flags & UPL_KEEPCACHED)) {
753 if ((*io_error = errno)) {
754 break;
755 }
756 }
757 cnt -= size;
758 offset += size;
759 }
760 if (resid_offset) {
761 *resid_offset = offset;
762 }
763 } else {
764 vm_object_offset_t vnode_size;
765 vm_object_offset_t base_offset;
766
767 /*
768 * this is the pageout path
769 */
770 vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle);
771
772 if (vnode_size > (offset + PAGE_SIZE)) {
773 /*
774 * preset the maximum size of the cluster
775 * and put us on a nice cluster boundary...
776 * and then clip the size to insure we
777 * don't request past the end of the underlying file
778 */
779 size = MAX_UPL_TRANSFER_BYTES;
780 base_offset = offset & ~((signed)(size - 1));
781
782 if ((base_offset + size) > vnode_size) {
783 size = round_page(((vm_size_t)(vnode_size - base_offset)));
784 }
785 } else {
786 /*
787 * we've been requested to page out a page beyond the current
788 * end of the 'file'... don't try to cluster in this case...
789 * we still need to send this page through because it might
790 * be marked precious and the underlying filesystem may need
791 * to do something with it (besides page it out)...
792 */
793 base_offset = offset;
794 size = PAGE_SIZE;
795 }
796 assert((upl_size_t) size == size);
797 vnode_pageout(vnode_object->vnode_handle,
798 NULL, (upl_offset_t)(offset - base_offset), base_offset, (upl_size_t) size,
799 (upl_flags & UPL_IOSYNC) | UPL_VNODE_PAGER, NULL);
800 }
801 }
802
803
804 /*
805 *
806 */
807 kern_return_t
vnode_pager_cluster_read(vnode_pager_t vnode_object,vm_object_offset_t base_offset,vm_object_offset_t offset,uint32_t io_streaming,vm_size_t cnt)808 vnode_pager_cluster_read(
809 vnode_pager_t vnode_object,
810 vm_object_offset_t base_offset,
811 vm_object_offset_t offset,
812 uint32_t io_streaming,
813 vm_size_t cnt)
814 {
815 int local_error = 0;
816 int kret;
817 int flags = 0;
818
819 assert(!(cnt & PAGE_MASK));
820
821 if (io_streaming) {
822 flags |= UPL_IOSTREAMING;
823 }
824
825 assert((upl_size_t) cnt == cnt);
826 kret = vnode_pagein(vnode_object->vnode_handle,
827 (upl_t) NULL,
828 (upl_offset_t) (offset - base_offset),
829 base_offset,
830 (upl_size_t) cnt,
831 flags,
832 &local_error);
833 /*
834 * if(kret == PAGER_ABSENT) {
835 * Need to work out the defs here, 1 corresponds to PAGER_ABSENT
836 * defined in bsd/vm/vm_pager.h However, we should not be including
837 * that file here it is a layering violation.
838 */
839 if (kret == 1) {
840 int uplflags;
841 upl_t upl = NULL;
842 unsigned int count = 0;
843 kern_return_t kr;
844
845 uplflags = (UPL_NO_SYNC |
846 UPL_CLEAN_IN_PLACE |
847 UPL_SET_INTERNAL);
848 count = 0;
849 assert((upl_size_t) cnt == cnt);
850 kr = memory_object_upl_request(vnode_object->vn_pgr_hdr.mo_control,
851 base_offset, (upl_size_t) cnt,
852 &upl, NULL, &count, uplflags, VM_KERN_MEMORY_NONE);
853 if (kr == KERN_SUCCESS) {
854 upl_abort(upl, 0);
855 upl_deallocate(upl);
856 } else {
857 /*
858 * We couldn't gather the page list, probably
859 * because the memory object doesn't have a link
860 * to a VM object anymore (forced unmount, for
861 * example). Just return an error to the vm_fault()
862 * path and let it handle it.
863 */
864 }
865
866 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_VNODEPAGER_CLREAD_NO_UPL), 0 /* arg */);
867 return KERN_FAILURE;
868 }
869
870 return KERN_SUCCESS;
871 }
872
873 /*
874 *
875 */
876 vnode_pager_t
vnode_object_create(struct vnode * vp)877 vnode_object_create(
878 struct vnode *vp)
879 {
880 vnode_pager_t vnode_object;
881
882 vnode_object = zalloc_flags(vnode_pager_zone, Z_WAITOK | Z_NOFAIL);
883
884 /*
885 * The vm_map call takes both named entry ports and raw memory
886 * objects in the same parameter. We need to make sure that
887 * vm_map does not see this object as a named entry port. So,
888 * we reserve the first word in the object for a fake ip_kotype
889 * setting - that will tell vm_map to use it as a memory object.
890 */
891 vnode_object->vn_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
892 vnode_object->vn_pgr_hdr.mo_pager_ops = &vnode_pager_ops;
893 vnode_object->vn_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
894
895 os_ref_init_raw(&vnode_object->vn_pgr_hdr_ref, NULL);
896 vnode_object->vnode_handle = vp;
897
898 return vnode_object;
899 }
900
901 /*
902 *
903 */
904 vnode_pager_t
vnode_pager_lookup(memory_object_t name)905 vnode_pager_lookup(
906 memory_object_t name)
907 {
908 vnode_pager_t vnode_object;
909
910 vnode_object = (vnode_pager_t)name;
911 assert(vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops);
912 return vnode_object;
913 }
914
915
916 struct vnode *
vnode_pager_lookup_vnode(memory_object_t name)917 vnode_pager_lookup_vnode(
918 memory_object_t name)
919 {
920 vnode_pager_t vnode_object;
921 vnode_object = (vnode_pager_t)name;
922 if (vnode_object->vn_pgr_hdr.mo_pager_ops == &vnode_pager_ops) {
923 return vnode_object->vnode_handle;
924 } else {
925 return NULL;
926 }
927 }
928
929 /*********************** proc_info implementation *************/
930
931 #include <sys/bsdtask_info.h>
932
933 static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uintptr_t * vnodeaddr, uint32_t * vid);
934
935 int
fill_procregioninfo(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)936 fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
937 {
938 vm_map_t map;
939 vm_map_offset_t address = (vm_map_offset_t)arg;
940 vm_map_entry_t tmp_entry;
941 vm_map_entry_t entry;
942 vm_map_offset_t start;
943 vm_region_extended_info_data_t extended;
944 vm_region_top_info_data_t top;
945 boolean_t do_region_footprint;
946 int effective_page_shift, effective_page_size;
947
948 task_lock(task);
949 map = task->map;
950 if (map == VM_MAP_NULL) {
951 task_unlock(task);
952 return 0;
953 }
954
955 effective_page_shift = vm_self_region_page_shift(map);
956 effective_page_size = (1 << effective_page_shift);
957
958 vm_map_reference(map);
959 task_unlock(task);
960
961 do_region_footprint = task_self_region_footprint();
962
963 vm_map_lock_read(map);
964
965 start = address;
966
967 if (!vm_map_lookup_entry_allow_pgz(map, start, &tmp_entry)) {
968 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
969 if (do_region_footprint &&
970 address == tmp_entry->vme_end) {
971 ledger_amount_t ledger_resident;
972 ledger_amount_t ledger_compressed;
973
974 /*
975 * This request is right after the last valid
976 * memory region; instead of reporting the
977 * end of the address space, report a fake
978 * memory region to account for non-volatile
979 * purgeable and/or ledger-tagged memory
980 * owned by this task.
981 */
982 task_ledgers_footprint(task->ledger,
983 &ledger_resident,
984 &ledger_compressed);
985 if (ledger_resident + ledger_compressed == 0) {
986 /* nothing to report */
987 vm_map_unlock_read(map);
988 vm_map_deallocate(map);
989 return 0;
990 }
991
992 /* provide fake region for purgeable */
993 pinfo->pri_offset = address;
994 pinfo->pri_protection = VM_PROT_DEFAULT;
995 pinfo->pri_max_protection = VM_PROT_DEFAULT;
996 pinfo->pri_inheritance = VM_INHERIT_NONE;
997 pinfo->pri_behavior = VM_BEHAVIOR_DEFAULT;
998 pinfo->pri_user_wired_count = 0;
999 pinfo->pri_user_tag = -1;
1000 pinfo->pri_pages_resident =
1001 (uint32_t) (ledger_resident / effective_page_size);
1002 pinfo->pri_pages_shared_now_private = 0;
1003 pinfo->pri_pages_swapped_out =
1004 (uint32_t) (ledger_compressed / effective_page_size);
1005 pinfo->pri_pages_dirtied =
1006 (uint32_t) (ledger_resident / effective_page_size);
1007 pinfo->pri_ref_count = 1;
1008 pinfo->pri_shadow_depth = 0;
1009 pinfo->pri_share_mode = SM_PRIVATE;
1010 pinfo->pri_private_pages_resident =
1011 (uint32_t) (ledger_resident / effective_page_size);
1012 pinfo->pri_shared_pages_resident = 0;
1013 pinfo->pri_obj_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile);
1014 pinfo->pri_address = address;
1015 pinfo->pri_size =
1016 (uint64_t) (ledger_resident + ledger_compressed);
1017 pinfo->pri_depth = 0;
1018
1019 vm_map_unlock_read(map);
1020 vm_map_deallocate(map);
1021 return 1;
1022 }
1023 vm_map_unlock_read(map);
1024 vm_map_deallocate(map);
1025 return 0;
1026 }
1027 } else {
1028 entry = tmp_entry;
1029 }
1030
1031 start = entry->vme_start;
1032
1033 pinfo->pri_offset = VME_OFFSET(entry);
1034 pinfo->pri_protection = entry->protection;
1035 pinfo->pri_max_protection = entry->max_protection;
1036 pinfo->pri_inheritance = entry->inheritance;
1037 pinfo->pri_behavior = entry->behavior;
1038 pinfo->pri_user_wired_count = entry->user_wired_count;
1039 pinfo->pri_user_tag = VME_ALIAS(entry);
1040
1041 if (entry->is_sub_map) {
1042 pinfo->pri_flags |= PROC_REGION_SUBMAP;
1043 } else {
1044 if (entry->is_shared) {
1045 pinfo->pri_flags |= PROC_REGION_SHARED;
1046 }
1047 }
1048
1049
1050 extended.protection = entry->protection;
1051 extended.user_tag = VME_ALIAS(entry);
1052 extended.pages_resident = 0;
1053 extended.pages_swapped_out = 0;
1054 extended.pages_shared_now_private = 0;
1055 extended.pages_dirtied = 0;
1056 extended.external_pager = 0;
1057 extended.shadow_depth = 0;
1058
1059 vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, &extended, TRUE, VM_REGION_EXTENDED_INFO_COUNT);
1060
1061 if (extended.external_pager && extended.ref_count == 2 && extended.share_mode == SM_SHARED) {
1062 extended.share_mode = SM_PRIVATE;
1063 }
1064
1065 top.private_pages_resident = 0;
1066 top.shared_pages_resident = 0;
1067 vm_map_region_top_walk(entry, &top);
1068
1069
1070 pinfo->pri_pages_resident = extended.pages_resident;
1071 pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private;
1072 pinfo->pri_pages_swapped_out = extended.pages_swapped_out;
1073 pinfo->pri_pages_dirtied = extended.pages_dirtied;
1074 pinfo->pri_ref_count = extended.ref_count;
1075 pinfo->pri_shadow_depth = extended.shadow_depth;
1076 pinfo->pri_share_mode = extended.share_mode;
1077
1078 pinfo->pri_private_pages_resident = top.private_pages_resident;
1079 pinfo->pri_shared_pages_resident = top.shared_pages_resident;
1080 pinfo->pri_obj_id = top.obj_id;
1081
1082 pinfo->pri_address = (uint64_t)start;
1083 pinfo->pri_size = (uint64_t)(entry->vme_end - start);
1084 pinfo->pri_depth = 0;
1085
1086 if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) {
1087 *vnodeaddr = (uintptr_t)0;
1088
1089 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid) == 0) {
1090 vm_map_unlock_read(map);
1091 vm_map_deallocate(map);
1092 return 1;
1093 }
1094 }
1095
1096 vm_map_unlock_read(map);
1097 vm_map_deallocate(map);
1098 return 1;
1099 }
1100
1101 int
fill_procregioninfo_onlymappedvnodes(task_t task,uint64_t arg,struct proc_regioninfo_internal * pinfo,uintptr_t * vnodeaddr,uint32_t * vid)1102 fill_procregioninfo_onlymappedvnodes(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uintptr_t *vnodeaddr, uint32_t *vid)
1103 {
1104 vm_map_t map;
1105 vm_map_offset_t address = (vm_map_offset_t)arg;
1106 vm_map_entry_t tmp_entry;
1107 vm_map_entry_t entry;
1108
1109 task_lock(task);
1110 map = task->map;
1111 if (map == VM_MAP_NULL) {
1112 task_unlock(task);
1113 return 0;
1114 }
1115 vm_map_reference(map);
1116 task_unlock(task);
1117
1118 vm_map_lock_read(map);
1119
1120 if (!vm_map_lookup_entry_allow_pgz(map, address, &tmp_entry)) {
1121 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1122 vm_map_unlock_read(map);
1123 vm_map_deallocate(map);
1124 return 0;
1125 }
1126 } else {
1127 entry = tmp_entry;
1128 }
1129
1130 while (entry != vm_map_to_entry(map)) {
1131 *vnodeaddr = 0;
1132 *vid = 0;
1133
1134 if (entry->is_sub_map == 0) {
1135 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid)) {
1136 pinfo->pri_offset = VME_OFFSET(entry);
1137 pinfo->pri_protection = entry->protection;
1138 pinfo->pri_max_protection = entry->max_protection;
1139 pinfo->pri_inheritance = entry->inheritance;
1140 pinfo->pri_behavior = entry->behavior;
1141 pinfo->pri_user_wired_count = entry->user_wired_count;
1142 pinfo->pri_user_tag = VME_ALIAS(entry);
1143
1144 if (entry->is_shared) {
1145 pinfo->pri_flags |= PROC_REGION_SHARED;
1146 }
1147
1148 pinfo->pri_pages_resident = 0;
1149 pinfo->pri_pages_shared_now_private = 0;
1150 pinfo->pri_pages_swapped_out = 0;
1151 pinfo->pri_pages_dirtied = 0;
1152 pinfo->pri_ref_count = 0;
1153 pinfo->pri_shadow_depth = 0;
1154 pinfo->pri_share_mode = 0;
1155
1156 pinfo->pri_private_pages_resident = 0;
1157 pinfo->pri_shared_pages_resident = 0;
1158 pinfo->pri_obj_id = 0;
1159
1160 pinfo->pri_address = (uint64_t)entry->vme_start;
1161 pinfo->pri_size = (uint64_t)(entry->vme_end - entry->vme_start);
1162 pinfo->pri_depth = 0;
1163
1164 vm_map_unlock_read(map);
1165 vm_map_deallocate(map);
1166 return 1;
1167 }
1168 }
1169
1170 /* Keep searching for a vnode-backed mapping */
1171 entry = entry->vme_next;
1172 }
1173
1174 vm_map_unlock_read(map);
1175 vm_map_deallocate(map);
1176 return 0;
1177 }
1178
1179 int
find_region_details(task_t task,vm_map_offset_t offset,uintptr_t * vnodeaddr,uint32_t * vid,uint64_t * start,uint64_t * len)1180 find_region_details(task_t task, vm_map_offset_t offset,
1181 uintptr_t *vnodeaddr, uint32_t *vid,
1182 uint64_t *start, uint64_t *len)
1183 {
1184 vm_map_t map;
1185 vm_map_entry_t tmp_entry, entry;
1186 int rc = 0;
1187
1188 task_lock(task);
1189 map = task->map;
1190 if (map == VM_MAP_NULL) {
1191 task_unlock(task);
1192 return 0;
1193 }
1194 vm_map_reference(map);
1195 task_unlock(task);
1196
1197 vm_map_lock_read(map);
1198 if (!vm_map_lookup_entry_allow_pgz(map, offset, &tmp_entry)) {
1199 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
1200 rc = 0;
1201 goto ret;
1202 }
1203 } else {
1204 entry = tmp_entry;
1205 }
1206
1207 while (entry != vm_map_to_entry(map)) {
1208 *vnodeaddr = 0;
1209 *vid = 0;
1210 *start = 0;
1211 *len = 0;
1212
1213 if (entry->is_sub_map == 0) {
1214 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid)) {
1215 *start = entry->vme_start;
1216 *len = entry->vme_end - entry->vme_start;
1217 rc = 1;
1218 goto ret;
1219 }
1220 }
1221
1222 entry = entry->vme_next;
1223 }
1224
1225 ret:
1226 vm_map_unlock_read(map);
1227 vm_map_deallocate(map);
1228 return rc;
1229 }
1230
1231 static int
fill_vnodeinfoforaddr(vm_map_entry_t entry,uintptr_t * vnodeaddr,uint32_t * vid)1232 fill_vnodeinfoforaddr(
1233 vm_map_entry_t entry,
1234 uintptr_t * vnodeaddr,
1235 uint32_t * vid)
1236 {
1237 vm_object_t top_object, object;
1238 memory_object_t memory_object;
1239 memory_object_pager_ops_t pager_ops;
1240 kern_return_t kr;
1241 int shadow_depth;
1242
1243
1244 if (entry->is_sub_map) {
1245 return 0;
1246 } else {
1247 /*
1248 * The last object in the shadow chain has the
1249 * relevant pager information.
1250 */
1251 top_object = VME_OBJECT(entry);
1252 if (top_object == VM_OBJECT_NULL) {
1253 object = VM_OBJECT_NULL;
1254 shadow_depth = 0;
1255 } else {
1256 vm_object_lock(top_object);
1257 for (object = top_object, shadow_depth = 0;
1258 object->shadow != VM_OBJECT_NULL;
1259 object = object->shadow, shadow_depth++) {
1260 vm_object_lock(object->shadow);
1261 vm_object_unlock(object);
1262 }
1263 }
1264 }
1265
1266 if (object == VM_OBJECT_NULL) {
1267 return 0;
1268 } else if (object->internal) {
1269 vm_object_unlock(object);
1270 return 0;
1271 } else if (!object->pager_ready ||
1272 object->terminating ||
1273 !object->alive ||
1274 object->pager == NULL) {
1275 vm_object_unlock(object);
1276 return 0;
1277 } else {
1278 memory_object = object->pager;
1279 pager_ops = memory_object->mo_pager_ops;
1280 if (pager_ops == &vnode_pager_ops) {
1281 kr = vnode_pager_get_object_vnode(
1282 memory_object,
1283 vnodeaddr, vid);
1284 if (kr != KERN_SUCCESS) {
1285 vm_object_unlock(object);
1286 return 0;
1287 }
1288 } else {
1289 vm_object_unlock(object);
1290 return 0;
1291 }
1292 }
1293 vm_object_unlock(object);
1294 return 1;
1295 }
1296
1297 kern_return_t
vnode_pager_get_object_vnode(memory_object_t mem_obj,uintptr_t * vnodeaddr,uint32_t * vid)1298 vnode_pager_get_object_vnode(
1299 memory_object_t mem_obj,
1300 uintptr_t * vnodeaddr,
1301 uint32_t * vid)
1302 {
1303 vnode_pager_t vnode_object;
1304
1305 vnode_object = vnode_pager_lookup(mem_obj);
1306 if (vnode_object->vnode_handle) {
1307 *vnodeaddr = (uintptr_t)vnode_object->vnode_handle;
1308 *vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle);
1309
1310 return KERN_SUCCESS;
1311 }
1312
1313 return KERN_FAILURE;
1314 }
1315
1316 #if CONFIG_IOSCHED
1317 kern_return_t
vnode_pager_get_object_devvp(memory_object_t mem_obj,uintptr_t * devvp)1318 vnode_pager_get_object_devvp(
1319 memory_object_t mem_obj,
1320 uintptr_t *devvp)
1321 {
1322 struct vnode *vp;
1323 uint32_t vid;
1324
1325 if (vnode_pager_get_object_vnode(mem_obj, (uintptr_t *)&vp, (uint32_t *)&vid) != KERN_SUCCESS) {
1326 return KERN_FAILURE;
1327 }
1328 *devvp = (uintptr_t)vnode_mountdevvp(vp);
1329 if (*devvp) {
1330 return KERN_SUCCESS;
1331 }
1332 return KERN_FAILURE;
1333 }
1334 #endif
1335
1336 /*
1337 * Find the underlying vnode object for the given vm_map_entry. If found, return with the
1338 * object locked, otherwise return NULL with nothing locked.
1339 */
1340
1341 vm_object_t
find_vnode_object(vm_map_entry_t entry)1342 find_vnode_object(
1343 vm_map_entry_t entry
1344 )
1345 {
1346 vm_object_t top_object, object;
1347 memory_object_t memory_object;
1348 memory_object_pager_ops_t pager_ops;
1349
1350 if (!entry->is_sub_map) {
1351 /*
1352 * The last object in the shadow chain has the
1353 * relevant pager information.
1354 */
1355
1356 top_object = VME_OBJECT(entry);
1357
1358 if (top_object) {
1359 vm_object_lock(top_object);
1360
1361 for (object = top_object; object->shadow != VM_OBJECT_NULL; object = object->shadow) {
1362 vm_object_lock(object->shadow);
1363 vm_object_unlock(object);
1364 }
1365
1366 if (object &&
1367 !object->internal &&
1368 object->pager_ready &&
1369 !object->terminating &&
1370 object->alive &&
1371 object->pager != NULL) {
1372 memory_object = object->pager;
1373 pager_ops = memory_object->mo_pager_ops;
1374
1375 /*
1376 * If this object points to the vnode_pager_ops, then we found what we're
1377 * looking for. Otherwise, this vm_map_entry doesn't have an underlying
1378 * vnode and so we fall through to the bottom and return NULL.
1379 */
1380
1381 if (pager_ops == &vnode_pager_ops) {
1382 return object; /* we return with the object locked */
1383 }
1384 }
1385
1386 vm_object_unlock(object);
1387 }
1388 }
1389
1390 return VM_OBJECT_NULL;
1391 }
1392