1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/memory_object.c
60 * Author: Michael Wayne Young
61 *
62 * External memory management interface control functions.
63 */
64
65 /*
66 * Interface dependencies:
67 */
68
69 #include <mach/std_types.h> /* For pointer_t */
70 #include <mach/mach_types.h>
71
72 #include <mach/mig.h>
73 #include <mach/kern_return.h>
74 #include <mach/memory_object.h>
75 #include <mach/memory_object_default.h>
76 #include <mach/memory_object_control_server.h>
77 #include <mach/host_priv_server.h>
78 #include <mach/boolean.h>
79 #include <mach/vm_prot.h>
80 #include <mach/message.h>
81
82 /*
83 * Implementation dependencies:
84 */
85 #include <string.h> /* For memcpy() */
86
87 #include <kern/host.h>
88 #include <kern/thread.h> /* For current_thread() */
89 #include <kern/ipc_mig.h>
90 #include <kern/misc_protos.h>
91
92 #include <vm/vm_object.h>
93 #include <vm/vm_fault.h>
94 #include <vm/memory_object.h>
95 #include <vm/vm_page.h>
96 #include <vm/vm_pageout.h>
97 #include <vm/pmap.h> /* For pmap_clear_modify */
98 #include <vm/vm_kern.h> /* For kernel_map, vm_move */
99 #include <vm/vm_map.h> /* For vm_map_pageable */
100 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
101 #include <vm/vm_shared_region.h>
102
103 #include <vm/vm_external.h>
104
105 #include <vm/vm_protos.h>
106
107 memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
108 LCK_MTX_EARLY_DECLARE(memory_manager_default_lock, &vm_object_lck_grp);
109
110
111 /*
112 * Routine: memory_object_should_return_page
113 *
114 * Description:
115 * Determine whether the given page should be returned,
116 * based on the page's state and on the given return policy.
117 *
118 * We should return the page if one of the following is true:
119 *
120 * 1. Page is dirty and should_return is not RETURN_NONE.
121 * 2. Page is precious and should_return is RETURN_ALL.
122 * 3. Should_return is RETURN_ANYTHING.
123 *
124 * As a side effect, m->vmp_dirty will be made consistent
125 * with pmap_is_modified(m), if should_return is not
126 * MEMORY_OBJECT_RETURN_NONE.
127 */
128
129 #define memory_object_should_return_page(m, should_return) \
130 (should_return != MEMORY_OBJECT_RETURN_NONE && \
131 (((m)->vmp_dirty || ((m)->vmp_dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m)))) || \
132 ((m)->vmp_precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \
133 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING))
134
135 typedef int memory_object_lock_result_t;
136
137 #define MEMORY_OBJECT_LOCK_RESULT_DONE 0
138 #define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1
139 #define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2
140 #define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3
141
142 memory_object_lock_result_t memory_object_lock_page(
143 vm_page_t m,
144 memory_object_return_t should_return,
145 boolean_t should_flush,
146 vm_prot_t prot);
147
148 /*
149 * Routine: memory_object_lock_page
150 *
151 * Description:
152 * Perform the appropriate lock operations on the
153 * given page. See the description of
154 * "memory_object_lock_request" for the meanings
155 * of the arguments.
156 *
157 * Returns an indication that the operation
158 * completed, blocked, or that the page must
159 * be cleaned.
160 */
161 memory_object_lock_result_t
memory_object_lock_page(vm_page_t m,memory_object_return_t should_return,boolean_t should_flush,vm_prot_t prot)162 memory_object_lock_page(
163 vm_page_t m,
164 memory_object_return_t should_return,
165 boolean_t should_flush,
166 vm_prot_t prot)
167 {
168 if (prot == VM_PROT_NO_CHANGE_LEGACY) {
169 prot = VM_PROT_NO_CHANGE;
170 }
171
172 if (m->vmp_busy || m->vmp_cleaning) {
173 return MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK;
174 }
175
176 if (m->vmp_laundry) {
177 vm_pageout_steal_laundry(m, FALSE);
178 }
179
180 /*
181 * Don't worry about pages for which the kernel
182 * does not have any data.
183 */
184 if (m->vmp_absent || m->vmp_error || m->vmp_restart) {
185 if (m->vmp_error && should_flush && !VM_PAGE_WIRED(m)) {
186 /*
187 * dump the page, pager wants us to
188 * clean it up and there is no
189 * relevant data to return
190 */
191 return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE;
192 }
193 return MEMORY_OBJECT_LOCK_RESULT_DONE;
194 }
195 assert(!m->vmp_fictitious);
196
197 if (VM_PAGE_WIRED(m)) {
198 /*
199 * The page is wired... just clean or return the page if needed.
200 * Wired pages don't get flushed or disconnected from the pmap.
201 */
202 if (memory_object_should_return_page(m, should_return)) {
203 return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN;
204 }
205
206 return MEMORY_OBJECT_LOCK_RESULT_DONE;
207 }
208
209 if (should_flush) {
210 /*
211 * must do the pmap_disconnect before determining the
212 * need to return the page... otherwise it's possible
213 * for the page to go from the clean to the dirty state
214 * after we've made our decision
215 */
216 if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
217 SET_PAGE_DIRTY(m, FALSE);
218 }
219 } else {
220 /*
221 * If we are decreasing permission, do it now;
222 * let the fault handler take care of increases
223 * (pmap_page_protect may not increase protection).
224 */
225 if (prot != VM_PROT_NO_CHANGE) {
226 pmap_page_protect(VM_PAGE_GET_PHYS_PAGE(m), VM_PROT_ALL & ~prot);
227 }
228 }
229 /*
230 * Handle returning dirty or precious pages
231 */
232 if (memory_object_should_return_page(m, should_return)) {
233 /*
234 * we use to do a pmap_disconnect here in support
235 * of memory_object_lock_request, but that routine
236 * no longer requires this... in any event, in
237 * our world, it would turn into a big noop since
238 * we don't lock the page in any way and as soon
239 * as we drop the object lock, the page can be
240 * faulted back into an address space
241 *
242 * if (!should_flush)
243 * pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
244 */
245 return MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN;
246 }
247
248 /*
249 * Handle flushing clean pages
250 */
251 if (should_flush) {
252 return MEMORY_OBJECT_LOCK_RESULT_MUST_FREE;
253 }
254
255 /*
256 * we use to deactivate clean pages at this point,
257 * but we do not believe that an msync should change
258 * the 'age' of a page in the cache... here is the
259 * original comment and code concerning this...
260 *
261 * XXX Make clean but not flush a paging hint,
262 * and deactivate the pages. This is a hack
263 * because it overloads flush/clean with
264 * implementation-dependent meaning. This only
265 * happens to pages that are already clean.
266 *
267 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE))
268 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE);
269 */
270
271 return MEMORY_OBJECT_LOCK_RESULT_DONE;
272 }
273
274
275
276 /*
277 * Routine: memory_object_lock_request [user interface]
278 *
279 * Description:
280 * Control use of the data associated with the given
281 * memory object. For each page in the given range,
282 * perform the following operations, in order:
283 * 1) restrict access to the page (disallow
284 * forms specified by "prot");
285 * 2) return data to the manager (if "should_return"
286 * is RETURN_DIRTY and the page is dirty, or
287 * "should_return" is RETURN_ALL and the page
288 * is either dirty or precious); and,
289 * 3) flush the cached copy (if "should_flush"
290 * is asserted).
291 * The set of pages is defined by a starting offset
292 * ("offset") and size ("size"). Only pages with the
293 * same page alignment as the starting offset are
294 * considered.
295 *
296 * A single acknowledgement is sent (to the "reply_to"
297 * port) when these actions are complete. If successful,
298 * the naked send right for reply_to is consumed.
299 */
300
301 kern_return_t
memory_object_lock_request(memory_object_control_t control,memory_object_offset_t offset,memory_object_size_t size,memory_object_offset_t * resid_offset,int * io_errno,memory_object_return_t should_return,int flags,vm_prot_t prot)302 memory_object_lock_request(
303 memory_object_control_t control,
304 memory_object_offset_t offset,
305 memory_object_size_t size,
306 memory_object_offset_t * resid_offset,
307 int * io_errno,
308 memory_object_return_t should_return,
309 int flags,
310 vm_prot_t prot)
311 {
312 vm_object_t object;
313
314 if (prot == VM_PROT_NO_CHANGE_LEGACY) {
315 prot = VM_PROT_NO_CHANGE;
316 }
317
318 /*
319 * Check for bogus arguments.
320 */
321 object = memory_object_control_to_vm_object(control);
322 if (object == VM_OBJECT_NULL) {
323 return KERN_INVALID_ARGUMENT;
324 }
325
326 if ((prot & ~(VM_PROT_ALL | VM_PROT_ALLEXEC)) != 0 && prot != VM_PROT_NO_CHANGE) {
327 return KERN_INVALID_ARGUMENT;
328 }
329
330 size = round_page_64(size);
331
332 /*
333 * Lock the object, and acquire a paging reference to
334 * prevent the memory_object reference from being released.
335 */
336 vm_object_lock(object);
337 vm_object_paging_begin(object);
338
339 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
340 if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) {
341 flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL;
342 flags |= MEMORY_OBJECT_DATA_FLUSH;
343 }
344 }
345 offset -= object->paging_offset;
346
347 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) {
348 vm_object_reap_pages(object, REAP_DATA_FLUSH);
349 } else {
350 (void)vm_object_update(object, offset, size, resid_offset,
351 io_errno, should_return, flags, prot);
352 }
353
354 vm_object_paging_end(object);
355 vm_object_unlock(object);
356
357 return KERN_SUCCESS;
358 }
359
360 /*
361 * memory_object_release_name: [interface]
362 *
363 * Enforces name semantic on memory_object reference count decrement
364 * This routine should not be called unless the caller holds a name
365 * reference gained through the memory_object_named_create or the
366 * memory_object_rename call.
367 * If the TERMINATE_IDLE flag is set, the call will return if the
368 * reference count is not 1. i.e. idle with the only remaining reference
369 * being the name.
370 * If the decision is made to proceed the name field flag is set to
371 * false and the reference count is decremented. If the RESPECT_CACHE
372 * flag is set and the reference count has gone to zero, the
373 * memory_object is checked to see if it is cacheable otherwise when
374 * the reference count is zero, it is simply terminated.
375 */
376
377 kern_return_t
memory_object_release_name(memory_object_control_t control,int flags)378 memory_object_release_name(
379 memory_object_control_t control,
380 int flags)
381 {
382 vm_object_t object;
383
384 object = memory_object_control_to_vm_object(control);
385 if (object == VM_OBJECT_NULL) {
386 return KERN_INVALID_ARGUMENT;
387 }
388
389 return vm_object_release_name(object, flags);
390 }
391
392
393
394 /*
395 * Routine: memory_object_destroy [user interface]
396 * Purpose:
397 * Shut down a memory object, despite the
398 * presence of address map (or other) references
399 * to the vm_object.
400 */
401 kern_return_t
memory_object_destroy(memory_object_control_t control,kern_return_t reason)402 memory_object_destroy(
403 memory_object_control_t control,
404 kern_return_t reason)
405 {
406 vm_object_t object;
407
408 object = memory_object_control_to_vm_object(control);
409 if (object == VM_OBJECT_NULL) {
410 return KERN_INVALID_ARGUMENT;
411 }
412
413 return vm_object_destroy(object, reason);
414 }
415
416 /*
417 * Routine: vm_object_sync
418 *
419 * Kernel internal function to synch out pages in a given
420 * range within an object to its memory manager. Much the
421 * same as memory_object_lock_request but page protection
422 * is not changed.
423 *
424 * If the should_flush and should_return flags are true pages
425 * are flushed, that is dirty & precious pages are written to
426 * the memory manager and then discarded. If should_return
427 * is false, only precious pages are returned to the memory
428 * manager.
429 *
430 * If should flush is false and should_return true, the memory
431 * manager's copy of the pages is updated. If should_return
432 * is also false, only the precious pages are updated. This
433 * last option is of limited utility.
434 *
435 * Returns:
436 * FALSE if no pages were returned to the pager
437 * TRUE otherwise.
438 */
439
440 boolean_t
vm_object_sync(vm_object_t object,vm_object_offset_t offset,vm_object_size_t size,boolean_t should_flush,boolean_t should_return,boolean_t should_iosync)441 vm_object_sync(
442 vm_object_t object,
443 vm_object_offset_t offset,
444 vm_object_size_t size,
445 boolean_t should_flush,
446 boolean_t should_return,
447 boolean_t should_iosync)
448 {
449 boolean_t rv;
450 int flags;
451
452 /*
453 * Lock the object, and acquire a paging reference to
454 * prevent the memory_object and control ports from
455 * being destroyed.
456 */
457 vm_object_lock(object);
458 vm_object_paging_begin(object);
459
460 if (should_flush) {
461 flags = MEMORY_OBJECT_DATA_FLUSH;
462 /*
463 * This flush is from an msync(), not a truncate(), so the
464 * contents of the file are not affected.
465 * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know
466 * that the data is not changed and that there's no need to
467 * push the old contents to a copy object.
468 */
469 flags |= MEMORY_OBJECT_DATA_NO_CHANGE;
470 } else {
471 flags = 0;
472 }
473
474 if (should_iosync) {
475 flags |= MEMORY_OBJECT_IO_SYNC;
476 }
477
478 rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL,
479 (should_return) ?
480 MEMORY_OBJECT_RETURN_ALL :
481 MEMORY_OBJECT_RETURN_NONE,
482 flags,
483 VM_PROT_NO_CHANGE);
484
485
486 vm_object_paging_end(object);
487 vm_object_unlock(object);
488 return rv;
489 }
490
491
492
493 #define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \
494 MACRO_BEGIN \
495 \
496 int upl_flags; \
497 memory_object_t pager; \
498 \
499 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \
500 vm_object_paging_begin(object); \
501 vm_object_unlock(object); \
502 \
503 if (iosync) \
504 upl_flags = UPL_MSYNC | UPL_IOSYNC; \
505 else \
506 upl_flags = UPL_MSYNC; \
507 \
508 (void) memory_object_data_return(pager, \
509 po, \
510 (memory_object_cluster_size_t)data_cnt, \
511 ro, \
512 ioerr, \
513 FALSE, \
514 FALSE, \
515 upl_flags); \
516 \
517 vm_object_lock(object); \
518 vm_object_paging_end(object); \
519 } \
520 MACRO_END
521
522 extern struct vnode *
523 vnode_pager_lookup_vnode(memory_object_t);
524
525 static int
vm_object_update_extent(vm_object_t object,vm_object_offset_t offset,vm_object_offset_t offset_end,vm_object_offset_t * offset_resid,int * io_errno,boolean_t should_flush,memory_object_return_t should_return,boolean_t should_iosync,vm_prot_t prot)526 vm_object_update_extent(
527 vm_object_t object,
528 vm_object_offset_t offset,
529 vm_object_offset_t offset_end,
530 vm_object_offset_t *offset_resid,
531 int *io_errno,
532 boolean_t should_flush,
533 memory_object_return_t should_return,
534 boolean_t should_iosync,
535 vm_prot_t prot)
536 {
537 vm_page_t m;
538 int retval = 0;
539 vm_object_offset_t paging_offset = 0;
540 vm_object_offset_t next_offset = offset;
541 memory_object_lock_result_t page_lock_result;
542 memory_object_cluster_size_t data_cnt = 0;
543 struct vm_page_delayed_work dw_array;
544 struct vm_page_delayed_work *dwp, *dwp_start;
545 bool dwp_finish_ctx = TRUE;
546 int dw_count;
547 int dw_limit;
548 int dirty_count;
549
550 dwp_start = dwp = NULL;
551 dw_count = 0;
552 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
553 dwp_start = vm_page_delayed_work_get_ctx();
554 if (dwp_start == NULL) {
555 dwp_start = &dw_array;
556 dw_limit = 1;
557 dwp_finish_ctx = FALSE;
558 }
559 dwp = dwp_start;
560
561 dirty_count = 0;
562
563 for (;
564 offset < offset_end && object->resident_page_count;
565 offset += PAGE_SIZE_64) {
566 /*
567 * Limit the number of pages to be cleaned at once to a contiguous
568 * run, or at most MAX_UPL_TRANSFER_BYTES
569 */
570 if (data_cnt) {
571 if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) {
572 if (dw_count) {
573 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
574 dwp = dwp_start;
575 dw_count = 0;
576 }
577 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
578 paging_offset, offset_resid, io_errno, should_iosync);
579 data_cnt = 0;
580 }
581 }
582 while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
583 dwp->dw_mask = 0;
584
585 page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot);
586
587 if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) {
588 /*
589 * End of a run of dirty/precious pages.
590 */
591 if (dw_count) {
592 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
593 dwp = dwp_start;
594 dw_count = 0;
595 }
596 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
597 paging_offset, offset_resid, io_errno, should_iosync);
598 /*
599 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will
600 * allow the state of page 'm' to change... we need to re-lookup
601 * the current offset
602 */
603 data_cnt = 0;
604 continue;
605 }
606
607 switch (page_lock_result) {
608 case MEMORY_OBJECT_LOCK_RESULT_DONE:
609 break;
610
611 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE:
612 if (m->vmp_dirty == TRUE) {
613 dirty_count++;
614 }
615 dwp->dw_mask |= DW_vm_page_free;
616 break;
617
618 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK:
619 PAGE_SLEEP(object, m, THREAD_UNINT);
620 continue;
621
622 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN:
623 if (data_cnt == 0) {
624 paging_offset = offset;
625 }
626
627 data_cnt += PAGE_SIZE;
628 next_offset = offset + PAGE_SIZE_64;
629
630 /*
631 * wired pages shouldn't be flushed and
632 * since they aren't on any queue,
633 * no need to remove them
634 */
635 if (!VM_PAGE_WIRED(m)) {
636 if (should_flush) {
637 /*
638 * add additional state for the flush
639 */
640 m->vmp_free_when_done = TRUE;
641 }
642 /*
643 * we use to remove the page from the queues at this
644 * point, but we do not believe that an msync
645 * should cause the 'age' of a page to be changed
646 *
647 * else
648 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE;
649 */
650 }
651 retval = 1;
652 break;
653 }
654 if (dwp->dw_mask) {
655 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
656
657 if (dw_count >= dw_limit) {
658 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
659 dwp = dwp_start;
660 dw_count = 0;
661 }
662 }
663 break;
664 }
665 }
666
667 if (object->pager) {
668 task_update_logical_writes(current_task(), (dirty_count * PAGE_SIZE), TASK_WRITE_INVALIDATED, vnode_pager_lookup_vnode(object->pager));
669 }
670 /*
671 * We have completed the scan for applicable pages.
672 * Clean any pages that have been saved.
673 */
674 if (dw_count) {
675 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
676 }
677
678 if (data_cnt) {
679 LIST_REQ_PAGEOUT_PAGES(object, data_cnt,
680 paging_offset, offset_resid, io_errno, should_iosync);
681 }
682
683 if (dwp_start && dwp_finish_ctx) {
684 vm_page_delayed_work_finish_ctx(dwp_start);
685 dwp_start = dwp = NULL;
686 }
687
688 return retval;
689 }
690
691
692
693 /*
694 * Routine: vm_object_update
695 * Description:
696 * Work function for m_o_lock_request(), vm_o_sync().
697 *
698 * Called with object locked and paging ref taken.
699 */
700 kern_return_t
vm_object_update(vm_object_t object,vm_object_offset_t offset,vm_object_size_t size,vm_object_offset_t * resid_offset,int * io_errno,memory_object_return_t should_return,int flags,vm_prot_t protection)701 vm_object_update(
702 vm_object_t object,
703 vm_object_offset_t offset,
704 vm_object_size_t size,
705 vm_object_offset_t *resid_offset,
706 int *io_errno,
707 memory_object_return_t should_return,
708 int flags,
709 vm_prot_t protection)
710 {
711 vm_object_t copy_object = VM_OBJECT_NULL;
712 boolean_t data_returned = FALSE;
713 boolean_t update_cow;
714 boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
715 boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE;
716 vm_fault_return_t result;
717 int num_of_extents;
718 int n;
719 #define MAX_EXTENTS 8
720 #define EXTENT_SIZE (1024 * 1024 * 256)
721 #define RESIDENT_LIMIT (1024 * 32)
722 struct extent {
723 vm_object_offset_t e_base;
724 vm_object_offset_t e_min;
725 vm_object_offset_t e_max;
726 } extents[MAX_EXTENTS];
727
728 /*
729 * To avoid blocking while scanning for pages, save
730 * dirty pages to be cleaned all at once.
731 *
732 * XXXO A similar strategy could be used to limit the
733 * number of times that a scan must be restarted for
734 * other reasons. Those pages that would require blocking
735 * could be temporarily collected in another list, or
736 * their offsets could be recorded in a small array.
737 */
738
739 /*
740 * XXX NOTE: May want to consider converting this to a page list
741 * XXX vm_map_copy interface. Need to understand object
742 * XXX coalescing implications before doing so.
743 */
744
745 update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH)
746 && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) &&
747 !(flags & MEMORY_OBJECT_DATA_PURGE)))
748 || (flags & MEMORY_OBJECT_COPY_SYNC);
749
750 if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
751 int collisions = 0;
752
753 while ((copy_object = object->copy) != VM_OBJECT_NULL) {
754 /*
755 * need to do a try here since we're swimming upstream
756 * against the normal lock ordering... however, we need
757 * to hold the object stable until we gain control of the
758 * copy object so we have to be careful how we approach this
759 */
760 if (vm_object_lock_try(copy_object)) {
761 /*
762 * we 'won' the lock on the copy object...
763 * no need to hold the object lock any longer...
764 * take a real reference on the copy object because
765 * we're going to call vm_fault_page on it which may
766 * under certain conditions drop the lock and the paging
767 * reference we're about to take... the reference
768 * will keep the copy object from going away if that happens
769 */
770 vm_object_unlock(object);
771 vm_object_reference_locked(copy_object);
772 break;
773 }
774 vm_object_unlock(object);
775
776 collisions++;
777 mutex_pause(collisions);
778
779 vm_object_lock(object);
780 }
781 }
782 if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
783 vm_object_offset_t i;
784 vm_object_size_t copy_size;
785 vm_object_offset_t copy_offset;
786 vm_prot_t prot;
787 vm_page_t page;
788 vm_page_t top_page;
789 kern_return_t error = 0;
790 struct vm_object_fault_info fault_info = {};
791
792 if (copy_object != VM_OBJECT_NULL) {
793 /*
794 * translate offset with respect to shadow's offset
795 */
796 copy_offset = (offset >= copy_object->vo_shadow_offset) ?
797 (offset - copy_object->vo_shadow_offset) : 0;
798
799 if (copy_offset > copy_object->vo_size) {
800 copy_offset = copy_object->vo_size;
801 }
802
803 /*
804 * clip size with respect to shadow offset
805 */
806 if (offset >= copy_object->vo_shadow_offset) {
807 copy_size = size;
808 } else if (size >= copy_object->vo_shadow_offset - offset) {
809 copy_size = (size - (copy_object->vo_shadow_offset - offset));
810 } else {
811 copy_size = 0;
812 }
813
814 if (copy_offset + copy_size > copy_object->vo_size) {
815 if (copy_object->vo_size >= copy_offset) {
816 copy_size = copy_object->vo_size - copy_offset;
817 } else {
818 copy_size = 0;
819 }
820 }
821 copy_size += copy_offset;
822 } else {
823 copy_object = object;
824
825 copy_size = offset + size;
826 copy_offset = offset;
827 }
828 fault_info.interruptible = THREAD_UNINT;
829 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
830 fault_info.lo_offset = copy_offset;
831 fault_info.hi_offset = copy_size;
832 fault_info.stealth = TRUE;
833 assert(fault_info.cs_bypass == FALSE);
834 assert(fault_info.pmap_cs_associated == FALSE);
835
836 vm_object_paging_begin(copy_object);
837
838 for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
839 RETRY_COW_OF_LOCK_REQUEST:
840 fault_info.cluster_size = (vm_size_t) (copy_size - i);
841 assert(fault_info.cluster_size == copy_size - i);
842
843 prot = VM_PROT_WRITE | VM_PROT_READ;
844 page = VM_PAGE_NULL;
845 result = vm_fault_page(copy_object, i,
846 VM_PROT_WRITE | VM_PROT_READ,
847 FALSE,
848 FALSE, /* page not looked up */
849 &prot,
850 &page,
851 &top_page,
852 (int *)0,
853 &error,
854 FALSE,
855 &fault_info);
856
857 switch (result) {
858 case VM_FAULT_SUCCESS:
859 if (top_page) {
860 vm_fault_cleanup(
861 VM_PAGE_OBJECT(page), top_page);
862 vm_object_lock(copy_object);
863 vm_object_paging_begin(copy_object);
864 }
865 if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {
866 vm_page_lockspin_queues();
867
868 if ((!VM_PAGE_NON_SPECULATIVE_PAGEABLE(page))) {
869 vm_page_deactivate(page);
870 }
871 vm_page_unlock_queues();
872 }
873 PAGE_WAKEUP_DONE(page);
874 break;
875 case VM_FAULT_RETRY:
876 prot = VM_PROT_WRITE | VM_PROT_READ;
877 vm_object_lock(copy_object);
878 vm_object_paging_begin(copy_object);
879 goto RETRY_COW_OF_LOCK_REQUEST;
880 case VM_FAULT_INTERRUPTED:
881 prot = VM_PROT_WRITE | VM_PROT_READ;
882 vm_object_lock(copy_object);
883 vm_object_paging_begin(copy_object);
884 goto RETRY_COW_OF_LOCK_REQUEST;
885 case VM_FAULT_MEMORY_SHORTAGE:
886 VM_PAGE_WAIT();
887 prot = VM_PROT_WRITE | VM_PROT_READ;
888 vm_object_lock(copy_object);
889 vm_object_paging_begin(copy_object);
890 goto RETRY_COW_OF_LOCK_REQUEST;
891 case VM_FAULT_SUCCESS_NO_VM_PAGE:
892 /* success but no VM page: fail */
893 vm_object_paging_end(copy_object);
894 vm_object_unlock(copy_object);
895 OS_FALLTHROUGH;
896 case VM_FAULT_MEMORY_ERROR:
897 if (object != copy_object) {
898 vm_object_deallocate(copy_object);
899 }
900 vm_object_lock(object);
901 goto BYPASS_COW_COPYIN;
902 default:
903 panic("vm_object_update: unexpected error 0x%x"
904 " from vm_fault_page()\n", result);
905 }
906 }
907 vm_object_paging_end(copy_object);
908 }
909 if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
910 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
911 vm_object_unlock(copy_object);
912 vm_object_deallocate(copy_object);
913 vm_object_lock(object);
914 }
915 return KERN_SUCCESS;
916 }
917 if (copy_object != VM_OBJECT_NULL && copy_object != object) {
918 if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
919 vm_object_lock_assert_exclusive(copy_object);
920 copy_object->shadow_severed = TRUE;
921 copy_object->shadowed = FALSE;
922 copy_object->shadow = NULL;
923 /*
924 * delete the ref the COW was holding on the target object
925 */
926 vm_object_deallocate(object);
927 }
928 vm_object_unlock(copy_object);
929 vm_object_deallocate(copy_object);
930 vm_object_lock(object);
931 }
932 BYPASS_COW_COPYIN:
933
934 /*
935 * when we have a really large range to check relative
936 * to the number of actual resident pages, we'd like
937 * to use the resident page list to drive our checks
938 * however, the object lock will get dropped while processing
939 * the page which means the resident queue can change which
940 * means we can't walk the queue as we process the pages
941 * we also want to do the processing in offset order to allow
942 * 'runs' of pages to be collected if we're being told to
943 * flush to disk... the resident page queue is NOT ordered.
944 *
945 * a temporary solution (until we figure out how to deal with
946 * large address spaces more generically) is to pre-flight
947 * the resident page queue (if it's small enough) and develop
948 * a collection of extents (that encompass actual resident pages)
949 * to visit. This will at least allow us to deal with some of the
950 * more pathological cases in a more efficient manner. The current
951 * worst case (a single resident page at the end of an extremely large
952 * range) can take minutes to complete for ranges in the terrabyte
953 * category... since this routine is called when truncating a file,
954 * and we currently support files up to 16 Tbytes in size, this
955 * is not a theoretical problem
956 */
957
958 if ((object->resident_page_count < RESIDENT_LIMIT) &&
959 (atop_64(size) > (unsigned)(object->resident_page_count / (8 * MAX_EXTENTS)))) {
960 vm_page_t next;
961 vm_object_offset_t start;
962 vm_object_offset_t end;
963 vm_object_size_t e_mask;
964 vm_page_t m;
965
966 start = offset;
967 end = offset + size;
968 num_of_extents = 0;
969 e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1));
970
971 m = (vm_page_t) vm_page_queue_first(&object->memq);
972
973 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) m)) {
974 next = (vm_page_t) vm_page_queue_next(&m->vmp_listq);
975
976 if ((m->vmp_offset >= start) && (m->vmp_offset < end)) {
977 /*
978 * this is a page we're interested in
979 * try to fit it into a current extent
980 */
981 for (n = 0; n < num_of_extents; n++) {
982 if ((m->vmp_offset & e_mask) == extents[n].e_base) {
983 /*
984 * use (PAGE_SIZE - 1) to determine the
985 * max offset so that we don't wrap if
986 * we're at the last page of the space
987 */
988 if (m->vmp_offset < extents[n].e_min) {
989 extents[n].e_min = m->vmp_offset;
990 } else if ((m->vmp_offset + (PAGE_SIZE - 1)) > extents[n].e_max) {
991 extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1);
992 }
993 break;
994 }
995 }
996 if (n == num_of_extents) {
997 /*
998 * didn't find a current extent that can encompass
999 * this page
1000 */
1001 if (n < MAX_EXTENTS) {
1002 /*
1003 * if we still have room,
1004 * create a new extent
1005 */
1006 extents[n].e_base = m->vmp_offset & e_mask;
1007 extents[n].e_min = m->vmp_offset;
1008 extents[n].e_max = m->vmp_offset + (PAGE_SIZE - 1);
1009
1010 num_of_extents++;
1011 } else {
1012 /*
1013 * no room to create a new extent...
1014 * fall back to a single extent based
1015 * on the min and max page offsets
1016 * we find in the range we're interested in...
1017 * first, look through the extent list and
1018 * develop the overall min and max for the
1019 * pages we've looked at up to this point
1020 */
1021 for (n = 1; n < num_of_extents; n++) {
1022 if (extents[n].e_min < extents[0].e_min) {
1023 extents[0].e_min = extents[n].e_min;
1024 }
1025 if (extents[n].e_max > extents[0].e_max) {
1026 extents[0].e_max = extents[n].e_max;
1027 }
1028 }
1029 /*
1030 * now setup to run through the remaining pages
1031 * to determine the overall min and max
1032 * offset for the specified range
1033 */
1034 extents[0].e_base = 0;
1035 e_mask = 0;
1036 num_of_extents = 1;
1037
1038 /*
1039 * by continuing, we'll reprocess the
1040 * page that forced us to abandon trying
1041 * to develop multiple extents
1042 */
1043 continue;
1044 }
1045 }
1046 }
1047 m = next;
1048 }
1049 } else {
1050 extents[0].e_min = offset;
1051 extents[0].e_max = offset + (size - 1);
1052
1053 num_of_extents = 1;
1054 }
1055 for (n = 0; n < num_of_extents; n++) {
1056 if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno,
1057 should_flush, should_return, should_iosync, protection)) {
1058 data_returned = TRUE;
1059 }
1060 }
1061 return data_returned;
1062 }
1063
1064
1065 static kern_return_t
vm_object_set_attributes_common(vm_object_t object,boolean_t may_cache,memory_object_copy_strategy_t copy_strategy)1066 vm_object_set_attributes_common(
1067 vm_object_t object,
1068 boolean_t may_cache,
1069 memory_object_copy_strategy_t copy_strategy)
1070 {
1071 boolean_t object_became_ready;
1072
1073 if (object == VM_OBJECT_NULL) {
1074 return KERN_INVALID_ARGUMENT;
1075 }
1076
1077 /*
1078 * Verify the attributes of importance
1079 */
1080
1081 switch (copy_strategy) {
1082 case MEMORY_OBJECT_COPY_NONE:
1083 case MEMORY_OBJECT_COPY_DELAY:
1084 break;
1085 default:
1086 return KERN_INVALID_ARGUMENT;
1087 }
1088
1089 if (may_cache) {
1090 may_cache = TRUE;
1091 }
1092
1093 vm_object_lock(object);
1094
1095 /*
1096 * Copy the attributes
1097 */
1098 assert(!object->internal);
1099 object_became_ready = !object->pager_ready;
1100 object->copy_strategy = copy_strategy;
1101 object->can_persist = may_cache;
1102
1103 /*
1104 * Wake up anyone waiting for the ready attribute
1105 * to become asserted.
1106 */
1107
1108 if (object_became_ready) {
1109 object->pager_ready = TRUE;
1110 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
1111 }
1112
1113 vm_object_unlock(object);
1114
1115 return KERN_SUCCESS;
1116 }
1117
1118
1119 kern_return_t
memory_object_synchronize_completed(__unused memory_object_control_t control,__unused memory_object_offset_t offset,__unused memory_object_size_t length)1120 memory_object_synchronize_completed(
1121 __unused memory_object_control_t control,
1122 __unused memory_object_offset_t offset,
1123 __unused memory_object_size_t length)
1124 {
1125 panic("memory_object_synchronize_completed no longer supported");
1126 return KERN_FAILURE;
1127 }
1128
1129
1130 /*
1131 * Set the memory object attribute as provided.
1132 *
1133 * XXX This routine cannot be completed until the vm_msync, clean
1134 * in place, and cluster work is completed. See ifdef notyet
1135 * below and note that vm_object_set_attributes_common()
1136 * may have to be expanded.
1137 */
1138 kern_return_t
memory_object_change_attributes(memory_object_control_t control,memory_object_flavor_t flavor,memory_object_info_t attributes,mach_msg_type_number_t count)1139 memory_object_change_attributes(
1140 memory_object_control_t control,
1141 memory_object_flavor_t flavor,
1142 memory_object_info_t attributes,
1143 mach_msg_type_number_t count)
1144 {
1145 vm_object_t object;
1146 kern_return_t result = KERN_SUCCESS;
1147 boolean_t may_cache;
1148 boolean_t invalidate;
1149 memory_object_copy_strategy_t copy_strategy;
1150
1151 object = memory_object_control_to_vm_object(control);
1152 if (object == VM_OBJECT_NULL) {
1153 return KERN_INVALID_ARGUMENT;
1154 }
1155
1156 vm_object_lock(object);
1157
1158 may_cache = object->can_persist;
1159 copy_strategy = object->copy_strategy;
1160 #if notyet
1161 invalidate = object->invalidate;
1162 #endif
1163 vm_object_unlock(object);
1164
1165 switch (flavor) {
1166 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1167 {
1168 old_memory_object_behave_info_t behave;
1169
1170 if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1171 result = KERN_INVALID_ARGUMENT;
1172 break;
1173 }
1174
1175 behave = (old_memory_object_behave_info_t) attributes;
1176
1177 invalidate = behave->invalidate;
1178 copy_strategy = behave->copy_strategy;
1179
1180 break;
1181 }
1182
1183 case MEMORY_OBJECT_BEHAVIOR_INFO:
1184 {
1185 memory_object_behave_info_t behave;
1186
1187 if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1188 result = KERN_INVALID_ARGUMENT;
1189 break;
1190 }
1191
1192 behave = (memory_object_behave_info_t) attributes;
1193
1194 invalidate = behave->invalidate;
1195 copy_strategy = behave->copy_strategy;
1196 break;
1197 }
1198
1199 case MEMORY_OBJECT_PERFORMANCE_INFO:
1200 {
1201 memory_object_perf_info_t perf;
1202
1203 if (count != MEMORY_OBJECT_PERF_INFO_COUNT) {
1204 result = KERN_INVALID_ARGUMENT;
1205 break;
1206 }
1207
1208 perf = (memory_object_perf_info_t) attributes;
1209
1210 may_cache = perf->may_cache;
1211
1212 break;
1213 }
1214
1215 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1216 {
1217 old_memory_object_attr_info_t attr;
1218
1219 if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1220 result = KERN_INVALID_ARGUMENT;
1221 break;
1222 }
1223
1224 attr = (old_memory_object_attr_info_t) attributes;
1225
1226 may_cache = attr->may_cache;
1227 copy_strategy = attr->copy_strategy;
1228
1229 break;
1230 }
1231
1232 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1233 {
1234 memory_object_attr_info_t attr;
1235
1236 if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) {
1237 result = KERN_INVALID_ARGUMENT;
1238 break;
1239 }
1240
1241 attr = (memory_object_attr_info_t) attributes;
1242
1243 copy_strategy = attr->copy_strategy;
1244 may_cache = attr->may_cache_object;
1245
1246 break;
1247 }
1248
1249 default:
1250 result = KERN_INVALID_ARGUMENT;
1251 break;
1252 }
1253
1254 if (result != KERN_SUCCESS) {
1255 return result;
1256 }
1257
1258 if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) {
1259 copy_strategy = MEMORY_OBJECT_COPY_DELAY;
1260 }
1261
1262 /*
1263 * XXX may_cache may become a tri-valued variable to handle
1264 * XXX uncache if not in use.
1265 */
1266 return vm_object_set_attributes_common(object,
1267 may_cache,
1268 copy_strategy);
1269 }
1270
1271 kern_return_t
memory_object_get_attributes(memory_object_control_t control,memory_object_flavor_t flavor,memory_object_info_t attributes,mach_msg_type_number_t * count)1272 memory_object_get_attributes(
1273 memory_object_control_t control,
1274 memory_object_flavor_t flavor,
1275 memory_object_info_t attributes, /* pointer to OUT array */
1276 mach_msg_type_number_t *count) /* IN/OUT */
1277 {
1278 kern_return_t ret = KERN_SUCCESS;
1279 vm_object_t object;
1280
1281 object = memory_object_control_to_vm_object(control);
1282 if (object == VM_OBJECT_NULL) {
1283 return KERN_INVALID_ARGUMENT;
1284 }
1285
1286 vm_object_lock(object);
1287
1288 switch (flavor) {
1289 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO:
1290 {
1291 old_memory_object_behave_info_t behave;
1292
1293 if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1294 ret = KERN_INVALID_ARGUMENT;
1295 break;
1296 }
1297
1298 behave = (old_memory_object_behave_info_t) attributes;
1299 behave->copy_strategy = object->copy_strategy;
1300 behave->temporary = FALSE;
1301 #if notyet /* remove when vm_msync complies and clean in place fini */
1302 behave->invalidate = object->invalidate;
1303 #else
1304 behave->invalidate = FALSE;
1305 #endif
1306
1307 *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1308 break;
1309 }
1310
1311 case MEMORY_OBJECT_BEHAVIOR_INFO:
1312 {
1313 memory_object_behave_info_t behave;
1314
1315 if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) {
1316 ret = KERN_INVALID_ARGUMENT;
1317 break;
1318 }
1319
1320 behave = (memory_object_behave_info_t) attributes;
1321 behave->copy_strategy = object->copy_strategy;
1322 behave->temporary = FALSE;
1323 #if notyet /* remove when vm_msync complies and clean in place fini */
1324 behave->invalidate = object->invalidate;
1325 #else
1326 behave->invalidate = FALSE;
1327 #endif
1328 behave->advisory_pageout = FALSE;
1329 behave->silent_overwrite = FALSE;
1330 *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT;
1331 break;
1332 }
1333
1334 case MEMORY_OBJECT_PERFORMANCE_INFO:
1335 {
1336 memory_object_perf_info_t perf;
1337
1338 if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) {
1339 ret = KERN_INVALID_ARGUMENT;
1340 break;
1341 }
1342
1343 perf = (memory_object_perf_info_t) attributes;
1344 perf->cluster_size = PAGE_SIZE;
1345 perf->may_cache = object->can_persist;
1346
1347 *count = MEMORY_OBJECT_PERF_INFO_COUNT;
1348 break;
1349 }
1350
1351 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO:
1352 {
1353 old_memory_object_attr_info_t attr;
1354
1355 if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) {
1356 ret = KERN_INVALID_ARGUMENT;
1357 break;
1358 }
1359
1360 attr = (old_memory_object_attr_info_t) attributes;
1361 attr->may_cache = object->can_persist;
1362 attr->copy_strategy = object->copy_strategy;
1363
1364 *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT;
1365 break;
1366 }
1367
1368 case MEMORY_OBJECT_ATTRIBUTE_INFO:
1369 {
1370 memory_object_attr_info_t attr;
1371
1372 if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) {
1373 ret = KERN_INVALID_ARGUMENT;
1374 break;
1375 }
1376
1377 attr = (memory_object_attr_info_t) attributes;
1378 attr->copy_strategy = object->copy_strategy;
1379 attr->cluster_size = PAGE_SIZE;
1380 attr->may_cache_object = object->can_persist;
1381 attr->temporary = FALSE;
1382
1383 *count = MEMORY_OBJECT_ATTR_INFO_COUNT;
1384 break;
1385 }
1386
1387 default:
1388 ret = KERN_INVALID_ARGUMENT;
1389 break;
1390 }
1391
1392 vm_object_unlock(object);
1393
1394 return ret;
1395 }
1396
1397
1398 kern_return_t
memory_object_iopl_request(ipc_port_t port,memory_object_offset_t offset,upl_size_t * upl_size,upl_t * upl_ptr,upl_page_info_array_t user_page_list,unsigned int * page_list_count,upl_control_flags_t * flags,vm_tag_t tag)1399 memory_object_iopl_request(
1400 ipc_port_t port,
1401 memory_object_offset_t offset,
1402 upl_size_t *upl_size,
1403 upl_t *upl_ptr,
1404 upl_page_info_array_t user_page_list,
1405 unsigned int *page_list_count,
1406 upl_control_flags_t *flags,
1407 vm_tag_t tag)
1408 {
1409 vm_object_t object;
1410 kern_return_t ret;
1411 upl_control_flags_t caller_flags;
1412 vm_named_entry_t named_entry;
1413
1414 caller_flags = *flags;
1415
1416 if (caller_flags & ~UPL_VALID_FLAGS) {
1417 /*
1418 * For forward compatibility's sake,
1419 * reject any unknown flag.
1420 */
1421 return KERN_INVALID_VALUE;
1422 }
1423
1424 named_entry = mach_memory_entry_from_port(port);
1425 if (named_entry != NULL) {
1426 /* a few checks to make sure user is obeying rules */
1427 if (*upl_size == 0) {
1428 if (offset >= named_entry->size) {
1429 return KERN_INVALID_RIGHT;
1430 }
1431 *upl_size = (upl_size_t)(named_entry->size - offset);
1432 if (*upl_size != named_entry->size - offset) {
1433 return KERN_INVALID_ARGUMENT;
1434 }
1435 }
1436 if (caller_flags & UPL_COPYOUT_FROM) {
1437 if ((named_entry->protection & VM_PROT_READ)
1438 != VM_PROT_READ) {
1439 return KERN_INVALID_RIGHT;
1440 }
1441 } else {
1442 if ((named_entry->protection &
1443 (VM_PROT_READ | VM_PROT_WRITE))
1444 != (VM_PROT_READ | VM_PROT_WRITE)) {
1445 return KERN_INVALID_RIGHT;
1446 }
1447 }
1448 if (named_entry->size < (offset + *upl_size)) {
1449 return KERN_INVALID_ARGUMENT;
1450 }
1451
1452 /* the callers parameter offset is defined to be the */
1453 /* offset from beginning of named entry offset in object */
1454 offset = offset + named_entry->offset;
1455 offset += named_entry->data_offset;
1456
1457 if (named_entry->is_sub_map ||
1458 named_entry->is_copy) {
1459 return KERN_INVALID_ARGUMENT;
1460 }
1461 if (!named_entry->is_object) {
1462 return KERN_INVALID_ARGUMENT;
1463 }
1464
1465 named_entry_lock(named_entry);
1466
1467 object = vm_named_entry_to_vm_object(named_entry);
1468 assert(object != VM_OBJECT_NULL);
1469 vm_object_reference(object);
1470 named_entry_unlock(named_entry);
1471 } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
1472 panic("unexpected IKOT_MEM_OBJ_CONTROL: %p", port);
1473 } else {
1474 return KERN_INVALID_ARGUMENT;
1475 }
1476 if (object == VM_OBJECT_NULL) {
1477 return KERN_INVALID_ARGUMENT;
1478 }
1479
1480 if (!object->private) {
1481 if (object->phys_contiguous) {
1482 *flags = UPL_PHYS_CONTIG;
1483 } else {
1484 *flags = 0;
1485 }
1486 } else {
1487 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
1488 }
1489
1490 ret = vm_object_iopl_request(object,
1491 offset,
1492 *upl_size,
1493 upl_ptr,
1494 user_page_list,
1495 page_list_count,
1496 caller_flags,
1497 tag);
1498 vm_object_deallocate(object);
1499 return ret;
1500 }
1501
1502 /*
1503 * Routine: memory_object_upl_request [interface]
1504 * Purpose:
1505 * Cause the population of a portion of a vm_object.
1506 * Depending on the nature of the request, the pages
1507 * returned may be contain valid data or be uninitialized.
1508 *
1509 */
1510
1511 kern_return_t
memory_object_upl_request(memory_object_control_t control,memory_object_offset_t offset,upl_size_t size,upl_t * upl_ptr,upl_page_info_array_t user_page_list,unsigned int * page_list_count,int cntrl_flags,int tag)1512 memory_object_upl_request(
1513 memory_object_control_t control,
1514 memory_object_offset_t offset,
1515 upl_size_t size,
1516 upl_t *upl_ptr,
1517 upl_page_info_array_t user_page_list,
1518 unsigned int *page_list_count,
1519 int cntrl_flags,
1520 int tag)
1521 {
1522 vm_object_t object;
1523 vm_tag_t vmtag = (vm_tag_t)tag;
1524 assert(vmtag == tag);
1525
1526 object = memory_object_control_to_vm_object(control);
1527 if (object == VM_OBJECT_NULL) {
1528 return KERN_TERMINATED;
1529 }
1530
1531 return vm_object_upl_request(object,
1532 offset,
1533 size,
1534 upl_ptr,
1535 user_page_list,
1536 page_list_count,
1537 (upl_control_flags_t)(unsigned int) cntrl_flags,
1538 vmtag);
1539 }
1540
1541 /*
1542 * Routine: memory_object_super_upl_request [interface]
1543 * Purpose:
1544 * Cause the population of a portion of a vm_object
1545 * in much the same way as memory_object_upl_request.
1546 * Depending on the nature of the request, the pages
1547 * returned may be contain valid data or be uninitialized.
1548 * However, the region may be expanded up to the super
1549 * cluster size provided.
1550 */
1551
1552 kern_return_t
memory_object_super_upl_request(memory_object_control_t control,memory_object_offset_t offset,upl_size_t size,upl_size_t super_cluster,upl_t * upl,upl_page_info_t * user_page_list,unsigned int * page_list_count,int cntrl_flags,int tag)1553 memory_object_super_upl_request(
1554 memory_object_control_t control,
1555 memory_object_offset_t offset,
1556 upl_size_t size,
1557 upl_size_t super_cluster,
1558 upl_t *upl,
1559 upl_page_info_t *user_page_list,
1560 unsigned int *page_list_count,
1561 int cntrl_flags,
1562 int tag)
1563 {
1564 vm_object_t object;
1565 vm_tag_t vmtag = (vm_tag_t)tag;
1566 assert(vmtag == tag);
1567
1568 object = memory_object_control_to_vm_object(control);
1569 if (object == VM_OBJECT_NULL) {
1570 return KERN_INVALID_ARGUMENT;
1571 }
1572
1573 return vm_object_super_upl_request(object,
1574 offset,
1575 size,
1576 super_cluster,
1577 upl,
1578 user_page_list,
1579 page_list_count,
1580 (upl_control_flags_t)(unsigned int) cntrl_flags,
1581 vmtag);
1582 }
1583
1584 kern_return_t
memory_object_cluster_size(memory_object_control_t control,memory_object_offset_t * start,vm_size_t * length,uint32_t * io_streaming,memory_object_fault_info_t mo_fault_info)1585 memory_object_cluster_size(
1586 memory_object_control_t control,
1587 memory_object_offset_t *start,
1588 vm_size_t *length,
1589 uint32_t *io_streaming,
1590 memory_object_fault_info_t mo_fault_info)
1591 {
1592 vm_object_t object;
1593 vm_object_fault_info_t fault_info;
1594
1595 object = memory_object_control_to_vm_object(control);
1596
1597 if (object == VM_OBJECT_NULL || object->paging_offset > *start) {
1598 return KERN_INVALID_ARGUMENT;
1599 }
1600
1601 *start -= object->paging_offset;
1602
1603 fault_info = (vm_object_fault_info_t)(uintptr_t) mo_fault_info;
1604 vm_object_cluster_size(object,
1605 (vm_object_offset_t *)start,
1606 length,
1607 fault_info,
1608 io_streaming);
1609
1610 *start += object->paging_offset;
1611
1612 return KERN_SUCCESS;
1613 }
1614
1615
1616 /*
1617 * Routine: host_default_memory_manager [interface]
1618 * Purpose:
1619 * set/get the default memory manager port and default cluster
1620 * size.
1621 *
1622 * If successful, consumes the supplied naked send right.
1623 */
1624 kern_return_t
host_default_memory_manager(host_priv_t host_priv,memory_object_default_t * default_manager,__unused memory_object_cluster_size_t cluster_size)1625 host_default_memory_manager(
1626 host_priv_t host_priv,
1627 memory_object_default_t *default_manager,
1628 __unused memory_object_cluster_size_t cluster_size)
1629 {
1630 memory_object_default_t current_manager;
1631 memory_object_default_t new_manager;
1632 memory_object_default_t returned_manager;
1633 kern_return_t result = KERN_SUCCESS;
1634
1635 if (host_priv == HOST_PRIV_NULL) {
1636 return KERN_INVALID_HOST;
1637 }
1638
1639 new_manager = *default_manager;
1640 lck_mtx_lock(&memory_manager_default_lock);
1641 current_manager = memory_manager_default;
1642 returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
1643
1644 if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1645 /*
1646 * Retrieve the current value.
1647 */
1648 returned_manager = ipc_port_make_send(current_manager);
1649 } else {
1650 /*
1651 * Only allow the kernel to change the value.
1652 */
1653 extern task_t kernel_task;
1654 if (current_task() != kernel_task) {
1655 result = KERN_NO_ACCESS;
1656 goto out;
1657 }
1658
1659 /*
1660 * If this is the first non-null manager, start
1661 * up the internal pager support.
1662 */
1663 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1664 result = vm_pageout_internal_start();
1665 if (result != KERN_SUCCESS) {
1666 goto out;
1667 }
1668 }
1669
1670 /*
1671 * Retrieve the current value,
1672 * and replace it with the supplied value.
1673 * We return the old reference to the caller
1674 * but we have to take a reference on the new
1675 * one.
1676 */
1677 returned_manager = current_manager;
1678 memory_manager_default = ipc_port_make_send(new_manager);
1679
1680 /*
1681 * In case anyone's been waiting for a memory
1682 * manager to be established, wake them up.
1683 */
1684
1685 thread_wakeup((event_t) &memory_manager_default);
1686
1687 /*
1688 * Now that we have a default pager for anonymous memory,
1689 * reactivate all the throttled pages (i.e. dirty pages with
1690 * no pager).
1691 */
1692 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1693 vm_page_reactivate_all_throttled();
1694 }
1695 }
1696 out:
1697 lck_mtx_unlock(&memory_manager_default_lock);
1698
1699 *default_manager = returned_manager;
1700 return result;
1701 }
1702
1703 /*
1704 * Routine: memory_manager_default_reference
1705 * Purpose:
1706 * Returns a naked send right for the default
1707 * memory manager. The returned right is always
1708 * valid (not IP_NULL or IP_DEAD).
1709 */
1710
1711 __private_extern__ memory_object_default_t
memory_manager_default_reference(void)1712 memory_manager_default_reference(void)
1713 {
1714 memory_object_default_t current_manager;
1715
1716 lck_mtx_lock(&memory_manager_default_lock);
1717 current_manager = memory_manager_default;
1718 while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
1719 wait_result_t res;
1720
1721 res = lck_mtx_sleep(&memory_manager_default_lock,
1722 LCK_SLEEP_DEFAULT,
1723 (event_t) &memory_manager_default,
1724 THREAD_UNINT);
1725 assert(res == THREAD_AWAKENED);
1726 current_manager = memory_manager_default;
1727 }
1728 current_manager = ipc_port_make_send(current_manager);
1729 lck_mtx_unlock(&memory_manager_default_lock);
1730
1731 return current_manager;
1732 }
1733
1734 /*
1735 * Routine: memory_manager_default_check
1736 *
1737 * Purpose:
1738 * Check whether a default memory manager has been set
1739 * up yet, or not. Returns KERN_SUCCESS if dmm exists,
1740 * and KERN_FAILURE if dmm does not exist.
1741 *
1742 * If there is no default memory manager, log an error,
1743 * but only the first time.
1744 *
1745 */
1746 __private_extern__ kern_return_t
memory_manager_default_check(void)1747 memory_manager_default_check(void)
1748 {
1749 memory_object_default_t current;
1750
1751 lck_mtx_lock(&memory_manager_default_lock);
1752 current = memory_manager_default;
1753 if (current == MEMORY_OBJECT_DEFAULT_NULL) {
1754 static boolean_t logged; /* initialized to 0 */
1755 boolean_t complain = !logged;
1756 logged = TRUE;
1757 lck_mtx_unlock(&memory_manager_default_lock);
1758 if (complain) {
1759 printf("Warning: No default memory manager\n");
1760 }
1761 return KERN_FAILURE;
1762 } else {
1763 lck_mtx_unlock(&memory_manager_default_lock);
1764 return KERN_SUCCESS;
1765 }
1766 }
1767
1768 /* Allow manipulation of individual page state. This is actually part of */
1769 /* the UPL regimen but takes place on the object rather than on a UPL */
1770
1771 kern_return_t
memory_object_page_op(memory_object_control_t control,memory_object_offset_t offset,int ops,ppnum_t * phys_entry,int * flags)1772 memory_object_page_op(
1773 memory_object_control_t control,
1774 memory_object_offset_t offset,
1775 int ops,
1776 ppnum_t *phys_entry,
1777 int *flags)
1778 {
1779 vm_object_t object;
1780
1781 object = memory_object_control_to_vm_object(control);
1782 if (object == VM_OBJECT_NULL) {
1783 return KERN_INVALID_ARGUMENT;
1784 }
1785
1786 return vm_object_page_op(object, offset, ops, phys_entry, flags);
1787 }
1788
1789 /*
1790 * memory_object_range_op offers performance enhancement over
1791 * memory_object_page_op for page_op functions which do not require page
1792 * level state to be returned from the call. Page_op was created to provide
1793 * a low-cost alternative to page manipulation via UPLs when only a single
1794 * page was involved. The range_op call establishes the ability in the _op
1795 * family of functions to work on multiple pages where the lack of page level
1796 * state handling allows the caller to avoid the overhead of the upl structures.
1797 */
1798
1799 kern_return_t
memory_object_range_op(memory_object_control_t control,memory_object_offset_t offset_beg,memory_object_offset_t offset_end,int ops,int * range)1800 memory_object_range_op(
1801 memory_object_control_t control,
1802 memory_object_offset_t offset_beg,
1803 memory_object_offset_t offset_end,
1804 int ops,
1805 int *range)
1806 {
1807 vm_object_t object;
1808
1809 object = memory_object_control_to_vm_object(control);
1810 if (object == VM_OBJECT_NULL) {
1811 return KERN_INVALID_ARGUMENT;
1812 }
1813
1814 return vm_object_range_op(object,
1815 offset_beg,
1816 offset_end,
1817 ops,
1818 (uint32_t *) range);
1819 }
1820
1821
1822 void
memory_object_mark_used(memory_object_control_t control)1823 memory_object_mark_used(
1824 memory_object_control_t control)
1825 {
1826 vm_object_t object;
1827
1828 if (control == NULL) {
1829 return;
1830 }
1831
1832 object = memory_object_control_to_vm_object(control);
1833
1834 if (object != VM_OBJECT_NULL) {
1835 vm_object_cache_remove(object);
1836 }
1837 }
1838
1839
1840 void
memory_object_mark_unused(memory_object_control_t control,__unused boolean_t rage)1841 memory_object_mark_unused(
1842 memory_object_control_t control,
1843 __unused boolean_t rage)
1844 {
1845 vm_object_t object;
1846
1847 if (control == NULL) {
1848 return;
1849 }
1850
1851 object = memory_object_control_to_vm_object(control);
1852
1853 if (object != VM_OBJECT_NULL) {
1854 vm_object_cache_add(object);
1855 }
1856 }
1857
1858 void
memory_object_mark_io_tracking(memory_object_control_t control)1859 memory_object_mark_io_tracking(
1860 memory_object_control_t control)
1861 {
1862 vm_object_t object;
1863
1864 if (control == NULL) {
1865 return;
1866 }
1867 object = memory_object_control_to_vm_object(control);
1868
1869 if (object != VM_OBJECT_NULL) {
1870 vm_object_lock(object);
1871 object->io_tracking = TRUE;
1872 vm_object_unlock(object);
1873 }
1874 }
1875
1876 void
memory_object_mark_trusted(memory_object_control_t control)1877 memory_object_mark_trusted(
1878 memory_object_control_t control)
1879 {
1880 vm_object_t object;
1881
1882 if (control == NULL) {
1883 return;
1884 }
1885 object = memory_object_control_to_vm_object(control);
1886
1887 if (object != VM_OBJECT_NULL) {
1888 vm_object_lock(object);
1889 object->pager_trusted = TRUE;
1890 vm_object_unlock(object);
1891 }
1892 }
1893
1894 #if CONFIG_SECLUDED_MEMORY
1895 void
memory_object_mark_eligible_for_secluded(memory_object_control_t control,boolean_t eligible_for_secluded)1896 memory_object_mark_eligible_for_secluded(
1897 memory_object_control_t control,
1898 boolean_t eligible_for_secluded)
1899 {
1900 vm_object_t object;
1901
1902 if (control == NULL) {
1903 return;
1904 }
1905 object = memory_object_control_to_vm_object(control);
1906
1907 if (object == VM_OBJECT_NULL) {
1908 return;
1909 }
1910
1911 vm_object_lock(object);
1912 if (eligible_for_secluded &&
1913 secluded_for_filecache && /* global boot-arg */
1914 !object->eligible_for_secluded) {
1915 object->eligible_for_secluded = TRUE;
1916 vm_page_secluded.eligible_for_secluded += object->resident_page_count;
1917 } else if (!eligible_for_secluded &&
1918 object->eligible_for_secluded) {
1919 object->eligible_for_secluded = FALSE;
1920 vm_page_secluded.eligible_for_secluded -= object->resident_page_count;
1921 if (object->resident_page_count) {
1922 /* XXX FBDP TODO: flush pages from secluded queue? */
1923 // printf("FBDP TODO: flush %d pages from %p from secluded queue\n", object->resident_page_count, object);
1924 }
1925 }
1926 vm_object_unlock(object);
1927 }
1928 #endif /* CONFIG_SECLUDED_MEMORY */
1929
1930 kern_return_t
memory_object_pages_resident(memory_object_control_t control,boolean_t * has_pages_resident)1931 memory_object_pages_resident(
1932 memory_object_control_t control,
1933 boolean_t * has_pages_resident)
1934 {
1935 vm_object_t object;
1936
1937 *has_pages_resident = FALSE;
1938
1939 object = memory_object_control_to_vm_object(control);
1940 if (object == VM_OBJECT_NULL) {
1941 return KERN_INVALID_ARGUMENT;
1942 }
1943
1944 if (object->resident_page_count) {
1945 *has_pages_resident = TRUE;
1946 }
1947
1948 return KERN_SUCCESS;
1949 }
1950
1951 kern_return_t
memory_object_signed(memory_object_control_t control,boolean_t is_signed)1952 memory_object_signed(
1953 memory_object_control_t control,
1954 boolean_t is_signed)
1955 {
1956 vm_object_t object;
1957
1958 object = memory_object_control_to_vm_object(control);
1959 if (object == VM_OBJECT_NULL) {
1960 return KERN_INVALID_ARGUMENT;
1961 }
1962
1963 vm_object_lock(object);
1964 object->code_signed = is_signed;
1965 vm_object_unlock(object);
1966
1967 return KERN_SUCCESS;
1968 }
1969
1970 boolean_t
memory_object_is_signed(memory_object_control_t control)1971 memory_object_is_signed(
1972 memory_object_control_t control)
1973 {
1974 boolean_t is_signed;
1975 vm_object_t object;
1976
1977 object = memory_object_control_to_vm_object(control);
1978 if (object == VM_OBJECT_NULL) {
1979 return FALSE;
1980 }
1981
1982 vm_object_lock_shared(object);
1983 is_signed = object->code_signed;
1984 vm_object_unlock(object);
1985
1986 return is_signed;
1987 }
1988
1989 boolean_t
memory_object_is_shared_cache(memory_object_control_t control)1990 memory_object_is_shared_cache(
1991 memory_object_control_t control)
1992 {
1993 vm_object_t object = VM_OBJECT_NULL;
1994
1995 object = memory_object_control_to_vm_object(control);
1996 if (object == VM_OBJECT_NULL) {
1997 return FALSE;
1998 }
1999
2000 return object->object_is_shared_cache;
2001 }
2002
2003 __private_extern__ memory_object_control_t
memory_object_control_allocate(vm_object_t object)2004 memory_object_control_allocate(
2005 vm_object_t object)
2006 {
2007 return object;
2008 }
2009
2010 __private_extern__ void
memory_object_control_collapse(memory_object_control_t * control,vm_object_t object)2011 memory_object_control_collapse(
2012 memory_object_control_t *control,
2013 vm_object_t object)
2014 {
2015 *control = object;
2016 }
2017
2018 __private_extern__ vm_object_t
memory_object_control_to_vm_object(memory_object_control_t control)2019 memory_object_control_to_vm_object(
2020 memory_object_control_t control)
2021 {
2022 return control;
2023 }
2024
2025 __private_extern__ vm_object_t
memory_object_to_vm_object(memory_object_t mem_obj)2026 memory_object_to_vm_object(
2027 memory_object_t mem_obj)
2028 {
2029 memory_object_control_t mo_control;
2030
2031 if (mem_obj == MEMORY_OBJECT_NULL) {
2032 return VM_OBJECT_NULL;
2033 }
2034 mo_control = mem_obj->mo_control;
2035 if (mo_control == NULL) {
2036 return VM_OBJECT_NULL;
2037 }
2038 return memory_object_control_to_vm_object(mo_control);
2039 }
2040
2041 memory_object_control_t
convert_port_to_mo_control(__unused mach_port_t port)2042 convert_port_to_mo_control(
2043 __unused mach_port_t port)
2044 {
2045 return MEMORY_OBJECT_CONTROL_NULL;
2046 }
2047
2048
2049 mach_port_t
convert_mo_control_to_port(__unused memory_object_control_t control)2050 convert_mo_control_to_port(
2051 __unused memory_object_control_t control)
2052 {
2053 return MACH_PORT_NULL;
2054 }
2055
2056 void
memory_object_control_reference(__unused memory_object_control_t control)2057 memory_object_control_reference(
2058 __unused memory_object_control_t control)
2059 {
2060 return;
2061 }
2062
2063 /*
2064 * We only every issue one of these references, so kill it
2065 * when that gets released (should switch the real reference
2066 * counting in true port-less EMMI).
2067 */
2068 void
memory_object_control_deallocate(__unused memory_object_control_t control)2069 memory_object_control_deallocate(
2070 __unused memory_object_control_t control)
2071 {
2072 }
2073
2074 void
memory_object_control_disable(memory_object_control_t * control)2075 memory_object_control_disable(
2076 memory_object_control_t *control)
2077 {
2078 assert(*control != VM_OBJECT_NULL);
2079 *control = VM_OBJECT_NULL;
2080 }
2081
2082 memory_object_t
convert_port_to_memory_object(__unused mach_port_t port)2083 convert_port_to_memory_object(
2084 __unused mach_port_t port)
2085 {
2086 return MEMORY_OBJECT_NULL;
2087 }
2088
2089
2090 mach_port_t
convert_memory_object_to_port(__unused memory_object_t object)2091 convert_memory_object_to_port(
2092 __unused memory_object_t object)
2093 {
2094 return MACH_PORT_NULL;
2095 }
2096
2097
2098 /* Routine memory_object_reference */
2099 void
memory_object_reference(memory_object_t memory_object)2100 memory_object_reference(
2101 memory_object_t memory_object)
2102 {
2103 (memory_object->mo_pager_ops->memory_object_reference)(
2104 memory_object);
2105 }
2106
2107 /* Routine memory_object_deallocate */
2108 void
memory_object_deallocate(memory_object_t memory_object)2109 memory_object_deallocate(
2110 memory_object_t memory_object)
2111 {
2112 (memory_object->mo_pager_ops->memory_object_deallocate)(
2113 memory_object);
2114 }
2115
2116
2117 /* Routine memory_object_init */
2118 kern_return_t
memory_object_init(memory_object_t memory_object,memory_object_control_t memory_control,memory_object_cluster_size_t memory_object_page_size)2119 memory_object_init
2120 (
2121 memory_object_t memory_object,
2122 memory_object_control_t memory_control,
2123 memory_object_cluster_size_t memory_object_page_size
2124 )
2125 {
2126 return (memory_object->mo_pager_ops->memory_object_init)(
2127 memory_object,
2128 memory_control,
2129 memory_object_page_size);
2130 }
2131
2132 /* Routine memory_object_terminate */
2133 kern_return_t
memory_object_terminate(memory_object_t memory_object)2134 memory_object_terminate
2135 (
2136 memory_object_t memory_object
2137 )
2138 {
2139 return (memory_object->mo_pager_ops->memory_object_terminate)(
2140 memory_object);
2141 }
2142
2143 /* Routine memory_object_data_request */
2144 kern_return_t
memory_object_data_request(memory_object_t memory_object,memory_object_offset_t offset,memory_object_cluster_size_t length,vm_prot_t desired_access,memory_object_fault_info_t fault_info)2145 memory_object_data_request
2146 (
2147 memory_object_t memory_object,
2148 memory_object_offset_t offset,
2149 memory_object_cluster_size_t length,
2150 vm_prot_t desired_access,
2151 memory_object_fault_info_t fault_info
2152 )
2153 {
2154 return (memory_object->mo_pager_ops->memory_object_data_request)(
2155 memory_object,
2156 offset,
2157 length,
2158 desired_access,
2159 fault_info);
2160 }
2161
2162 /* Routine memory_object_data_return */
2163 kern_return_t
memory_object_data_return(memory_object_t memory_object,memory_object_offset_t offset,memory_object_cluster_size_t size,memory_object_offset_t * resid_offset,int * io_error,boolean_t dirty,boolean_t kernel_copy,int upl_flags)2164 memory_object_data_return
2165 (
2166 memory_object_t memory_object,
2167 memory_object_offset_t offset,
2168 memory_object_cluster_size_t size,
2169 memory_object_offset_t *resid_offset,
2170 int *io_error,
2171 boolean_t dirty,
2172 boolean_t kernel_copy,
2173 int upl_flags
2174 )
2175 {
2176 return (memory_object->mo_pager_ops->memory_object_data_return)(
2177 memory_object,
2178 offset,
2179 size,
2180 resid_offset,
2181 io_error,
2182 dirty,
2183 kernel_copy,
2184 upl_flags);
2185 }
2186
2187 /* Routine memory_object_data_initialize */
2188 kern_return_t
memory_object_data_initialize(memory_object_t memory_object,memory_object_offset_t offset,memory_object_cluster_size_t size)2189 memory_object_data_initialize
2190 (
2191 memory_object_t memory_object,
2192 memory_object_offset_t offset,
2193 memory_object_cluster_size_t size
2194 )
2195 {
2196 return (memory_object->mo_pager_ops->memory_object_data_initialize)(
2197 memory_object,
2198 offset,
2199 size);
2200 }
2201
2202 /* Routine memory_object_data_unlock */
2203 kern_return_t
memory_object_data_unlock(memory_object_t memory_object,memory_object_offset_t offset,memory_object_size_t size,vm_prot_t desired_access)2204 memory_object_data_unlock
2205 (
2206 memory_object_t memory_object,
2207 memory_object_offset_t offset,
2208 memory_object_size_t size,
2209 vm_prot_t desired_access
2210 )
2211 {
2212 return (memory_object->mo_pager_ops->memory_object_data_unlock)(
2213 memory_object,
2214 offset,
2215 size,
2216 desired_access);
2217 }
2218
2219 /* Routine memory_object_synchronize */
2220 kern_return_t
memory_object_synchronize(memory_object_t memory_object,memory_object_offset_t offset,memory_object_size_t size,vm_sync_t sync_flags)2221 memory_object_synchronize
2222 (
2223 memory_object_t memory_object,
2224 memory_object_offset_t offset,
2225 memory_object_size_t size,
2226 vm_sync_t sync_flags
2227 )
2228 {
2229 panic("memory_object_syncrhonize no longer supported");
2230
2231 return (memory_object->mo_pager_ops->memory_object_synchronize)(
2232 memory_object,
2233 offset,
2234 size,
2235 sync_flags);
2236 }
2237
2238
2239 /*
2240 * memory_object_map() is called by VM (in vm_map_enter() and its variants)
2241 * each time a "named" VM object gets mapped directly or indirectly
2242 * (copy-on-write mapping). A "named" VM object has an extra reference held
2243 * by the pager to keep it alive until the pager decides that the
2244 * memory object (and its VM object) can be reclaimed.
2245 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all
2246 * the mappings of that memory object have been removed.
2247 *
2248 * For a given VM object, calls to memory_object_map() and memory_object_unmap()
2249 * are serialized (through object->mapping_in_progress), to ensure that the
2250 * pager gets a consistent view of the mapping status of the memory object.
2251 *
2252 * This allows the pager to keep track of how many times a memory object
2253 * has been mapped and with which protections, to decide when it can be
2254 * reclaimed.
2255 */
2256
2257 /* Routine memory_object_map */
2258 kern_return_t
memory_object_map(memory_object_t memory_object,vm_prot_t prot)2259 memory_object_map
2260 (
2261 memory_object_t memory_object,
2262 vm_prot_t prot
2263 )
2264 {
2265 return (memory_object->mo_pager_ops->memory_object_map)(
2266 memory_object,
2267 prot);
2268 }
2269
2270 /* Routine memory_object_last_unmap */
2271 kern_return_t
memory_object_last_unmap(memory_object_t memory_object)2272 memory_object_last_unmap
2273 (
2274 memory_object_t memory_object
2275 )
2276 {
2277 return (memory_object->mo_pager_ops->memory_object_last_unmap)(
2278 memory_object);
2279 }
2280
2281 /* Routine memory_object_data_reclaim */
2282 kern_return_t
memory_object_data_reclaim(memory_object_t memory_object,boolean_t reclaim_backing_store)2283 memory_object_data_reclaim
2284 (
2285 memory_object_t memory_object,
2286 boolean_t reclaim_backing_store
2287 )
2288 {
2289 if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) {
2290 return KERN_NOT_SUPPORTED;
2291 }
2292 return (memory_object->mo_pager_ops->memory_object_data_reclaim)(
2293 memory_object,
2294 reclaim_backing_store);
2295 }
2296
2297 boolean_t
memory_object_backing_object(memory_object_t memory_object,memory_object_offset_t offset,vm_object_t * backing_object,vm_object_offset_t * backing_offset)2298 memory_object_backing_object
2299 (
2300 memory_object_t memory_object,
2301 memory_object_offset_t offset,
2302 vm_object_t *backing_object,
2303 vm_object_offset_t *backing_offset)
2304 {
2305 if (memory_object->mo_pager_ops->memory_object_backing_object == NULL) {
2306 return FALSE;
2307 }
2308 return (memory_object->mo_pager_ops->memory_object_backing_object)(
2309 memory_object,
2310 offset,
2311 backing_object,
2312 backing_offset);
2313 }
2314
2315 upl_t
convert_port_to_upl(__unused ipc_port_t port)2316 convert_port_to_upl(
2317 __unused ipc_port_t port)
2318 {
2319 return NULL;
2320 }
2321
2322 mach_port_t
convert_upl_to_port(__unused upl_t upl)2323 convert_upl_to_port(
2324 __unused upl_t upl)
2325 {
2326 return MACH_PORT_NULL;
2327 }
2328