1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_pageout.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * The proverbial page-out daemon.
64 */
65
66 #include "mach/kern_return.h"
67 #include <stdint.h>
68 #include <ptrauth.h>
69
70 #include <debug.h>
71
72 #include <mach/mach_types.h>
73 #include <mach/memory_object.h>
74 #include <mach/mach_host_server.h>
75 #include <mach/upl.h>
76 #include <mach/vm_map.h>
77 #include <mach/vm_param.h>
78 #include <mach/vm_statistics.h>
79 #include <mach/sdt.h>
80
81 #include <kern/kern_types.h>
82 #include <kern/counter.h>
83 #include <kern/host_statistics.h>
84 #include <kern/machine.h>
85 #include <kern/misc_protos.h>
86 #include <kern/sched.h>
87 #include <kern/thread.h>
88 #include <kern/kalloc.h>
89 #include <kern/zalloc_internal.h>
90 #include <kern/policy_internal.h>
91 #include <kern/thread_group.h>
92
93 #include <os/atomic_private.h>
94 #include <os/log.h>
95
96 #include <machine/vm_tuning.h>
97 #include <machine/commpage.h>
98
99 #include <vm/pmap.h>
100 #include <vm/vm_compressor_pager_internal.h>
101 #include <vm/vm_fault_internal.h>
102 #include <vm/vm_log.h>
103 #include <vm/vm_map_internal.h>
104 #include <vm/vm_object_internal.h>
105 #include <vm/vm_page_internal.h>
106 #include <vm/vm_pageout_internal.h>
107 #include <vm/vm_protos_internal.h> /* must be last */
108 #include <vm/memory_object.h>
109 #include <vm/vm_purgeable_internal.h>
110 #include <vm/vm_shared_region.h>
111 #include <vm/vm_compressor_internal.h>
112 #include <vm/vm_kern_xnu.h>
113 #include <vm/vm_iokit.h>
114 #include <vm/vm_ubc.h>
115 #include <vm/vm_reclaim_xnu.h>
116
117 #include <san/kasan.h>
118 #include <sys/kdebug_triage.h>
119 #include <sys/kern_memorystatus_xnu.h>
120 #include <sys/kdebug.h>
121
122 #if CONFIG_PHANTOM_CACHE
123 #include <vm/vm_phantom_cache_internal.h>
124 #endif
125
126 #if HAS_MTE
127 #include <vm/vm_mteinfo_internal.h>
128 #endif /* HAS_MTE */
129
130 #if UPL_DEBUG
131 #include <libkern/OSDebug.h>
132 #endif
133
134 os_log_t vm_log_handle = OS_LOG_DEFAULT;
135 TUNABLE(bool, vm_log_to_serial, "vm_log_to_serial", false);
136 TUNABLE(bool, vm_log_debug_enabled, "vm_log_debug", false);
137
138 extern int cs_debug;
139
140 #if CONFIG_MBUF_MCACHE
141 extern void mbuf_drain(boolean_t);
142 #endif /* CONFIG_MBUF_MCACHE */
143
144 #if CONFIG_FREEZE
145 extern unsigned int memorystatus_frozen_count;
146 extern unsigned int memorystatus_suspended_count;
147 #endif /* CONFIG_FREEZE */
148 extern vm_pressure_level_t memorystatus_vm_pressure_level;
149
150 extern lck_mtx_t memorystatus_jetsam_broadcast_lock;
151 extern uint32_t memorystatus_jetsam_fg_band_waiters;
152 extern uint32_t memorystatus_jetsam_bg_band_waiters;
153
154 void vm_pressure_response(void);
155 extern void consider_vm_pressure_events(void);
156
157 #define MEMORYSTATUS_SUSPENDED_THRESHOLD 4
158
159 SECURITY_READ_ONLY_LATE(thread_t) vm_pageout_scan_thread;
160 SECURITY_READ_ONLY_LATE(thread_t) vm_pageout_gc_thread;
161 sched_cond_atomic_t vm_pageout_gc_cond;
162 #if CONFIG_VPS_DYNAMIC_PRIO
163 TUNABLE(bool, vps_dynamic_priority_enabled, "vps_dynamic_priority_enabled", false);
164 #else
165 const bool vps_dynamic_priority_enabled = false;
166 #endif
167 boolean_t vps_yield_for_pgqlockwaiters = TRUE;
168
169 #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
170 #if !XNU_TARGET_OS_OSX
171 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
172 #else /* !XNU_TARGET_OS_OSX */
173 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
174 #endif /* !XNU_TARGET_OS_OSX */
175 #endif
176
177 #ifndef VM_PAGEOUT_DEADLOCK_RELIEF
178 #define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
179 #endif
180
181 #ifndef VM_PAGE_LAUNDRY_MAX
182 #define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */
183 #endif /* VM_PAGEOUT_LAUNDRY_MAX */
184
185 #ifndef VM_PAGEOUT_BURST_WAIT
186 #define VM_PAGEOUT_BURST_WAIT 1 /* milliseconds */
187 #endif /* VM_PAGEOUT_BURST_WAIT */
188
189 #ifndef VM_PAGEOUT_EMPTY_WAIT
190 #define VM_PAGEOUT_EMPTY_WAIT 50 /* milliseconds */
191 #endif /* VM_PAGEOUT_EMPTY_WAIT */
192
193 #ifndef VM_PAGEOUT_DEADLOCK_WAIT
194 #define VM_PAGEOUT_DEADLOCK_WAIT 100 /* milliseconds */
195 #endif /* VM_PAGEOUT_DEADLOCK_WAIT */
196
197 #ifndef VM_PAGEOUT_IDLE_WAIT
198 #define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */
199 #endif /* VM_PAGEOUT_IDLE_WAIT */
200
201 #ifndef VM_PAGEOUT_SWAP_WAIT
202 #define VM_PAGEOUT_SWAP_WAIT 10 /* milliseconds */
203 #endif /* VM_PAGEOUT_SWAP_WAIT */
204
205 /*
206 * vm_page_max_speculative_age_q should be less than or equal to
207 * VM_PAGE_RESERVED_SPECULATIVE_AGE_Q which is number of allocated
208 * vm_page_queue_speculative entries.
209 */
210
211 TUNABLE_DEV_WRITEABLE(unsigned int, vm_page_max_speculative_age_q, "vm_page_max_speculative_age_q", VM_PAGE_DEFAULT_MAX_SPECULATIVE_AGE_Q);
212 #ifndef VM_PAGE_SPECULATIVE_TARGET
213 #define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_pageout_state.vm_page_speculative_percentage))
214 #endif /* VM_PAGE_SPECULATIVE_TARGET */
215
216
217 /*
218 * To obtain a reasonable LRU approximation, the inactive queue
219 * needs to be large enough to give pages on it a chance to be
220 * referenced a second time. This macro defines the fraction
221 * of active+inactive pages that should be inactive.
222 * The pageout daemon uses it to update vm_page_inactive_target.
223 *
224 * If vm_page_free_count falls below vm_page_free_target and
225 * vm_page_inactive_count is below vm_page_inactive_target,
226 * then the pageout daemon starts running.
227 */
228
229 #ifndef VM_PAGE_INACTIVE_TARGET
230 #define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2)
231 #endif /* VM_PAGE_INACTIVE_TARGET */
232
233 /*
234 * Once the pageout daemon starts running, it keeps going
235 * until vm_page_free_count meets or exceeds vm_page_free_target.
236 */
237
238 #ifndef VM_PAGE_FREE_TARGET
239 #if !XNU_TARGET_OS_OSX
240 #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100)
241 #else /* !XNU_TARGET_OS_OSX */
242 #define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
243 #endif /* !XNU_TARGET_OS_OSX */
244 #endif /* VM_PAGE_FREE_TARGET */
245
246
247 /*
248 * The pageout daemon always starts running once vm_page_free_count
249 * falls below vm_page_free_min.
250 */
251
252 #ifndef VM_PAGE_FREE_MIN
253 #if !XNU_TARGET_OS_OSX
254 #define VM_PAGE_FREE_MIN(free) (10 + (free) / 200)
255 #else /* !XNU_TARGET_OS_OSX */
256 #define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
257 #endif /* !XNU_TARGET_OS_OSX */
258 #endif /* VM_PAGE_FREE_MIN */
259
260 #if !XNU_TARGET_OS_OSX
261 #define VM_PAGE_FREE_RESERVED_LIMIT 100
262 #define VM_PAGE_FREE_MIN_LIMIT 1500
263 #define VM_PAGE_FREE_TARGET_LIMIT 2000
264 #else /* !XNU_TARGET_OS_OSX */
265 #define VM_PAGE_FREE_RESERVED_LIMIT 1700
266 #define VM_PAGE_FREE_MIN_LIMIT 3500
267 #define VM_PAGE_FREE_TARGET_LIMIT 4000
268 #endif /* !XNU_TARGET_OS_OSX */
269
270 /*
271 * When vm_page_free_count falls below vm_page_free_reserved,
272 * only vm-privileged threads can allocate pages. vm-privilege
273 * allows the pageout daemon and default pager (and any other
274 * associated threads needed for default pageout) to continue
275 * operation by dipping into the reserved pool of pages.
276 */
277
278 #ifndef VM_PAGE_FREE_RESERVED
279 #define VM_PAGE_FREE_RESERVED(n) \
280 ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n))
281 #endif /* VM_PAGE_FREE_RESERVED */
282
283 /*
284 * When we dequeue pages from the inactive list, they are
285 * reactivated (ie, put back on the active queue) if referenced.
286 * However, it is possible to starve the free list if other
287 * processors are referencing pages faster than we can turn off
288 * the referenced bit. So we limit the number of reactivations
289 * we will make per call of vm_pageout_scan().
290 */
291 #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
292
293 #ifndef VM_PAGE_REACTIVATE_LIMIT
294 #if !XNU_TARGET_OS_OSX
295 #define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)
296 #else /* !XNU_TARGET_OS_OSX */
297 #define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
298 #endif /* !XNU_TARGET_OS_OSX */
299 #endif /* VM_PAGE_REACTIVATE_LIMIT */
300 #define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000
301
302 int vm_pageout_protect_realtime = true;
303
304 extern boolean_t hibernate_cleaning_in_progress;
305
306 struct pgo_iothread_state pgo_iothread_internal_state[MAX_COMPRESSOR_THREAD_COUNT];
307 struct pgo_iothread_state pgo_iothread_external_state;
308
309 #if VM_PRESSURE_EVENTS
310 void vm_pressure_thread(void);
311
312 boolean_t VM_PRESSURE_NORMAL_TO_WARNING(void);
313 boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void);
314
315 boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
316 boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
317 #endif
318
319 static void vm_pageout_iothread_external(struct pgo_iothread_state *, wait_result_t);
320 static void vm_pageout_iothread_internal(struct pgo_iothread_state *, wait_result_t);
321 static void vm_pageout_adjust_eq_iothrottle(struct pgo_iothread_state *, boolean_t);
322
323 extern void vm_pageout_continue(void);
324 extern void vm_pageout_scan(void);
325
326 boolean_t vm_pageout_running = FALSE;
327
328 uint32_t vm_page_upl_tainted = 0;
329 uint32_t vm_page_iopl_tainted = 0;
330
331 #if XNU_TARGET_OS_OSX
332 static boolean_t vm_pageout_waiter = FALSE;
333 #endif /* XNU_TARGET_OS_OSX */
334
335
336 #if DEVELOPMENT || DEBUG
337 struct vm_pageout_debug vm_pageout_debug;
338 #endif
339 struct vm_pageout_vminfo vm_pageout_vminfo;
340 struct vm_pageout_state vm_pageout_state;
341 struct vm_config vm_config;
342
343 struct vm_pageout_queue vm_pageout_queue_internal VM_PAGE_PACKED_ALIGNED;
344 struct vm_pageout_queue vm_pageout_queue_external VM_PAGE_PACKED_ALIGNED;
345 #if DEVELOPMENT || DEBUG
346 struct vm_pageout_queue vm_pageout_queue_benchmark VM_PAGE_PACKED_ALIGNED;
347 #endif /* DEVELOPMENT || DEBUG */
348
349 int vm_upl_wait_for_pages = 0;
350 vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL;
351
352 boolean_t(*volatile consider_buffer_cache_collect)(int) = NULL;
353
354 int vm_debug_events = 0;
355
356 LCK_GRP_DECLARE(vm_pageout_lck_grp, "vm_pageout");
357
358 #if CONFIG_MEMORYSTATUS
359 uint32_t vm_pageout_memorystatus_fb_factor_nr = 5;
360 uint32_t vm_pageout_memorystatus_fb_factor_dr = 2;
361 #endif
362
363 #if __AMP__
364
365
366 /*
367 * Bind compressor threads to e-cores unless there are multiple non-e clusters
368 */
369 #if (MAX_CPU_CLUSTERS > 2)
370 #define VM_COMPRESSOR_EBOUND_DEFAULT false
371 #elif defined(XNU_TARGET_OS_XR)
372 #define VM_COMPRESSOR_EBOUND_DEFAULT false
373 #else
374 #define VM_COMPRESSOR_EBOUND_DEFAULT true
375 #endif
376
377 TUNABLE(bool, vm_compressor_ebound, "vmcomp_ecluster", VM_COMPRESSOR_EBOUND_DEFAULT);
378 int vm_pgo_pbound = 0;
379 extern kern_return_t thread_soft_bind_cluster_type(thread_t, char);
380
381 #endif /* __AMP__ */
382
383
384 /*
385 * Routine: vm_pageout_object_terminate
386 * Purpose:
387 * Destroy the pageout_object, and perform all of the
388 * required cleanup actions.
389 *
390 * In/Out conditions:
391 * The object must be locked, and will be returned locked.
392 */
393 void
vm_pageout_object_terminate(vm_object_t object)394 vm_pageout_object_terminate(
395 vm_object_t object)
396 {
397 vm_object_t shadow_object;
398
399 /*
400 * Deal with the deallocation (last reference) of a pageout object
401 * (used for cleaning-in-place) by dropping the paging references/
402 * freeing pages in the original object.
403 */
404
405 assert(object->pageout);
406 shadow_object = object->shadow;
407 vm_object_lock(shadow_object);
408
409 while (!vm_page_queue_empty(&object->memq)) {
410 vm_page_t p, m;
411 vm_object_offset_t offset;
412
413 p = (vm_page_t) vm_page_queue_first(&object->memq);
414
415 assert(vm_page_is_private(p));
416 assert(p->vmp_free_when_done);
417 p->vmp_free_when_done = FALSE;
418 assert(!p->vmp_cleaning);
419 assert(!p->vmp_laundry);
420
421 offset = p->vmp_offset;
422 VM_PAGE_FREE(p);
423 p = VM_PAGE_NULL;
424
425 m = vm_page_lookup(shadow_object,
426 offset + object->vo_shadow_offset);
427
428 if (m == VM_PAGE_NULL) {
429 continue;
430 }
431
432 assert((m->vmp_dirty) || (m->vmp_precious) ||
433 (m->vmp_busy && m->vmp_cleaning));
434
435 /*
436 * Handle the trusted pager throttle.
437 * Also decrement the burst throttle (if external).
438 */
439 vm_page_lock_queues();
440 if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
441 vm_pageout_throttle_up(m);
442 }
443
444 /*
445 * Handle the "target" page(s). These pages are to be freed if
446 * successfully cleaned. Target pages are always busy, and are
447 * wired exactly once. The initial target pages are not mapped,
448 * (so cannot be referenced or modified) but converted target
449 * pages may have been modified between the selection as an
450 * adjacent page and conversion to a target.
451 */
452 if (m->vmp_free_when_done) {
453 assert(m->vmp_busy);
454 assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
455 assert(m->vmp_wire_count == 1);
456 m->vmp_cleaning = FALSE;
457 m->vmp_free_when_done = FALSE;
458 /*
459 * Revoke all access to the page. Since the object is
460 * locked, and the page is busy, this prevents the page
461 * from being dirtied after the pmap_disconnect() call
462 * returns.
463 *
464 * Since the page is left "dirty" but "not modifed", we
465 * can detect whether the page was redirtied during
466 * pageout by checking the modify state.
467 */
468 if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
469 SET_PAGE_DIRTY(m, FALSE);
470 } else {
471 m->vmp_dirty = FALSE;
472 }
473
474 if (m->vmp_dirty) {
475 vm_page_unwire(m, TRUE); /* reactivates */
476 counter_inc(&vm_statistics_reactivations);
477 vm_page_wakeup_done(object, m);
478 } else {
479 vm_page_free(m); /* clears busy, etc. */
480 }
481 vm_page_unlock_queues();
482 continue;
483 }
484 /*
485 * Handle the "adjacent" pages. These pages were cleaned in
486 * place, and should be left alone.
487 * If prep_pin_count is nonzero, then someone is using the
488 * page, so make it active.
489 */
490 if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) && !vm_page_is_private(m)) {
491 if (m->vmp_reference) {
492 vm_page_activate(m);
493 } else {
494 vm_page_deactivate(m);
495 }
496 }
497 if (m->vmp_overwriting) {
498 /*
499 * the (COPY_OUT_FROM == FALSE) request_page_list case
500 */
501 if (m->vmp_busy) {
502 /*
503 * We do not re-set m->vmp_dirty !
504 * The page was busy so no extraneous activity
505 * could have occurred. COPY_INTO is a read into the
506 * new pages. CLEAN_IN_PLACE does actually write
507 * out the pages but handling outside of this code
508 * will take care of resetting dirty. We clear the
509 * modify however for the Programmed I/O case.
510 */
511 pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
512
513 m->vmp_busy = FALSE;
514 m->vmp_absent = FALSE;
515 } else {
516 /*
517 * alternate (COPY_OUT_FROM == FALSE) request_page_list case
518 * Occurs when the original page was wired
519 * at the time of the list request
520 */
521 assert(VM_PAGE_WIRED(m));
522 vm_page_unwire(m, TRUE); /* reactivates */
523 }
524 m->vmp_overwriting = FALSE;
525 } else {
526 m->vmp_dirty = FALSE;
527 }
528 m->vmp_cleaning = FALSE;
529
530 /*
531 * Wakeup any thread waiting for the page to be un-cleaning.
532 */
533 vm_page_wakeup(object, m);
534 vm_page_unlock_queues();
535 }
536 /*
537 * Account for the paging reference taken in vm_paging_object_allocate.
538 */
539 vm_object_activity_end(shadow_object);
540 vm_object_unlock(shadow_object);
541
542 assert(os_ref_get_count_raw(&object->ref_count) == 0);
543 assert(object->paging_in_progress == 0);
544 assert(object->activity_in_progress == 0);
545 assert(object->resident_page_count == 0);
546 return;
547 }
548
549 /*
550 * Routine: vm_pageclean_setup
551 *
552 * Purpose: setup a page to be cleaned (made non-dirty), but not
553 * necessarily flushed from the VM page cache.
554 * This is accomplished by cleaning in place.
555 *
556 * The page must not be busy, and new_object
557 * must be locked.
558 *
559 */
560 static void
vm_pageclean_setup(vm_page_t m,vm_page_t new_m,vm_object_t new_object,vm_object_offset_t new_offset)561 vm_pageclean_setup(
562 vm_page_t m,
563 vm_page_t new_m,
564 vm_object_t new_object,
565 vm_object_offset_t new_offset)
566 {
567 assert(!m->vmp_busy);
568 #if 0
569 assert(!m->vmp_cleaning);
570 #endif
571
572 pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
573
574 /*
575 * Mark original page as cleaning in place.
576 */
577 m->vmp_cleaning = TRUE;
578 SET_PAGE_DIRTY(m, FALSE);
579 m->vmp_precious = FALSE;
580
581 /*
582 * Convert the fictitious page to a private shadow of
583 * the real page.
584 */
585 new_m->vmp_free_when_done = TRUE;
586
587 vm_page_lockspin_queues();
588 vm_page_make_private(new_m, VM_PAGE_GET_PHYS_PAGE(m));
589 vm_page_wire(new_m, VM_KERN_MEMORY_NONE, TRUE);
590 vm_page_unlock_queues();
591
592 vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE);
593 assert(!new_m->vmp_wanted);
594 new_m->vmp_busy = FALSE;
595 }
596
597 /*
598 * Routine: vm_pageout_initialize_page
599 * Purpose:
600 * Causes the specified page to be initialized in
601 * the appropriate memory object. This routine is used to push
602 * pages into a copy-object when they are modified in the
603 * permanent object.
604 *
605 * The page is moved to a temporary object and paged out.
606 *
607 * In/out conditions:
608 * The page in question must not be on any pageout queues.
609 * The object to which it belongs must be locked.
610 * The page must be busy, but not hold a paging reference.
611 *
612 * Implementation:
613 * Move this page to a completely new object.
614 */
615 void
vm_pageout_initialize_page(vm_page_t m)616 vm_pageout_initialize_page(
617 vm_page_t m)
618 {
619 vm_object_t object;
620 vm_object_offset_t paging_offset;
621 memory_object_t pager;
622
623 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
624
625 object = VM_PAGE_OBJECT(m);
626
627 assert(m->vmp_busy);
628 assert(object->internal);
629
630 /*
631 * Verify that we really want to clean this page
632 */
633 assert(!m->vmp_absent);
634 assert(m->vmp_dirty);
635
636 /*
637 * Create a paging reference to let us play with the object.
638 */
639 paging_offset = m->vmp_offset + object->paging_offset;
640
641 if (m->vmp_absent || VMP_ERROR_GET(m) || m->vmp_restart || (!m->vmp_dirty && !m->vmp_precious)) {
642 panic("reservation without pageout?"); /* alan */
643
644 VM_PAGE_FREE(m);
645 vm_object_unlock(object);
646
647 return;
648 }
649
650 /*
651 * If there's no pager, then we can't clean the page. This should
652 * never happen since this should be a copy object and therefore not
653 * an external object, so the pager should always be there.
654 */
655
656 pager = object->pager;
657
658 if (pager == MEMORY_OBJECT_NULL) {
659 panic("missing pager for copy object");
660
661 VM_PAGE_FREE(m);
662 return;
663 }
664
665 /*
666 * set the page for future call to vm_fault_list_request
667 */
668 pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
669 SET_PAGE_DIRTY(m, FALSE);
670
671 /*
672 * keep the object from collapsing or terminating
673 */
674 vm_object_paging_begin(object);
675 vm_object_unlock(object);
676
677 /*
678 * Write the data to its pager.
679 * Note that the data is passed by naming the new object,
680 * not a virtual address; the pager interface has been
681 * manipulated to use the "internal memory" data type.
682 * [The object reference from its allocation is donated
683 * to the eventual recipient.]
684 */
685 memory_object_data_initialize(pager, paging_offset, PAGE_SIZE);
686
687 vm_object_lock(object);
688 vm_object_paging_end(object);
689 }
690
691
692 /*
693 * vm_pageout_cluster:
694 *
695 * Given a page, queue it to the appropriate I/O thread,
696 * which will page it out and attempt to clean adjacent pages
697 * in the same operation.
698 *
699 * The object and queues must be locked. We will take a
700 * paging reference to prevent deallocation or collapse when we
701 * release the object lock back at the call site. The I/O thread
702 * is responsible for consuming this reference
703 *
704 * The page must not be on any pageout queue.
705 */
706 #if DEVELOPMENT || DEBUG
707 vmct_stats_t vmct_stats;
708
709 int32_t vmct_active = 0;
710 uint64_t vm_compressor_epoch_start = 0;
711 uint64_t vm_compressor_epoch_stop = 0;
712
713 typedef enum vmct_state_t {
714 VMCT_IDLE,
715 VMCT_AWAKENED,
716 VMCT_ACTIVE,
717 } vmct_state_t;
718 vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT];
719 #endif
720
721
722
723 static void
vm_pageout_cluster_to_queue(vm_page_t m,struct vm_pageout_queue * q)724 vm_pageout_cluster_to_queue(vm_page_t m, struct vm_pageout_queue *q)
725 {
726 vm_object_t object = VM_PAGE_OBJECT(m);
727
728 VM_PAGE_CHECK(m);
729 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
730 vm_object_lock_assert_exclusive(object);
731
732 /*
733 * Make sure it's OK to page this out.
734 */
735 assert((m->vmp_dirty || m->vmp_precious) && (!VM_PAGE_WIRED(m)));
736 assert(!m->vmp_cleaning && !m->vmp_laundry);
737 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
738
739 /*
740 * protect the object from collapse or termination
741 */
742 vm_object_activity_begin(object);
743
744
745 /*
746 * pgo_laundry count is tied to the laundry bit
747 */
748 m->vmp_laundry = TRUE;
749 q->pgo_laundry++;
750
751 m->vmp_q_state = VM_PAGE_ON_PAGEOUT_Q;
752 vm_page_queue_enter(&q->pgo_pending, m, vmp_pageq);
753
754 if (object->internal == TRUE) {
755 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
756 m->vmp_busy = TRUE;
757 #if DEVELOPMENT || DEBUG
758 /*
759 * The benchmark queue will be woken up independently by the benchmark
760 * itself.
761 */
762 if (q != &vm_pageout_queue_benchmark) {
763 #else /* DEVELOPMENT || DEBUG */
764 if (true) {
765 #endif /* DEVELOPMENT || DEBUG */
766 /*
767 * Wake up the first compressor thread. It will wake subsequent
768 * threads if necessary.
769 */
770 sched_cond_signal(&pgo_iothread_internal_state[0].pgo_wakeup,
771 pgo_iothread_internal_state[0].pgo_iothread);
772 }
773 } else {
774 sched_cond_signal(&pgo_iothread_external_state.pgo_wakeup, pgo_iothread_external_state.pgo_iothread);
775 }
776 VM_PAGE_CHECK(m);
777 }
778
779 void
780 vm_pageout_cluster(vm_page_t m)
781 {
782 struct vm_pageout_queue *q;
783 vm_object_t object = VM_PAGE_OBJECT(m);
784 if (object->internal) {
785 q = &vm_pageout_queue_internal;
786 } else {
787 q = &vm_pageout_queue_external;
788 }
789 vm_pageout_cluster_to_queue(m, q);
790 }
791
792
793 /*
794 * A page is back from laundry or we are stealing it back from
795 * the laundering state. See if there are some pages waiting to
796 * go to laundry and if we can let some of them go now.
797 *
798 * Object and page queues must be locked.
799 */
800 void
801 vm_pageout_throttle_up(
802 vm_page_t m)
803 {
804 struct vm_pageout_queue *q;
805 vm_object_t m_object;
806
807 m_object = VM_PAGE_OBJECT(m);
808
809 assert(m_object != VM_OBJECT_NULL);
810 assert(!is_kernel_object(m_object));
811
812 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
813 vm_object_lock_assert_exclusive(m_object);
814
815 if (m_object->internal == TRUE) {
816 q = &vm_pageout_queue_internal;
817 } else {
818 q = &vm_pageout_queue_external;
819 }
820
821 if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
822 vm_page_queue_remove(&q->pgo_pending, m, vmp_pageq);
823 m->vmp_q_state = VM_PAGE_NOT_ON_Q;
824
825 VM_PAGE_ZERO_PAGEQ_ENTRY(m);
826
827 vm_object_activity_end(m_object);
828
829 VM_PAGEOUT_DEBUG(vm_page_steal_pageout_page, 1);
830 }
831 if (m->vmp_laundry == TRUE) {
832 m->vmp_laundry = FALSE;
833 q->pgo_laundry--;
834
835 if (q->pgo_throttled == TRUE) {
836 q->pgo_throttled = FALSE;
837 thread_wakeup((event_t) &q->pgo_laundry);
838 }
839 if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
840 q->pgo_draining = FALSE;
841 thread_wakeup((event_t) (&q->pgo_laundry + 1));
842 }
843 VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, 1);
844 }
845 }
846
847
848 static void
849 vm_pageout_throttle_up_batch(
850 struct vm_pageout_queue *q,
851 int batch_cnt)
852 {
853 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
854
855 VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, batch_cnt);
856
857 q->pgo_laundry -= batch_cnt;
858
859 if (q->pgo_throttled == TRUE) {
860 q->pgo_throttled = FALSE;
861 thread_wakeup((event_t) &q->pgo_laundry);
862 }
863 if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
864 q->pgo_draining = FALSE;
865 thread_wakeup((event_t) (&q->pgo_laundry + 1));
866 }
867 }
868
869
870
871 /*
872 * VM memory pressure monitoring.
873 *
874 * vm_pageout_scan() keeps track of the number of pages it considers and
875 * reclaims, in the currently active vm_pageout_stat[vm_pageout_stat_now].
876 *
877 * compute_memory_pressure() is called every second from compute_averages()
878 * and moves "vm_pageout_stat_now" forward, to start accumulating the number
879 * of recalimed pages in a new vm_pageout_stat[] bucket.
880 *
881 * mach_vm_pressure_monitor() collects past statistics about memory pressure.
882 * The caller provides the number of seconds ("nsecs") worth of statistics
883 * it wants, up to 30 seconds.
884 * It computes the number of pages reclaimed in the past "nsecs" seconds and
885 * also returns the number of pages the system still needs to reclaim at this
886 * moment in time.
887 */
888 #if DEVELOPMENT || DEBUG
889 #define VM_PAGEOUT_STAT_SIZE (30 * 8) + 1
890 #else
891 #define VM_PAGEOUT_STAT_SIZE (1 * 8) + 1
892 #endif
893 struct vm_pageout_stat {
894 unsigned long vm_page_active_count;
895 unsigned long vm_page_speculative_count;
896 unsigned long vm_page_inactive_count;
897 unsigned long vm_page_anonymous_count;
898
899 unsigned long vm_page_free_count;
900 unsigned long vm_page_wire_count;
901 unsigned long vm_page_compressor_count;
902
903 unsigned long vm_page_pages_compressed;
904 unsigned long vm_page_pageable_internal_count;
905 unsigned long vm_page_pageable_external_count;
906 unsigned long vm_page_xpmapped_external_count;
907
908 unsigned long vm_page_swapped_count;
909 uint64_t swapouts;
910 uint64_t swapins;
911
912 unsigned int pages_grabbed;
913 unsigned int pages_freed;
914
915 unsigned int pages_compressed;
916 unsigned int pages_grabbed_by_compressor;
917 unsigned int failed_compressions;
918
919 unsigned int pages_evicted;
920 unsigned int pages_purged;
921
922 unsigned int considered;
923 unsigned int considered_bq_internal;
924 unsigned int considered_bq_external;
925
926 unsigned int skipped_external;
927 unsigned int skipped_internal;
928 unsigned int filecache_min_reactivations;
929
930 unsigned int freed_speculative;
931 unsigned int freed_cleaned;
932 unsigned int freed_internal;
933 unsigned int freed_external;
934
935 unsigned int cleaned_dirty_external;
936 unsigned int cleaned_dirty_internal;
937
938 unsigned int inactive_referenced;
939 unsigned int inactive_nolock;
940 unsigned int reactivation_limit_exceeded;
941 unsigned int forced_inactive_reclaim;
942
943 unsigned int throttled_internal_q;
944 unsigned int throttled_external_q;
945
946 unsigned int phantom_ghosts_found;
947 unsigned int phantom_ghosts_added;
948
949 unsigned int vm_page_realtime_count;
950 unsigned int forcereclaimed_sharedcache;
951 unsigned int forcereclaimed_realtime;
952 unsigned int protected_sharedcache;
953 unsigned int protected_realtime;
954
955 unsigned long cswap_unripe_under_30s;
956 unsigned long cswap_unripe_under_60s;
957 unsigned long cswap_unripe_under_300s;
958 unsigned long cswap_reclaim_swapins;
959 unsigned long cswap_defrag_swapins;
960 unsigned long cswap_swap_threshold_exceeded;
961 unsigned long cswap_external_q_throttled;
962 unsigned long cswap_free_below_reserve;
963 unsigned long cswap_thrashing_detected;
964 unsigned long cswap_fragmentation_detected;
965
966 unsigned long major_compactions_considered;
967 unsigned long major_compactions_completed;
968 unsigned long major_compactions_bailed;
969 unsigned long major_compaction_bytes_freed;
970 unsigned long major_compaction_bytes_moved;
971 unsigned long major_compaction_slots_moved;
972 unsigned long major_compaction_segments_freed;
973 unsigned long swapouts_queued;
974 unsigned long swapout_bytes_wasted;
975 } vm_pageout_stats[VM_PAGEOUT_STAT_SIZE];
976
977 unsigned int vm_pageout_stat_now = 0;
978
979 #define VM_PAGEOUT_STAT_BEFORE(i) \
980 (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1)
981 #define VM_PAGEOUT_STAT_AFTER(i) \
982 (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
983
984 #if VM_PAGE_BUCKETS_CHECK
985 int vm_page_buckets_check_interval = 80; /* in eighths of a second */
986 #endif /* VM_PAGE_BUCKETS_CHECK */
987
988
989 void
990 record_memory_pressure(void);
991 void
992 record_memory_pressure(void)
993 {
994 unsigned int vm_pageout_next;
995
996 #if VM_PAGE_BUCKETS_CHECK
997 /* check the consistency of VM page buckets at regular interval */
998 static int counter = 0;
999 if ((++counter % vm_page_buckets_check_interval) == 0) {
1000 vm_page_buckets_check();
1001 }
1002 #endif /* VM_PAGE_BUCKETS_CHECK */
1003
1004 vm_pageout_state.vm_memory_pressure =
1005 vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_speculative +
1006 vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_cleaned +
1007 vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_internal +
1008 vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_external;
1009
1010 commpage_set_memory_pressure((unsigned int)vm_pageout_state.vm_memory_pressure );
1011
1012 /* move "now" forward */
1013 vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);
1014
1015 bzero(&vm_pageout_stats[vm_pageout_next], sizeof(struct vm_pageout_stat));
1016
1017 vm_pageout_stat_now = vm_pageout_next;
1018 }
1019
1020
1021 /*
1022 * IMPORTANT
1023 * mach_vm_ctl_page_free_wanted() is called indirectly, via
1024 * mach_vm_pressure_monitor(), when taking a stackshot. Therefore,
1025 * it must be safe in the restricted stackshot context. Locks and/or
1026 * blocking are not allowable.
1027 */
1028 unsigned int
1029 mach_vm_ctl_page_free_wanted(void)
1030 {
1031 unsigned int page_free_target, page_free_count, page_free_wanted;
1032
1033 page_free_target = vm_page_free_target;
1034 page_free_count = vm_page_free_count;
1035 if (page_free_target > page_free_count) {
1036 page_free_wanted = page_free_target - page_free_count;
1037 } else {
1038 page_free_wanted = 0;
1039 }
1040
1041 return page_free_wanted;
1042 }
1043
1044
1045 /*
1046 * IMPORTANT:
1047 * mach_vm_pressure_monitor() is called when taking a stackshot, with
1048 * wait_for_pressure FALSE, so that code path must remain safe in the
1049 * restricted stackshot context. No blocking or locks are allowable.
1050 * on that code path.
1051 */
1052
1053 kern_return_t
1054 mach_vm_pressure_monitor(
1055 boolean_t wait_for_pressure,
1056 unsigned int nsecs_monitored,
1057 unsigned int *pages_reclaimed_p,
1058 unsigned int *pages_wanted_p)
1059 {
1060 wait_result_t wr;
1061 unsigned int vm_pageout_then, vm_pageout_now;
1062 unsigned int pages_reclaimed;
1063 unsigned int units_of_monitor;
1064
1065 units_of_monitor = 8 * nsecs_monitored;
1066 /*
1067 * We don't take the vm_page_queue_lock here because we don't want
1068 * vm_pressure_monitor() to get in the way of the vm_pageout_scan()
1069 * thread when it's trying to reclaim memory. We don't need fully
1070 * accurate monitoring anyway...
1071 */
1072
1073 if (wait_for_pressure) {
1074 /* wait until there's memory pressure */
1075 while (vm_page_free_count >= vm_page_free_target) {
1076 wr = assert_wait((event_t) &vm_page_free_wanted,
1077 THREAD_INTERRUPTIBLE);
1078 if (wr == THREAD_WAITING) {
1079 wr = thread_block(THREAD_CONTINUE_NULL);
1080 }
1081 if (wr == THREAD_INTERRUPTED) {
1082 return KERN_ABORTED;
1083 }
1084 if (wr == THREAD_AWAKENED) {
1085 /*
1086 * The memory pressure might have already
1087 * been relieved but let's not block again
1088 * and let's report that there was memory
1089 * pressure at some point.
1090 */
1091 break;
1092 }
1093 }
1094 }
1095
1096 /* provide the number of pages the system wants to reclaim */
1097 if (pages_wanted_p != NULL) {
1098 *pages_wanted_p = mach_vm_ctl_page_free_wanted();
1099 }
1100
1101 if (pages_reclaimed_p == NULL) {
1102 return KERN_SUCCESS;
1103 }
1104
1105 /* provide number of pages reclaimed in the last "nsecs_monitored" */
1106 vm_pageout_now = vm_pageout_stat_now;
1107 pages_reclaimed = 0;
1108 for (vm_pageout_then =
1109 VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
1110 vm_pageout_then != vm_pageout_now &&
1111 units_of_monitor-- != 0;
1112 vm_pageout_then =
1113 VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
1114 pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_speculative;
1115 pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_cleaned;
1116 pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_internal;
1117 pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_external;
1118 }
1119 *pages_reclaimed_p = pages_reclaimed;
1120
1121 return KERN_SUCCESS;
1122 }
1123
1124
1125
1126 #if DEVELOPMENT || DEBUG
1127
1128 static void
1129 vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int);
1130
1131 /*
1132 * condition variable used to make sure there is
1133 * only a single sweep going on at a time
1134 */
1135 bool vm_pageout_disconnect_all_pages_active = false;
1136
1137 void
1138 vm_pageout_disconnect_all_pages()
1139 {
1140 vm_page_lock_queues();
1141
1142 if (vm_pageout_disconnect_all_pages_active) {
1143 vm_page_unlock_queues();
1144 return;
1145 }
1146 vm_pageout_disconnect_all_pages_active = true;
1147
1148 vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_throttled,
1149 vm_page_throttled_count);
1150 vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_anonymous,
1151 vm_page_anonymous_count);
1152 vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_inactive,
1153 (vm_page_inactive_count - vm_page_anonymous_count));
1154 vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_active,
1155 vm_page_active_count);
1156 #ifdef CONFIG_SECLUDED_MEMORY
1157 vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_secluded,
1158 vm_page_secluded_count);
1159 #endif /* CONFIG_SECLUDED_MEMORY */
1160 vm_page_unlock_queues();
1161
1162 vm_pageout_disconnect_all_pages_active = false;
1163 }
1164
1165 /* NB: assumes the page_queues lock is held on entry, returns with page queue lock held */
1166 void
1167 vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
1168 {
1169 vm_page_t m;
1170 vm_object_t t_object = NULL;
1171 vm_object_t l_object = NULL;
1172 vm_object_t m_object = NULL;
1173 int delayed_unlock = 0;
1174 int try_failed_count = 0;
1175 int disconnected_count = 0;
1176 int paused_count = 0;
1177 int object_locked_count = 0;
1178
1179 KDBG((MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS) |
1180 DBG_FUNC_START),
1181 q, qcount);
1182
1183 while (qcount && !vm_page_queue_empty(q)) {
1184 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
1185
1186 m = (vm_page_t) vm_page_queue_first(q);
1187 m_object = VM_PAGE_OBJECT(m);
1188
1189 if (m_object == VM_OBJECT_NULL) {
1190 /*
1191 * Bumped into a free page. This should only happen on the
1192 * secluded queue
1193 */
1194 #if CONFIG_SECLUDED_MEMORY
1195 assert(q == &vm_page_queue_secluded);
1196 #endif /* CONFIG_SECLUDED_MEMORY */
1197 goto reenter_pg_on_q;
1198 }
1199
1200 /*
1201 * check to see if we currently are working
1202 * with the same object... if so, we've
1203 * already got the lock
1204 */
1205 if (m_object != l_object) {
1206 /*
1207 * the object associated with candidate page is
1208 * different from the one we were just working
1209 * with... dump the lock if we still own it
1210 */
1211 if (l_object != NULL) {
1212 vm_object_unlock(l_object);
1213 l_object = NULL;
1214 }
1215 if (m_object != t_object) {
1216 try_failed_count = 0;
1217 }
1218
1219 /*
1220 * Try to lock object; since we've alread got the
1221 * page queues lock, we can only 'try' for this one.
1222 * if the 'try' fails, we need to do a mutex_pause
1223 * to allow the owner of the object lock a chance to
1224 * run...
1225 */
1226 if (!vm_object_lock_try_scan(m_object)) {
1227 if (try_failed_count > 20) {
1228 goto reenter_pg_on_q;
1229 }
1230 vm_page_unlock_queues();
1231 mutex_pause(try_failed_count++);
1232 vm_page_lock_queues();
1233 delayed_unlock = 0;
1234
1235 paused_count++;
1236
1237 t_object = m_object;
1238 continue;
1239 }
1240 object_locked_count++;
1241
1242 l_object = m_object;
1243 }
1244 if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry ||
1245 m->vmp_busy || m->vmp_absent || VMP_ERROR_GET(m) ||
1246 m->vmp_free_when_done) {
1247 /*
1248 * put it back on the head of its queue
1249 */
1250 goto reenter_pg_on_q;
1251 }
1252 if (m->vmp_pmapped == TRUE) {
1253 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
1254
1255 disconnected_count++;
1256 }
1257 reenter_pg_on_q:
1258 vm_page_queue_remove(q, m, vmp_pageq);
1259 vm_page_queue_enter(q, m, vmp_pageq);
1260
1261 qcount--;
1262 try_failed_count = 0;
1263
1264 if (delayed_unlock++ > 128) {
1265 if (l_object != NULL) {
1266 vm_object_unlock(l_object);
1267 l_object = NULL;
1268 }
1269 lck_mtx_yield(&vm_page_queue_lock);
1270 delayed_unlock = 0;
1271 }
1272 }
1273 if (l_object != NULL) {
1274 vm_object_unlock(l_object);
1275 l_object = NULL;
1276 }
1277
1278 KDBG((MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS) |
1279 DBG_FUNC_END),
1280 q, disconnected_count, object_locked_count, paused_count);
1281 }
1282
1283 extern const char *proc_best_name(struct proc* proc);
1284
1285 int
1286 vm_toggle_task_selfdonate_pages(task_t task)
1287 {
1288 int state = 0;
1289 if (vm_page_donate_mode == VM_PAGE_DONATE_DISABLED) {
1290 printf("VM Donation mode is OFF on the system\n");
1291 return state;
1292 }
1293 if (task != kernel_task) {
1294 task_lock(task);
1295 if (!task->donates_own_pages) {
1296 printf("SELF DONATE for %s ON\n", proc_best_name(get_bsdtask_info(task)));
1297 task->donates_own_pages = true;
1298 state = 1;
1299 } else if (task->donates_own_pages) {
1300 printf("SELF DONATE for %s OFF\n", proc_best_name(get_bsdtask_info(task)));
1301 task->donates_own_pages = false;
1302 state = 0;
1303 }
1304 task_unlock(task);
1305 }
1306 return state;
1307 }
1308 #endif /* DEVELOPMENT || DEBUG */
1309
1310 void
1311 vm_task_set_selfdonate_pages(task_t task, bool donate)
1312 {
1313 assert(vm_page_donate_mode != VM_PAGE_DONATE_DISABLED);
1314 assert(task != kernel_task);
1315
1316 task_lock(task);
1317 task->donates_own_pages = donate;
1318 task_unlock(task);
1319 }
1320
1321
1322
1323 static size_t
1324 vm_pageout_page_queue(vm_page_queue_head_t *, size_t, bool);
1325
1326 /*
1327 * condition variable used to make sure there is
1328 * only a single sweep going on at a time
1329 */
1330 boolean_t vm_pageout_anonymous_pages_active = FALSE;
1331
1332
1333 kern_return_t
1334 vm_pageout_anonymous_pages()
1335 {
1336 if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
1337 size_t throttled_pages_moved, anonymous_pages_moved, active_pages_moved;
1338 vm_page_lock_queues();
1339
1340 if (vm_pageout_anonymous_pages_active == TRUE) {
1341 vm_page_unlock_queues();
1342 return KERN_RESOURCE_SHORTAGE;
1343 }
1344 vm_pageout_anonymous_pages_active = TRUE;
1345 vm_page_unlock_queues();
1346
1347 throttled_pages_moved = vm_pageout_page_queue(&vm_page_queue_throttled, vm_page_throttled_count, false);
1348 anonymous_pages_moved = vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count, false);
1349 active_pages_moved = vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count, false);
1350
1351 os_log(OS_LOG_DEFAULT,
1352 "%s: throttled pages moved: %zu, anonymous pages moved: %zu, active pages moved: %zu",
1353 __func__, throttled_pages_moved, anonymous_pages_moved, active_pages_moved);
1354
1355 if (VM_CONFIG_SWAP_IS_PRESENT) {
1356 vm_consider_swapping();
1357 }
1358
1359 vm_page_lock_queues();
1360 vm_pageout_anonymous_pages_active = FALSE;
1361 vm_page_unlock_queues();
1362 return KERN_SUCCESS;
1363 } else {
1364 return KERN_NOT_SUPPORTED;
1365 }
1366 }
1367
1368
1369 size_t
1370 vm_pageout_page_queue(vm_page_queue_head_t *q, size_t qcount, bool perf_test)
1371 {
1372 vm_page_t m;
1373 vm_object_t t_object = NULL;
1374 vm_object_t l_object = NULL;
1375 vm_object_t m_object = NULL;
1376 int delayed_unlock = 0;
1377 int try_failed_count = 0;
1378 int refmod_state;
1379 int pmap_options;
1380 struct vm_pageout_queue *iq;
1381 ppnum_t phys_page;
1382 size_t pages_moved = 0;
1383
1384
1385 iq = &vm_pageout_queue_internal;
1386
1387 vm_page_lock_queues();
1388
1389 #if DEVELOPMENT || DEBUG
1390 if (perf_test) {
1391 iq = &vm_pageout_queue_benchmark;
1392 // ensure the benchmark queue isn't throttled
1393 iq->pgo_maxlaundry = (unsigned int) qcount;
1394 }
1395 #endif /* DEVELOPMENT ||DEBUG */
1396
1397 while (qcount && !vm_page_queue_empty(q)) {
1398 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
1399
1400 if (VM_PAGE_Q_THROTTLED(iq)) {
1401 if (l_object != NULL) {
1402 vm_object_unlock(l_object);
1403 l_object = NULL;
1404 }
1405 iq->pgo_draining = TRUE;
1406
1407 assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE);
1408 vm_page_unlock_queues();
1409
1410 thread_block(THREAD_CONTINUE_NULL);
1411
1412 vm_page_lock_queues();
1413 delayed_unlock = 0;
1414 continue;
1415 }
1416 m = (vm_page_t) vm_page_queue_first(q);
1417 m_object = VM_PAGE_OBJECT(m);
1418
1419 /*
1420 * check to see if we currently are working
1421 * with the same object... if so, we've
1422 * already got the lock
1423 */
1424 if (m_object != l_object) {
1425 if (!m_object->internal) {
1426 goto reenter_pg_on_q;
1427 }
1428
1429 /*
1430 * the object associated with candidate page is
1431 * different from the one we were just working
1432 * with... dump the lock if we still own it
1433 */
1434 if (l_object != NULL) {
1435 vm_object_unlock(l_object);
1436 l_object = NULL;
1437 }
1438 if (m_object != t_object) {
1439 try_failed_count = 0;
1440 }
1441
1442 /*
1443 * Try to lock object; since we've alread got the
1444 * page queues lock, we can only 'try' for this one.
1445 * if the 'try' fails, we need to do a mutex_pause
1446 * to allow the owner of the object lock a chance to
1447 * run...
1448 */
1449 if (!vm_object_lock_try_scan(m_object)) {
1450 if (try_failed_count > 20) {
1451 goto reenter_pg_on_q;
1452 }
1453 vm_page_unlock_queues();
1454 mutex_pause(try_failed_count++);
1455 vm_page_lock_queues();
1456 delayed_unlock = 0;
1457
1458 t_object = m_object;
1459 continue;
1460 }
1461 l_object = m_object;
1462 }
1463 if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || VMP_ERROR_GET(m) || m->vmp_free_when_done) {
1464 /*
1465 * page is not to be cleaned
1466 * put it back on the head of its queue
1467 */
1468 goto reenter_pg_on_q;
1469 }
1470 phys_page = VM_PAGE_GET_PHYS_PAGE(m);
1471
1472 if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {
1473 refmod_state = pmap_get_refmod(phys_page);
1474
1475 if (refmod_state & VM_MEM_REFERENCED) {
1476 m->vmp_reference = TRUE;
1477 }
1478 if (refmod_state & VM_MEM_MODIFIED) {
1479 SET_PAGE_DIRTY(m, FALSE);
1480 }
1481 }
1482 if (m->vmp_reference == TRUE) {
1483 m->vmp_reference = FALSE;
1484 pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
1485 goto reenter_pg_on_q;
1486 }
1487 if (m->vmp_pmapped == TRUE) {
1488 if (m->vmp_dirty || m->vmp_precious) {
1489 pmap_options = PMAP_OPTIONS_COMPRESSOR;
1490 } else {
1491 pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
1492 }
1493 refmod_state = pmap_disconnect_options(phys_page, pmap_options, NULL);
1494 if (refmod_state & VM_MEM_MODIFIED) {
1495 SET_PAGE_DIRTY(m, FALSE);
1496 }
1497 }
1498
1499 if (!m->vmp_dirty && !m->vmp_precious) {
1500 vm_page_unlock_queues();
1501 VM_PAGE_FREE(m);
1502 vm_page_lock_queues();
1503 delayed_unlock = 0;
1504
1505 goto next_pg;
1506 }
1507 if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {
1508 if (!m_object->pager_initialized) {
1509 vm_page_unlock_queues();
1510
1511 vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE);
1512
1513 if (!m_object->pager_initialized) {
1514 vm_object_compressor_pager_create(m_object);
1515 }
1516
1517 vm_page_lock_queues();
1518 delayed_unlock = 0;
1519 }
1520 if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {
1521 /*
1522 * We dropped the page queues lock above, so
1523 * "m" might no longer be on this queue...
1524 */
1525 if (m != (vm_page_t) vm_page_queue_first(q)) {
1526 continue;
1527 }
1528 goto reenter_pg_on_q;
1529 }
1530 /*
1531 * vm_object_compressor_pager_create will drop the object lock
1532 * which means 'm' may no longer be valid to use
1533 */
1534 continue;
1535 }
1536
1537 if (!perf_test) {
1538 /*
1539 * we've already factored out pages in the laundry which
1540 * means this page can't be on the pageout queue so it's
1541 * safe to do the vm_page_queues_remove
1542 */
1543 bool donate = (m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE);
1544 vm_page_queues_remove(m, TRUE);
1545 if (donate) {
1546 /*
1547 * The compressor needs to see this bit to know
1548 * where this page needs to land. Also if stolen,
1549 * this bit helps put the page back in the right
1550 * special queue where it belongs.
1551 */
1552 m->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
1553 }
1554 } else {
1555 vm_page_queue_remove(q, m, vmp_pageq);
1556 }
1557
1558 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
1559
1560 vm_pageout_cluster_to_queue(m, iq);
1561
1562 pages_moved++;
1563 goto next_pg;
1564
1565 reenter_pg_on_q:
1566 vm_page_queue_remove(q, m, vmp_pageq);
1567 vm_page_queue_enter(q, m, vmp_pageq);
1568 next_pg:
1569 qcount--;
1570 try_failed_count = 0;
1571
1572 if (delayed_unlock++ > 128) {
1573 if (l_object != NULL) {
1574 vm_object_unlock(l_object);
1575 l_object = NULL;
1576 }
1577 lck_mtx_yield(&vm_page_queue_lock);
1578 delayed_unlock = 0;
1579 }
1580 }
1581 if (l_object != NULL) {
1582 vm_object_unlock(l_object);
1583 l_object = NULL;
1584 }
1585 vm_page_unlock_queues();
1586 return pages_moved;
1587 }
1588
1589
1590
1591 /*
1592 * function in BSD to apply I/O throttle to the pageout thread
1593 */
1594 extern void vm_pageout_io_throttle(void);
1595
1596 #define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \
1597 MACRO_BEGIN \
1598 /* \
1599 * If a "reusable" page somehow made it back into \
1600 * the active queue, it's been re-used and is not \
1601 * quite re-usable. \
1602 * If the VM object was "all_reusable", consider it \
1603 * as "all re-used" instead of converting it to \
1604 * "partially re-used", which could be expensive. \
1605 */ \
1606 assert(VM_PAGE_OBJECT((m)) == (obj)); \
1607 if ((m)->vmp_reusable || \
1608 (obj)->all_reusable) { \
1609 vm_object_reuse_pages((obj), \
1610 (m)->vmp_offset, \
1611 (m)->vmp_offset + PAGE_SIZE_64, \
1612 FALSE); \
1613 } \
1614 MACRO_END
1615
1616
1617 #define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64
1618 #define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024
1619
1620 #define FCS_IDLE 0
1621 #define FCS_DELAYED 1
1622 #define FCS_DEADLOCK_DETECTED 2
1623
1624 struct flow_control {
1625 int state;
1626 mach_timespec_t ts;
1627 };
1628
1629
1630 uint64_t vm_pageout_rejected_bq_internal = 0;
1631 uint64_t vm_pageout_rejected_bq_external = 0;
1632 uint64_t vm_pageout_skipped_bq_internal = 0;
1633 uint64_t vm_pageout_skipped_bq_external = 0;
1634
1635 #define ANONS_GRABBED_LIMIT 2
1636
1637
1638 #if 0
1639 static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *);
1640 #endif
1641 static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int);
1642
1643 #define VM_PAGEOUT_PB_NO_ACTION 0
1644 #define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1
1645 #define VM_PAGEOUT_PB_THREAD_YIELD 2
1646
1647
1648 #if 0
1649 static void
1650 vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq)
1651 {
1652 if (*local_freeq) {
1653 vm_page_unlock_queues();
1654
1655 VM_DEBUG_CONSTANT_EVENT(
1656 vm_pageout_freelist, DBG_VM_PAGEOUT_FREELIST, DBG_FUNC_START,
1657 vm_page_free_count, 0, 0, 1);
1658
1659 vm_page_free_list(*local_freeq, TRUE);
1660
1661 VM_DEBUG_CONSTANT_EVENT(vm_pageout_freelist, DBG_VM_PAGEOUT_FREELIST, DBG_FUNC_END,
1662 vm_page_free_count, *local_freed, 0, 1);
1663
1664 *local_freeq = NULL;
1665 *local_freed = 0;
1666
1667 vm_page_lock_queues();
1668 } else {
1669 lck_mtx_yield(&vm_page_queue_lock);
1670 }
1671 *delayed_unlock = 1;
1672 }
1673 #endif
1674
1675
1676 static void
1677 vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
1678 vm_page_t *local_freeq, int *local_freed, int action)
1679 {
1680 vm_page_unlock_queues();
1681
1682 if (*object != NULL) {
1683 vm_object_unlock(*object);
1684 *object = NULL;
1685 }
1686 if (*local_freeq) {
1687 vm_page_free_list(*local_freeq, TRUE);
1688
1689 *local_freeq = NULL;
1690 *local_freed = 0;
1691 }
1692 *delayed_unlock = 1;
1693
1694 switch (action) {
1695 case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER:
1696 vm_consider_waking_compactor_swapper();
1697 break;
1698 case VM_PAGEOUT_PB_THREAD_YIELD:
1699 thread_yield_internal(1);
1700 break;
1701 case VM_PAGEOUT_PB_NO_ACTION:
1702 default:
1703 break;
1704 }
1705 vm_page_lock_queues();
1706 }
1707
1708
1709 static struct vm_pageout_vminfo last;
1710 static struct vm_compressor_swapper_stats last_vmcs;
1711 static uint64_t last_swapouts;
1712 static uint64_t last_swapins;
1713
1714 uint64_t last_vm_page_pages_grabbed = 0;
1715
1716 extern uint32_t c_segment_pages_compressed;
1717
1718 extern uint64_t shared_region_pager_reclaimed;
1719 extern struct memory_object_pager_ops shared_region_pager_ops;
1720
1721 void
1722 update_vm_info(void)
1723 {
1724 unsigned long tmp;
1725 uint64_t tmp64;
1726
1727 vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count = vm_page_active_count;
1728 vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count = vm_page_speculative_count;
1729 vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count = vm_page_inactive_count;
1730 vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count = vm_page_anonymous_count;
1731
1732 vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count = vm_page_free_count;
1733 vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count = vm_page_wire_count;
1734 vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count = VM_PAGE_COMPRESSOR_COUNT;
1735 vm_pageout_stats[vm_pageout_stat_now].vm_page_swapped_count = os_atomic_load(&vm_page_swapped_count, relaxed);
1736
1737 vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed = c_segment_pages_compressed;
1738 vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count = vm_page_pageable_internal_count;
1739 vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count = vm_page_pageable_external_count;
1740 vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count = vm_page_xpmapped_external_count;
1741 vm_pageout_stats[vm_pageout_stat_now].vm_page_realtime_count = vm_page_realtime_count;
1742
1743 tmp = vm_pageout_vminfo.vm_pageout_considered_page;
1744 vm_pageout_stats[vm_pageout_stat_now].considered = (unsigned int)(tmp - last.vm_pageout_considered_page);
1745 last.vm_pageout_considered_page = tmp;
1746
1747 tmp64 = vm_pageout_vminfo.vm_pageout_compressions;
1748 vm_pageout_stats[vm_pageout_stat_now].pages_compressed = (unsigned int)(tmp64 - last.vm_pageout_compressions);
1749 last.vm_pageout_compressions = tmp64;
1750
1751 tmp = vm_pageout_vminfo.vm_compressor_failed;
1752 vm_pageout_stats[vm_pageout_stat_now].failed_compressions = (unsigned int)(tmp - last.vm_compressor_failed);
1753 last.vm_compressor_failed = tmp;
1754
1755 tmp64 = vm_pageout_vminfo.vm_compressor_pages_grabbed;
1756 vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor = (unsigned int)(tmp64 - last.vm_compressor_pages_grabbed);
1757 last.vm_compressor_pages_grabbed = tmp64;
1758
1759 tmp = vm_pageout_vminfo.vm_phantom_cache_found_ghost;
1760 vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found = (unsigned int)(tmp - last.vm_phantom_cache_found_ghost);
1761 last.vm_phantom_cache_found_ghost = tmp;
1762
1763 tmp = vm_pageout_vminfo.vm_phantom_cache_added_ghost;
1764 vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added = (unsigned int)(tmp - last.vm_phantom_cache_added_ghost);
1765 last.vm_phantom_cache_added_ghost = tmp;
1766
1767 tmp64 = counter_load(&vm_page_grab_count);
1768 vm_pageout_stats[vm_pageout_stat_now].pages_grabbed = (unsigned int)(tmp64 - last_vm_page_pages_grabbed);
1769 last_vm_page_pages_grabbed = tmp64;
1770
1771 tmp = vm_pageout_vminfo.vm_page_pages_freed;
1772 vm_pageout_stats[vm_pageout_stat_now].pages_freed = (unsigned int)(tmp - last.vm_page_pages_freed);
1773 last.vm_page_pages_freed = tmp;
1774
1775 tmp64 = counter_load(&vm_statistics_swapouts);
1776 vm_pageout_stats[vm_pageout_stat_now].swapouts = tmp64 - last_swapouts;
1777 last_swapouts = tmp64;
1778
1779 tmp64 = counter_load(&vm_statistics_swapins);
1780 vm_pageout_stats[vm_pageout_stat_now].swapins = tmp64 - last_swapins;
1781 last_swapins = tmp64;
1782
1783 if (vm_pageout_stats[vm_pageout_stat_now].considered) {
1784 tmp = vm_pageout_vminfo.vm_pageout_pages_evicted;
1785 vm_pageout_stats[vm_pageout_stat_now].pages_evicted = (unsigned int)(tmp - last.vm_pageout_pages_evicted);
1786 last.vm_pageout_pages_evicted = tmp;
1787
1788 tmp = vm_pageout_vminfo.vm_pageout_pages_purged;
1789 vm_pageout_stats[vm_pageout_stat_now].pages_purged = (unsigned int)(tmp - last.vm_pageout_pages_purged);
1790 last.vm_pageout_pages_purged = tmp;
1791
1792 tmp = vm_pageout_vminfo.vm_pageout_freed_speculative;
1793 vm_pageout_stats[vm_pageout_stat_now].freed_speculative = (unsigned int)(tmp - last.vm_pageout_freed_speculative);
1794 last.vm_pageout_freed_speculative = tmp;
1795
1796 tmp = vm_pageout_vminfo.vm_pageout_freed_external;
1797 vm_pageout_stats[vm_pageout_stat_now].freed_external = (unsigned int)(tmp - last.vm_pageout_freed_external);
1798 last.vm_pageout_freed_external = tmp;
1799
1800 tmp = vm_pageout_vminfo.vm_pageout_inactive_referenced;
1801 vm_pageout_stats[vm_pageout_stat_now].inactive_referenced = (unsigned int)(tmp - last.vm_pageout_inactive_referenced);
1802 last.vm_pageout_inactive_referenced = tmp;
1803
1804 tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external;
1805 vm_pageout_stats[vm_pageout_stat_now].throttled_external_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_external);
1806 last.vm_pageout_scan_inactive_throttled_external = tmp;
1807
1808 tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_external;
1809 vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_external);
1810 last.vm_pageout_inactive_dirty_external = tmp;
1811
1812 tmp = vm_pageout_vminfo.vm_pageout_freed_cleaned;
1813 vm_pageout_stats[vm_pageout_stat_now].freed_cleaned = (unsigned int)(tmp - last.vm_pageout_freed_cleaned);
1814 last.vm_pageout_freed_cleaned = tmp;
1815
1816 tmp = vm_pageout_vminfo.vm_pageout_inactive_nolock;
1817 vm_pageout_stats[vm_pageout_stat_now].inactive_nolock = (unsigned int)(tmp - last.vm_pageout_inactive_nolock);
1818 last.vm_pageout_inactive_nolock = tmp;
1819
1820 tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal;
1821 vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_internal);
1822 last.vm_pageout_scan_inactive_throttled_internal = tmp;
1823
1824 tmp = vm_pageout_vminfo.vm_pageout_skipped_external;
1825 vm_pageout_stats[vm_pageout_stat_now].skipped_external = (unsigned int)(tmp - last.vm_pageout_skipped_external);
1826 last.vm_pageout_skipped_external = tmp;
1827
1828 tmp = vm_pageout_vminfo.vm_pageout_skipped_internal;
1829 vm_pageout_stats[vm_pageout_stat_now].skipped_internal = (unsigned int)(tmp - last.vm_pageout_skipped_internal);
1830 last.vm_pageout_skipped_internal = tmp;
1831
1832 tmp = vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded;
1833 vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded = (unsigned int)(tmp - last.vm_pageout_reactivation_limit_exceeded);
1834 last.vm_pageout_reactivation_limit_exceeded = tmp;
1835
1836 tmp = vm_pageout_vminfo.vm_pageout_inactive_force_reclaim;
1837 vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim = (unsigned int)(tmp - last.vm_pageout_inactive_force_reclaim);
1838 last.vm_pageout_inactive_force_reclaim = tmp;
1839
1840 tmp = vm_pageout_vminfo.vm_pageout_freed_internal;
1841 vm_pageout_stats[vm_pageout_stat_now].freed_internal = (unsigned int)(tmp - last.vm_pageout_freed_internal);
1842 last.vm_pageout_freed_internal = tmp;
1843
1844 tmp = vm_pageout_vminfo.vm_pageout_considered_bq_internal;
1845 vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal = (unsigned int)(tmp - last.vm_pageout_considered_bq_internal);
1846 last.vm_pageout_considered_bq_internal = tmp;
1847
1848 tmp = vm_pageout_vminfo.vm_pageout_considered_bq_external;
1849 vm_pageout_stats[vm_pageout_stat_now].considered_bq_external = (unsigned int)(tmp - last.vm_pageout_considered_bq_external);
1850 last.vm_pageout_considered_bq_external = tmp;
1851
1852 tmp = vm_pageout_vminfo.vm_pageout_filecache_min_reactivated;
1853 vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations = (unsigned int)(tmp - last.vm_pageout_filecache_min_reactivated);
1854 last.vm_pageout_filecache_min_reactivated = tmp;
1855
1856 tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_internal;
1857 vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_internal);
1858 last.vm_pageout_inactive_dirty_internal = tmp;
1859
1860 tmp = vm_pageout_vminfo.vm_pageout_forcereclaimed_sharedcache;
1861 vm_pageout_stats[vm_pageout_stat_now].forcereclaimed_sharedcache = (unsigned int)(tmp - last.vm_pageout_forcereclaimed_sharedcache);
1862 last.vm_pageout_forcereclaimed_sharedcache = tmp;
1863
1864 tmp = vm_pageout_vminfo.vm_pageout_forcereclaimed_realtime;
1865 vm_pageout_stats[vm_pageout_stat_now].forcereclaimed_realtime = (unsigned int)(tmp - last.vm_pageout_forcereclaimed_realtime);
1866 last.vm_pageout_forcereclaimed_realtime = tmp;
1867
1868 tmp = vm_pageout_vminfo.vm_pageout_protected_sharedcache;
1869 vm_pageout_stats[vm_pageout_stat_now].protected_sharedcache = (unsigned int)(tmp - last.vm_pageout_protected_sharedcache);
1870 last.vm_pageout_protected_sharedcache = tmp;
1871
1872 tmp = vm_pageout_vminfo.vm_pageout_protected_realtime;
1873 vm_pageout_stats[vm_pageout_stat_now].protected_realtime = (unsigned int)(tmp - last.vm_pageout_protected_realtime);
1874 last.vm_pageout_protected_realtime = tmp;
1875 }
1876
1877 tmp64 = vm_pageout_vminfo.vm_compactor_major_compactions_considered;
1878 vm_pageout_stats[vm_pageout_stat_now].major_compactions_considered = (unsigned int)(tmp - last.vm_compactor_major_compactions_considered);
1879 last.vm_compactor_major_compactions_considered = tmp64;
1880
1881 if (vm_pageout_stats[vm_pageout_stat_now].major_compactions_considered) {
1882 tmp64 = vm_pageout_vminfo.vm_compactor_major_compactions_completed;
1883 vm_pageout_stats[vm_pageout_stat_now].major_compactions_completed = (unsigned int)(tmp - last.vm_compactor_major_compactions_completed);
1884 last.vm_compactor_major_compactions_completed = tmp64;
1885
1886 tmp64 = vm_pageout_vminfo.vm_compactor_major_compactions_bailed;
1887 vm_pageout_stats[vm_pageout_stat_now].major_compactions_bailed = (unsigned int)(tmp - last.vm_compactor_major_compactions_bailed);
1888 last.vm_compactor_major_compactions_bailed = tmp64;
1889
1890 tmp64 = vm_pageout_vminfo.vm_compactor_major_compaction_bytes_freed;
1891 vm_pageout_stats[vm_pageout_stat_now].major_compaction_bytes_freed = (unsigned int)(tmp - last.vm_compactor_major_compaction_bytes_freed);
1892 last.vm_compactor_major_compaction_bytes_freed = tmp64;
1893
1894 tmp64 = vm_pageout_vminfo.vm_compactor_swapouts_queued;
1895 vm_pageout_stats[vm_pageout_stat_now].swapouts_queued = (unsigned int)(tmp - last.vm_compactor_swapouts_queued);
1896 last.vm_compactor_swapouts_queued = tmp64;
1897
1898 tmp64 = vm_pageout_vminfo.vm_compactor_swapout_bytes_wasted;
1899 vm_pageout_stats[vm_pageout_stat_now].swapout_bytes_wasted = (unsigned int)(tmp - last.vm_compactor_swapout_bytes_wasted);
1900 last.vm_compactor_swapout_bytes_wasted = tmp64;
1901
1902 tmp64 = vm_pageout_vminfo.vm_compactor_major_compaction_bytes_moved;
1903 vm_pageout_stats[vm_pageout_stat_now].major_compaction_bytes_moved = (unsigned int)(tmp - last.vm_compactor_major_compaction_bytes_moved);
1904 last.vm_compactor_major_compaction_bytes_moved = tmp64;
1905
1906 tmp64 = vm_pageout_vminfo.vm_compactor_major_compaction_slots_moved;
1907 vm_pageout_stats[vm_pageout_stat_now].major_compaction_slots_moved = (unsigned int)(tmp - last.vm_compactor_major_compaction_slots_moved);
1908 last.vm_compactor_major_compaction_slots_moved = tmp64;
1909
1910 tmp64 = vm_pageout_vminfo.vm_compactor_major_compaction_segments_freed;
1911 vm_pageout_stats[vm_pageout_stat_now].major_compaction_segments_freed = (unsigned int)(tmp - last.vm_compactor_major_compaction_segments_freed);
1912 last.vm_compactor_major_compaction_segments_freed = tmp64;
1913 }
1914
1915 tmp64 = vmcs_stats.unripe_under_30s;
1916 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_30s = (unsigned int)(tmp - last_vmcs.unripe_under_30s);
1917 last_vmcs.unripe_under_30s = tmp64;
1918
1919 tmp64 = vmcs_stats.unripe_under_60s;
1920 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_60s = (unsigned int)(tmp - last_vmcs.unripe_under_60s);
1921 last_vmcs.unripe_under_60s = tmp64;
1922
1923 tmp64 = vmcs_stats.unripe_under_300s;
1924 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_300s = (unsigned int)(tmp - last_vmcs.unripe_under_300s);
1925 last_vmcs.unripe_under_300s = tmp64;
1926
1927 tmp64 = vmcs_stats.reclaim_swapins;
1928 vm_pageout_stats[vm_pageout_stat_now].cswap_reclaim_swapins = (unsigned int)(tmp - last_vmcs.reclaim_swapins);
1929 last_vmcs.reclaim_swapins = tmp64;
1930
1931 tmp64 = vmcs_stats.defrag_swapins;
1932 vm_pageout_stats[vm_pageout_stat_now].cswap_defrag_swapins = (unsigned int)(tmp - last_vmcs.defrag_swapins);
1933 last_vmcs.defrag_swapins = tmp64;
1934
1935 tmp64 = vmcs_stats.compressor_swap_threshold_exceeded;
1936 vm_pageout_stats[vm_pageout_stat_now].cswap_swap_threshold_exceeded = (unsigned int)(tmp - last_vmcs.compressor_swap_threshold_exceeded);
1937 last_vmcs.compressor_swap_threshold_exceeded = tmp64;
1938
1939 tmp64 = vmcs_stats.external_q_throttled;
1940 vm_pageout_stats[vm_pageout_stat_now].cswap_external_q_throttled = (unsigned int)(tmp - last_vmcs.external_q_throttled);
1941 last_vmcs.external_q_throttled = tmp64;
1942
1943 tmp64 = vmcs_stats.free_count_below_reserve;
1944 vm_pageout_stats[vm_pageout_stat_now].cswap_free_below_reserve = (unsigned int)(tmp - last_vmcs.free_count_below_reserve);
1945 last_vmcs.free_count_below_reserve = tmp64;
1946
1947 tmp64 = vmcs_stats.thrashing_detected;
1948 vm_pageout_stats[vm_pageout_stat_now].cswap_thrashing_detected = (unsigned int)(tmp - last_vmcs.thrashing_detected);
1949 last_vmcs.thrashing_detected = tmp64;
1950
1951 tmp64 = vmcs_stats.fragmentation_detected;
1952 vm_pageout_stats[vm_pageout_stat_now].cswap_fragmentation_detected = (unsigned int)(tmp - last_vmcs.fragmentation_detected);
1953 last_vmcs.fragmentation_detected = tmp64;
1954
1955 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT1) | DBG_FUNC_NONE,
1956 vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count,
1957 vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count,
1958 vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count,
1959 vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count);
1960
1961 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT2) | DBG_FUNC_NONE,
1962 vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count,
1963 vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count,
1964 vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count);
1965
1966 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT3) | DBG_FUNC_NONE,
1967 vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed,
1968 vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count,
1969 vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count,
1970 vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count);
1971
1972 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT4) | DBG_FUNC_NONE,
1973 vm_pageout_stats[vm_pageout_stat_now].vm_page_swapped_count);
1974
1975 #if HAS_MTE
1976 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT5) | DBG_FUNC_NONE,
1977 mte_info_lists[MTE_LIST_INACTIVE_IDX].count,
1978 mte_info_lists[MTE_LIST_ACTIVE_IDX].count,
1979 vm_page_free_taggable_count,
1980 vm_page_tagged_count);
1981 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT6) | DBG_FUNC_NONE,
1982 mte_info_lists[MTE_LIST_PINNED_IDX].count +
1983 mte_info_lists[MTE_LIST_CLAIMED_IDX].count,
1984 mte_info_lists[MTE_LIST_RECLAIMING_IDX].count,
1985 vm_page_wired_tag_storage_count,
1986 mte_info_lists[MTE_LIST_ACTIVE_0_IDX].count);
1987 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT7) | DBG_FUNC_NONE,
1988 counter_load(&vm_cpu_free_claimed_count),
1989 vm_page_free_unmanaged_tag_storage_count,
1990 counter_load(&compressor_tag_storage_pages_in_pool),
1991 counter_load(&compressor_tagged_pages));
1992 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGCNT8) | DBG_FUNC_NONE,
1993 mte_claimable_queue.vmpfq_count,
1994 mte_info_lists[MTE_LIST_PINNED_IDX].count);
1995 #endif /* HAS_MTE */
1996
1997 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSEG1) | DBG_FUNC_NONE,
1998 c_segment_count,
1999 c_empty_count,
2000 c_bad_count,
2001 c_age_count);
2002 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSEG2) | DBG_FUNC_NONE,
2003 c_major_count,
2004 c_minor_count,
2005 c_swappedout_count,
2006 c_swappedout_sparse_count);
2007 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSEG3) | DBG_FUNC_NONE,
2008 c_early_swapout_count,
2009 c_regular_swapout_count,
2010 c_late_swapout_count,
2011 c_swapio_count);
2012 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSEG4) | DBG_FUNC_NONE,
2013 c_early_swappedin_count,
2014 c_regular_swappedin_count,
2015 c_late_swappedin_count,
2016 c_filling_count);
2017
2018 if (vm_pageout_stats[vm_pageout_stat_now].considered ||
2019 vm_pageout_stats[vm_pageout_stat_now].pages_compressed ||
2020 vm_pageout_stats[vm_pageout_stat_now].failed_compressions) {
2021 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT1) | DBG_FUNC_NONE,
2022 vm_pageout_stats[vm_pageout_stat_now].considered,
2023 vm_pageout_stats[vm_pageout_stat_now].freed_speculative,
2024 vm_pageout_stats[vm_pageout_stat_now].freed_external,
2025 vm_pageout_stats[vm_pageout_stat_now].inactive_referenced);
2026
2027 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT2) | DBG_FUNC_NONE,
2028 vm_pageout_stats[vm_pageout_stat_now].throttled_external_q,
2029 vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external,
2030 vm_pageout_stats[vm_pageout_stat_now].freed_cleaned,
2031 vm_pageout_stats[vm_pageout_stat_now].inactive_nolock);
2032
2033 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT3) | DBG_FUNC_NONE,
2034 vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q,
2035 vm_pageout_stats[vm_pageout_stat_now].pages_compressed,
2036 vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor,
2037 vm_pageout_stats[vm_pageout_stat_now].skipped_external);
2038
2039 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT4) | DBG_FUNC_NONE,
2040 vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded,
2041 vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim,
2042 vm_pageout_stats[vm_pageout_stat_now].failed_compressions,
2043 vm_pageout_stats[vm_pageout_stat_now].freed_internal);
2044
2045 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT5) | DBG_FUNC_NONE,
2046 vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal,
2047 vm_pageout_stats[vm_pageout_stat_now].considered_bq_external,
2048 vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations,
2049 vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal);
2050
2051 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_PGOUT6) | DBG_FUNC_NONE,
2052 vm_pageout_stats[vm_pageout_stat_now].forcereclaimed_sharedcache,
2053 vm_pageout_stats[vm_pageout_stat_now].forcereclaimed_realtime,
2054 vm_pageout_stats[vm_pageout_stat_now].protected_sharedcache,
2055 vm_pageout_stats[vm_pageout_stat_now].protected_realtime);
2056 }
2057
2058 if (vm_pageout_stats[vm_pageout_stat_now].major_compactions_considered) {
2059 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_COMPACTOR1) | DBG_FUNC_NONE,
2060 vm_pageout_stats[vm_pageout_stat_now].major_compactions_considered,
2061 vm_pageout_stats[vm_pageout_stat_now].major_compactions_completed,
2062 vm_pageout_stats[vm_pageout_stat_now].major_compactions_bailed,
2063 vm_pageout_stats[vm_pageout_stat_now].major_compaction_bytes_freed);
2064 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_COMPACTOR2) | DBG_FUNC_NONE,
2065 vm_pageout_stats[vm_pageout_stat_now].major_compaction_segments_freed,
2066 vm_pageout_stats[vm_pageout_stat_now].major_compaction_bytes_moved,
2067 vm_pageout_stats[vm_pageout_stat_now].major_compaction_slots_moved);
2068 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_COMPACTOR3) | DBG_FUNC_NONE,
2069 vm_pageout_stats[vm_pageout_stat_now].swapouts_queued,
2070 vm_pageout_stats[vm_pageout_stat_now].swapout_bytes_wasted);
2071 }
2072
2073 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSWAP1) | DBG_FUNC_NONE,
2074 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_30s,
2075 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_60s,
2076 vm_pageout_stats[vm_pageout_stat_now].cswap_unripe_under_300s,
2077 vm_pageout_stats[vm_pageout_stat_now].cswap_reclaim_swapins);
2078 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSWAP2) | DBG_FUNC_NONE,
2079 vm_pageout_stats[vm_pageout_stat_now].cswap_defrag_swapins,
2080 vm_pageout_stats[vm_pageout_stat_now].cswap_swap_threshold_exceeded,
2081 vm_pageout_stats[vm_pageout_stat_now].cswap_external_q_throttled,
2082 vm_pageout_stats[vm_pageout_stat_now].cswap_free_below_reserve);
2083 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_CSWAP3) | DBG_FUNC_NONE,
2084 vm_pageout_stats[vm_pageout_stat_now].cswap_thrashing_detected,
2085 vm_pageout_stats[vm_pageout_stat_now].cswap_fragmentation_detected);
2086
2087 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_DEMAND1) | DBG_FUNC_NONE,
2088 vm_pageout_stats[vm_pageout_stat_now].pages_grabbed,
2089 vm_pageout_stats[vm_pageout_stat_now].pages_freed,
2090 vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found,
2091 vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added);
2092
2093 KDBG_RELEASE(MEMINFO_CODE(DBG_MEMINFO_DEMAND2) | DBG_FUNC_NONE,
2094 vm_pageout_stats[vm_pageout_stat_now].swapouts,
2095 vm_pageout_stats[vm_pageout_stat_now].swapins);
2096
2097 record_memory_pressure();
2098 }
2099
2100 extern boolean_t hibernation_vmqueues_inspection;
2101
2102 /*
2103 * Return values for functions called by vm_pageout_scan
2104 * that control its flow.
2105 *
2106 * PROCEED -- vm_pageout_scan will keep making forward progress.
2107 * DONE_RETURN -- page demand satisfied, work is done -> vm_pageout_scan returns.
2108 * NEXT_ITERATION -- restart the 'for' loop in vm_pageout_scan aka continue.
2109 */
2110
2111 #define VM_PAGEOUT_SCAN_PROCEED (0)
2112 #define VM_PAGEOUT_SCAN_DONE_RETURN (1)
2113 #define VM_PAGEOUT_SCAN_NEXT_ITERATION (2)
2114
2115 /*
2116 * This function is called only from vm_pageout_scan and
2117 * it moves overflow secluded pages (one-at-a-time) to the
2118 * batched 'local' free Q or active Q.
2119 */
2120 static void
2121 vps_deal_with_secluded_page_overflow(vm_page_t *local_freeq, int *local_freed)
2122 {
2123 #if CONFIG_SECLUDED_MEMORY
2124 /*
2125 * Deal with secluded_q overflow.
2126 */
2127 if (vm_page_secluded_count > vm_page_secluded_target) {
2128 vm_page_t secluded_page;
2129
2130 /*
2131 * SECLUDED_AGING_BEFORE_ACTIVE:
2132 * Excess secluded pages go to the active queue and
2133 * will later go to the inactive queue.
2134 */
2135 assert((vm_page_secluded_count_free +
2136 vm_page_secluded_count_inuse) ==
2137 vm_page_secluded_count);
2138 secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
2139 assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
2140
2141 vm_page_queues_remove(secluded_page, FALSE);
2142 assert(!vm_page_is_fictitious(secluded_page));
2143 assert(!VM_PAGE_WIRED(secluded_page));
2144
2145 if (secluded_page->vmp_object == 0) {
2146 /* transfer to free queue */
2147 assert(secluded_page->vmp_busy);
2148 secluded_page->vmp_snext = *local_freeq;
2149 *local_freeq = secluded_page;
2150 *local_freed += 1;
2151 } else {
2152 /* transfer to head of active queue */
2153 vm_page_enqueue_active(secluded_page, FALSE);
2154 secluded_page = VM_PAGE_NULL;
2155 }
2156 }
2157 #else /* CONFIG_SECLUDED_MEMORY */
2158
2159 #pragma unused(local_freeq)
2160 #pragma unused(local_freed)
2161
2162 return;
2163
2164 #endif /* CONFIG_SECLUDED_MEMORY */
2165 }
2166
2167 /*
2168 * This function is called only from vm_pageout_scan and
2169 * it initializes the loop targets for vm_pageout_scan().
2170 */
2171 static void
2172 vps_init_page_targets(void)
2173 {
2174 /*
2175 * LD TODO: Other page targets should be calculated here too.
2176 */
2177 vm_page_anonymous_min = vm_page_inactive_target / 20;
2178
2179 if (vm_pageout_state.vm_page_speculative_percentage > 50) {
2180 vm_pageout_state.vm_page_speculative_percentage = 50;
2181 } else if (vm_pageout_state.vm_page_speculative_percentage <= 0) {
2182 vm_pageout_state.vm_page_speculative_percentage = 1;
2183 }
2184
2185 vm_pageout_state.vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
2186 vm_page_inactive_count);
2187 }
2188
2189 /*
2190 * This function is called only from vm_pageout_scan and
2191 * it purges a single VM object at-a-time and will either
2192 * make vm_pageout_scan() restart the loop or keeping moving forward.
2193 */
2194 static int
2195 vps_purge_object()
2196 {
2197 int force_purge;
2198
2199 assert(available_for_purge >= 0);
2200 force_purge = 0; /* no force-purging */
2201
2202 #if VM_PRESSURE_EVENTS
2203 vm_pressure_level_t pressure_level;
2204
2205 pressure_level = memorystatus_vm_pressure_level;
2206
2207 if (pressure_level > kVMPressureNormal) {
2208 if (pressure_level >= kVMPressureCritical) {
2209 force_purge = vm_pageout_state.memorystatus_purge_on_critical;
2210 } else if (pressure_level >= kVMPressureUrgent) {
2211 force_purge = vm_pageout_state.memorystatus_purge_on_urgent;
2212 } else if (pressure_level >= kVMPressureWarning) {
2213 force_purge = vm_pageout_state.memorystatus_purge_on_warning;
2214 }
2215 }
2216 #endif /* VM_PRESSURE_EVENTS */
2217
2218 if (available_for_purge || force_purge) {
2219 memoryshot(DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_START);
2220
2221 VM_DEBUG_EVENT(vm_pageout_purgeone, DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
2222 if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
2223 VM_PAGEOUT_DEBUG(vm_pageout_purged_objects, 1);
2224 VM_DEBUG_EVENT(vm_pageout_purgeone, DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
2225 memoryshot(DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
2226
2227 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
2228 }
2229 VM_DEBUG_EVENT(vm_pageout_purgeone, DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1);
2230 memoryshot(DBG_VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
2231 }
2232
2233 return VM_PAGEOUT_SCAN_PROCEED;
2234 }
2235
2236 /*
2237 * This function is called only from vm_pageout_scan and
2238 * it will try to age the next speculative Q if the oldest
2239 * one is empty.
2240 */
2241 static int
2242 vps_age_speculative_queue(boolean_t force_speculative_aging)
2243 {
2244 #define DELAY_SPECULATIVE_AGE 1000
2245
2246 /*
2247 * try to pull pages from the aging bins...
2248 * see vm_page_internal.h for an explanation of how
2249 * this mechanism works
2250 */
2251 boolean_t can_steal = FALSE;
2252 int num_scanned_queues;
2253 static int delay_speculative_age = 0; /* depends the # of times we go through the main pageout_scan loop.*/
2254 mach_timespec_t ts;
2255 struct vm_speculative_age_q *aq;
2256 struct vm_speculative_age_q *sq;
2257
2258 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
2259
2260 aq = &vm_page_queue_speculative[speculative_steal_index];
2261
2262 num_scanned_queues = 0;
2263 while (vm_page_queue_empty(&aq->age_q) &&
2264 num_scanned_queues++ != vm_page_max_speculative_age_q) {
2265 speculative_steal_index++;
2266
2267 if (speculative_steal_index > vm_page_max_speculative_age_q) {
2268 speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
2269 }
2270
2271 aq = &vm_page_queue_speculative[speculative_steal_index];
2272 }
2273
2274 if (num_scanned_queues == vm_page_max_speculative_age_q + 1) {
2275 /*
2276 * XXX We've scanned all the speculative
2277 * queues but still haven't found one
2278 * that is not empty, even though
2279 * vm_page_speculative_count is not 0.
2280 */
2281 if (!vm_page_queue_empty(&sq->age_q)) {
2282 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
2283 }
2284 #if DEVELOPMENT || DEBUG
2285 panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count);
2286 #endif
2287 /* readjust... */
2288 vm_page_speculative_count = 0;
2289 /* ... and continue */
2290 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
2291 }
2292
2293 if (vm_page_speculative_count > vm_pageout_state.vm_page_speculative_target || force_speculative_aging == TRUE) {
2294 can_steal = TRUE;
2295 } else {
2296 if (!delay_speculative_age) {
2297 mach_timespec_t ts_fully_aged;
2298
2299 ts_fully_aged.tv_sec = (vm_page_max_speculative_age_q * vm_pageout_state.vm_page_speculative_q_age_ms) / 1000;
2300 ts_fully_aged.tv_nsec = ((vm_page_max_speculative_age_q * vm_pageout_state.vm_page_speculative_q_age_ms) % 1000)
2301 * 1000 * NSEC_PER_USEC;
2302
2303 ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
2304
2305 clock_sec_t sec;
2306 clock_nsec_t nsec;
2307 clock_get_system_nanotime(&sec, &nsec);
2308 ts.tv_sec = (unsigned int) sec;
2309 ts.tv_nsec = nsec;
2310
2311 if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) {
2312 can_steal = TRUE;
2313 } else {
2314 delay_speculative_age++;
2315 }
2316 } else {
2317 delay_speculative_age++;
2318 if (delay_speculative_age == DELAY_SPECULATIVE_AGE) {
2319 delay_speculative_age = 0;
2320 }
2321 }
2322 }
2323 if (can_steal == TRUE) {
2324 vm_page_speculate_ageit(aq);
2325 }
2326
2327 return VM_PAGEOUT_SCAN_PROCEED;
2328 }
2329
2330 /*
2331 * This function is called only from vm_pageout_scan and
2332 * it evicts a single VM object from the cache.
2333 */
2334 static int inline
2335 vps_object_cache_evict(vm_object_t *object_to_unlock)
2336 {
2337 static int cache_evict_throttle = 0;
2338 struct vm_speculative_age_q *sq;
2339
2340 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
2341
2342 if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) {
2343 int pages_evicted;
2344
2345 if (*object_to_unlock != NULL) {
2346 vm_object_unlock(*object_to_unlock);
2347 *object_to_unlock = NULL;
2348 }
2349 KDBG(0x13001ec | DBG_FUNC_START);
2350
2351 pages_evicted = vm_object_cache_evict(100, 10);
2352
2353 KDBG(0x13001ec | DBG_FUNC_END, pages_evicted);
2354
2355 if (pages_evicted) {
2356 vm_pageout_vminfo.vm_pageout_pages_evicted += pages_evicted;
2357
2358 VM_DEBUG_EVENT(vm_pageout_cache_evict, DBG_VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
2359 vm_page_free_count, pages_evicted, vm_pageout_vminfo.vm_pageout_pages_evicted, 0);
2360 memoryshot(DBG_VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
2361
2362 /*
2363 * we just freed up to 100 pages,
2364 * so go back to the top of the main loop
2365 * and re-evaulate the memory situation
2366 */
2367 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
2368 } else {
2369 cache_evict_throttle = 1000;
2370 }
2371 }
2372 if (cache_evict_throttle) {
2373 cache_evict_throttle--;
2374 }
2375
2376 return VM_PAGEOUT_SCAN_PROCEED;
2377 }
2378
2379
2380 /*
2381 * This function is called only from vm_pageout_scan and
2382 * it calculates the filecache min. that needs to be maintained
2383 * as we start to steal pages.
2384 */
2385 static void
2386 vps_calculate_filecache_min(void)
2387 {
2388 int divisor = vm_pageout_state.vm_page_filecache_min_divisor;
2389
2390 #if CONFIG_JETSAM
2391 /*
2392 * don't let the filecache_min fall below 15% of available memory
2393 * on systems with an active compressor that isn't nearing its
2394 * limits w/r to accepting new data
2395 *
2396 * on systems w/o the compressor/swapper, the filecache is always
2397 * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY
2398 * since most (if not all) of the anonymous pages are in the
2399 * throttled queue (which isn't counted as available) which
2400 * effectively disables this filter
2401 */
2402 if (vm_compressor_low_on_space() || divisor == 0) {
2403 vm_pageout_state.vm_page_filecache_min = 0;
2404 } else {
2405 vm_pageout_state.vm_page_filecache_min =
2406 ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;
2407 }
2408 #else
2409 if (vm_compressor_out_of_space() || divisor == 0) {
2410 vm_pageout_state.vm_page_filecache_min = 0;
2411 } else {
2412 /*
2413 * don't let the filecache_min fall below the specified critical level
2414 */
2415 vm_pageout_state.vm_page_filecache_min =
2416 ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;
2417 }
2418 #endif
2419 if (vm_page_free_count < (vm_page_free_reserved / 4)) {
2420 vm_pageout_state.vm_page_filecache_min = 0;
2421 }
2422 }
2423
2424 /*
2425 * This function is called only from vm_pageout_scan and
2426 * it updates the flow control time to detect if VM pageoutscan
2427 * isn't making progress.
2428 */
2429 static void
2430 vps_flow_control_reset_deadlock_timer(struct flow_control *flow_control)
2431 {
2432 mach_timespec_t ts;
2433 clock_sec_t sec;
2434 clock_nsec_t nsec;
2435
2436 ts.tv_sec = vm_pageout_state.vm_pageout_deadlock_wait / 1000;
2437 ts.tv_nsec = (vm_pageout_state.vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
2438 clock_get_system_nanotime(&sec, &nsec);
2439 flow_control->ts.tv_sec = (unsigned int) sec;
2440 flow_control->ts.tv_nsec = nsec;
2441 ADD_MACH_TIMESPEC(&flow_control->ts, &ts);
2442
2443 flow_control->state = FCS_DELAYED;
2444
2445 vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal++;
2446 }
2447
2448 /*
2449 * This function is called only from vm_pageout_scan and
2450 * it is the flow control logic of VM pageout scan which
2451 * controls if it should block and for how long.
2452 * Any blocking of vm_pageout_scan happens ONLY in this function.
2453 */
2454 static int
2455 vps_flow_control(struct flow_control *flow_control, int *anons_grabbed, vm_object_t *object, int *delayed_unlock,
2456 vm_page_t *local_freeq, int *local_freed, int *vm_pageout_deadlock_target, unsigned int inactive_burst_count)
2457 {
2458 boolean_t exceeded_burst_throttle = FALSE;
2459 unsigned int msecs = 0;
2460 uint32_t inactive_external_count;
2461 mach_timespec_t ts;
2462 struct vm_pageout_queue *iq;
2463 struct vm_pageout_queue *eq;
2464 struct vm_speculative_age_q *sq;
2465
2466 iq = &vm_pageout_queue_internal;
2467 eq = &vm_pageout_queue_external;
2468 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
2469
2470 /*
2471 * Sometimes we have to pause:
2472 * 1) No inactive pages - nothing to do.
2473 * 2) Loop control - no acceptable pages found on the inactive queue
2474 * within the last vm_pageout_burst_inactive_throttle iterations
2475 * 3) Flow control - default pageout queue is full
2476 */
2477 if (vm_page_queue_empty(&vm_page_queue_inactive) &&
2478 vm_page_queue_empty(&vm_page_queue_anonymous) &&
2479 vm_page_queue_empty(&vm_page_queue_cleaned) &&
2480 vm_page_queue_empty(&sq->age_q)) {
2481 VM_PAGEOUT_DEBUG(vm_pageout_scan_empty_throttle, 1);
2482 msecs = vm_pageout_state.vm_pageout_empty_wait;
2483 } else if (inactive_burst_count >=
2484 MIN(vm_pageout_state.vm_pageout_burst_inactive_throttle,
2485 (vm_page_inactive_count +
2486 vm_page_speculative_count))) {
2487 VM_PAGEOUT_DEBUG(vm_pageout_scan_burst_throttle, 1);
2488 msecs = vm_pageout_state.vm_pageout_burst_wait;
2489
2490 exceeded_burst_throttle = TRUE;
2491 } else if (VM_PAGE_Q_THROTTLED(iq) &&
2492 VM_DYNAMIC_PAGING_ENABLED()) {
2493 clock_sec_t sec;
2494 clock_nsec_t nsec;
2495
2496 switch (flow_control->state) {
2497 case FCS_IDLE:
2498 if ((vm_page_free_count + *local_freed) < vm_page_free_target &&
2499 vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
2500 /*
2501 * since the compressor is running independently of vm_pageout_scan
2502 * let's not wait for it just yet... as long as we have a healthy supply
2503 * of filecache pages to work with, let's keep stealing those.
2504 */
2505 inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
2506
2507 if (vm_page_pageable_external_count > vm_pageout_state.vm_page_filecache_min &&
2508 (inactive_external_count >= VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {
2509 *anons_grabbed = ANONS_GRABBED_LIMIT;
2510 VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle_deferred, 1);
2511 return VM_PAGEOUT_SCAN_PROCEED;
2512 }
2513 }
2514
2515 vps_flow_control_reset_deadlock_timer(flow_control);
2516 msecs = vm_pageout_state.vm_pageout_deadlock_wait;
2517
2518 break;
2519
2520 case FCS_DELAYED:
2521 clock_get_system_nanotime(&sec, &nsec);
2522 ts.tv_sec = (unsigned int) sec;
2523 ts.tv_nsec = nsec;
2524
2525 if (CMP_MACH_TIMESPEC(&ts, &flow_control->ts) >= 0) {
2526 /*
2527 * the pageout thread for the default pager is potentially
2528 * deadlocked since the
2529 * default pager queue has been throttled for more than the
2530 * allowable time... we need to move some clean pages or dirty
2531 * pages belonging to the external pagers if they aren't throttled
2532 * vm_page_free_wanted represents the number of threads currently
2533 * blocked waiting for pages... we'll move one page for each of
2534 * these plus a fixed amount to break the logjam... once we're done
2535 * moving this number of pages, we'll re-enter the FSC_DELAYED state
2536 * with a new timeout target since we have no way of knowing
2537 * whether we've broken the deadlock except through observation
2538 * of the queue associated with the default pager... we need to
2539 * stop moving pages and allow the system to run to see what
2540 * state it settles into.
2541 */
2542
2543 *vm_pageout_deadlock_target = vm_pageout_state.vm_pageout_deadlock_relief +
2544 vm_page_free_wanted + vm_page_free_wanted_privileged;
2545 VM_PAGEOUT_DEBUG(vm_pageout_scan_deadlock_detected, 1);
2546 flow_control->state = FCS_DEADLOCK_DETECTED;
2547 sched_cond_signal(&vm_pageout_gc_cond, vm_pageout_gc_thread);
2548 return VM_PAGEOUT_SCAN_PROCEED;
2549 }
2550 /*
2551 * just resniff instead of trying
2552 * to compute a new delay time... we're going to be
2553 * awakened immediately upon a laundry completion,
2554 * so we won't wait any longer than necessary
2555 */
2556 msecs = vm_pageout_state.vm_pageout_idle_wait;
2557 break;
2558
2559 case FCS_DEADLOCK_DETECTED:
2560 if (*vm_pageout_deadlock_target) {
2561 return VM_PAGEOUT_SCAN_PROCEED;
2562 }
2563
2564 vps_flow_control_reset_deadlock_timer(flow_control);
2565 msecs = vm_pageout_state.vm_pageout_deadlock_wait;
2566
2567 break;
2568 }
2569 } else {
2570 /*
2571 * No need to pause...
2572 */
2573 return VM_PAGEOUT_SCAN_PROCEED;
2574 }
2575
2576 vm_pageout_scan_wants_object = VM_OBJECT_NULL;
2577
2578 vm_pageout_prepare_to_block(object, delayed_unlock, local_freeq, local_freed,
2579 VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
2580
2581 if (vm_page_free_count >= vm_page_free_target) {
2582 /*
2583 * we're here because
2584 * 1) someone else freed up some pages while we had
2585 * the queues unlocked above
2586 * and we've hit one of the 3 conditions that
2587 * cause us to pause the pageout scan thread
2588 *
2589 * since we already have enough free pages,
2590 * let's avoid stalling and return normally
2591 *
2592 * before we return, make sure the pageout I/O threads
2593 * are running throttled in case there are still requests
2594 * in the laundry... since we have enough free pages
2595 * we don't need the laundry to be cleaned in a timely
2596 * fashion... so let's avoid interfering with foreground
2597 * activity
2598 *
2599 * we don't want to hold vm_page_queue_free_lock when
2600 * calling vm_pageout_adjust_eq_iothrottle (since it
2601 * may cause other locks to be taken), we do the intitial
2602 * check outside of the lock. Once we take the lock,
2603 * we recheck the condition since it may have changed.
2604 * if it has, no problem, we will make the threads
2605 * non-throttled before actually blocking
2606 */
2607 vm_pageout_adjust_eq_iothrottle(&pgo_iothread_external_state, TRUE);
2608 }
2609 vm_free_page_lock();
2610
2611 if (vm_page_free_count >= vm_page_free_target &&
2612 (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
2613 return VM_PAGEOUT_SCAN_DONE_RETURN;
2614 }
2615 vm_free_page_unlock();
2616
2617 if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
2618 /*
2619 * we're most likely about to block due to one of
2620 * the 3 conditions that cause vm_pageout_scan to
2621 * not be able to make forward progress w/r
2622 * to providing new pages to the free queue,
2623 * so unthrottle the I/O threads in case we
2624 * have laundry to be cleaned... it needs
2625 * to be completed ASAP.
2626 *
2627 * even if we don't block, we want the io threads
2628 * running unthrottled since the sum of free +
2629 * clean pages is still under our free target
2630 */
2631 vm_pageout_adjust_eq_iothrottle(&pgo_iothread_external_state, FALSE);
2632 }
2633 if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) {
2634 /*
2635 * if we get here we're below our free target and
2636 * we're stalling due to a full laundry queue or
2637 * we don't have any inactive pages other then
2638 * those in the clean queue...
2639 * however, we have pages on the clean queue that
2640 * can be moved to the free queue, so let's not
2641 * stall the pageout scan
2642 */
2643 flow_control->state = FCS_IDLE;
2644 return VM_PAGEOUT_SCAN_PROCEED;
2645 }
2646 if (flow_control->state == FCS_DELAYED && !VM_PAGE_Q_THROTTLED(iq)) {
2647 flow_control->state = FCS_IDLE;
2648 return VM_PAGEOUT_SCAN_PROCEED;
2649 }
2650
2651 VM_CHECK_MEMORYSTATUS;
2652
2653 if (flow_control->state != FCS_IDLE) {
2654 VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle, 1);
2655 }
2656
2657 iq->pgo_throttled = TRUE;
2658 assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000 * NSEC_PER_USEC);
2659
2660 vm_page_unlock_queues();
2661
2662 assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
2663
2664 VM_DEBUG_EVENT(vm_pageout_thread_block, DBG_VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
2665 iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
2666 memoryshot(DBG_VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
2667
2668 thread_block(THREAD_CONTINUE_NULL);
2669
2670 VM_DEBUG_EVENT(vm_pageout_thread_block, DBG_VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
2671 iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
2672 memoryshot(DBG_VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
2673
2674 vm_page_lock_queues();
2675
2676 iq->pgo_throttled = FALSE;
2677
2678 vps_init_page_targets();
2679
2680 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
2681 }
2682
2683 extern boolean_t vm_darkwake_mode;
2684 /*
2685 * This function is called only from vm_pageout_scan and
2686 * it will find and return the most appropriate page to be
2687 * reclaimed.
2688 */
2689 static int
2690 vps_choose_victim_page(vm_page_t *victim_page, int *anons_grabbed, boolean_t *grab_anonymous, boolean_t force_anonymous,
2691 boolean_t *is_page_from_bg_q, unsigned int *reactivated_this_call)
2692 {
2693 vm_page_t m = NULL;
2694 vm_object_t m_object = VM_OBJECT_NULL;
2695 uint32_t inactive_external_count;
2696 struct vm_speculative_age_q *sq;
2697 struct vm_pageout_queue *iq;
2698 int retval = VM_PAGEOUT_SCAN_PROCEED;
2699
2700 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
2701 iq = &vm_pageout_queue_internal;
2702
2703 *is_page_from_bg_q = FALSE;
2704
2705 m = NULL;
2706 m_object = VM_OBJECT_NULL;
2707
2708 if (VM_DYNAMIC_PAGING_ENABLED()) {
2709 assert(vm_page_throttled_count == 0);
2710 assert(vm_page_queue_empty(&vm_page_queue_throttled));
2711 }
2712
2713 /*
2714 * Try for a clean-queue inactive page.
2715 * These are pages that vm_pageout_scan tried to steal earlier, but
2716 * were dirty and had to be cleaned. Pick them up now that they are clean.
2717 */
2718 if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
2719 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
2720
2721 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
2722
2723 goto found_page;
2724 }
2725
2726 /*
2727 * The next most eligible pages are ones we paged in speculatively,
2728 * but which have not yet been touched and have been aged out.
2729 */
2730 if (!vm_page_queue_empty(&sq->age_q)) {
2731 m = (vm_page_t) vm_page_queue_first(&sq->age_q);
2732
2733 assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
2734
2735 if (!m->vmp_dirty || force_anonymous == FALSE) {
2736 goto found_page;
2737 } else {
2738 m = NULL;
2739 }
2740 }
2741
2742 #if !CONFIG_JETSAM
2743 if (vm_page_donate_mode != VM_PAGE_DONATE_DISABLED) {
2744 if (vm_page_donate_queue_ripe && !vm_page_queue_empty(&vm_page_queue_donate)) {
2745 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_donate);
2746 assert(m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE);
2747 goto found_page;
2748 }
2749 }
2750 #endif /* !CONFIG_JETSAM */
2751
2752 if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) {
2753 vm_object_t bg_m_object = NULL;
2754
2755 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
2756
2757 bg_m_object = VM_PAGE_OBJECT(m);
2758
2759 if (!VM_PAGE_PAGEABLE(m) || (vm_darkwake_mode && m->vmp_busy)) {
2760 /*
2761 * This page is on the background queue
2762 * but not on a pageable queue OR is busy during
2763 * darkwake mode when the target is artificially lowered.
2764 * If it is busy during darkwake mode, and we don't skip it,
2765 * we will just swing back around and try again with the same
2766 * queue and might hit the same page or its neighbor in a
2767 * similar state. Both of these are transient states and will
2768 * get resolved, but, at this point let's ignore this page.
2769 */
2770 if (vm_darkwake_mode && m->vmp_busy) {
2771 if (bg_m_object->internal) {
2772 vm_pageout_skipped_bq_internal++;
2773 } else {
2774 vm_pageout_skipped_bq_external++;
2775 }
2776 }
2777 } else if (force_anonymous == FALSE || bg_m_object->internal) {
2778 if (bg_m_object->internal &&
2779 (VM_PAGE_Q_THROTTLED(iq) ||
2780 vm_compressor_out_of_space() == TRUE ||
2781 vm_page_free_count < (vm_page_free_reserved / 4))) {
2782 vm_pageout_skipped_bq_internal++;
2783 } else {
2784 *is_page_from_bg_q = TRUE;
2785
2786 if (bg_m_object->internal) {
2787 vm_pageout_vminfo.vm_pageout_considered_bq_internal++;
2788 } else {
2789 vm_pageout_vminfo.vm_pageout_considered_bq_external++;
2790 }
2791 goto found_page;
2792 }
2793 }
2794 }
2795
2796 inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
2797
2798 if ((vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min || force_anonymous == TRUE) ||
2799 (inactive_external_count < VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {
2800 *grab_anonymous = TRUE;
2801 *anons_grabbed = 0;
2802
2803 if (VM_CONFIG_SWAP_IS_ACTIVE) {
2804 vm_pageout_vminfo.vm_pageout_skipped_external++;
2805 } else {
2806 if (vm_page_free_count < (COMPRESSOR_FREE_RESERVED_LIMIT * 2)) {
2807 /*
2808 * No swap and we are in dangerously low levels of free memory.
2809 * If we keep going ahead with anonymous pages, we are going to run into a situation
2810 * where the compressor will be stuck waiting for free pages (if it isn't already).
2811 *
2812 * So, pick a file backed page...
2813 */
2814 *grab_anonymous = FALSE;
2815 *anons_grabbed = ANONS_GRABBED_LIMIT;
2816 vm_pageout_vminfo.vm_pageout_skipped_internal++;
2817 }
2818 }
2819 goto want_anonymous;
2820 }
2821 *grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
2822
2823 #if CONFIG_JETSAM
2824 /* If the file-backed pool has accumulated
2825 * significantly more pages than the jetsam
2826 * threshold, prefer to reclaim those
2827 * inline to minimise compute overhead of reclaiming
2828 * anonymous pages.
2829 * This calculation does not account for the CPU local
2830 * external page queues, as those are expected to be
2831 * much smaller relative to the global pools.
2832 */
2833
2834 struct vm_pageout_queue *eq = &vm_pageout_queue_external;
2835
2836 if (*grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) {
2837 if (vm_page_pageable_external_count >
2838 vm_pageout_state.vm_page_filecache_min) {
2839 if ((vm_page_pageable_external_count *
2840 vm_pageout_memorystatus_fb_factor_dr) >
2841 (memorystatus_get_critical_page_shortage_threshold() *
2842 vm_pageout_memorystatus_fb_factor_nr)) {
2843 *grab_anonymous = FALSE;
2844
2845 VM_PAGEOUT_DEBUG(vm_grab_anon_overrides, 1);
2846 }
2847 }
2848 if (*grab_anonymous) {
2849 VM_PAGEOUT_DEBUG(vm_grab_anon_nops, 1);
2850 }
2851 }
2852 #endif /* CONFIG_JETSAM */
2853
2854 want_anonymous:
2855 if (*grab_anonymous == FALSE || *anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
2856 if (!vm_page_queue_empty(&vm_page_queue_inactive)) {
2857 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
2858
2859 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
2860 *anons_grabbed = 0;
2861
2862 if (vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min) {
2863 if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
2864 if ((++(*reactivated_this_call) % 100)) {
2865 vm_pageout_vminfo.vm_pageout_filecache_min_reactivated++;
2866
2867 vm_page_activate(m);
2868 counter_inc(&vm_statistics_reactivations);
2869 #if DEVELOPMENT || DEBUG
2870 if (*is_page_from_bg_q == TRUE) {
2871 if (m_object->internal) {
2872 vm_pageout_rejected_bq_internal++;
2873 } else {
2874 vm_pageout_rejected_bq_external++;
2875 }
2876 }
2877 #endif /* DEVELOPMENT || DEBUG */
2878 vm_pageout_state.vm_pageout_inactive_used++;
2879
2880 m = NULL;
2881 retval = VM_PAGEOUT_SCAN_NEXT_ITERATION;
2882
2883 goto found_page;
2884 }
2885
2886 /*
2887 * steal 1 of the file backed pages even if
2888 * we are under the limit that has been set
2889 * for a healthy filecache
2890 */
2891 }
2892 }
2893 goto found_page;
2894 }
2895 }
2896 if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
2897 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
2898
2899 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
2900 *anons_grabbed += 1;
2901
2902 goto found_page;
2903 }
2904
2905 m = NULL;
2906
2907 found_page:
2908 *victim_page = m;
2909
2910 return retval;
2911 }
2912
2913 /*
2914 * This function is called only from vm_pageout_scan and
2915 * it will put a page back on the active/inactive queue
2916 * if we can't reclaim it for some reason.
2917 */
2918 static void
2919 vps_requeue_page(vm_page_t m, int page_prev_q_state, __unused boolean_t page_from_bg_q)
2920 {
2921 if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
2922 vm_page_enqueue_inactive(m, FALSE);
2923 } else {
2924 vm_page_activate(m);
2925 }
2926
2927 #if DEVELOPMENT || DEBUG
2928 vm_object_t m_object = VM_PAGE_OBJECT(m);
2929
2930 if (page_from_bg_q == TRUE) {
2931 if (m_object->internal) {
2932 vm_pageout_rejected_bq_internal++;
2933 } else {
2934 vm_pageout_rejected_bq_external++;
2935 }
2936 }
2937 #endif /* DEVELOPMENT || DEBUG */
2938 }
2939
2940 /*
2941 * This function is called only from vm_pageout_scan and
2942 * it will try to grab the victim page's VM object (m_object)
2943 * which differs from the previous victim page's object (object).
2944 */
2945 static int
2946 vps_switch_object(vm_page_t m, vm_object_t m_object, vm_object_t *object, int page_prev_q_state, boolean_t avoid_anon_pages, boolean_t page_from_bg_q)
2947 {
2948 struct vm_speculative_age_q *sq;
2949
2950 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
2951
2952 /*
2953 * the object associated with candidate page is
2954 * different from the one we were just working
2955 * with... dump the lock if we still own it
2956 */
2957 if (*object != NULL) {
2958 vm_object_unlock(*object);
2959 *object = NULL;
2960 }
2961 /*
2962 * Try to lock object; since we've alread got the
2963 * page queues lock, we can only 'try' for this one.
2964 * if the 'try' fails, we need to do a mutex_pause
2965 * to allow the owner of the object lock a chance to
2966 * run... otherwise, we're likely to trip over this
2967 * object in the same state as we work our way through
2968 * the queue... clumps of pages associated with the same
2969 * object are fairly typical on the inactive and active queues
2970 */
2971 if (!vm_object_lock_try_scan(m_object)) {
2972 vm_page_t m_want = NULL;
2973
2974 vm_pageout_vminfo.vm_pageout_inactive_nolock++;
2975
2976 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
2977 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_nolock, 1);
2978 }
2979
2980 pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
2981
2982 m->vmp_reference = FALSE;
2983
2984 if (!m_object->object_is_shared_cache) {
2985 /*
2986 * don't apply this optimization if this is the shared cache
2987 * object, it's too easy to get rid of very hot and important
2988 * pages...
2989 * m->vmp_object must be stable since we hold the page queues lock...
2990 * we can update the scan_collisions field sans the object lock
2991 * since it is a separate field and this is the only spot that does
2992 * a read-modify-write operation and it is never executed concurrently...
2993 * we can asynchronously set this field to 0 when creating a UPL, so it
2994 * is possible for the value to be a bit non-determistic, but that's ok
2995 * since it's only used as a hint
2996 */
2997 m_object->scan_collisions = 1;
2998 }
2999 if (page_from_bg_q) {
3000 m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
3001 } else if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
3002 m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
3003 } else if (!vm_page_queue_empty(&sq->age_q)) {
3004 m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
3005 } else if ((avoid_anon_pages || vm_page_queue_empty(&vm_page_queue_anonymous)) &&
3006 !vm_page_queue_empty(&vm_page_queue_inactive)) {
3007 m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
3008 } else if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
3009 m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
3010 }
3011
3012 /*
3013 * this is the next object we're going to be interested in
3014 * try to make sure its available after the mutex_pause
3015 * returns control
3016 */
3017 if (m_want) {
3018 vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want);
3019 }
3020
3021 vps_requeue_page(m, page_prev_q_state, page_from_bg_q);
3022
3023 return VM_PAGEOUT_SCAN_NEXT_ITERATION;
3024 } else {
3025 *object = m_object;
3026 vm_pageout_scan_wants_object = VM_OBJECT_NULL;
3027 }
3028
3029 return VM_PAGEOUT_SCAN_PROCEED;
3030 }
3031
3032 /*
3033 * This function is called only from vm_pageout_scan and
3034 * it notices that pageout scan may be rendered ineffective
3035 * due to a FS deadlock and will jetsam a process if possible.
3036 * If jetsam isn't supported, it'll move the page to the active
3037 * queue to try and get some different pages pushed onwards so
3038 * we can try to get out of this scenario.
3039 */
3040 static void
3041 vps_deal_with_throttled_queues(vm_page_t m, vm_object_t *object, uint32_t *vm_pageout_inactive_external_forced_reactivate_limit,
3042 boolean_t *force_anonymous, __unused boolean_t is_page_from_bg_q)
3043 {
3044 struct vm_pageout_queue *eq;
3045 vm_object_t cur_object = VM_OBJECT_NULL;
3046
3047 cur_object = *object;
3048
3049 eq = &vm_pageout_queue_external;
3050
3051 if (cur_object->internal == FALSE) {
3052 /*
3053 * we need to break up the following potential deadlock case...
3054 * a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written.
3055 * b) The thread doing the writing is waiting for pages while holding the truncate lock
3056 * c) Most of the pages in the inactive queue belong to this file.
3057 *
3058 * we are potentially in this deadlock because...
3059 * a) the external pageout queue is throttled
3060 * b) we're done with the active queue and moved on to the inactive queue
3061 * c) we've got a dirty external page
3062 *
3063 * since we don't know the reason for the external pageout queue being throttled we
3064 * must suspect that we are deadlocked, so move the current page onto the active queue
3065 * in an effort to cause a page from the active queue to 'age' to the inactive queue
3066 *
3067 * if we don't have jetsam configured (i.e. we have a dynamic pager), set
3068 * 'force_anonymous' to TRUE to cause us to grab a page from the cleaned/anonymous
3069 * pool the next time we select a victim page... if we can make enough new free pages,
3070 * the deadlock will break, the external pageout queue will empty and it will no longer
3071 * be throttled
3072 *
3073 * if we have jetsam configured, keep a count of the pages reactivated this way so
3074 * that we can try to find clean pages in the active/inactive queues before
3075 * deciding to jetsam a process
3076 */
3077 vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external++;
3078
3079 vm_page_check_pageable_safe(m);
3080 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
3081 vm_page_queue_enter(&vm_page_queue_active, m, vmp_pageq);
3082 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
3083 vm_page_active_count++;
3084 vm_page_pageable_external_count++;
3085
3086 vm_pageout_adjust_eq_iothrottle(&pgo_iothread_external_state, FALSE);
3087
3088 #if CONFIG_MEMORYSTATUS && CONFIG_JETSAM
3089
3090 #pragma unused(force_anonymous)
3091
3092 *vm_pageout_inactive_external_forced_reactivate_limit -= 1;
3093
3094 if (*vm_pageout_inactive_external_forced_reactivate_limit <= 0) {
3095 *vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
3096 /*
3097 * Possible deadlock scenario so request jetsam action
3098 */
3099 memorystatus_kill_on_vps_starvation();
3100 VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, DBG_VM_PAGEOUT_JETSAM, DBG_FUNC_NONE,
3101 vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
3102 }
3103 #else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
3104
3105 #pragma unused(vm_pageout_inactive_external_forced_reactivate_limit)
3106
3107 *force_anonymous = TRUE;
3108 #endif /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
3109 } else {
3110 vm_page_activate(m);
3111 counter_inc(&vm_statistics_reactivations);
3112
3113 #if DEVELOPMENT || DEBUG
3114 if (is_page_from_bg_q == TRUE) {
3115 if (cur_object->internal) {
3116 vm_pageout_rejected_bq_internal++;
3117 } else {
3118 vm_pageout_rejected_bq_external++;
3119 }
3120 }
3121 #endif /* DEVELOPMENT || DEBUG */
3122
3123 vm_pageout_state.vm_pageout_inactive_used++;
3124 }
3125 }
3126
3127
3128 void
3129 vm_page_balance_inactive(int max_to_move)
3130 {
3131 vm_page_t m;
3132
3133 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3134
3135 if (hibernation_vmqueues_inspection || hibernate_cleaning_in_progress) {
3136 /*
3137 * It is likely that the hibernation code path is
3138 * dealing with these very queues as we are about
3139 * to move pages around in/from them and completely
3140 * change the linkage of the pages.
3141 *
3142 * And so we skip the rebalancing of these queues.
3143 */
3144 return;
3145 }
3146 vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
3147 vm_page_inactive_count +
3148 vm_page_speculative_count);
3149
3150 while (max_to_move-- && (vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) {
3151 VM_PAGEOUT_DEBUG(vm_pageout_balanced, 1);
3152
3153 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
3154
3155 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
3156 assert(!m->vmp_laundry);
3157 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
3158 assert(!vm_page_is_guard(m));
3159
3160 DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
3161
3162 /*
3163 * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
3164 *
3165 * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
3166 * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
3167 * new reference happens. If no futher references happen on the page after that remote TLB flushes
3168 * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
3169 * by pageout_scan, which is just fine since the last reference would have happened quite far
3170 * in the past (TLB caches don't hang around for very long), and of course could just as easily
3171 * have happened before we moved the page
3172 */
3173 if (m->vmp_pmapped == TRUE) {
3174 /*
3175 * We might be holding the page queue lock as a
3176 * spin lock and clearing the "referenced" bit could
3177 * take a while if there are lots of mappings of
3178 * that page, so make sure we acquire the lock as
3179 * as mutex to avoid a spinlock timeout.
3180 */
3181 vm_page_lockconvert_queues();
3182 pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
3183 }
3184
3185 /*
3186 * The page might be absent or busy,
3187 * but vm_page_deactivate can handle that.
3188 * FALSE indicates that we don't want a H/W clear reference
3189 */
3190 vm_page_deactivate_internal(m, FALSE);
3191 }
3192 }
3193
3194 /*
3195 * vm_pageout_scan does the dirty work for the pageout daemon.
3196 * It returns with both vm_page_queue_free_lock and vm_page_queue_lock
3197 * held and vm_page_free_wanted == 0.
3198 */
3199 void
3200 vm_pageout_scan(void)
3201 {
3202 unsigned int loop_count = 0;
3203 unsigned int inactive_burst_count = 0;
3204 unsigned int reactivated_this_call;
3205 unsigned int reactivate_limit;
3206 vm_page_t local_freeq = NULL;
3207 int local_freed = 0;
3208 int delayed_unlock;
3209 int delayed_unlock_limit = 0;
3210 int refmod_state = 0;
3211 int vm_pageout_deadlock_target = 0;
3212 struct vm_pageout_queue *iq;
3213 struct vm_pageout_queue *eq;
3214 struct vm_speculative_age_q *sq;
3215 struct flow_control flow_control = { .state = 0, .ts = { .tv_sec = 0, .tv_nsec = 0 } };
3216 boolean_t inactive_throttled = FALSE;
3217 vm_object_t object = NULL; /* object that we're currently working on from previous iterations */
3218 uint32_t inactive_reclaim_run;
3219 boolean_t grab_anonymous = FALSE;
3220 boolean_t force_anonymous = FALSE;
3221 boolean_t force_speculative_aging = FALSE;
3222 int anons_grabbed = 0;
3223 int page_prev_q_state = 0;
3224 boolean_t page_from_bg_q = FALSE;
3225 uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0;
3226 vm_object_t m_object = VM_OBJECT_NULL; /* object of the current page (m) */
3227 int retval = 0;
3228 boolean_t lock_yield_check = FALSE;
3229
3230
3231 VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, DBG_VM_PAGEOUT_SCAN, DBG_FUNC_START,
3232 vm_pageout_vminfo.vm_pageout_freed_speculative,
3233 vm_pageout_state.vm_pageout_inactive_clean,
3234 vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
3235 vm_pageout_vminfo.vm_pageout_inactive_dirty_external);
3236
3237 flow_control.state = FCS_IDLE;
3238 iq = &vm_pageout_queue_internal;
3239 eq = &vm_pageout_queue_external;
3240 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
3241
3242 /* Ask the pmap layer to return any pages it no longer needs. */
3243 pmap_release_pages_fast();
3244
3245 vm_page_lock_queues();
3246
3247 delayed_unlock = 1;
3248
3249 /*
3250 * Calculate the max number of referenced pages on the inactive
3251 * queue that we will reactivate.
3252 */
3253 reactivated_this_call = 0;
3254 reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
3255 vm_page_inactive_count);
3256 inactive_reclaim_run = 0;
3257
3258 vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
3259
3260 /*
3261 * We must limit the rate at which we send pages to the pagers
3262 * so that we don't tie up too many pages in the I/O queues.
3263 * We implement a throttling mechanism using the laundry count
3264 * to limit the number of pages outstanding to the default
3265 * and external pagers. We can bypass the throttles and look
3266 * for clean pages if the pageout queues don't drain in a timely
3267 * fashion since this may indicate that the pageout paths are
3268 * stalled waiting for memory, which only we can provide.
3269 */
3270
3271 vps_init_page_targets();
3272 assert(object == NULL);
3273 assert(delayed_unlock != 0);
3274
3275 for (;;) {
3276 vm_page_t m;
3277
3278 DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
3279
3280 if (lock_yield_check) {
3281 lock_yield_check = FALSE;
3282
3283 if (delayed_unlock++ > delayed_unlock_limit) {
3284 vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
3285 VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
3286 } else if (vm_pageout_scan_wants_object) {
3287 vm_page_unlock_queues();
3288 mutex_pause(0);
3289 vm_page_lock_queues();
3290 } else if (vps_yield_for_pgqlockwaiters && lck_mtx_yield(&vm_page_queue_lock)) {
3291 VM_PAGEOUT_DEBUG(vm_pageout_yield_for_free_pages, 1);
3292 }
3293 }
3294
3295 if (vm_upl_wait_for_pages < 0) {
3296 vm_upl_wait_for_pages = 0;
3297 }
3298
3299 delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
3300
3301 if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) {
3302 delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
3303 }
3304
3305 vps_deal_with_secluded_page_overflow(&local_freeq, &local_freed);
3306
3307 assert(delayed_unlock);
3308
3309 /*
3310 * maintain our balance
3311 */
3312 vm_page_balance_inactive(1);
3313
3314
3315 /**********************************************************************
3316 * above this point we're playing with the active and secluded queues
3317 * below this point we're playing with the throttling mechanisms
3318 * and the inactive queue
3319 **********************************************************************/
3320
3321 if (vm_page_free_count + local_freed >= vm_page_free_target) {
3322 vm_pageout_scan_wants_object = VM_OBJECT_NULL;
3323
3324 vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
3325 VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
3326 /*
3327 * make sure the pageout I/O threads are running
3328 * throttled in case there are still requests
3329 * in the laundry... since we have met our targets
3330 * we don't need the laundry to be cleaned in a timely
3331 * fashion... so let's avoid interfering with foreground
3332 * activity
3333 */
3334 vm_pageout_adjust_eq_iothrottle(&pgo_iothread_external_state, TRUE);
3335
3336 vm_free_page_lock();
3337
3338 if ((vm_page_free_count >= vm_page_free_target) &&
3339 (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
3340 /*
3341 * done - we have met our target *and*
3342 * there is no one waiting for a page.
3343 */
3344 return_from_scan:
3345 assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
3346
3347 VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, DBG_VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
3348 vm_pageout_state.vm_pageout_inactive,
3349 vm_pageout_state.vm_pageout_inactive_used, 0, 0);
3350 VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, DBG_VM_PAGEOUT_SCAN, DBG_FUNC_END,
3351 vm_pageout_vminfo.vm_pageout_freed_speculative,
3352 vm_pageout_state.vm_pageout_inactive_clean,
3353 vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
3354 vm_pageout_vminfo.vm_pageout_inactive_dirty_external);
3355
3356 return;
3357 }
3358 vm_free_page_unlock();
3359 }
3360
3361 /*
3362 * Before anything, we check if we have any ripe volatile
3363 * objects around. If so, try to purge the first object.
3364 * If the purge fails, fall through to reclaim a page instead.
3365 * If the purge succeeds, go back to the top and reevalute
3366 * the new memory situation.
3367 */
3368 retval = vps_purge_object();
3369
3370 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3371 /*
3372 * Success
3373 */
3374 if (object != NULL) {
3375 vm_object_unlock(object);
3376 object = NULL;
3377 }
3378
3379 lock_yield_check = FALSE;
3380 continue;
3381 }
3382
3383 /*
3384 * If our 'aged' queue is empty and we have some speculative pages
3385 * in the other queues, let's go through and see if we need to age
3386 * them.
3387 *
3388 * If we succeeded in aging a speculative Q or just that everything
3389 * looks normal w.r.t queue age and queue counts, we keep going onward.
3390 *
3391 * If, for some reason, we seem to have a mismatch between the spec.
3392 * page count and the page queues, we reset those variables and
3393 * restart the loop (LD TODO: Track this better?).
3394 */
3395 if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) {
3396 retval = vps_age_speculative_queue(force_speculative_aging);
3397
3398 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3399 lock_yield_check = FALSE;
3400 continue;
3401 }
3402 }
3403 force_speculative_aging = FALSE;
3404
3405 /*
3406 * Check to see if we need to evict objects from the cache.
3407 *
3408 * Note: 'object' here doesn't have anything to do with
3409 * the eviction part. We just need to make sure we have dropped
3410 * any object lock we might be holding if we need to go down
3411 * into the eviction logic.
3412 */
3413 retval = vps_object_cache_evict(&object);
3414
3415 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3416 lock_yield_check = FALSE;
3417 continue;
3418 }
3419
3420
3421 /*
3422 * Calculate our filecache_min that will affect the loop
3423 * going forward.
3424 */
3425 vps_calculate_filecache_min();
3426
3427 /*
3428 * LD TODO: Use a structure to hold all state variables for a single
3429 * vm_pageout_scan iteration and pass that structure to this function instead.
3430 */
3431 retval = vps_flow_control(&flow_control, &anons_grabbed, &object,
3432 &delayed_unlock, &local_freeq, &local_freed,
3433 &vm_pageout_deadlock_target, inactive_burst_count);
3434
3435 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3436 if (loop_count >= vm_page_inactive_count) {
3437 loop_count = 0;
3438 }
3439
3440 inactive_burst_count = 0;
3441
3442 assert(object == NULL);
3443 assert(delayed_unlock != 0);
3444
3445 lock_yield_check = FALSE;
3446 continue;
3447 } else if (retval == VM_PAGEOUT_SCAN_DONE_RETURN) {
3448 goto return_from_scan;
3449 }
3450
3451 flow_control.state = FCS_IDLE;
3452
3453 vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
3454 vm_pageout_inactive_external_forced_reactivate_limit);
3455 loop_count++;
3456 inactive_burst_count++;
3457 vm_pageout_state.vm_pageout_inactive++;
3458
3459 /*
3460 * Choose a victim.
3461 */
3462
3463 m = NULL;
3464 retval = vps_choose_victim_page(&m, &anons_grabbed, &grab_anonymous, force_anonymous, &page_from_bg_q, &reactivated_this_call);
3465
3466 if (m == NULL) {
3467 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3468 inactive_burst_count = 0;
3469
3470 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3471 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
3472 }
3473
3474 lock_yield_check = TRUE;
3475 continue;
3476 }
3477
3478 /*
3479 * if we've gotten here, we have no victim page.
3480 * check to see if we've not finished balancing the queues
3481 * or we have a page on the aged speculative queue that we
3482 * skipped due to force_anonymous == TRUE.. or we have
3483 * speculative pages that we can prematurely age... if
3484 * one of these cases we'll keep going, else panic
3485 */
3486 force_anonymous = FALSE;
3487 VM_PAGEOUT_DEBUG(vm_pageout_no_victim, 1);
3488
3489 if (!vm_page_queue_empty(&sq->age_q)) {
3490 lock_yield_check = TRUE;
3491 continue;
3492 }
3493
3494 if (vm_page_speculative_count) {
3495 force_speculative_aging = TRUE;
3496 lock_yield_check = TRUE;
3497 continue;
3498 }
3499 panic("vm_pageout: no victim");
3500
3501 /* NOTREACHED */
3502 }
3503
3504 assert(VM_PAGE_PAGEABLE(m));
3505 m_object = VM_PAGE_OBJECT(m);
3506 force_anonymous = FALSE;
3507
3508 page_prev_q_state = m->vmp_q_state;
3509 /*
3510 * we just found this page on one of our queues...
3511 * it can't also be on the pageout queue, so safe
3512 * to call vm_page_queues_remove
3513 */
3514 bool donate = (m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE);
3515 vm_page_queues_remove(m, TRUE);
3516 if (donate) {
3517 /*
3518 * The compressor needs to see this bit to know
3519 * where this page needs to land. Also if stolen,
3520 * this bit helps put the page back in the right
3521 * special queue where it belongs.
3522 */
3523 m->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
3524 }
3525
3526 assert(!m->vmp_laundry);
3527 assert(vm_page_is_canonical(m));
3528 assert(!is_kernel_object(m_object));
3529
3530 vm_pageout_vminfo.vm_pageout_considered_page++;
3531
3532 DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
3533
3534 /*
3535 * check to see if we currently are working
3536 * with the same object... if so, we've
3537 * already got the lock
3538 */
3539 if (m_object != object) {
3540 boolean_t avoid_anon_pages = (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT);
3541
3542 /*
3543 * vps_switch_object() will always drop the 'object' lock first
3544 * and then try to acquire the 'm_object' lock. So 'object' has to point to
3545 * either 'm_object' or NULL.
3546 */
3547 retval = vps_switch_object(m, m_object, &object, page_prev_q_state, avoid_anon_pages, page_from_bg_q);
3548
3549 if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
3550 lock_yield_check = TRUE;
3551 continue;
3552 }
3553 }
3554 assert(m_object == object);
3555 assert(VM_PAGE_OBJECT(m) == m_object);
3556
3557 if (m->vmp_busy) {
3558 /*
3559 * Somebody is already playing with this page.
3560 * Put it back on the appropriate queue
3561 *
3562 */
3563 VM_PAGEOUT_DEBUG(vm_pageout_inactive_busy, 1);
3564
3565 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3566 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_busy, 1);
3567 }
3568
3569 vps_requeue_page(m, page_prev_q_state, page_from_bg_q);
3570
3571 lock_yield_check = TRUE;
3572 continue;
3573 }
3574
3575 /*
3576 * if (m->vmp_cleaning && !m->vmp_free_when_done)
3577 * If already cleaning this page in place
3578 * just leave if off the paging queues.
3579 * We can leave the page mapped, and upl_commit_range
3580 * will put it on the clean queue.
3581 *
3582 * if (m->vmp_free_when_done && !m->vmp_cleaning)
3583 * an msync INVALIDATE is in progress...
3584 * this page has been marked for destruction
3585 * after it has been cleaned,
3586 * but not yet gathered into a UPL
3587 * where 'cleaning' will be set...
3588 * just leave it off the paging queues
3589 *
3590 * if (m->vmp_free_when_done && m->vmp_clenaing)
3591 * an msync INVALIDATE is in progress
3592 * and the UPL has already gathered this page...
3593 * just leave it off the paging queues
3594 */
3595 if (m->vmp_free_when_done || m->vmp_cleaning) {
3596 lock_yield_check = TRUE;
3597 continue;
3598 }
3599
3600
3601 /*
3602 * If it's absent, in error or the object is no longer alive,
3603 * we can reclaim the page... in the no longer alive case,
3604 * there are 2 states the page can be in that preclude us
3605 * from reclaiming it - busy or cleaning - that we've already
3606 * dealt with
3607 */
3608 if (m->vmp_absent || VMP_ERROR_GET(m) || !object->alive ||
3609 (!object->internal && object->pager == MEMORY_OBJECT_NULL)) {
3610 if (m->vmp_absent) {
3611 VM_PAGEOUT_DEBUG(vm_pageout_inactive_absent, 1);
3612 } else if (!object->alive ||
3613 (!object->internal &&
3614 object->pager == MEMORY_OBJECT_NULL)) {
3615 VM_PAGEOUT_DEBUG(vm_pageout_inactive_notalive, 1);
3616 } else {
3617 VM_PAGEOUT_DEBUG(vm_pageout_inactive_error, 1);
3618 }
3619 if (m->vmp_pmapped) {
3620 int refmod;
3621
3622 /*
3623 * If this page was file-backed and wired while its pager
3624 * was lost (during a forced unmount, for example), there
3625 * could still be some pmap mappings that need to be
3626 * cleaned up before we can free the page.
3627 */
3628 refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
3629 if ((refmod & VM_MEM_MODIFIED) &&
3630 !m->vmp_dirty) {
3631 SET_PAGE_DIRTY(m, FALSE);
3632 }
3633 }
3634 reclaim_page:
3635 if (vm_pageout_deadlock_target) {
3636 VM_PAGEOUT_DEBUG(vm_pageout_scan_inactive_throttle_success, 1);
3637 vm_pageout_deadlock_target--;
3638 }
3639
3640 DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
3641
3642 if (object->internal) {
3643 DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
3644 #if HAS_MTE
3645 if (vm_object_is_mte_mappable(object)) {
3646 KDBG(VMDBG_CODE(DBG_VM_PAGEOUT_FREE_MTE) | DBG_FUNC_NONE,
3647 VM_KERNEL_ADDRHIDE(m), VM_KERNEL_ADDRHIDE(object),
3648 m->vmp_offset,
3649 mteinfo_tag_storage_free_pages_for_covered(m));
3650 }
3651 #endif /* HAS_MTE */
3652 } else {
3653 DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
3654 }
3655 assert(!m->vmp_cleaning);
3656 assert(!m->vmp_laundry);
3657
3658 if (!object->internal &&
3659 object->pager != NULL &&
3660 object->pager->mo_pager_ops == &shared_region_pager_ops) {
3661 shared_region_pager_reclaimed++;
3662 }
3663
3664 m->vmp_busy = TRUE;
3665
3666 /*
3667 * remove page from object here since we're already
3668 * behind the object lock... defer the rest of the work
3669 * we'd normally do in vm_page_free_prepare_object
3670 * until 'vm_page_free_list' is called
3671 */
3672 if (m->vmp_tabled) {
3673 vm_page_remove(m, TRUE);
3674 }
3675
3676 assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
3677 m->vmp_snext = local_freeq;
3678 local_freeq = m;
3679 local_freed++;
3680
3681 if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
3682 vm_pageout_vminfo.vm_pageout_freed_speculative++;
3683 } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3684 vm_pageout_vminfo.vm_pageout_freed_cleaned++;
3685 } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) {
3686 vm_pageout_vminfo.vm_pageout_freed_internal++;
3687 } else {
3688 vm_pageout_vminfo.vm_pageout_freed_external++;
3689 }
3690
3691 inactive_burst_count = 0;
3692
3693 lock_yield_check = TRUE;
3694 continue;
3695 }
3696 if (object->vo_copy == VM_OBJECT_NULL) {
3697 /*
3698 * No one else can have any interest in this page.
3699 * If this is an empty purgable object, the page can be
3700 * reclaimed even if dirty.
3701 * If the page belongs to a volatile purgable object, we
3702 * reactivate it if the compressor isn't active.
3703 */
3704 if (object->purgable == VM_PURGABLE_EMPTY) {
3705 if (m->vmp_pmapped == TRUE) {
3706 /* unmap the page */
3707 refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
3708 if (refmod_state & VM_MEM_MODIFIED) {
3709 SET_PAGE_DIRTY(m, FALSE);
3710 }
3711 }
3712 if (m->vmp_dirty || m->vmp_precious) {
3713 /* we saved the cost of cleaning this page ! */
3714 vm_page_purged_count++;
3715 }
3716 goto reclaim_page;
3717 }
3718
3719 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
3720 /*
3721 * With the VM compressor, the cost of
3722 * reclaiming a page is much lower (no I/O),
3723 * so if we find a "volatile" page, it's better
3724 * to let it get compressed rather than letting
3725 * it occupy a full page until it gets purged.
3726 * So no need to check for "volatile" here.
3727 */
3728 } else if (object->purgable == VM_PURGABLE_VOLATILE) {
3729 /*
3730 * Avoid cleaning a "volatile" page which might
3731 * be purged soon.
3732 */
3733
3734 /* if it's wired, we can't put it on our queue */
3735 assert(!VM_PAGE_WIRED(m));
3736
3737 /* just stick it back on! */
3738 reactivated_this_call++;
3739
3740 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3741 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_volatile_reactivated, 1);
3742 }
3743
3744 goto reactivate_page;
3745 }
3746 } /* vo_copy NULL */
3747 /*
3748 * If it's being used, reactivate.
3749 * (Fictitious pages are either busy or absent.)
3750 * First, update the reference and dirty bits
3751 * to make sure the page is unreferenced.
3752 */
3753 refmod_state = -1;
3754
3755 if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {
3756 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
3757
3758 if (refmod_state & VM_MEM_REFERENCED) {
3759 m->vmp_reference = TRUE;
3760 }
3761 if (refmod_state & VM_MEM_MODIFIED) {
3762 SET_PAGE_DIRTY(m, FALSE);
3763 }
3764 }
3765
3766 if (m->vmp_reference || m->vmp_dirty) {
3767 /* deal with a rogue "reusable" page */
3768 VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object);
3769 }
3770
3771 if (vm_pageout_state.vm_page_xpmapped_min_divisor == 0) {
3772 vm_pageout_state.vm_page_xpmapped_min = 0;
3773 } else {
3774 vm_pageout_state.vm_page_xpmapped_min = (vm_page_pageable_external_count * 10) /
3775 vm_pageout_state.vm_page_xpmapped_min_divisor;
3776 }
3777
3778 if (!m->vmp_no_cache &&
3779 page_from_bg_q == FALSE &&
3780 (m->vmp_reference || (m->vmp_xpmapped && !object->internal &&
3781 (vm_page_xpmapped_external_count < vm_pageout_state.vm_page_xpmapped_min)))) {
3782 /*
3783 * The page we pulled off the inactive list has
3784 * been referenced. It is possible for other
3785 * processors to be touching pages faster than we
3786 * can clear the referenced bit and traverse the
3787 * inactive queue, so we limit the number of
3788 * reactivations.
3789 */
3790 if (++reactivated_this_call >= reactivate_limit &&
3791 !object->object_is_shared_cache &&
3792 !((m->vmp_realtime ||
3793 object->for_realtime) &&
3794 vm_pageout_protect_realtime)) {
3795 vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded++;
3796 } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
3797 vm_pageout_vminfo.vm_pageout_inactive_force_reclaim++;
3798 if (object->object_is_shared_cache) {
3799 vm_pageout_vminfo.vm_pageout_forcereclaimed_sharedcache++;
3800 } else if (m->vmp_realtime ||
3801 object->for_realtime) {
3802 vm_pageout_vminfo.vm_pageout_forcereclaimed_realtime++;
3803 }
3804 } else {
3805 uint32_t isinuse;
3806
3807 if (reactivated_this_call >= reactivate_limit) {
3808 if (object->object_is_shared_cache) {
3809 vm_pageout_vminfo.vm_pageout_protected_sharedcache++;
3810 } else if ((m->vmp_realtime ||
3811 object->for_realtime) &&
3812 vm_pageout_protect_realtime) {
3813 vm_pageout_vminfo.vm_pageout_protected_realtime++;
3814 }
3815 }
3816 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3817 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reference_reactivated, 1);
3818 }
3819
3820 vm_pageout_vminfo.vm_pageout_inactive_referenced++;
3821 reactivate_page:
3822 if (!object->internal && object->pager != MEMORY_OBJECT_NULL &&
3823 vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
3824 /*
3825 * no explict mappings of this object exist
3826 * and it's not open via the filesystem
3827 */
3828 vm_page_deactivate(m);
3829 VM_PAGEOUT_DEBUG(vm_pageout_inactive_deactivated, 1);
3830 } else {
3831 /*
3832 * The page was/is being used, so put back on active list.
3833 */
3834 vm_page_activate(m);
3835 counter_inc(&vm_statistics_reactivations);
3836 inactive_burst_count = 0;
3837 }
3838 #if DEVELOPMENT || DEBUG
3839 if (page_from_bg_q == TRUE) {
3840 if (m_object->internal) {
3841 vm_pageout_rejected_bq_internal++;
3842 } else {
3843 vm_pageout_rejected_bq_external++;
3844 }
3845 }
3846 #endif /* DEVELOPMENT || DEBUG */
3847
3848 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3849 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
3850 }
3851 vm_pageout_state.vm_pageout_inactive_used++;
3852
3853 lock_yield_check = TRUE;
3854 continue;
3855 }
3856 /*
3857 * Make sure we call pmap_get_refmod() if it
3858 * wasn't already called just above, to update
3859 * the dirty bit.
3860 */
3861 if ((refmod_state == -1) && !m->vmp_dirty && m->vmp_pmapped) {
3862 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
3863 if (refmod_state & VM_MEM_MODIFIED) {
3864 SET_PAGE_DIRTY(m, FALSE);
3865 }
3866 }
3867 }
3868
3869 /*
3870 * we've got a candidate page to steal...
3871 *
3872 * m->vmp_dirty is up to date courtesy of the
3873 * preceding check for m->vmp_reference... if
3874 * we get here, then m->vmp_reference had to be
3875 * FALSE (or possibly "reactivate_limit" was
3876 * exceeded), but in either case we called
3877 * pmap_get_refmod() and updated both
3878 * m->vmp_reference and m->vmp_dirty
3879 *
3880 * if it's dirty or precious we need to
3881 * see if the target queue is throtttled
3882 * it if is, we need to skip over it by moving it back
3883 * to the end of the inactive queue
3884 */
3885
3886 inactive_throttled = FALSE;
3887
3888 if (m->vmp_dirty || m->vmp_precious) {
3889 if (object->internal) {
3890 if (VM_PAGE_Q_THROTTLED(iq)) {
3891 inactive_throttled = TRUE;
3892 }
3893 } else if (VM_PAGE_Q_THROTTLED(eq)) {
3894 inactive_throttled = TRUE;
3895 }
3896 }
3897 throttle_inactive:
3898 if (!VM_DYNAMIC_PAGING_ENABLED() &&
3899 object->internal && m->vmp_dirty &&
3900 (object->purgable == VM_PURGABLE_DENY ||
3901 object->purgable == VM_PURGABLE_NONVOLATILE ||
3902 object->purgable == VM_PURGABLE_VOLATILE)) {
3903 vm_page_check_pageable_safe(m);
3904 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
3905 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
3906 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
3907 vm_page_throttled_count++;
3908
3909 VM_PAGEOUT_DEBUG(vm_pageout_scan_reclaimed_throttled, 1);
3910
3911 inactive_burst_count = 0;
3912
3913 lock_yield_check = TRUE;
3914 continue;
3915 }
3916 if (inactive_throttled == TRUE) {
3917 vps_deal_with_throttled_queues(m, &object, &vm_pageout_inactive_external_forced_reactivate_limit,
3918 &force_anonymous, page_from_bg_q);
3919
3920 inactive_burst_count = 0;
3921
3922 if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3923 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
3924 }
3925
3926 lock_yield_check = TRUE;
3927 continue;
3928 }
3929
3930 /*
3931 * we've got a page that we can steal...
3932 * eliminate all mappings and make sure
3933 * we have the up-to-date modified state
3934 *
3935 * if we need to do a pmap_disconnect then we
3936 * need to re-evaluate m->vmp_dirty since the pmap_disconnect
3937 * provides the true state atomically... the
3938 * page was still mapped up to the pmap_disconnect
3939 * and may have been dirtied at the last microsecond
3940 *
3941 * Note that if 'pmapped' is FALSE then the page is not
3942 * and has not been in any map, so there is no point calling
3943 * pmap_disconnect(). m->vmp_dirty could have been set in anticipation
3944 * of likely usage of the page.
3945 */
3946 if (m->vmp_pmapped == TRUE) {
3947 int pmap_options;
3948
3949 /*
3950 * Don't count this page as going into the compressor
3951 * if any of these are true:
3952 * 1) compressed pager isn't enabled
3953 * 2) Freezer enabled device with compressed pager
3954 * backend (exclusive use) i.e. most of the VM system
3955 * (including vm_pageout_scan) has no knowledge of
3956 * the compressor
3957 * 3) This page belongs to a file and hence will not be
3958 * sent into the compressor
3959 */
3960 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE ||
3961 object->internal == FALSE) {
3962 pmap_options = 0;
3963 } else if (m->vmp_dirty || m->vmp_precious) {
3964 /*
3965 * VM knows that this page is dirty (or
3966 * precious) and needs to be compressed
3967 * rather than freed.
3968 * Tell the pmap layer to count this page
3969 * as "compressed".
3970 */
3971 pmap_options = PMAP_OPTIONS_COMPRESSOR;
3972 } else {
3973 /*
3974 * VM does not know if the page needs to
3975 * be preserved but the pmap layer might tell
3976 * us if any mapping has "modified" it.
3977 * Let's the pmap layer to count this page
3978 * as compressed if and only if it has been
3979 * modified.
3980 */
3981 pmap_options =
3982 PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
3983 }
3984 refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),
3985 pmap_options,
3986 NULL);
3987 if (refmod_state & VM_MEM_MODIFIED) {
3988 SET_PAGE_DIRTY(m, FALSE);
3989 }
3990 }
3991
3992 /*
3993 * reset our count of pages that have been reclaimed
3994 * since the last page was 'stolen'
3995 */
3996 inactive_reclaim_run = 0;
3997
3998 /*
3999 * If it's clean and not precious, we can free the page.
4000 */
4001 if (!m->vmp_dirty && !m->vmp_precious) {
4002 vm_pageout_state.vm_pageout_inactive_clean++;
4003
4004 /*
4005 * OK, at this point we have found a page we are going to free.
4006 */
4007 #if CONFIG_PHANTOM_CACHE
4008 if (!object->internal) {
4009 vm_phantom_cache_add_ghost(m);
4010 }
4011 #endif
4012 goto reclaim_page;
4013 }
4014
4015 /*
4016 * The page may have been dirtied since the last check
4017 * for a throttled target queue (which may have been skipped
4018 * if the page was clean then). With the dirty page
4019 * disconnected here, we can make one final check.
4020 */
4021 if (object->internal) {
4022 if (VM_PAGE_Q_THROTTLED(iq)) {
4023 inactive_throttled = TRUE;
4024 }
4025 } else if (VM_PAGE_Q_THROTTLED(eq)) {
4026 inactive_throttled = TRUE;
4027 }
4028
4029 if (inactive_throttled == TRUE) {
4030 goto throttle_inactive;
4031 }
4032 #if !CONFIG_JETSAM
4033 memorystatus_update_available_page_count(AVAILABLE_NON_COMPRESSED_MEMORY);
4034 #endif /* !CONFIG_JETSAM */
4035
4036 if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
4037 VM_PAGEOUT_DEBUG(vm_pageout_speculative_dirty, 1);
4038 }
4039
4040 if (object->internal) {
4041 vm_pageout_vminfo.vm_pageout_inactive_dirty_internal++;
4042 } else {
4043 vm_pageout_vminfo.vm_pageout_inactive_dirty_external++;
4044 }
4045
4046 /*
4047 * internal pages will go to the compressor...
4048 * external pages will go to the appropriate pager to be cleaned
4049 * and upon completion will end up on 'vm_page_queue_cleaned' which
4050 * is a preferred queue to steal from
4051 */
4052 vm_pageout_cluster(m);
4053 inactive_burst_count = 0;
4054
4055 /*
4056 * back to top of pageout scan loop
4057 */
4058 }
4059 }
4060
4061
4062 void
4063 vm_page_free_reserve(
4064 int pages)
4065 {
4066 int free_after_reserve;
4067
4068 if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
4069 if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) {
4070 vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
4071 } else {
4072 vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);
4073 }
4074 } else {
4075 if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) {
4076 vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;
4077 } else {
4078 vm_page_free_reserved += pages;
4079 }
4080 }
4081 free_after_reserve = vm_pageout_state.vm_page_free_count_init - vm_page_free_reserved;
4082
4083 vm_page_free_min = vm_page_free_reserved +
4084 VM_PAGE_FREE_MIN(free_after_reserve);
4085
4086 if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) {
4087 vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
4088 }
4089
4090 vm_page_free_target = vm_page_free_reserved +
4091 VM_PAGE_FREE_TARGET(free_after_reserve);
4092
4093 if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) {
4094 vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
4095 }
4096
4097 if (vm_page_free_target < vm_page_free_min + 5) {
4098 vm_page_free_target = vm_page_free_min + 5;
4099 }
4100
4101 vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
4102 }
4103
4104 /*
4105 * vm_pageout is the high level pageout daemon.
4106 */
4107
4108 void
4109 vm_pageout_continue(void)
4110 {
4111 DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
4112 VM_PAGEOUT_DEBUG(vm_pageout_scan_event_counter, 1);
4113
4114 vm_free_page_lock();
4115 vm_pageout_running = TRUE;
4116 vm_free_page_unlock();
4117
4118 vm_pageout_scan();
4119 /*
4120 * we hold both the vm_page_queue_free_lock
4121 * and the vm_page_queues_lock at this point
4122 */
4123 assert(vm_page_free_wanted == 0);
4124 assert(vm_page_free_wanted_privileged == 0);
4125 assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
4126
4127 vm_pageout_running = FALSE;
4128 #if XNU_TARGET_OS_OSX
4129 if (vm_pageout_waiter) {
4130 vm_pageout_waiter = FALSE;
4131 thread_wakeup((event_t)&vm_pageout_waiter);
4132 }
4133 #endif /* XNU_TARGET_OS_OSX */
4134
4135 vm_free_page_unlock();
4136 vm_page_unlock_queues();
4137
4138 thread_block((thread_continue_t)vm_pageout_continue);
4139 /*NOTREACHED*/
4140 }
4141
4142 #if XNU_TARGET_OS_OSX
4143 kern_return_t
4144 vm_pageout_wait(uint64_t deadline)
4145 {
4146 kern_return_t kr;
4147
4148 vm_free_page_lock();
4149 for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr);) {
4150 vm_pageout_waiter = TRUE;
4151 if (THREAD_AWAKENED != lck_mtx_sleep_deadline(
4152 &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
4153 (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {
4154 kr = KERN_OPERATION_TIMED_OUT;
4155 }
4156 }
4157 vm_free_page_unlock();
4158
4159 return kr;
4160 }
4161 #endif /* XNU_TARGET_OS_OSX */
4162
4163 OS_NORETURN
4164 static void
4165 vm_pageout_iothread_external_continue(struct pgo_iothread_state *ethr, __unused wait_result_t w)
4166 {
4167 vm_page_t m = NULL;
4168 vm_object_t object;
4169 vm_object_offset_t offset;
4170 memory_object_t pager;
4171 struct vm_pageout_queue *q = ethr->q;
4172
4173 /* On systems with a compressor, the external IO thread clears its
4174 * VM privileged bit to accommodate large allocations (e.g. bulk UPL
4175 * creation)
4176 */
4177 if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
4178 current_thread()->options &= ~TH_OPT_VMPRIV;
4179 }
4180
4181 sched_cond_ack(&(ethr->pgo_wakeup));
4182
4183 while (true) {
4184 vm_page_lockspin_queues();
4185
4186 while (!vm_page_queue_empty(&q->pgo_pending)) {
4187 q->pgo_busy = TRUE;
4188 vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
4189
4190 assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
4191 VM_PAGE_CHECK(m);
4192 /*
4193 * grab a snapshot of the object and offset this
4194 * page is tabled in so that we can relookup this
4195 * page after we've taken the object lock - these
4196 * fields are stable while we hold the page queues lock
4197 * but as soon as we drop it, there is nothing to keep
4198 * this page in this object... we hold an activity_in_progress
4199 * on this object which will keep it from terminating
4200 */
4201 object = VM_PAGE_OBJECT(m);
4202 offset = m->vmp_offset;
4203
4204 m->vmp_q_state = VM_PAGE_NOT_ON_Q;
4205 VM_PAGE_ZERO_PAGEQ_ENTRY(m);
4206
4207 vm_page_unlock_queues();
4208
4209 vm_object_lock(object);
4210
4211 m = vm_page_lookup(object, offset);
4212
4213 if (m == NULL || m->vmp_busy || m->vmp_cleaning ||
4214 !m->vmp_laundry || (m->vmp_q_state != VM_PAGE_NOT_ON_Q)) {
4215 /*
4216 * it's either the same page that someone else has
4217 * started cleaning (or it's finished cleaning or
4218 * been put back on the pageout queue), or
4219 * the page has been freed or we have found a
4220 * new page at this offset... in all of these cases
4221 * we merely need to release the activity_in_progress
4222 * we took when we put the page on the pageout queue
4223 */
4224 vm_object_activity_end(object);
4225 vm_object_unlock(object);
4226
4227 vm_page_lockspin_queues();
4228 continue;
4229 }
4230 pager = object->pager;
4231
4232 if (pager == MEMORY_OBJECT_NULL) {
4233 /*
4234 * This pager has been destroyed by either
4235 * memory_object_destroy or vm_object_destroy, and
4236 * so there is nowhere for the page to go.
4237 */
4238 if (m->vmp_free_when_done) {
4239 /*
4240 * Just free the page... VM_PAGE_FREE takes
4241 * care of cleaning up all the state...
4242 * including doing the vm_pageout_throttle_up
4243 */
4244 VM_PAGE_FREE(m);
4245 } else {
4246 vm_page_lockspin_queues();
4247
4248 vm_pageout_throttle_up(m);
4249 vm_page_activate(m);
4250
4251 vm_page_unlock_queues();
4252
4253 /*
4254 * And we are done with it.
4255 */
4256 }
4257 vm_object_activity_end(object);
4258 vm_object_unlock(object);
4259
4260 vm_page_lockspin_queues();
4261 continue;
4262 }
4263 #if 0
4264 /*
4265 * we don't hold the page queue lock
4266 * so this check isn't safe to make
4267 */
4268 VM_PAGE_CHECK(m);
4269 #endif
4270 /*
4271 * give back the activity_in_progress reference we
4272 * took when we queued up this page and replace it
4273 * it with a paging_in_progress reference that will
4274 * also hold the paging offset from changing and
4275 * prevent the object from terminating
4276 */
4277 vm_object_activity_end(object);
4278 vm_object_paging_begin(object);
4279 vm_object_unlock(object);
4280
4281 /*
4282 * Send the data to the pager.
4283 * any pageout clustering happens there
4284 */
4285 memory_object_data_return(pager,
4286 m->vmp_offset + object->paging_offset,
4287 PAGE_SIZE,
4288 NULL,
4289 NULL,
4290 FALSE,
4291 FALSE,
4292 0);
4293
4294 vm_object_lock(object);
4295 vm_object_paging_end(object);
4296 vm_object_unlock(object);
4297
4298 vm_pageout_io_throttle();
4299
4300 vm_page_lockspin_queues();
4301 }
4302 q->pgo_busy = FALSE;
4303
4304 vm_page_unlock_queues();
4305 sched_cond_wait_parameter(&(ethr->pgo_wakeup), THREAD_UNINT, (thread_continue_t)vm_pageout_iothread_external_continue, ethr);
4306 }
4307 /*NOTREACHED*/
4308 }
4309
4310 uint32_t vm_compressor_time_thread; /* Set via sysctl 'vm.compressor_timing_enabled' to record time accrued by this thread. */
4311
4312 #if DEVELOPMENT || DEBUG
4313 static void
4314 vm_pageout_record_thread_time(int cqid, int ncomps)
4315 {
4316 if (__improbable(vm_compressor_time_thread)) {
4317 vmct_stats.vmct_runtimes[cqid] = thread_get_runtime_self();
4318 vmct_stats.vmct_pages[cqid] += ncomps;
4319 vmct_stats.vmct_iterations[cqid]++;
4320 if (ncomps > vmct_stats.vmct_maxpages[cqid]) {
4321 vmct_stats.vmct_maxpages[cqid] = ncomps;
4322 }
4323 if (ncomps < vmct_stats.vmct_minpages[cqid]) {
4324 vmct_stats.vmct_minpages[cqid] = ncomps;
4325 }
4326 }
4327 }
4328 #endif
4329
4330 static void *
4331 vm_pageout_select_filling_chead(struct pgo_iothread_state *cq, vm_page_t m)
4332 {
4333 /*
4334 * Technically we need the pageq locks to manipulate the vmp_on_specialq field.
4335 * However, this page has been removed from all queues and is only
4336 * known to this compressor thread dealing with this local queue.
4337 *
4338 * TODO: Add a second localq that is the early localq and
4339 * put special pages like this one on that queue in the block above
4340 * under the pageq lock to avoid this 'works but not clean' logic.
4341 */
4342 void *donate_queue_head;
4343 #if XNU_TARGET_OS_OSX /* tag:DONATE */
4344 donate_queue_head = &cq->current_early_swapout_chead;
4345 #else /* XNU_TARGET_OS_OSX */
4346 donate_queue_head = &cq->current_late_swapout_chead;
4347 #endif /* XNU_TARGET_OS_OSX */
4348 if (m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE) {
4349 m->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
4350 return donate_queue_head;
4351 }
4352
4353 uint32_t sel_i = 0;
4354 #if COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT > 1
4355 vm_object_t object = VM_PAGE_OBJECT(m);
4356 sel_i = object->vo_chead_hint;
4357 #endif
4358 assert(sel_i < COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT);
4359 return &cq->current_regular_swapout_cheads[sel_i];
4360 }
4361
4362 #define MAX_FREE_BATCH 32
4363
4364 OS_NORETURN
4365 static void
4366 vm_pageout_iothread_internal_continue(struct pgo_iothread_state *cq, __unused wait_result_t w)
4367 {
4368 struct vm_pageout_queue *q;
4369 vm_page_t m = NULL;
4370 boolean_t pgo_draining;
4371 vm_page_t local_q;
4372 int local_cnt;
4373 vm_page_t local_freeq = NULL;
4374 int local_freed = 0;
4375 int local_batch_size;
4376 #if DEVELOPMENT || DEBUG
4377 int ncomps = 0;
4378 boolean_t marked_active = FALSE;
4379 int num_pages_processed = 0;
4380 #endif
4381 void *chead = NULL;
4382
4383 KDBG_FILTERED(0xe040000c | DBG_FUNC_END);
4384
4385 sched_cond_ack(&(cq->pgo_wakeup));
4386
4387 q = cq->q;
4388
4389 while (true) { /* this top loop is for the compressor_running_perf_test running a full speed without blocking */
4390 #if DEVELOPMENT || DEBUG
4391 bool benchmark_accounting = false;
4392 /* If we're running the compressor perf test, only process the benchmark pages.
4393 * We'll get back to our regular queue once the benchmark is done */
4394 if (compressor_running_perf_test) {
4395 q = cq->benchmark_q;
4396 if (!vm_page_queue_empty(&q->pgo_pending)) {
4397 benchmark_accounting = true;
4398 } else {
4399 q = cq->q;
4400 benchmark_accounting = false;
4401 }
4402 }
4403 #endif /* DEVELOPMENT || DEBUG */
4404
4405 #if __AMP__
4406 if (vm_compressor_ebound && (vm_pageout_state.vm_compressor_thread_count > 1)) {
4407 local_batch_size = (q->pgo_maxlaundry >> 3);
4408 local_batch_size = MAX(local_batch_size, 16);
4409 } else {
4410 local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
4411 }
4412 #else
4413 local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
4414 #endif
4415
4416 #if RECORD_THE_COMPRESSED_DATA
4417 if (q->pgo_laundry) {
4418 c_compressed_record_init();
4419 }
4420 #endif
4421 while (true) { /* this loop is for working though all the pages in the pending queue */
4422 int pages_left_on_q = 0;
4423
4424 local_cnt = 0;
4425 local_q = NULL;
4426
4427 KDBG_FILTERED(0xe0400014 | DBG_FUNC_START);
4428
4429 vm_page_lock_queues();
4430 #if DEVELOPMENT || DEBUG
4431 if (marked_active == FALSE) {
4432 vmct_active++;
4433 vmct_state[cq->id] = VMCT_ACTIVE;
4434 marked_active = TRUE;
4435 if (vmct_active == 1) {
4436 vm_compressor_epoch_start = mach_absolute_time();
4437 }
4438 }
4439 #endif
4440 KDBG_FILTERED(0xe0400014 | DBG_FUNC_END);
4441
4442 KDBG_FILTERED(0xe0400018 | DBG_FUNC_START, q->pgo_laundry);
4443
4444 /* empty the entire content of the thread input q to local_q, but not more than local_batch_size pages */
4445 while (!vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
4446 vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
4447 assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
4448 VM_PAGE_CHECK(m);
4449
4450 m->vmp_q_state = VM_PAGE_NOT_ON_Q;
4451 VM_PAGE_ZERO_PAGEQ_ENTRY(m);
4452 m->vmp_laundry = FALSE;
4453
4454 m->vmp_snext = local_q;
4455 local_q = m;
4456 local_cnt++;
4457 }
4458 if (local_q == NULL) {
4459 break;
4460 }
4461
4462 q->pgo_busy = TRUE;
4463
4464 if ((pgo_draining = q->pgo_draining) == FALSE) {
4465 vm_pageout_throttle_up_batch(q, local_cnt);
4466 pages_left_on_q = q->pgo_laundry;
4467 } else {
4468 pages_left_on_q = q->pgo_laundry - local_cnt;
4469 }
4470
4471 vm_page_unlock_queues();
4472
4473 #if !RECORD_THE_COMPRESSED_DATA
4474 /* if we have lots to compress, wake up the other thread to help.
4475 * disabled when recording data since record data is not protected with a mutex so this may cause races */
4476 if (pages_left_on_q >= local_batch_size && cq->id < (vm_pageout_state.vm_compressor_thread_count - 1)) {
4477 // wake up the next compressor thread
4478 sched_cond_signal(&pgo_iothread_internal_state[cq->id + 1].pgo_wakeup,
4479 pgo_iothread_internal_state[cq->id + 1].pgo_iothread);
4480 }
4481 #endif
4482 KDBG_FILTERED(0xe0400018 | DBG_FUNC_END, q->pgo_laundry);
4483
4484 while (local_q) {
4485 KDBG_FILTERED(0xe0400024 | DBG_FUNC_START, local_cnt);
4486
4487 m = local_q;
4488 local_q = m->vmp_snext;
4489 m->vmp_snext = NULL;
4490
4491
4492 chead = vm_pageout_select_filling_chead(cq, m);
4493
4494 if (vm_pageout_compress_page(chead, cq->scratch_buf, m) == KERN_SUCCESS) {
4495 #if DEVELOPMENT || DEBUG
4496 ncomps++;
4497 #endif
4498 KDBG_FILTERED(0xe0400024 | DBG_FUNC_END, local_cnt);
4499
4500 m->vmp_snext = local_freeq;
4501 local_freeq = m;
4502 local_freed++;
4503
4504 /* if we gathered enough free pages, free them now */
4505 if (local_freed >= MAX_FREE_BATCH) {
4506 OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
4507
4508 vm_page_free_list(local_freeq, TRUE);
4509
4510 local_freeq = NULL;
4511 local_freed = 0;
4512 }
4513 }
4514 #if DEVELOPMENT || DEBUG
4515 num_pages_processed++;
4516 #endif /* DEVELOPMENT || DEBUG */
4517 #if !CONFIG_JETSAM /* Maybe: if there's no JETSAM, be more proactive in waking up anybody that needs free pages */
4518 while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
4519 kern_return_t wait_result;
4520 int need_wakeup = 0;
4521
4522 if (local_freeq) {
4523 OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
4524
4525 vm_page_free_list(local_freeq, TRUE);
4526 local_freeq = NULL;
4527 local_freed = 0;
4528
4529 continue;
4530 }
4531 vm_free_page_lock_spin();
4532
4533 if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
4534 if (vm_page_free_wanted_privileged++ == 0) {
4535 need_wakeup = 1;
4536 }
4537 wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
4538
4539 vm_free_page_unlock();
4540
4541 if (need_wakeup) {
4542 thread_wakeup((event_t)&vm_page_free_wanted);
4543 }
4544
4545 if (wait_result == THREAD_WAITING) {
4546 thread_block(THREAD_CONTINUE_NULL);
4547 }
4548 } else {
4549 vm_free_page_unlock();
4550 }
4551 }
4552 #endif
4553 } /* while (local_q) */
4554 /* free any leftovers in the freeq */
4555 if (local_freeq) {
4556 OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
4557
4558 vm_page_free_list(local_freeq, TRUE);
4559 local_freeq = NULL;
4560 local_freed = 0;
4561 }
4562 if (pgo_draining == TRUE) {
4563 vm_page_lockspin_queues();
4564 vm_pageout_throttle_up_batch(q, local_cnt);
4565 vm_page_unlock_queues();
4566 }
4567 }
4568 KDBG_FILTERED(0xe040000c | DBG_FUNC_START);
4569
4570 /*
4571 * queue lock is held and our q is empty
4572 */
4573 q->pgo_busy = FALSE;
4574 #if DEVELOPMENT || DEBUG
4575 if (marked_active == TRUE) {
4576 vmct_active--;
4577 vmct_state[cq->id] = VMCT_IDLE;
4578
4579 if (vmct_active == 0) {
4580 vm_compressor_epoch_stop = mach_absolute_time();
4581 assertf(vm_compressor_epoch_stop >= vm_compressor_epoch_start,
4582 "Compressor epoch non-monotonic: 0x%llx -> 0x%llx",
4583 vm_compressor_epoch_start, vm_compressor_epoch_stop);
4584 /* This interval includes intervals where one or more
4585 * compressor threads were pre-empted
4586 */
4587 vmct_stats.vmct_cthreads_total += vm_compressor_epoch_stop - vm_compressor_epoch_start;
4588 }
4589 }
4590 if (compressor_running_perf_test && benchmark_accounting) {
4591 /*
4592 * We could turn ON compressor_running_perf_test while still processing
4593 * regular non-benchmark pages. We shouldn't count them here else we
4594 * could overshoot. We might also still be populating that benchmark Q
4595 * and be under pressure. So we will go back to the regular queues. And
4596 * benchmark accounting will be off for that case too.
4597 */
4598 compressor_perf_test_pages_processed += num_pages_processed;
4599 thread_wakeup(&compressor_perf_test_pages_processed);
4600 }
4601 #endif
4602 vm_page_unlock_queues();
4603 #if DEVELOPMENT || DEBUG
4604 vm_pageout_record_thread_time(cq->id, ncomps);
4605 #endif
4606
4607 KDBG_FILTERED(0xe0400018 | DBG_FUNC_END);
4608 #if DEVELOPMENT || DEBUG
4609 if (compressor_running_perf_test && benchmark_accounting) {
4610 /*
4611 * We've been exclusively compressing pages from the benchmark queue,
4612 * do 1 pass over the internal queue before blocking.
4613 */
4614 continue;
4615 }
4616 #endif
4617
4618 sched_cond_wait_parameter(&(cq->pgo_wakeup), THREAD_UNINT, (thread_continue_t)vm_pageout_iothread_internal_continue, (void *) cq);
4619 }
4620 /*NOTREACHED*/
4621 }
4622
4623 /* resolves the pager and maintain stats in the pager and in the vm_object */
4624 kern_return_t
4625 vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m)
4626 {
4627 vm_object_t object;
4628 memory_object_t pager;
4629 int compressed_count_delta;
4630 kern_return_t retval;
4631
4632 object = VM_PAGE_OBJECT(m);
4633
4634 assert(!m->vmp_free_when_done);
4635 assert(!m->vmp_laundry);
4636
4637 pager = object->pager;
4638
4639 if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
4640 KDBG_FILTERED(0xe0400010 | DBG_FUNC_START, object, pager);
4641
4642 vm_object_lock(object);
4643
4644 /*
4645 * If there is no memory object for the page, create
4646 * one and hand it to the compression pager.
4647 */
4648
4649 if (!object->pager_initialized) {
4650 vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
4651 }
4652 if (!object->pager_initialized) {
4653 vm_object_compressor_pager_create(object);
4654 }
4655
4656 pager = object->pager;
4657
4658 if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
4659 /*
4660 * Still no pager for the object,
4661 * or the pager has been destroyed.
4662 * Reactivate the page.
4663 *
4664 * Should only happen if there is no
4665 * compression pager
4666 */
4667 vm_page_wakeup_done(object, m);
4668
4669 vm_page_lockspin_queues();
4670 vm_page_activate(m);
4671 VM_PAGEOUT_DEBUG(vm_pageout_dirty_no_pager, 1);
4672 vm_page_unlock_queues();
4673
4674 /*
4675 * And we are done with it.
4676 */
4677 vm_object_activity_end(object);
4678 vm_object_unlock(object);
4679
4680 return KERN_FAILURE;
4681 }
4682 vm_object_unlock(object);
4683
4684 KDBG_FILTERED(0xe0400010 | DBG_FUNC_END, object, pager);
4685 }
4686 assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);
4687 assert(object->activity_in_progress > 0);
4688
4689 #if CONFIG_TRACK_UNMODIFIED_ANON_PAGES
4690 if (m->vmp_unmodified_ro == true) {
4691 os_atomic_inc(&compressor_ro_uncompressed_total_returned, relaxed);
4692 }
4693 #endif /* CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
4694
4695 vm_compressor_options_t flags = 0;
4696
4697 #if CONFIG_TRACK_UNMODIFIED_ANON_PAGES
4698 if (m->vmp_unmodified_ro) {
4699 flags |= C_PAGE_UNMODIFIED;
4700 }
4701 #endif /* CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
4702
4703 #if HAS_MTE
4704 if (vm_object_is_mte_mappable(object)) {
4705 flags |= C_MTE;
4706 }
4707 #endif /* HAS_MTE */
4708
4709 retval = vm_compressor_pager_put(
4710 pager,
4711 m->vmp_offset + object->paging_offset,
4712 VM_PAGE_GET_PHYS_PAGE(m),
4713 current_chead,
4714 scratch_buf,
4715 &compressed_count_delta,
4716 flags);
4717
4718 vm_object_lock(object);
4719
4720 assert(object->activity_in_progress > 0);
4721 assert(VM_PAGE_OBJECT(m) == object);
4722 assert( !VM_PAGE_WIRED(m));
4723
4724 vm_compressor_pager_count(pager,
4725 compressed_count_delta,
4726 FALSE, /* shared_lock */
4727 object);
4728
4729 if (retval == KERN_SUCCESS) {
4730 /*
4731 * If the object is purgeable, its owner's
4732 * purgeable ledgers will be updated in
4733 * vm_page_remove() but the page still
4734 * contributes to the owner's memory footprint,
4735 * so account for it as such.
4736 */
4737 if (m->vmp_tabled) {
4738 vm_page_remove(m, TRUE);
4739 }
4740 if ((object->purgable != VM_PURGABLE_DENY ||
4741 object->vo_ledger_tag) &&
4742 object->vo_owner != NULL) {
4743 /* one more compressed purgeable/tagged page */
4744 vm_object_owner_compressed_update(object,
4745 compressed_count_delta);
4746 }
4747 counter_inc(&vm_statistics_compressions);
4748 } else {
4749 vm_page_wakeup_done(object, m);
4750
4751 vm_page_lockspin_queues();
4752
4753 vm_page_activate(m);
4754 vm_pageout_vminfo.vm_compressor_failed++;
4755
4756 vm_page_unlock_queues();
4757 }
4758 vm_object_activity_end(object);
4759 vm_object_unlock(object);
4760
4761 return retval;
4762 }
4763
4764
4765 static void
4766 vm_pageout_adjust_eq_iothrottle(struct pgo_iothread_state *ethr, boolean_t req_lowpriority)
4767 {
4768 uint32_t policy;
4769
4770 if (hibernate_cleaning_in_progress == TRUE) {
4771 req_lowpriority = FALSE;
4772 }
4773
4774 if (ethr->q->pgo_inited == TRUE && ethr->q->pgo_lowpriority != req_lowpriority) {
4775 vm_page_unlock_queues();
4776
4777 if (req_lowpriority == TRUE) {
4778 policy = THROTTLE_LEVEL_PAGEOUT_THROTTLED;
4779 DTRACE_VM(laundrythrottle);
4780 } else {
4781 policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED;
4782 DTRACE_VM(laundryunthrottle);
4783 }
4784 proc_set_thread_policy(ethr->pgo_iothread,
4785 TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
4786
4787 vm_page_lock_queues();
4788 ethr->q->pgo_lowpriority = req_lowpriority;
4789 }
4790 }
4791
4792 OS_NORETURN
4793 static void
4794 vm_pageout_iothread_external(struct pgo_iothread_state *ethr, __unused wait_result_t w)
4795 {
4796 thread_t self = current_thread();
4797
4798 self->options |= TH_OPT_VMPRIV;
4799
4800 DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
4801
4802 proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,
4803 TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
4804
4805 vm_page_lock_queues();
4806
4807 vm_pageout_queue_external.pgo_lowpriority = TRUE;
4808 vm_pageout_queue_external.pgo_inited = TRUE;
4809
4810 vm_page_unlock_queues();
4811
4812 #if CONFIG_THREAD_GROUPS
4813 thread_group_vm_add();
4814 #endif /* CONFIG_THREAD_GROUPS */
4815
4816 vm_pageout_iothread_external_continue(ethr, 0);
4817 /*NOTREACHED*/
4818 }
4819
4820
4821 OS_NORETURN
4822 static void
4823 vm_pageout_iothread_internal(struct pgo_iothread_state *cthr, __unused wait_result_t w)
4824 {
4825 thread_t self = current_thread();
4826
4827 self->options |= TH_OPT_VMPRIV;
4828
4829 vm_page_lock_queues();
4830
4831 vm_pageout_queue_internal.pgo_lowpriority = TRUE;
4832 vm_pageout_queue_internal.pgo_inited = TRUE;
4833
4834 #if DEVELOPMENT || DEBUG
4835 vm_pageout_queue_benchmark.pgo_lowpriority = vm_pageout_queue_internal.pgo_lowpriority;
4836 vm_pageout_queue_benchmark.pgo_inited = vm_pageout_queue_internal.pgo_inited;
4837 vm_pageout_queue_benchmark.pgo_busy = FALSE;
4838 #endif /* DEVELOPMENT || DEBUG */
4839
4840 vm_page_unlock_queues();
4841
4842 if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
4843 thread_vm_bind_group_add();
4844 }
4845
4846 #if CONFIG_THREAD_GROUPS
4847 thread_group_vm_add();
4848 #endif /* CONFIG_THREAD_GROUPS */
4849
4850 #if __AMP__
4851 if (vm_compressor_ebound) {
4852 /*
4853 * Use the soft bound option for vm_compressor to allow it to run on
4854 * P-cores if E-cluster is unavailable.
4855 */
4856 kern_return_t kr = thread_soft_bind_cluster_type(self, 'E');
4857 if (kr != KERN_SUCCESS) {
4858 printf("%s: WARN: failed to bind thread to cluster type; does the hardware topology match expectations?\n", __FUNCTION__);
4859 }
4860 }
4861 #endif /* __AMP__ */
4862
4863 thread_set_thread_name(current_thread(), "VM_compressor");
4864 #if DEVELOPMENT || DEBUG
4865 vmct_stats.vmct_minpages[cthr->id] = INT32_MAX;
4866 #endif
4867 vm_pageout_iothread_internal_continue(cthr, 0);
4868
4869 /*NOTREACHED*/
4870 }
4871
4872 kern_return_t
4873 vm_set_buffer_cleanup_callout(boolean_t (*func)(int))
4874 {
4875 if (OSCompareAndSwapPtr(NULL, ptrauth_nop_cast(void *, func), (void * volatile *) &consider_buffer_cache_collect)) {
4876 return KERN_SUCCESS;
4877 } else {
4878 return KERN_FAILURE; /* Already set */
4879 }
4880 }
4881
4882 extern boolean_t memorystatus_manual_testing_on;
4883 extern unsigned int memorystatus_level;
4884
4885
4886 #if VM_PRESSURE_EVENTS
4887
4888 boolean_t vm_pressure_events_enabled = FALSE;
4889
4890 extern uint64_t next_warning_notification_sent_at_ts;
4891 extern uint64_t next_critical_notification_sent_at_ts;
4892
4893 #define PRESSURE_LEVEL_STUCK_THRESHOLD_MINS (30) /* 30 minutes. */
4894
4895 /*
4896 * The last time there was change in pressure level OR we forced a check
4897 * because the system is stuck in a non-normal pressure level.
4898 */
4899 uint64_t vm_pressure_last_level_transition_abs = 0;
4900
4901 /*
4902 * This is how the long the system waits 'stuck' in an unchanged non-normal pressure
4903 * level before resending out notifications for that level again.
4904 */
4905 int vm_pressure_level_transition_threshold = PRESSURE_LEVEL_STUCK_THRESHOLD_MINS;
4906
4907 void
4908 vm_pressure_response()
4909 {
4910 vm_pressure_level_t old_level = kVMPressureNormal;
4911 int new_level = -1;
4912 unsigned int total_pages;
4913 uint64_t available_memory = 0;
4914 uint64_t curr_ts, abs_time_since_level_transition, time_in_ns;
4915 bool force_check = false;
4916 int time_in_mins;
4917
4918
4919 if (vm_pressure_events_enabled == FALSE) {
4920 return;
4921 }
4922
4923 available_memory = (uint64_t) memorystatus_get_available_page_count();
4924
4925 total_pages = (unsigned int) atop_64(max_mem);
4926 #if CONFIG_SECLUDED_MEMORY
4927 total_pages -= vm_page_secluded_count;
4928 #endif /* CONFIG_SECLUDED_MEMORY */
4929 memorystatus_level = (unsigned int) ((available_memory * 100) / total_pages);
4930
4931 if (memorystatus_manual_testing_on) {
4932 return;
4933 }
4934
4935 curr_ts = mach_absolute_time();
4936 abs_time_since_level_transition = curr_ts - vm_pressure_last_level_transition_abs;
4937
4938 absolutetime_to_nanoseconds(abs_time_since_level_transition, &time_in_ns);
4939 time_in_mins = (int) ((time_in_ns / NSEC_PER_SEC) / 60);
4940 force_check = (time_in_mins >= vm_pressure_level_transition_threshold);
4941
4942 old_level = memorystatus_vm_pressure_level;
4943
4944 switch (memorystatus_vm_pressure_level) {
4945 case kVMPressureNormal:
4946 {
4947 if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
4948 new_level = kVMPressureCritical;
4949 } else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
4950 new_level = kVMPressureWarning;
4951 }
4952 break;
4953 }
4954
4955 case kVMPressureWarning:
4956 case kVMPressureUrgent:
4957 {
4958 if (VM_PRESSURE_WARNING_TO_NORMAL()) {
4959 new_level = kVMPressureNormal;
4960 } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
4961 new_level = kVMPressureCritical;
4962 } else if (force_check) {
4963 new_level = kVMPressureWarning;
4964 next_warning_notification_sent_at_ts = curr_ts;
4965 }
4966 break;
4967 }
4968
4969 case kVMPressureCritical:
4970 {
4971 if (VM_PRESSURE_WARNING_TO_NORMAL()) {
4972 new_level = kVMPressureNormal;
4973 } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
4974 new_level = kVMPressureWarning;
4975 } else if (force_check) {
4976 new_level = kVMPressureCritical;
4977 next_critical_notification_sent_at_ts = curr_ts;
4978 }
4979 break;
4980 }
4981
4982 default:
4983 return;
4984 }
4985
4986 if (new_level != -1 || force_check) {
4987 if (new_level != -1) {
4988 memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
4989
4990 if (new_level != (int) old_level) {
4991 VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, DBG_VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE,
4992 new_level, old_level, 0, 0);
4993 }
4994 } else {
4995 VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, DBG_VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE,
4996 new_level, old_level, force_check, 0);
4997 }
4998
4999 if (hibernation_vmqueues_inspection || hibernate_cleaning_in_progress) {
5000 /*
5001 * We don't want to schedule a wakeup while hibernation is in progress
5002 * because that could collide with checks for non-monotonicity in the scheduler.
5003 * We do however do all the updates to memorystatus_vm_pressure_level because
5004 * we _might_ want to use that for decisions regarding which pages or how
5005 * many pages we want to dump in hibernation.
5006 */
5007 return;
5008 }
5009
5010 if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != memorystatus_vm_pressure_level) || force_check) {
5011 if (vm_pageout_state.vm_pressure_thread_running == FALSE) {
5012 thread_wakeup(&vm_pressure_thread);
5013 }
5014
5015 if (old_level != memorystatus_vm_pressure_level) {
5016 thread_wakeup(&vm_pageout_state.vm_pressure_changed);
5017 }
5018 vm_pressure_last_level_transition_abs = curr_ts; /* renew the window of observation for a stuck pressure level */
5019 }
5020 }
5021 }
5022 #endif /* VM_PRESSURE_EVENTS */
5023
5024
5025 /**
5026 * Called by a kernel thread to ask if a number of pages may be wired.
5027 */
5028 kern_return_t
5029 mach_vm_wire_level_monitor(int64_t requested_pages)
5030 {
5031 if (requested_pages <= 0) {
5032 return KERN_INVALID_ARGUMENT;
5033 }
5034
5035 const int64_t max_wire_pages = atop_64(vm_global_user_wire_limit);
5036 /**
5037 * Available pages can be negative in the case where more system memory is
5038 * wired than the threshold, so we must use a signed integer.
5039 */
5040 const int64_t available_pages = max_wire_pages - vm_page_wire_count;
5041
5042 if (requested_pages > available_pages) {
5043 return KERN_RESOURCE_SHORTAGE;
5044 }
5045 return KERN_SUCCESS;
5046 }
5047
5048 /*
5049 * Function called by a kernel thread to either get the current pressure level or
5050 * wait until memory pressure changes from a given level.
5051 */
5052 kern_return_t
5053 mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level)
5054 {
5055 #if !VM_PRESSURE_EVENTS
5056 (void)wait_for_pressure;
5057 (void)pressure_level;
5058 return KERN_NOT_SUPPORTED;
5059 #else /* VM_PRESSURE_EVENTS */
5060
5061 uint32_t *waiters = NULL;
5062 wait_result_t wr = 0;
5063 vm_pressure_level_t old_level = memorystatus_vm_pressure_level;
5064
5065 if (pressure_level == NULL) {
5066 return KERN_INVALID_ARGUMENT;
5067 }
5068 if (!wait_for_pressure && (*pressure_level == kVMPressureBackgroundJetsam ||
5069 *pressure_level == kVMPressureForegroundJetsam)) {
5070 return KERN_INVALID_ARGUMENT;
5071 }
5072
5073 if (wait_for_pressure) {
5074 switch (*pressure_level) {
5075 case kVMPressureForegroundJetsam:
5076 case kVMPressureBackgroundJetsam:
5077
5078 if (*pressure_level == kVMPressureForegroundJetsam) {
5079 waiters = &memorystatus_jetsam_fg_band_waiters;
5080 } else {
5081 /* kVMPressureBackgroundJetsam */
5082 waiters = &memorystatus_jetsam_bg_band_waiters;
5083 }
5084
5085 lck_mtx_lock(&memorystatus_jetsam_broadcast_lock);
5086 wr = assert_wait((event_t)waiters, THREAD_INTERRUPTIBLE);
5087 if (wr == THREAD_WAITING) {
5088 *waiters += 1;
5089 lck_mtx_unlock(&memorystatus_jetsam_broadcast_lock);
5090 wr = thread_block(THREAD_CONTINUE_NULL);
5091 } else {
5092 lck_mtx_unlock(&memorystatus_jetsam_broadcast_lock);
5093 }
5094
5095 if (wr != THREAD_AWAKENED) {
5096 return KERN_ABORTED;
5097 }
5098
5099 return KERN_SUCCESS;
5100 case kVMPressureNormal:
5101 case kVMPressureWarning:
5102 case kVMPressureUrgent:
5103 case kVMPressureCritical:
5104 while (old_level == *pressure_level) {
5105 wr = assert_wait((event_t) &vm_pageout_state.vm_pressure_changed,
5106 THREAD_INTERRUPTIBLE);
5107 if (wr == THREAD_WAITING) {
5108 wr = thread_block(THREAD_CONTINUE_NULL);
5109 }
5110 if (wr == THREAD_INTERRUPTED) {
5111 return KERN_ABORTED;
5112 }
5113
5114 if (wr == THREAD_AWAKENED) {
5115 old_level = memorystatus_vm_pressure_level;
5116 }
5117 }
5118 break;
5119 default:
5120 return KERN_INVALID_ARGUMENT;
5121 }
5122 }
5123
5124 *pressure_level = old_level;
5125 return KERN_SUCCESS;
5126 #endif /* VM_PRESSURE_EVENTS */
5127 }
5128
5129 #if VM_PRESSURE_EVENTS
5130 void
5131 vm_pressure_thread(void)
5132 {
5133 static boolean_t thread_initialized = FALSE;
5134
5135 if (thread_initialized == TRUE) {
5136 vm_pageout_state.vm_pressure_thread_running = TRUE;
5137 consider_vm_pressure_events();
5138 vm_pageout_state.vm_pressure_thread_running = FALSE;
5139 }
5140
5141 #if CONFIG_THREAD_GROUPS
5142 thread_group_vm_add();
5143 #endif /* CONFIG_THREAD_GROUPS */
5144
5145 thread_set_thread_name(current_thread(), "VM_pressure");
5146 thread_initialized = TRUE;
5147 assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
5148 thread_block((thread_continue_t)vm_pressure_thread);
5149 }
5150 #endif /* VM_PRESSURE_EVENTS */
5151
5152
5153 /*
5154 * called once per-second via "compute_averages"
5155 */
5156 void
5157 compute_pageout_gc_throttle(__unused void *arg)
5158 {
5159 if (vm_pageout_vminfo.vm_pageout_considered_page != vm_pageout_state.vm_pageout_considered_page_last) {
5160 vm_pageout_state.vm_pageout_considered_page_last = vm_pageout_vminfo.vm_pageout_considered_page;
5161 sched_cond_signal(&vm_pageout_gc_cond, vm_pageout_gc_thread);
5162 }
5163 }
5164
5165 /*
5166 * vm_pageout_garbage_collect can also be called when the zone allocator needs
5167 * to call zone_gc on a different thread in order to trigger zone-map-exhaustion
5168 * jetsams. We need to check if the zone map size is above its jetsam limit to
5169 * decide if this was indeed the case.
5170 *
5171 * We need to do this on a different thread because of the following reasons:
5172 *
5173 * 1. In the case of synchronous jetsams, the leaking process can try to jetsam
5174 * itself causing the system to hang. We perform synchronous jetsams if we're
5175 * leaking in the VM map entries zone, so the leaking process could be doing a
5176 * zalloc for a VM map entry while holding its vm_map lock, when it decides to
5177 * jetsam itself. We also need the vm_map lock on the process termination path,
5178 * which would now lead the dying process to deadlock against itself.
5179 *
5180 * 2. The jetsam path might need to allocate zone memory itself. We could try
5181 * using the non-blocking variant of zalloc for this path, but we can still
5182 * end up trying to do a kmem_alloc when the zone maps are almost full.
5183 */
5184 __dead2
5185 void
5186 vm_pageout_garbage_collect(void *step, wait_result_t wr __unused)
5187 {
5188 assert(step == VM_PAGEOUT_GC_INIT || step == VM_PAGEOUT_GC_COLLECT);
5189
5190 if (step != VM_PAGEOUT_GC_INIT) {
5191 sched_cond_ack(&vm_pageout_gc_cond);
5192 }
5193
5194 while (true) {
5195 if (step == VM_PAGEOUT_GC_INIT) {
5196 /* first time being called is not about GC */
5197 #if CONFIG_THREAD_GROUPS
5198 thread_group_vm_add();
5199 #endif /* CONFIG_THREAD_GROUPS */
5200 step = VM_PAGEOUT_GC_COLLECT;
5201 } else if (zone_map_nearing_exhaustion()) {
5202 /*
5203 * Woken up by the zone allocator for zone-map-exhaustion jetsams.
5204 *
5205 * Bail out after calling zone_gc (which triggers the
5206 * zone-map-exhaustion jetsams). If we fall through, the subsequent
5207 * operations that clear out a bunch of caches might allocate zone
5208 * memory themselves (for eg. vm_map operations would need VM map
5209 * entries). Since the zone map is almost full at this point, we
5210 * could end up with a panic. We just need to quickly jetsam a
5211 * process and exit here.
5212 *
5213 * It could so happen that we were woken up to relieve memory
5214 * pressure and the zone map also happened to be near its limit at
5215 * the time, in which case we'll skip out early. But that should be
5216 * ok; if memory pressure persists, the thread will simply be woken
5217 * up again.
5218 */
5219
5220 zone_gc(ZONE_GC_JETSAM);
5221 } else {
5222 /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
5223 boolean_t buf_large_zfree = FALSE;
5224 boolean_t first_try = TRUE;
5225
5226 stack_collect();
5227
5228 consider_machine_collect();
5229 #if CONFIG_DEFERRED_RECLAIM
5230 mach_vm_size_t bytes_reclaimed;
5231 vm_deferred_reclamation_gc(RECLAIM_GC_TRIM, &bytes_reclaimed, RECLAIM_OPTIONS_NONE);
5232 #endif /* CONFIG_DEFERRED_RECLAIM */
5233 #if CONFIG_MBUF_MCACHE
5234 mbuf_drain(FALSE);
5235 #endif /* CONFIG_MBUF_MCACHE */
5236
5237 do {
5238 if (consider_buffer_cache_collect != NULL) {
5239 buf_large_zfree = (*consider_buffer_cache_collect)(0);
5240 }
5241 if (first_try == TRUE || buf_large_zfree == TRUE) {
5242 /*
5243 * zone_gc should be last, because the other operations
5244 * might return memory to zones.
5245 */
5246 zone_gc(ZONE_GC_TRIM);
5247 }
5248 first_try = FALSE;
5249 } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
5250
5251 consider_machine_adjust();
5252 }
5253
5254 sched_cond_wait_parameter(&vm_pageout_gc_cond, THREAD_UNINT, vm_pageout_garbage_collect, VM_PAGEOUT_GC_COLLECT);
5255 }
5256 __builtin_unreachable();
5257 }
5258
5259
5260 #if VM_PAGE_BUCKETS_CHECK
5261 #if VM_PAGE_FAKE_BUCKETS
5262 extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
5263 #endif /* VM_PAGE_FAKE_BUCKETS */
5264 #endif /* VM_PAGE_BUCKETS_CHECK */
5265
5266
5267
5268 void
5269 vm_set_restrictions(unsigned int num_cpus)
5270 {
5271 int vm_restricted_to_single_processor = 0;
5272
5273 if (PE_parse_boot_argn("vm_restricted_to_single_processor", &vm_restricted_to_single_processor, sizeof(vm_restricted_to_single_processor))) {
5274 kprintf("Overriding vm_restricted_to_single_processor to %d\n", vm_restricted_to_single_processor);
5275 vm_pageout_state.vm_restricted_to_single_processor = (vm_restricted_to_single_processor ? TRUE : FALSE);
5276 } else {
5277 assert(num_cpus > 0);
5278
5279 if (num_cpus <= 3) {
5280 /*
5281 * on systems with a limited number of CPUS, bind the
5282 * 4 major threads that can free memory and that tend to use
5283 * a fair bit of CPU under pressured conditions to a single processor.
5284 * This insures that these threads don't hog all of the available CPUs
5285 * (important for camera launch), while allowing them to run independently
5286 * w/r to locks... the 4 threads are
5287 * vm_pageout_scan, vm_pageout_iothread_internal (compressor),
5288 * vm_compressor_swap_trigger_thread (minor and major compactions),
5289 * memorystatus_thread (jetsams).
5290 *
5291 * the first time the thread is run, it is responsible for checking the
5292 * state of vm_restricted_to_single_processor, and if TRUE it calls
5293 * thread_bind_master... someday this should be replaced with a group
5294 * scheduling mechanism and KPI.
5295 */
5296 vm_pageout_state.vm_restricted_to_single_processor = TRUE;
5297 } else {
5298 vm_pageout_state.vm_restricted_to_single_processor = FALSE;
5299 }
5300 }
5301 }
5302
5303 /*
5304 * Set up vm_config based on the vm_compressor_mode.
5305 * Must run BEFORE the pageout thread starts up.
5306 */
5307 __startup_func
5308 void
5309 vm_config_init(void)
5310 {
5311 bzero(&vm_config, sizeof(vm_config));
5312
5313 switch (vm_compressor_mode) {
5314 case VM_PAGER_DEFAULT:
5315 printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
5316 OS_FALLTHROUGH;
5317
5318 case VM_PAGER_COMPRESSOR_WITH_SWAP:
5319 vm_config.compressor_is_present = TRUE;
5320 vm_config.swap_is_present = TRUE;
5321 vm_config.compressor_is_active = TRUE;
5322 vm_config.swap_is_active = TRUE;
5323 break;
5324
5325 case VM_PAGER_COMPRESSOR_NO_SWAP:
5326 vm_config.compressor_is_present = TRUE;
5327 vm_config.swap_is_present = TRUE;
5328 vm_config.compressor_is_active = TRUE;
5329 break;
5330
5331 case VM_PAGER_FREEZER_DEFAULT:
5332 printf("mapping deprecated VM_PAGER_FREEZER_DEFAULT to VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP\n");
5333 OS_FALLTHROUGH;
5334
5335 case VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP:
5336 vm_config.compressor_is_present = TRUE;
5337 vm_config.swap_is_present = TRUE;
5338 break;
5339
5340 case VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP:
5341 vm_config.compressor_is_present = TRUE;
5342 vm_config.swap_is_present = TRUE;
5343 vm_config.compressor_is_active = TRUE;
5344 vm_config.freezer_swap_is_active = TRUE;
5345 break;
5346
5347 case VM_PAGER_NOT_CONFIGURED:
5348 break;
5349
5350 default:
5351 printf("unknown compressor mode - %x\n", vm_compressor_mode);
5352 break;
5353 }
5354 }
5355
5356 __startup_func
5357 static void
5358 vm_pageout_create_gc_thread(void)
5359 {
5360 thread_t thread;
5361
5362 sched_cond_init(&vm_pageout_gc_cond);
5363 if (kernel_thread_create(vm_pageout_garbage_collect,
5364 VM_PAGEOUT_GC_INIT, BASEPRI_DEFAULT, &thread) != KERN_SUCCESS) {
5365 panic("vm_pageout_garbage_collect: create failed");
5366 }
5367 thread_set_thread_name(thread, "VM_pageout_garbage_collect");
5368 if (thread->reserved_stack == 0) {
5369 assert(thread->kernel_stack);
5370 thread->reserved_stack = thread->kernel_stack;
5371 }
5372
5373 /* thread is started in vm_pageout() */
5374 vm_pageout_gc_thread = thread;
5375 }
5376 STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, vm_pageout_create_gc_thread);
5377
5378 void
5379 vm_pageout(void)
5380 {
5381 thread_t self = current_thread();
5382 thread_t thread;
5383 kern_return_t result;
5384 spl_t s;
5385
5386 /*
5387 * Set thread privileges.
5388 */
5389 s = splsched();
5390
5391 #if CONFIG_VPS_DYNAMIC_PRIO
5392 if (vps_dynamic_priority_enabled) {
5393 sched_set_kernel_thread_priority(self, MAXPRI_THROTTLE);
5394 thread_set_eager_preempt(self);
5395 } else {
5396 sched_set_kernel_thread_priority(self, BASEPRI_VM);
5397 }
5398 #else /* CONFIG_VPS_DYNAMIC_PRIO */
5399 sched_set_kernel_thread_priority(self, BASEPRI_VM);
5400 #endif /* CONFIG_VPS_DYNAMIC_PRIO */
5401
5402 thread_lock(self);
5403 self->options |= TH_OPT_VMPRIV;
5404 thread_unlock(self);
5405
5406 if (!self->reserved_stack) {
5407 self->reserved_stack = self->kernel_stack;
5408 }
5409
5410 if (vm_pageout_state.vm_restricted_to_single_processor == TRUE &&
5411 !vps_dynamic_priority_enabled) {
5412 thread_vm_bind_group_add();
5413 }
5414
5415
5416 #if CONFIG_THREAD_GROUPS
5417 thread_group_vm_add();
5418 #endif /* CONFIG_THREAD_GROUPS */
5419
5420 #if __AMP__
5421 PE_parse_boot_argn("vmpgo_pcluster", &vm_pgo_pbound, sizeof(vm_pgo_pbound));
5422 if (vm_pgo_pbound) {
5423 /*
5424 * Use the soft bound option for vm pageout to allow it to run on
5425 * E-cores if P-cluster is unavailable.
5426 */
5427 kern_return_t kr = thread_soft_bind_cluster_type(self, 'P');
5428 if (kr != KERN_SUCCESS) {
5429 printf("%s: WARN: failed to bind thread to cluster type; does the hardware topology match expectations?\n", __FUNCTION__);
5430 }
5431 }
5432 #endif /* __AMP__ */
5433
5434 PE_parse_boot_argn("vmpgo_protect_realtime",
5435 &vm_pageout_protect_realtime,
5436 sizeof(vm_pageout_protect_realtime));
5437 splx(s);
5438
5439 thread_set_thread_name(current_thread(), "VM_pageout_scan");
5440
5441 vm_log_handle = os_log_create("com.apple.xnu", "virtual-memory");
5442
5443 /*
5444 * Initialize some paging parameters.
5445 */
5446
5447 vm_pageout_state.vm_pressure_thread_running = FALSE;
5448 vm_pageout_state.vm_pressure_changed = FALSE;
5449 vm_pageout_state.memorystatus_purge_on_warning = 2;
5450 vm_pageout_state.memorystatus_purge_on_urgent = 5;
5451 vm_pageout_state.memorystatus_purge_on_critical = 8;
5452 vm_pageout_state.vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
5453 vm_pageout_state.vm_page_speculative_percentage = 5;
5454 vm_pageout_state.vm_page_speculative_target = 0;
5455
5456 vm_pageout_state.vm_pageout_swap_wait = 0;
5457 vm_pageout_state.vm_pageout_idle_wait = 0;
5458 vm_pageout_state.vm_pageout_empty_wait = 0;
5459 vm_pageout_state.vm_pageout_burst_wait = 0;
5460 vm_pageout_state.vm_pageout_deadlock_wait = 0;
5461 vm_pageout_state.vm_pageout_deadlock_relief = 0;
5462 vm_pageout_state.vm_pageout_burst_inactive_throttle = 0;
5463
5464 vm_pageout_state.vm_pageout_inactive = 0;
5465 vm_pageout_state.vm_pageout_inactive_used = 0;
5466 vm_pageout_state.vm_pageout_inactive_clean = 0;
5467
5468 vm_pageout_state.vm_memory_pressure = 0;
5469 vm_pageout_state.vm_page_filecache_min = 0;
5470 #if CONFIG_JETSAM
5471 vm_pageout_state.vm_page_filecache_min_divisor = 70;
5472 vm_pageout_state.vm_page_xpmapped_min_divisor = 40;
5473 #else
5474 vm_pageout_state.vm_page_filecache_min_divisor = 27;
5475 vm_pageout_state.vm_page_xpmapped_min_divisor = 36;
5476 #endif
5477 vm_pageout_state.vm_page_free_count_init = vm_page_free_count;
5478
5479 vm_pageout_state.vm_pageout_considered_page_last = 0;
5480
5481 if (vm_pageout_state.vm_pageout_swap_wait == 0) {
5482 vm_pageout_state.vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
5483 }
5484
5485 if (vm_pageout_state.vm_pageout_idle_wait == 0) {
5486 vm_pageout_state.vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
5487 }
5488
5489 if (vm_pageout_state.vm_pageout_burst_wait == 0) {
5490 vm_pageout_state.vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
5491 }
5492
5493 if (vm_pageout_state.vm_pageout_empty_wait == 0) {
5494 vm_pageout_state.vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
5495 }
5496
5497 if (vm_pageout_state.vm_pageout_deadlock_wait == 0) {
5498 vm_pageout_state.vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
5499 }
5500
5501 if (vm_pageout_state.vm_pageout_deadlock_relief == 0) {
5502 vm_pageout_state.vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
5503 }
5504
5505 if (vm_pageout_state.vm_pageout_burst_inactive_throttle == 0) {
5506 vm_pageout_state.vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
5507 }
5508 /*
5509 * even if we've already called vm_page_free_reserve
5510 * call it again here to insure that the targets are
5511 * accurately calculated (it uses vm_page_free_count_init)
5512 * calling it with an arg of 0 will not change the reserve
5513 * but will re-calculate free_min and free_target
5514 */
5515 if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
5516 vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);
5517 } else {
5518 vm_page_free_reserve(0);
5519 }
5520
5521 bzero(&vm_pageout_queue_external, sizeof(struct vm_pageout_queue));
5522 bzero(&vm_pageout_queue_internal, sizeof(struct vm_pageout_queue));
5523
5524 vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);
5525 vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
5526
5527 vm_page_queue_init(&vm_pageout_queue_internal.pgo_pending);
5528
5529 #if DEVELOPMENT || DEBUG
5530 bzero(&vm_pageout_queue_benchmark, sizeof(struct vm_pageout_queue));
5531 vm_page_queue_init(&vm_pageout_queue_benchmark.pgo_pending);
5532 #endif /* DEVELOPMENT || DEBUG */
5533
5534
5535 /* internal pageout thread started when default pager registered first time */
5536 /* external pageout and garbage collection threads started here */
5537 struct pgo_iothread_state *ethr = &pgo_iothread_external_state;
5538 ethr->id = 0;
5539 ethr->q = &vm_pageout_queue_external;
5540 /* in external_state these cheads are never used, they are used only in internal_state for te compressor */
5541 ethr->current_early_swapout_chead = NULL;
5542 for (int reg_i = 0; reg_i < COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT; ++reg_i) {
5543 ethr->current_regular_swapout_cheads[reg_i] = NULL;
5544 }
5545 ethr->current_late_swapout_chead = NULL;
5546 ethr->scratch_buf = NULL;
5547 #if DEVELOPMENT || DEBUG
5548 ethr->benchmark_q = NULL;
5549 #endif /* DEVELOPMENT || DEBUG */
5550 sched_cond_init(&(ethr->pgo_wakeup));
5551
5552 result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external,
5553 (void *)ethr, BASEPRI_VM,
5554 &(ethr->pgo_iothread));
5555 if (result != KERN_SUCCESS) {
5556 panic("vm_pageout: Unable to create external thread (%d)\n", result);
5557 }
5558 thread_set_thread_name(ethr->pgo_iothread, "VM_pageout_external_iothread");
5559
5560 thread_mtx_lock(vm_pageout_gc_thread );
5561 thread_start(vm_pageout_gc_thread );
5562 thread_mtx_unlock(vm_pageout_gc_thread);
5563
5564 #if VM_PRESSURE_EVENTS
5565 result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
5566 BASEPRI_DEFAULT,
5567 &thread);
5568
5569 if (result != KERN_SUCCESS) {
5570 panic("vm_pressure_thread: create failed");
5571 }
5572
5573 thread_deallocate(thread);
5574 #endif
5575
5576 vm_object_reaper_init();
5577
5578
5579 if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
5580 vm_compressor_init();
5581 }
5582
5583 #if VM_PRESSURE_EVENTS
5584 vm_pressure_events_enabled = TRUE;
5585 #endif /* VM_PRESSURE_EVENTS */
5586
5587 #if CONFIG_PHANTOM_CACHE
5588 vm_phantom_cache_init();
5589 #endif
5590 #if VM_PAGE_BUCKETS_CHECK
5591 #if VM_PAGE_FAKE_BUCKETS
5592 printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
5593 (uint64_t) vm_page_fake_buckets_start,
5594 (uint64_t) vm_page_fake_buckets_end);
5595 pmap_protect(kernel_pmap,
5596 vm_page_fake_buckets_start,
5597 vm_page_fake_buckets_end,
5598 VM_PROT_READ);
5599 // *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
5600 #endif /* VM_PAGE_FAKE_BUCKETS */
5601 #endif /* VM_PAGE_BUCKETS_CHECK */
5602
5603 #if VM_OBJECT_TRACKING
5604 vm_object_tracking_init();
5605 #endif /* VM_OBJECT_TRACKING */
5606
5607 #if __arm64__
5608 // vm_tests();
5609 #endif /* __arm64__ */
5610
5611 vm_pageout_continue();
5612
5613 /*
5614 * Unreached code!
5615 *
5616 * The vm_pageout_continue() call above never returns, so the code below is never
5617 * executed. We take advantage of this to declare several DTrace VM related probe
5618 * points that our kernel doesn't have an analog for. These are probe points that
5619 * exist in Solaris and are in the DTrace documentation, so people may have written
5620 * scripts that use them. Declaring the probe points here means their scripts will
5621 * compile and execute which we want for portability of the scripts, but since this
5622 * section of code is never reached, the probe points will simply never fire. Yes,
5623 * this is basically a hack. The problem is the DTrace probe points were chosen with
5624 * Solaris specific VM events in mind, not portability to different VM implementations.
5625 */
5626
5627 DTRACE_VM2(execfree, int, 1, (uint64_t *), NULL);
5628 DTRACE_VM2(execpgin, int, 1, (uint64_t *), NULL);
5629 DTRACE_VM2(execpgout, int, 1, (uint64_t *), NULL);
5630 DTRACE_VM2(pgswapin, int, 1, (uint64_t *), NULL);
5631 DTRACE_VM2(pgswapout, int, 1, (uint64_t *), NULL);
5632 DTRACE_VM2(swapin, int, 1, (uint64_t *), NULL);
5633 DTRACE_VM2(swapout, int, 1, (uint64_t *), NULL);
5634 /*NOTREACHED*/
5635 }
5636
5637
5638
5639 kern_return_t
5640 vm_pageout_internal_start(void)
5641 {
5642 kern_return_t result = KERN_SUCCESS;
5643 host_basic_info_data_t hinfo;
5644 vm_offset_t buf, bufsize;
5645
5646 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
5647
5648 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
5649 #define BSD_HOST 1
5650 host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
5651
5652 assert(hinfo.max_cpus > 0);
5653
5654 #if !XNU_TARGET_OS_OSX
5655 vm_pageout_state.vm_compressor_thread_count = 1;
5656 #else /* !XNU_TARGET_OS_OSX */
5657 if (hinfo.max_cpus > 4) {
5658 vm_pageout_state.vm_compressor_thread_count = 2;
5659 } else {
5660 vm_pageout_state.vm_compressor_thread_count = 1;
5661 }
5662 #endif /* !XNU_TARGET_OS_OSX */
5663 #if __AMP__
5664 if (vm_compressor_ebound) {
5665 vm_pageout_state.vm_compressor_thread_count = 2;
5666 }
5667 #endif
5668 PE_parse_boot_argn("vmcomp_threads", &vm_pageout_state.vm_compressor_thread_count,
5669 sizeof(vm_pageout_state.vm_compressor_thread_count));
5670
5671 /* did we get from the bootargs an unreasonable number? */
5672 if (vm_pageout_state.vm_compressor_thread_count >= hinfo.max_cpus) {
5673 vm_pageout_state.vm_compressor_thread_count = hinfo.max_cpus - 1;
5674 }
5675 if (vm_pageout_state.vm_compressor_thread_count <= 0) {
5676 vm_pageout_state.vm_compressor_thread_count = 1;
5677 } else if (vm_pageout_state.vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) {
5678 vm_pageout_state.vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
5679 }
5680
5681 vm_pageout_queue_internal.pgo_maxlaundry =
5682 (vm_pageout_state.vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
5683
5684 PE_parse_boot_argn("vmpgoi_maxlaundry",
5685 &vm_pageout_queue_internal.pgo_maxlaundry,
5686 sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
5687
5688 #if DEVELOPMENT || DEBUG
5689 // Note: this will be modified at enqueue-time such that the benchmark queue is never throttled
5690 vm_pageout_queue_benchmark.pgo_maxlaundry = vm_pageout_queue_internal.pgo_maxlaundry;
5691 #endif /* DEVELOPMENT || DEBUG */
5692
5693 bufsize = COMPRESSOR_SCRATCH_BUF_SIZE;
5694
5695 kmem_alloc(kernel_map, &buf,
5696 bufsize * vm_pageout_state.vm_compressor_thread_count,
5697 KMA_DATA_SHARED | KMA_NOFAIL | KMA_KOBJECT | KMA_PERMANENT,
5698 VM_KERN_MEMORY_COMPRESSOR);
5699
5700 for (int i = 0; i < vm_pageout_state.vm_compressor_thread_count; i++) {
5701 struct pgo_iothread_state *iq = &pgo_iothread_internal_state[i];
5702 iq->id = i;
5703 iq->q = &vm_pageout_queue_internal;
5704 iq->current_early_swapout_chead = NULL;
5705 for (int reg_i = 0; reg_i < COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT; ++reg_i) {
5706 iq->current_regular_swapout_cheads[reg_i] = NULL;
5707 }
5708 iq->current_late_swapout_chead = NULL;
5709 iq->scratch_buf = (char *)(buf + i * bufsize);
5710 #if DEVELOPMENT || DEBUG
5711 iq->benchmark_q = &vm_pageout_queue_benchmark;
5712 #endif /* DEVELOPMENT || DEBUG */
5713 sched_cond_init(&(iq->pgo_wakeup));
5714 result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal,
5715 (void *)iq, BASEPRI_VM,
5716 &(iq->pgo_iothread));
5717
5718 if (result != KERN_SUCCESS) {
5719 panic("vm_pageout: Unable to create compressor thread no. %d (%d)\n", i, result);
5720 }
5721 }
5722 return result;
5723 }
5724
5725 #if CONFIG_IOSCHED
5726 /*
5727 * To support I/O Expedite for compressed files we mark the upls with special flags.
5728 * The way decmpfs works is that we create a big upl which marks all the pages needed to
5729 * represent the compressed file as busy. We tag this upl with the flag UPL_DECMP_REQ. Decmpfs
5730 * then issues smaller I/Os for compressed I/Os, deflates them and puts the data into the pages
5731 * being held in the big original UPL. We mark each of these smaller UPLs with the flag
5732 * UPL_DECMP_REAL_IO. Any outstanding real I/O UPL is tracked by the big req upl using the
5733 * decmp_io_upl field (in the upl structure). This link is protected in the forward direction
5734 * by the req upl lock (the reverse link doesnt need synch. since we never inspect this link
5735 * unless the real I/O upl is being destroyed).
5736 */
5737
5738
5739 static void
5740 upl_set_decmp_info(upl_t upl, upl_t src_upl)
5741 {
5742 assert((src_upl->flags & UPL_DECMP_REQ) != 0);
5743
5744 upl_lock(src_upl);
5745 if (src_upl->decmp_io_upl) {
5746 /*
5747 * If there is already an alive real I/O UPL, ignore this new UPL.
5748 * This case should rarely happen and even if it does, it just means
5749 * that we might issue a spurious expedite which the driver is expected
5750 * to handle.
5751 */
5752 upl_unlock(src_upl);
5753 return;
5754 }
5755 src_upl->decmp_io_upl = (void *)upl;
5756 src_upl->ref_count++;
5757
5758 upl->flags |= UPL_DECMP_REAL_IO;
5759 upl->decmp_io_upl = (void *)src_upl;
5760 upl_unlock(src_upl);
5761 }
5762 #endif /* CONFIG_IOSCHED */
5763
5764 #if UPL_DEBUG
5765 int upl_debug_enabled = 1;
5766 #else
5767 int upl_debug_enabled = 0;
5768 #endif
5769
5770 static upl_t
5771 upl_create(int type, int flags, upl_size_t size)
5772 {
5773 uint32_t pages = (uint32_t)atop(round_page_32(size));
5774 upl_t upl;
5775
5776 assert(page_aligned(size));
5777
5778 /*
5779 * FIXME: this code assumes the allocation always succeeds,
5780 * however `pages` can be up to MAX_UPL_SIZE.
5781 *
5782 * The allocation size is above 32k (resp. 128k)
5783 * on 16k pages (resp. 4k), which kalloc might fail
5784 * to allocate.
5785 */
5786 upl = kalloc_type(struct upl, struct upl_page_info,
5787 (type & UPL_CREATE_INTERNAL) ? pages : 0, Z_WAITOK | Z_ZERO);
5788 if (type & UPL_CREATE_INTERNAL) {
5789 flags |= UPL_INTERNAL;
5790 }
5791
5792 if (type & UPL_CREATE_LITE) {
5793 flags |= UPL_LITE;
5794 if (pages) {
5795 upl->lite_list = bitmap_alloc(pages);
5796 }
5797 }
5798
5799 upl->flags = flags;
5800 upl->ref_count = 1;
5801 upl_lock_init(upl);
5802 #if CONFIG_IOSCHED
5803 if (type & UPL_CREATE_IO_TRACKING) {
5804 upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
5805 }
5806
5807 if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {
5808 /* Only support expedite on internal UPLs */
5809 thread_t curthread = current_thread();
5810 upl->upl_reprio_info = kalloc_data(sizeof(uint64_t) * pages,
5811 Z_WAITOK | Z_ZERO);
5812 upl->flags |= UPL_EXPEDITE_SUPPORTED;
5813 if (curthread->decmp_upl != NULL) {
5814 upl_set_decmp_info(upl, curthread->decmp_upl);
5815 }
5816 }
5817 #endif
5818 #if CONFIG_IOSCHED || UPL_DEBUG
5819 if ((type & UPL_CREATE_IO_TRACKING) || upl_debug_enabled) {
5820 upl->upl_creator = current_thread();
5821 upl->flags |= UPL_TRACKED_BY_OBJECT;
5822 }
5823 #endif
5824
5825 #if UPL_DEBUG
5826 upl->upl_create_btref = btref_get(__builtin_frame_address(0), 0);
5827 #endif /* UPL_DEBUG */
5828
5829 return upl;
5830 }
5831
5832 static void
5833 upl_destroy(upl_t upl)
5834 {
5835 uint32_t pages;
5836
5837 // DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object);
5838
5839 if (upl->ext_ref_count) {
5840 panic("upl(%p) ext_ref_count", upl);
5841 }
5842
5843 #if CONFIG_IOSCHED
5844 if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
5845 upl_t src_upl;
5846 src_upl = upl->decmp_io_upl;
5847 assert((src_upl->flags & UPL_DECMP_REQ) != 0);
5848 upl_lock(src_upl);
5849 src_upl->decmp_io_upl = NULL;
5850 upl_unlock(src_upl);
5851 upl_deallocate(src_upl);
5852 }
5853 #endif /* CONFIG_IOSCHED */
5854
5855 #if CONFIG_IOSCHED || UPL_DEBUG
5856 if (((upl->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) &&
5857 !(upl->flags & UPL_VECTOR)) {
5858 vm_object_t object;
5859
5860 if (upl->flags & UPL_SHADOWED) {
5861 object = upl->map_object->shadow;
5862 } else {
5863 object = upl->map_object;
5864 }
5865
5866 vm_object_lock(object);
5867 queue_remove(&object->uplq, upl, upl_t, uplq);
5868 vm_object_activity_end(object);
5869 vm_object_collapse(object, 0, TRUE);
5870 vm_object_unlock(object);
5871 }
5872 #endif
5873 /*
5874 * drop a reference on the map_object whether or
5875 * not a pageout object is inserted
5876 */
5877 if (upl->flags & UPL_SHADOWED) {
5878 vm_object_deallocate(upl->map_object);
5879 }
5880
5881 if (upl->flags & UPL_DEVICE_MEMORY) {
5882 pages = 1;
5883 } else {
5884 pages = (uint32_t)atop(upl_adjusted_size(upl, PAGE_MASK));
5885 }
5886
5887 upl_lock_destroy(upl);
5888
5889 #if CONFIG_IOSCHED
5890 if (upl->flags & UPL_EXPEDITE_SUPPORTED) {
5891 kfree_data(upl->upl_reprio_info, sizeof(uint64_t) * pages);
5892 }
5893 #endif
5894
5895 if (upl->flags & UPL_HAS_FS_VERIFY_INFO) {
5896 assert(upl->u_fs_un.verify_info && upl->u_fs_un.verify_info->verify_data_len > 0 &&
5897 upl->u_fs_un.verify_info->verify_data_len <= upl_adjusted_size(upl, PAGE_MASK));
5898
5899 kfree_data(upl->u_fs_un.verify_info->verify_data_ptr,
5900 upl->u_fs_un.verify_info->verify_data_len);
5901 kfree_type(struct upl_fs_verify_info, upl->u_fs_un.verify_info);
5902 }
5903
5904 #if UPL_DEBUG
5905 for (int i = 0; i < upl->upl_commit_index; i++) {
5906 btref_put(upl->upl_commit_records[i].c_btref);
5907 }
5908 btref_put(upl->upl_create_btref);
5909 #endif /* UPL_DEBUG */
5910
5911 if ((upl->flags & UPL_LITE) && pages) {
5912 bitmap_free(upl->lite_list, pages);
5913 }
5914 kfree_type(struct upl, struct upl_page_info,
5915 (upl->flags & UPL_INTERNAL) ? pages : 0, upl);
5916 }
5917
5918 void
5919 upl_deallocate(upl_t upl)
5920 {
5921 upl_lock(upl);
5922
5923 if (--upl->ref_count == 0) {
5924 if (vector_upl_is_valid(upl)) {
5925 vector_upl_deallocate(upl);
5926 }
5927 upl_unlock(upl);
5928
5929 if (upl->upl_iodone) {
5930 upl_callout_iodone(upl);
5931 }
5932
5933 upl_destroy(upl);
5934 } else {
5935 upl_unlock(upl);
5936 }
5937 }
5938
5939 #if CONFIG_IOSCHED
5940 void
5941 upl_mark_decmp(upl_t upl)
5942 {
5943 if (upl->flags & UPL_TRACKED_BY_OBJECT) {
5944 upl->flags |= UPL_DECMP_REQ;
5945 upl->upl_creator->decmp_upl = (void *)upl;
5946 }
5947 }
5948
5949 void
5950 upl_unmark_decmp(upl_t upl)
5951 {
5952 if (upl && (upl->flags & UPL_DECMP_REQ)) {
5953 upl->upl_creator->decmp_upl = NULL;
5954 }
5955 }
5956
5957 #endif /* CONFIG_IOSCHED */
5958
5959 #define VM_PAGE_Q_BACKING_UP(q) \
5960 ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))
5961
5962 static boolean_t
5963 must_throttle_writes()
5964 {
5965 if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&
5966 vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) {
5967 /*
5968 * The external pageout queue is saturated, and there is an abundance of
5969 * filecache on the system that VM_pageout still needs to get to. Likely the
5970 * pageout thread is contending at the filesystem or storage layers with a
5971 * high volume of other I/Os. Attempt to give the pageout thread a chance to
5972 * catch up by applying a blanket throttle to all outgoing I/Os.
5973 */
5974 return TRUE;
5975 }
5976
5977 return FALSE;
5978 }
5979
5980 int vm_page_delayed_work_ctx_needed = 0;
5981 KALLOC_TYPE_DEFINE(dw_ctx_zone, struct vm_page_delayed_work_ctx, KT_PRIV_ACCT);
5982
5983 __startup_func
5984 static void
5985 vm_page_delayed_work_init_ctx(void)
5986 {
5987 uint16_t min_delayed_work_ctx_allocated = 16;
5988
5989 /*
5990 * try really hard to always keep NCPU elements around in the zone
5991 * in order for the UPL code to almost always get an element.
5992 */
5993 if (min_delayed_work_ctx_allocated < zpercpu_count()) {
5994 min_delayed_work_ctx_allocated = (uint16_t)zpercpu_count();
5995 }
5996
5997 zone_raise_reserve(dw_ctx_zone, min_delayed_work_ctx_allocated);
5998 }
5999 STARTUP(ZALLOC, STARTUP_RANK_LAST, vm_page_delayed_work_init_ctx);
6000
6001 struct vm_page_delayed_work*
6002 vm_page_delayed_work_get_ctx(void)
6003 {
6004 struct vm_page_delayed_work_ctx * dw_ctx = NULL;
6005
6006 dw_ctx = zalloc_flags(dw_ctx_zone, Z_ZERO | Z_NOWAIT);
6007
6008 if (__probable(dw_ctx)) {
6009 dw_ctx->delayed_owner = current_thread();
6010 } else {
6011 vm_page_delayed_work_ctx_needed++;
6012 }
6013 return dw_ctx ? dw_ctx->dwp : NULL;
6014 }
6015
6016 void
6017 vm_page_delayed_work_finish_ctx(struct vm_page_delayed_work* dwp)
6018 {
6019 struct vm_page_delayed_work_ctx *ldw_ctx;
6020
6021 ldw_ctx = (struct vm_page_delayed_work_ctx *)dwp;
6022 ldw_ctx->delayed_owner = NULL;
6023
6024 zfree(dw_ctx_zone, ldw_ctx);
6025 }
6026
6027 uint64_t vm_object_upl_throttle_cnt;
6028
6029 TUNABLE(uint32_t, vm_object_throttle_delay_us,
6030 "vm_object_upl_throttle_delay_us", 1000); /* 1ms */
6031
6032 /*
6033 * @func vm_object_upl_throttle
6034 *
6035 * @brief
6036 * Throttle the current UPL request to give the external pageout thread
6037 * a chance to catch up to system I/O demand.
6038 *
6039 * @discussion
6040 * We may end up in a situation where the file-cache is large, and we need to
6041 * evict some of it. However, the external pageout thread either can't keep up
6042 * with demand or is contending with other I/Os for the storage device (see
6043 * @c must_throttle_writes()). In these situations, we apply a throttle to
6044 * outgoing writes to give the pageout thread a chance to catch up.
6045 */
6046 OS_NOINLINE OS_NOT_TAIL_CALLED
6047 static void
6048 vm_object_upl_throttle(vm_object_t object, upl_size_t size)
6049 {
6050 int delay_us = vm_object_throttle_delay_us;
6051 #if XNU_TARGET_OS_OSX
6052 if (memory_object_is_vnode_pager(object->pager)) {
6053 boolean_t isSSD = FALSE;
6054 __assert_only kern_return_t kr;
6055 kr = vnode_pager_get_isSSD(object->pager, &isSSD);
6056 assert3u(kr, ==, KERN_SUCCESS);
6057 if (!isSSD) {
6058 delay_us = 5000; /* 5 ms */
6059 }
6060 }
6061 #endif /* !XNU_TARGET_OS_OSX */
6062
6063 KDBG(VMDBG_CODE(DBG_VM_UPL_THROTTLE) | DBG_FUNC_START, VM_OBJECT_ID(object),
6064 size, delay_us);
6065
6066 if (delay_us == 0) {
6067 goto done;
6068 }
6069
6070 vm_object_unlock(object);
6071
6072 uint32_t size_pages = size >> PAGE_SHIFT;
6073 os_atomic_inc(&vm_object_upl_throttle_cnt, relaxed);
6074
6075 os_atomic_add(&vm_upl_wait_for_pages, size_pages, relaxed);
6076
6077 /*
6078 * Unconditionally block for a fixed delay interval.
6079 *
6080 * FIXME: This mechanism should likely be revisited. (rdar://157163748)
6081 *
6082 * Should there be a back-pressure mechanisms that un-throttles the I/O if the
6083 * situation resolves?
6084 *
6085 * Is 1ms long enough? The original mechanism scaled the delay with the I/O
6086 * size, but that overly penalized large I/Os (which are actually preferrable
6087 * if device contention is the problem).
6088 *
6089 * Can we isolate only I/Os which are to the same device that the external
6090 * pageout thread is stuck on? e.g. There is no reason to penalize I/Os to an
6091 * external drive if the pageout thread is gummed up on the internal drive.
6092 */
6093 delay(delay_us);
6094
6095 os_atomic_sub(&vm_upl_wait_for_pages, size_pages, relaxed);
6096
6097 vm_object_lock(object);
6098 done:
6099 KDBG(VMDBG_CODE(DBG_VM_UPL_THROTTLE) | DBG_FUNC_END);
6100 }
6101
6102
6103 /*
6104 * Routine: vm_object_upl_request
6105 * Purpose:
6106 * Cause the population of a portion of a vm_object.
6107 * Depending on the nature of the request, the pages
6108 * returned may be contain valid data or be uninitialized.
6109 * A page list structure, listing the physical pages
6110 * will be returned upon request.
6111 * This function is called by the file system or any other
6112 * supplier of backing store to a pager.
6113 * IMPORTANT NOTE: The caller must still respect the relationship
6114 * between the vm_object and its backing memory object. The
6115 * caller MUST NOT substitute changes in the backing file
6116 * without first doing a memory_object_lock_request on the
6117 * target range unless it is know that the pages are not
6118 * shared with another entity at the pager level.
6119 * Copy_in_to:
6120 * if a page list structure is present
6121 * return the mapped physical pages, where a
6122 * page is not present, return a non-initialized
6123 * one. If the no_sync bit is turned on, don't
6124 * call the pager unlock to synchronize with other
6125 * possible copies of the page. Leave pages busy
6126 * in the original object, if a page list structure
6127 * was specified. When a commit of the page list
6128 * pages is done, the dirty bit will be set for each one.
6129 * Copy_out_from:
6130 * If a page list structure is present, return
6131 * all mapped pages. Where a page does not exist
6132 * map a zero filled one. Leave pages busy in
6133 * the original object. If a page list structure
6134 * is not specified, this call is a no-op.
6135 *
6136 * Note: access of default pager objects has a rather interesting
6137 * twist. The caller of this routine, presumably the file system
6138 * page cache handling code, will never actually make a request
6139 * against a default pager backed object. Only the default
6140 * pager will make requests on backing store related vm_objects
6141 * In this way the default pager can maintain the relationship
6142 * between backing store files (abstract memory objects) and
6143 * the vm_objects (cache objects), they support.
6144 *
6145 */
6146
6147 __private_extern__ kern_return_t
6148 vm_object_upl_request(
6149 vm_object_t object,
6150 vm_object_offset_t offset,
6151 upl_size_t size,
6152 upl_t *upl_ptr,
6153 upl_page_info_array_t user_page_list,
6154 unsigned int *page_list_count,
6155 upl_control_flags_t cntrl_flags,
6156 vm_tag_t tag)
6157 {
6158 vm_page_t dst_page = VM_PAGE_NULL;
6159 vm_object_offset_t dst_offset;
6160 upl_size_t xfer_size;
6161 unsigned int size_in_pages;
6162 boolean_t dirty;
6163 boolean_t hw_dirty;
6164 upl_t upl = NULL;
6165 unsigned int entry;
6166 vm_page_t alias_page = NULL;
6167 int refmod_state = 0;
6168 vm_object_t last_copy_object;
6169 uint64_t last_copy_version;
6170 struct vm_page_delayed_work dw_array;
6171 struct vm_page_delayed_work *dwp, *dwp_start;
6172 bool dwp_finish_ctx = TRUE;
6173 int dw_count;
6174 int dw_limit;
6175 int io_tracking_flag = 0;
6176 vm_grab_options_t grab_options;
6177 int page_grab_count = 0;
6178 ppnum_t phys_page;
6179 pmap_flush_context pmap_flush_context_storage;
6180 boolean_t pmap_flushes_delayed = FALSE;
6181 task_t task = current_task();
6182 thread_pri_floor_t token;
6183
6184 dwp_start = dwp = NULL;
6185
6186 if (cntrl_flags & ~UPL_VALID_FLAGS) {
6187 /*
6188 * For forward compatibility's sake,
6189 * reject any unknown flag.
6190 */
6191 return KERN_INVALID_VALUE;
6192 }
6193 if ((!object->internal) && (object->paging_offset != 0)) {
6194 panic("vm_object_upl_request: external object with non-zero paging offset");
6195 }
6196 if (object->phys_contiguous) {
6197 panic("vm_object_upl_request: contiguous object specified");
6198 }
6199
6200 assertf(page_aligned(offset) && page_aligned(size),
6201 "offset 0x%llx size 0x%x",
6202 offset, size);
6203
6204 VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, DBG_VM_UPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, 0, 0);
6205
6206 dw_count = 0;
6207 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
6208 dwp_start = vm_page_delayed_work_get_ctx();
6209 if (dwp_start == NULL) {
6210 dwp_start = &dw_array;
6211 dw_limit = 1;
6212 dwp_finish_ctx = FALSE;
6213 }
6214
6215 dwp = dwp_start;
6216
6217 if (size > MAX_UPL_SIZE_BYTES) {
6218 size = MAX_UPL_SIZE_BYTES;
6219 }
6220
6221 if ((cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) {
6222 *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
6223 }
6224
6225 #if CONFIG_IOSCHED || UPL_DEBUG
6226 if (object->io_tracking || upl_debug_enabled) {
6227 io_tracking_flag |= UPL_CREATE_IO_TRACKING;
6228 }
6229 #endif
6230 #if CONFIG_IOSCHED
6231 if (object->io_tracking) {
6232 io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
6233 }
6234 #endif
6235
6236 if (cntrl_flags & UPL_SET_INTERNAL) {
6237 if (cntrl_flags & UPL_SET_LITE) {
6238 upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
6239 } else {
6240 upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);
6241 }
6242 user_page_list = size ? upl->page_list : NULL;
6243 } else {
6244 if (cntrl_flags & UPL_SET_LITE) {
6245 upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
6246 } else {
6247 upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);
6248 }
6249 }
6250 *upl_ptr = upl;
6251
6252 if (user_page_list) {
6253 user_page_list[0].device = FALSE;
6254 }
6255
6256 if (cntrl_flags & UPL_SET_LITE) {
6257 upl->map_object = object;
6258 } else {
6259 upl->map_object = vm_object_allocate(size, object->vmo_provenance);
6260 vm_object_lock(upl->map_object);
6261 /*
6262 * No neeed to lock the new object: nobody else knows
6263 * about it yet, so it's all ours so far.
6264 */
6265 upl->map_object->shadow = object;
6266 VM_OBJECT_SET_PAGEOUT(upl->map_object, TRUE);
6267 VM_OBJECT_SET_CAN_PERSIST(upl->map_object, FALSE);
6268 upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
6269 upl->map_object->vo_shadow_offset = offset;
6270 upl->map_object->wimg_bits = object->wimg_bits;
6271 assertf(page_aligned(upl->map_object->vo_shadow_offset),
6272 "object %p shadow_offset 0x%llx",
6273 upl->map_object, upl->map_object->vo_shadow_offset);
6274 vm_object_unlock(upl->map_object);
6275
6276 alias_page = vm_page_create_fictitious();
6277
6278 upl->flags |= UPL_SHADOWED;
6279 }
6280 if (cntrl_flags & UPL_FOR_PAGEOUT) {
6281 upl->flags |= UPL_PAGEOUT;
6282 }
6283 if ((cntrl_flags & UPL_RET_ONLY_ABSENT) &&
6284 !(cntrl_flags & UPL_FILE_IO)) {
6285 upl->flags |= UPL_PAGEIN;
6286 }
6287
6288 vm_object_lock(object);
6289 vm_object_activity_begin(object);
6290 if (cntrl_flags & UPL_WILL_MODIFY) {
6291 token = thread_priority_floor_start();
6292 vm_object_pl_req_begin(object);
6293 }
6294
6295 grab_options = VM_PAGE_GRAB_OPTIONS_NONE;
6296 #if CONFIG_SECLUDED_MEMORY
6297 if (object->can_grab_secluded) {
6298 grab_options |= VM_PAGE_GRAB_SECLUDED;
6299 }
6300 #endif /* CONFIG_SECLUDED_MEMORY */
6301
6302 /*
6303 * we can lock in the paging_offset once paging_in_progress is set
6304 */
6305 upl->u_size = size;
6306 upl->u_offset = offset + object->paging_offset;
6307
6308 #if CONFIG_IOSCHED || UPL_DEBUG
6309 if (object->io_tracking || upl_debug_enabled) {
6310 vm_object_activity_begin(object);
6311 queue_enter(&object->uplq, upl, upl_t, uplq);
6312 }
6313 #endif
6314
6315 /* remember which copy object we synchronized with */
6316 last_copy_object = object->vo_copy;
6317 last_copy_version = object->vo_copy_version;
6318 if ((cntrl_flags & UPL_WILL_MODIFY) && object->vo_copy != VM_OBJECT_NULL) {
6319 /*
6320 * Honor copy-on-write obligations
6321 *
6322 * The caller is gathering these pages and
6323 * might modify their contents. We need to
6324 * make sure that the copy object has its own
6325 * private copies of these pages before we let
6326 * the caller modify them.
6327 */
6328 vm_object_update(object,
6329 offset,
6330 size,
6331 NULL,
6332 NULL,
6333 FALSE, /* should_return */
6334 MEMORY_OBJECT_COPY_SYNC,
6335 VM_PROT_NO_CHANGE);
6336
6337 VM_PAGEOUT_DEBUG(upl_cow, 1);
6338 VM_PAGEOUT_DEBUG(upl_cow_pages, (size >> PAGE_SHIFT));
6339 }
6340 entry = 0;
6341
6342 xfer_size = size;
6343 dst_offset = offset;
6344 size_in_pages = size / PAGE_SIZE;
6345
6346 if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||
6347 object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) {
6348 object->scan_collisions = 0;
6349 }
6350
6351 if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
6352 vm_object_upl_throttle(object, size);
6353 }
6354
6355 while (xfer_size) {
6356 dwp->dw_mask = 0;
6357
6358 if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
6359 vm_object_unlock(object);
6360 alias_page = vm_page_create_fictitious();
6361 vm_object_lock(object);
6362 }
6363 if (cntrl_flags & UPL_COPYOUT_FROM) {
6364 upl->flags |= UPL_PAGE_SYNC_DONE;
6365
6366 if (((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
6367 vm_page_is_fictitious(dst_page) ||
6368 dst_page->vmp_absent ||
6369 VMP_ERROR_GET(dst_page) ||
6370 dst_page->vmp_cleaning ||
6371 (VM_PAGE_WIRED(dst_page))) {
6372 if (user_page_list) {
6373 user_page_list[entry].phys_addr = 0;
6374 }
6375
6376 goto try_next_page;
6377 }
6378 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
6379
6380 /*
6381 * grab this up front...
6382 * a high percentange of the time we're going to
6383 * need the hardware modification state a bit later
6384 * anyway... so we can eliminate an extra call into
6385 * the pmap layer by grabbing it here and recording it
6386 */
6387 if (dst_page->vmp_pmapped) {
6388 refmod_state = pmap_get_refmod(phys_page);
6389 } else {
6390 refmod_state = 0;
6391 }
6392
6393 if ((refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
6394 /*
6395 * page is on inactive list and referenced...
6396 * reactivate it now... this gets it out of the
6397 * way of vm_pageout_scan which would have to
6398 * reactivate it upon tripping over it
6399 */
6400 dwp->dw_mask |= DW_vm_page_activate;
6401 }
6402 if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
6403 /*
6404 * we're only asking for DIRTY pages to be returned
6405 */
6406 if (dst_page->vmp_laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
6407 /*
6408 * if we were the page stolen by vm_pageout_scan to be
6409 * cleaned (as opposed to a buddy being clustered in
6410 * or this request is not being driven by a PAGEOUT cluster
6411 * then we only need to check for the page being dirty or
6412 * precious to decide whether to return it
6413 */
6414 if (dst_page->vmp_dirty || dst_page->vmp_precious || (refmod_state & VM_MEM_MODIFIED)) {
6415 goto check_busy;
6416 }
6417 goto dont_return;
6418 }
6419 /*
6420 * this is a request for a PAGEOUT cluster and this page
6421 * is merely along for the ride as a 'buddy'... not only
6422 * does it have to be dirty to be returned, but it also
6423 * can't have been referenced recently...
6424 */
6425 if ((hibernate_cleaning_in_progress == TRUE ||
6426 (!((refmod_state & VM_MEM_REFERENCED) || dst_page->vmp_reference) ||
6427 (dst_page->vmp_q_state == VM_PAGE_ON_THROTTLED_Q))) &&
6428 ((refmod_state & VM_MEM_MODIFIED) || dst_page->vmp_dirty || dst_page->vmp_precious)) {
6429 goto check_busy;
6430 }
6431 dont_return:
6432 /*
6433 * if we reach here, we're not to return
6434 * the page... go on to the next one
6435 */
6436 if (dst_page->vmp_laundry == TRUE) {
6437 /*
6438 * if we get here, the page is not 'cleaning' (filtered out above).
6439 * since it has been referenced, remove it from the laundry
6440 * so we don't pay the cost of an I/O to clean a page
6441 * we're just going to take back
6442 */
6443 vm_page_lockspin_queues();
6444
6445 vm_pageout_steal_laundry(dst_page, TRUE);
6446 vm_page_activate(dst_page);
6447
6448 vm_page_unlock_queues();
6449 }
6450 if (user_page_list) {
6451 user_page_list[entry].phys_addr = 0;
6452 }
6453
6454 goto try_next_page;
6455 }
6456 check_busy:
6457 if (dst_page->vmp_busy) {
6458 if (cntrl_flags & UPL_NOBLOCK) {
6459 if (user_page_list) {
6460 user_page_list[entry].phys_addr = 0;
6461 }
6462 dwp->dw_mask = 0;
6463
6464 goto try_next_page;
6465 }
6466 /*
6467 * someone else is playing with the
6468 * page. We will have to wait.
6469 */
6470 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_EXCLUSIVE);
6471
6472 continue;
6473 }
6474 if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
6475 vm_page_lockspin_queues();
6476
6477 if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
6478 /*
6479 * we've buddied up a page for a clustered pageout
6480 * that has already been moved to the pageout
6481 * queue by pageout_scan... we need to remove
6482 * it from the queue and drop the laundry count
6483 * on that queue
6484 */
6485 vm_pageout_throttle_up(dst_page);
6486 }
6487 vm_page_unlock_queues();
6488 }
6489 hw_dirty = refmod_state & VM_MEM_MODIFIED;
6490 dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
6491
6492 if (phys_page > upl->highest_page) {
6493 upl->highest_page = phys_page;
6494 }
6495
6496 assert(!pmap_is_noencrypt(phys_page));
6497
6498 if (cntrl_flags & UPL_SET_LITE) {
6499 unsigned int pg_num;
6500
6501 pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
6502 assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
6503 bitmap_set(upl->lite_list, pg_num);
6504
6505 if (hw_dirty) {
6506 if (pmap_flushes_delayed == FALSE) {
6507 pmap_flush_context_init(&pmap_flush_context_storage);
6508 pmap_flushes_delayed = TRUE;
6509 }
6510 pmap_clear_refmod_options(phys_page,
6511 VM_MEM_MODIFIED,
6512 PMAP_OPTIONS_NOFLUSH | PMAP_OPTIONS_CLEAR_WRITE,
6513 &pmap_flush_context_storage);
6514 }
6515
6516 /*
6517 * Mark original page as cleaning
6518 * in place.
6519 */
6520 dst_page->vmp_cleaning = TRUE;
6521 dst_page->vmp_precious = FALSE;
6522 } else {
6523 /*
6524 * use pageclean setup, it is more
6525 * convenient even for the pageout
6526 * cases here
6527 */
6528 vm_object_lock(upl->map_object);
6529 vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
6530 vm_object_unlock(upl->map_object);
6531
6532 alias_page->vmp_absent = FALSE;
6533 alias_page = NULL;
6534 }
6535 if (dirty) {
6536 SET_PAGE_DIRTY(dst_page, FALSE);
6537 } else {
6538 dst_page->vmp_dirty = FALSE;
6539 }
6540
6541 if (!dirty) {
6542 dst_page->vmp_precious = TRUE;
6543 }
6544
6545 if (!(cntrl_flags & UPL_CLEAN_IN_PLACE)) {
6546 if (!VM_PAGE_WIRED(dst_page)) {
6547 dst_page->vmp_free_when_done = TRUE;
6548 }
6549 }
6550 } else {
6551 while ((cntrl_flags & UPL_WILL_MODIFY) &&
6552 (object->vo_copy != last_copy_object ||
6553 object->vo_copy_version != last_copy_version)) {
6554 /*
6555 * Honor copy-on-write obligations
6556 *
6557 * The copy object has changed since we
6558 * last synchronized for copy-on-write.
6559 * Another copy object might have been
6560 * inserted while we released the object's
6561 * lock. Since someone could have seen the
6562 * original contents of the remaining pages
6563 * through that new object, we have to
6564 * synchronize with it again for the remaining
6565 * pages only. The previous pages are "busy"
6566 * so they can not be seen through the new
6567 * mapping. The new mapping will see our
6568 * upcoming changes for those previous pages,
6569 * but that's OK since they couldn't see what
6570 * was there before. It's just a race anyway
6571 * and there's no guarantee of consistency or
6572 * atomicity. We just don't want new mappings
6573 * to see both the *before* and *after* pages.
6574 */
6575
6576 /* first remember the copy object we re-synced with */
6577 last_copy_object = object->vo_copy;
6578 last_copy_version = object->vo_copy_version;
6579 if (object->vo_copy != VM_OBJECT_NULL) {
6580 vm_object_update(
6581 object,
6582 dst_offset,/* current offset */
6583 xfer_size, /* remaining size */
6584 NULL,
6585 NULL,
6586 FALSE, /* should_return */
6587 MEMORY_OBJECT_COPY_SYNC,
6588 VM_PROT_NO_CHANGE);
6589
6590 VM_PAGEOUT_DEBUG(upl_cow_again, 1);
6591 VM_PAGEOUT_DEBUG(upl_cow_again_pages, (xfer_size >> PAGE_SHIFT));
6592 }
6593 }
6594 dst_page = vm_page_lookup(object, dst_offset);
6595
6596 if (dst_page != VM_PAGE_NULL) {
6597 if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
6598 /*
6599 * skip over pages already present in the cache
6600 */
6601 if (user_page_list) {
6602 user_page_list[entry].phys_addr = 0;
6603 }
6604
6605 goto try_next_page;
6606 }
6607 if (vm_page_is_fictitious(dst_page)) {
6608 panic("need corner case for fictitious page");
6609 }
6610
6611 if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
6612 /*
6613 * someone else is playing with the
6614 * page. We will have to wait.
6615 */
6616 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_EXCLUSIVE);
6617
6618 continue;
6619 }
6620 if (dst_page->vmp_laundry) {
6621 vm_pageout_steal_laundry(dst_page, FALSE);
6622 }
6623 } else {
6624 if (object->private) {
6625 /*
6626 * This is a nasty wrinkle for users
6627 * of upl who encounter device or
6628 * private memory however, it is
6629 * unavoidable, only a fault can
6630 * resolve the actual backing
6631 * physical page by asking the
6632 * backing device.
6633 */
6634 if (user_page_list) {
6635 user_page_list[entry].phys_addr = 0;
6636 }
6637
6638 goto try_next_page;
6639 }
6640 if (object->scan_collisions) {
6641 /*
6642 * the pageout_scan thread is trying to steal
6643 * pages from this object, but has run into our
6644 * lock... grab 2 pages from the head of the object...
6645 * the first is freed on behalf of pageout_scan, the
6646 * 2nd is for our own use... we use vm_object_page_grab
6647 * in both cases to avoid taking pages from the free
6648 * list since we are under memory pressure and our
6649 * lock on this object is getting in the way of
6650 * relieving it
6651 */
6652 dst_page = vm_object_page_grab(object);
6653
6654 if (dst_page != VM_PAGE_NULL) {
6655 vm_page_release(dst_page,
6656 VMP_RELEASE_NONE);
6657 }
6658
6659 dst_page = vm_object_page_grab(object);
6660 }
6661 if (dst_page == VM_PAGE_NULL) {
6662 /*
6663 * need to allocate a page
6664 */
6665 dst_page = vm_page_grab_options(grab_options);
6666 if (dst_page != VM_PAGE_NULL) {
6667 page_grab_count++;
6668 }
6669 }
6670 if (dst_page == VM_PAGE_NULL) {
6671 if ((cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
6672 /*
6673 * we don't want to stall waiting for pages to come onto the free list
6674 * while we're already holding absent pages in this UPL
6675 * the caller will deal with the empty slots
6676 */
6677 if (user_page_list) {
6678 user_page_list[entry].phys_addr = 0;
6679 }
6680
6681 goto try_next_page;
6682 }
6683 /*
6684 * no pages available... wait
6685 * then try again for the same
6686 * offset...
6687 */
6688 vm_object_unlock(object);
6689
6690 OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
6691
6692 VM_DEBUG_EVENT(vm_upl_page_wait, DBG_VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
6693
6694 VM_PAGE_WAIT();
6695 OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
6696
6697 VM_DEBUG_EVENT(vm_upl_page_wait, DBG_VM_UPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
6698
6699 vm_object_lock(object);
6700
6701 continue;
6702 }
6703 vm_page_insert(dst_page, object, dst_offset);
6704
6705 dst_page->vmp_absent = TRUE;
6706 dst_page->vmp_busy = FALSE;
6707
6708 if (cntrl_flags & UPL_RET_ONLY_ABSENT) {
6709 /*
6710 * if UPL_RET_ONLY_ABSENT was specified,
6711 * than we're definitely setting up a
6712 * upl for a clustered read/pagein
6713 * operation... mark the pages as clustered
6714 * so upl_commit_range can put them on the
6715 * speculative list
6716 */
6717 dst_page->vmp_clustered = TRUE;
6718
6719 if (!(cntrl_flags & UPL_FILE_IO)) {
6720 counter_inc(&vm_statistics_pageins);
6721 counter_inc(&vm_statistics_pageins_requested);
6722 }
6723 }
6724 }
6725 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
6726
6727 dst_page->vmp_overwriting = TRUE;
6728
6729 if (dst_page->vmp_pmapped) {
6730 #if CONFIG_SPTM
6731 if (__improbable(PMAP_PAGE_IS_USER_EXECUTABLE(dst_page))) {
6732 /*
6733 * Various buffer cache operations may need to reload the page contents
6734 * even though the page may have an executable frame type from prior use of
6735 * the vnode associated with the VM object. For those cases, we need to
6736 * disconnect all mappings and reset the frame type, regardless of whether
6737 * UPL_FILE_IO was passed here, as the SPTM will not allow writable CPU
6738 * or IOMMU mappings of exec-typed pages.
6739 * NOTE: It's theoretically possible that the retype here could race with
6740 * setup/teardown of IOMMU mappings by another thread that went through
6741 * the vm_object_iopl_request() path. I'm not sure that would ever be
6742 * expected to happen for an exec page in practice though. If it does
6743 * happen, we may need to change vm_page_do_delayed_work() to forbid all
6744 * IOPLs against executable pages rather than only writable ones.
6745 */
6746 refmod_state = pmap_disconnect_options(phys_page, PMAP_OPTIONS_RETYPE, NULL);
6747 } else
6748 #endif /* CONFIG_SPTM */
6749 if (!(cntrl_flags & UPL_FILE_IO)) {
6750 /*
6751 * eliminate all mappings from the
6752 * original object and its progeny
6753 */
6754 refmod_state = pmap_disconnect(phys_page);
6755 } else {
6756 refmod_state = pmap_get_refmod(phys_page);
6757 }
6758 } else {
6759 refmod_state = 0;
6760 }
6761
6762 hw_dirty = refmod_state & VM_MEM_MODIFIED;
6763 dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
6764
6765 if (cntrl_flags & UPL_SET_LITE) {
6766 unsigned int pg_num;
6767
6768 pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
6769 assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
6770 bitmap_set(upl->lite_list, pg_num);
6771
6772 if (hw_dirty) {
6773 pmap_clear_modify(phys_page);
6774 }
6775
6776 /*
6777 * Mark original page as cleaning
6778 * in place.
6779 */
6780 dst_page->vmp_cleaning = TRUE;
6781 dst_page->vmp_precious = FALSE;
6782 } else {
6783 /*
6784 * use pageclean setup, it is more
6785 * convenient even for the pageout
6786 * cases here
6787 */
6788 vm_object_lock(upl->map_object);
6789 vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
6790 vm_object_unlock(upl->map_object);
6791
6792 alias_page->vmp_absent = FALSE;
6793 alias_page = NULL;
6794 }
6795
6796 if (cntrl_flags & UPL_REQUEST_SET_DIRTY) {
6797 upl->flags &= ~UPL_CLEAR_DIRTY;
6798 upl->flags |= UPL_SET_DIRTY;
6799 dirty = TRUE;
6800 /*
6801 * Page belonging to a code-signed object is about to
6802 * be written. Mark it tainted and disconnect it from
6803 * all pmaps so processes have to fault it back in and
6804 * deal with the tainted bit.
6805 */
6806 if (object->code_signed && dst_page->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
6807 dst_page->vmp_cs_tainted = VMP_CS_ALL_TRUE;
6808 vm_page_upl_tainted++;
6809 if (dst_page->vmp_pmapped) {
6810 refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
6811 if (refmod_state & VM_MEM_REFERENCED) {
6812 dst_page->vmp_reference = TRUE;
6813 }
6814 }
6815 }
6816 } else if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
6817 /*
6818 * clean in place for read implies
6819 * that a write will be done on all
6820 * the pages that are dirty before
6821 * a upl commit is done. The caller
6822 * is obligated to preserve the
6823 * contents of all pages marked dirty
6824 */
6825 upl->flags |= UPL_CLEAR_DIRTY;
6826 }
6827 dst_page->vmp_dirty = dirty;
6828
6829 if (!dirty) {
6830 dst_page->vmp_precious = TRUE;
6831 }
6832
6833 if (!VM_PAGE_WIRED(dst_page)) {
6834 /*
6835 * deny access to the target page while
6836 * it is being worked on
6837 */
6838 dst_page->vmp_busy = TRUE;
6839 } else {
6840 dwp->dw_mask |= DW_vm_page_wire;
6841 }
6842
6843 /*
6844 * We might be about to satisfy a fault which has been
6845 * requested. So no need for the "restart" bit.
6846 */
6847 dst_page->vmp_restart = FALSE;
6848 if (!dst_page->vmp_absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
6849 /*
6850 * expect the page to be used
6851 */
6852 dwp->dw_mask |= DW_set_reference;
6853 }
6854 if (cntrl_flags & UPL_PRECIOUS) {
6855 if (object->internal) {
6856 SET_PAGE_DIRTY(dst_page, FALSE);
6857 dst_page->vmp_precious = FALSE;
6858 } else {
6859 dst_page->vmp_precious = TRUE;
6860 }
6861 } else {
6862 dst_page->vmp_precious = FALSE;
6863 }
6864 }
6865 if (dst_page->vmp_busy) {
6866 upl->flags |= UPL_HAS_BUSY;
6867 }
6868 if (VM_PAGE_WIRED(dst_page)) {
6869 upl->flags |= UPL_HAS_WIRED;
6870 }
6871
6872 if (phys_page > upl->highest_page) {
6873 upl->highest_page = phys_page;
6874 }
6875 assert(!pmap_is_noencrypt(phys_page));
6876 if (user_page_list) {
6877 user_page_list[entry].phys_addr = phys_page;
6878 user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
6879 user_page_list[entry].absent = dst_page->vmp_absent;
6880 user_page_list[entry].dirty = dst_page->vmp_dirty;
6881 user_page_list[entry].precious = dst_page->vmp_precious;
6882 user_page_list[entry].device = FALSE;
6883 user_page_list[entry].needed = FALSE;
6884 if (dst_page->vmp_clustered == TRUE) {
6885 user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
6886 } else {
6887 user_page_list[entry].speculative = FALSE;
6888 }
6889 user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
6890 user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
6891 user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;
6892 user_page_list[entry].mark = FALSE;
6893 }
6894 /*
6895 * if UPL_RET_ONLY_ABSENT is set, then
6896 * we are working with a fresh page and we've
6897 * just set the clustered flag on it to
6898 * indicate that it was drug in as part of a
6899 * speculative cluster... so leave it alone
6900 */
6901 if (!(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
6902 /*
6903 * someone is explicitly grabbing this page...
6904 * update clustered and speculative state
6905 *
6906 */
6907 if (dst_page->vmp_clustered) {
6908 VM_PAGE_CONSUME_CLUSTERED(dst_page);
6909 }
6910 }
6911 try_next_page:
6912 if (dwp->dw_mask) {
6913 if (dwp->dw_mask & DW_vm_page_activate) {
6914 counter_inc(&vm_statistics_reactivations);
6915 }
6916
6917 VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
6918
6919 if (dw_count >= dw_limit) {
6920 vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
6921
6922 dwp = dwp_start;
6923 dw_count = 0;
6924 }
6925 }
6926 entry++;
6927 dst_offset += PAGE_SIZE_64;
6928 xfer_size -= PAGE_SIZE;
6929 }
6930 if (dw_count) {
6931 vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
6932 dwp = dwp_start;
6933 dw_count = 0;
6934 }
6935
6936 if (alias_page != NULL) {
6937 VM_PAGE_FREE(alias_page);
6938 }
6939 if (pmap_flushes_delayed == TRUE) {
6940 pmap_flush(&pmap_flush_context_storage);
6941 }
6942
6943 if (page_list_count != NULL) {
6944 if (upl->flags & UPL_INTERNAL) {
6945 *page_list_count = 0;
6946 } else if (*page_list_count > entry) {
6947 *page_list_count = entry;
6948 }
6949 }
6950 #if UPL_DEBUG
6951 upl->upl_state = 1;
6952 #endif
6953 vm_object_unlock(object);
6954
6955 VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, DBG_VM_UPL_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
6956 if (task != NULL) {
6957 ledger_credit(task->ledger, task_ledgers.pages_grabbed_upl, page_grab_count);
6958 }
6959 counter_add(&vm_page_grab_count_upl, page_grab_count);
6960
6961 if (dwp_start && dwp_finish_ctx) {
6962 vm_page_delayed_work_finish_ctx(dwp_start);
6963 dwp_start = dwp = NULL;
6964 }
6965
6966 vm_object_lock(object);
6967 if (cntrl_flags & UPL_WILL_MODIFY) {
6968 vm_object_pl_req_end(object);
6969 thread_priority_floor_end(&token);
6970 }
6971 vm_object_unlock(object);
6972
6973 return KERN_SUCCESS;
6974 }
6975
6976 int cs_executable_create_upl = 0;
6977 extern int proc_selfpid(void);
6978 extern char *proc_name_address(void *p);
6979
6980 /**
6981 * Helper for determining whether a writable (!UPL_COPYOUT_FROM) UPL is allowed for a given VA region.
6982 * This is determined not only by the allowed permissions in the relevant vm_map_entry, but also by
6983 * the code integrity enforcement model present on the system.
6984 *
6985 * @param map VM map against which the UPL is being populated.
6986 * @param entry The source vm_map_entry in [map] against which the UPL is being populated.
6987 * @param offset Base offset of UPL request in [map], for debugging purposes.
6988 *
6989 * @return True if the writable UPL is allowed for [entry], false otherwise.
6990 */
6991 static bool
6992 vme_allows_upl_write(
6993 vm_map_t map __unused,
6994 vm_map_entry_t entry,
6995 vm_map_address_t offset __unused)
6996 {
6997 if (!(entry->protection & VM_PROT_WRITE)) {
6998 return false;
6999 }
7000 #if CONFIG_SPTM
7001 /*
7002 * For SPTM configurations, reject any attempt to create a writable UPL against any executable
7003 * region. Even in cases such as JIT/USER_DEBUG in which the vm_map_entry may allow write
7004 * access, the SPTM/TXM codesigning model still forbids writable DMA mappings of these pages.
7005 */
7006 if ((entry->protection & VM_PROT_EXECUTE) || entry->vme_xnu_user_debug) {
7007 vm_map_guard_exception(offset, kGUARD_EXC_SEC_UPL_WRITE_ON_EXEC_REGION);
7008 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM,
7009 KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_UPL_WRITE_ON_EXEC_REGION), (uintptr_t)offset);
7010 return false;
7011 }
7012 #endif /* CONFIG_SPTM */
7013 return true;
7014 }
7015
7016 /**
7017 * Helper for determining whether a read-only (UPL_COPYOUT_FROM) UPL is allowed for a given VA region,
7018 * possibly with the additional requirement of creating a kernel copy of the source buffer.
7019 * This is determined by the code integrity enforcement model present on the system.
7020 *
7021 * @param map VM map against which the UPL is being populated.
7022 * @param entry The source vm_map_entry in [map] against which the UPL is being populated.
7023 * @param offset Base offset of UPL request in [map], for debugging purposes.
7024 * @param copy_required Output parameter indicating whether the UPL should be created against a kernel
7025 * copy of the source data.
7026 *
7027 * @return True if the read-only UPL is allowed for [entry], false otherwise.
7028 */
7029 static bool
7030 vme_allows_upl_read(
7031 vm_map_t map __unused,
7032 vm_map_entry_t entry __unused,
7033 vm_map_address_t offset __unused,
7034 bool *copy_required)
7035 {
7036 assert(copy_required != NULL);
7037 *copy_required = false;
7038 #if CONFIG_SPTM
7039 /*
7040 * For SPTM configs, always create a copy when attempting a read-only I/O operation against an
7041 * executable or debug (which may become executable) mapping. The SPTM's stricter security
7042 * enforcements against DMA mappings of executable pages may otherwise trigger an SPTM violation
7043 * panic. We expect the added cost of this copy to be manageable as DMA mappings of executable
7044 * regions are rare in practice.
7045 */
7046 if ((map->pmap != kernel_pmap) &&
7047 ((entry->protection & VM_PROT_EXECUTE) || entry->vme_xnu_user_debug)) {
7048 *copy_required = true;
7049 }
7050 #endif /* CONFIG_SPTM */
7051 #if !XNU_TARGET_OS_OSX
7052 /*
7053 * For all non-Mac targets, create a copy when attempting a read-only I/O operation against a
7054 * read-only executable region. These regions are likely to be codesigned and are typically
7055 * mapped CoW; our wire operation will be treated as a proactive CoW fault which will copy the
7056 * backing pages and thus cause them to no longer be codesigned.
7057 */
7058 if (map->pmap != kernel_pmap &&
7059 (entry->protection & VM_PROT_EXECUTE) &&
7060 !(entry->protection & VM_PROT_WRITE)) {
7061 *copy_required = true;
7062 }
7063 #endif /* !XNU_TARGET_OS_OSX */
7064 return true;
7065 }
7066
7067 kern_return_t
7068 vm_map_create_upl(
7069 vm_map_t map,
7070 vm_map_address_t offset,
7071 upl_size_t *upl_size,
7072 upl_t *upl,
7073 upl_page_info_array_t page_list,
7074 unsigned int *count,
7075 upl_control_flags_t *flags,
7076 vm_tag_t tag)
7077 {
7078 vm_map_entry_t entry;
7079 upl_control_flags_t caller_flags;
7080 int force_data_sync;
7081 int sync_cow_data;
7082 vm_object_t local_object;
7083 vm_map_offset_t local_offset;
7084 vm_map_offset_t local_start;
7085 kern_return_t ret;
7086 vm_map_address_t original_offset;
7087 vm_map_size_t original_size, adjusted_size;
7088 vm_map_offset_t local_entry_start;
7089 vm_object_offset_t local_entry_offset;
7090 boolean_t release_map = FALSE;
7091
7092 vmlp_api_start(VM_MAP_CREATE_UPL);
7093 start_with_map:
7094 caller_flags = *flags;
7095
7096 if (caller_flags & ~UPL_VALID_FLAGS) {
7097 /*
7098 * For forward compatibility's sake,
7099 * reject any unknown flag.
7100 */
7101 ret = KERN_INVALID_VALUE;
7102 goto done;
7103 }
7104
7105 if (upl == NULL) {
7106 ret = KERN_INVALID_ARGUMENT;
7107 goto done;
7108 }
7109
7110 #if HAS_MTE || HAS_MTE_EMULATION_SHIMS
7111 /* We expect only canonical addresses down this path. */
7112 if (offset != vm_memtag_canonicalize(map, offset)) {
7113 #if HAS_MTE
7114 mte_report_non_canonical_address((caddr_t)offset, map, __func__);
7115 #endif /* HAS_MTE */
7116 ret = KERN_INVALID_ARGUMENT;
7117 goto done;
7118 }
7119 #endif /* HAS_MTE || HAS_MTE_EMULATION_SHIMS */
7120
7121 original_offset = offset;
7122 original_size = *upl_size;
7123 adjusted_size = original_size;
7124
7125 force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
7126 sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
7127
7128 REDISCOVER_ENTRY:
7129 vm_map_lock_read(map);
7130
7131 if (!vm_map_lookup_entry(map, offset, &entry)) {
7132 vm_map_unlock_read(map);
7133 ret = KERN_FAILURE;
7134 goto done;
7135 }
7136
7137 if (!entry->is_sub_map) {
7138 vmlp_range_event_entry(map, entry);
7139 }
7140
7141 local_entry_start = entry->vme_start;
7142 local_entry_offset = VME_OFFSET(entry);
7143
7144 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) {
7145 DEBUG4K_UPL("map %p (%d) offset 0x%llx size 0x%x flags 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint64_t)offset, *upl_size, *flags);
7146 }
7147
7148 if (entry->vme_end - original_offset < adjusted_size) {
7149 adjusted_size = entry->vme_end - original_offset;
7150 assert(adjusted_size > 0);
7151 *upl_size = (upl_size_t) adjusted_size;
7152 assert(*upl_size == adjusted_size);
7153 }
7154
7155 if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
7156 *flags = 0;
7157
7158 if (!entry->is_sub_map &&
7159 VME_OBJECT(entry) != VM_OBJECT_NULL) {
7160 if (VME_OBJECT(entry)->private) {
7161 *flags = UPL_DEV_MEMORY;
7162 }
7163
7164 if (VME_OBJECT(entry)->phys_contiguous) {
7165 *flags |= UPL_PHYS_CONTIG;
7166 }
7167 }
7168 vm_map_unlock_read(map);
7169 ret = KERN_SUCCESS;
7170 goto done;
7171 }
7172
7173 bool copy_required = false;
7174
7175 if (!entry->is_sub_map) {
7176 if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
7177 !VME_OBJECT(entry)->phys_contiguous) {
7178 if (*upl_size > MAX_UPL_SIZE_BYTES) {
7179 *upl_size = MAX_UPL_SIZE_BYTES;
7180 }
7181 }
7182
7183 /*
7184 * Create an object if necessary.
7185 */
7186 if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
7187 if (entry->max_protection == VM_PROT_NONE) {
7188 /* don't create an object for a reserved range */
7189 vm_map_unlock_read(map);
7190 ret = KERN_PROTECTION_FAILURE;
7191 goto done;
7192 }
7193
7194 if (vm_map_lock_read_to_write(map)) {
7195 goto REDISCOVER_ENTRY;
7196 }
7197
7198 VME_OBJECT_SET(entry,
7199 vm_object_allocate((vm_size_t)
7200 vm_object_round_page((entry->vme_end - entry->vme_start)), map->serial_id),
7201 false, 0);
7202 VME_OFFSET_SET(entry, 0);
7203 assert(entry->use_pmap);
7204
7205 vm_map_lock_write_to_read(map);
7206 }
7207
7208 if (((caller_flags & UPL_COPYOUT_FROM) && !vme_allows_upl_read(map, entry, offset, ©_required)) ||
7209 (!(caller_flags & UPL_COPYOUT_FROM) && !vme_allows_upl_write(map, entry, offset))) {
7210 vm_map_unlock_read(map);
7211 ret = KERN_PROTECTION_FAILURE;
7212 goto done;
7213 }
7214 }
7215
7216 if (__improbable(copy_required)) {
7217 vm_offset_t kaddr;
7218 vm_size_t ksize;
7219
7220 /*
7221 * Depending on the device configuration, wiring certain pages
7222 * for I/O may violate the security policy for codesigning-related
7223 * reasons.
7224 * Instead, let's copy the data into a kernel buffer and
7225 * create the UPL from this kernel buffer.
7226 * The kernel buffer is then freed, leaving the UPL holding
7227 * the last reference on the VM object, so the memory will
7228 * be released when the UPL is committed.
7229 */
7230
7231 vm_map_unlock_read(map);
7232 entry = VM_MAP_ENTRY_NULL;
7233 /* allocate kernel buffer */
7234 ksize = round_page(*upl_size);
7235 kaddr = 0;
7236 ret = kmem_alloc(kernel_map, &kaddr, ksize,
7237 KMA_PAGEABLE | KMA_DATA_SHARED, tag);
7238 if (ret == KERN_SUCCESS) {
7239 /* copyin the user data */
7240 ret = copyinmap(map, offset, (void *)kaddr, *upl_size);
7241 }
7242 if (ret == KERN_SUCCESS) {
7243 if (ksize > *upl_size) {
7244 /* zero out the extra space in kernel buffer */
7245 memset((void *)(kaddr + *upl_size),
7246 0,
7247 ksize - *upl_size);
7248 }
7249 /* create the UPL from the kernel buffer */
7250 ret = vm_map_create_upl(kernel_map,
7251 (vm_map_address_t)kaddr, upl_size, upl, page_list, count, flags, tag);
7252 }
7253 if (kaddr != 0) {
7254 /* free the kernel buffer */
7255 kmem_free(kernel_map, kaddr, ksize);
7256 kaddr = 0;
7257 ksize = 0;
7258 }
7259 #if DEVELOPMENT || DEBUG
7260 DTRACE_VM4(create_upl_from_executable,
7261 vm_map_t, map,
7262 vm_map_address_t, offset,
7263 upl_size_t, *upl_size,
7264 kern_return_t, ret);
7265 #endif /* DEVELOPMENT || DEBUG */
7266 goto done;
7267 }
7268
7269 if (!entry->is_sub_map) {
7270 local_object = VME_OBJECT(entry);
7271 assert(local_object != VM_OBJECT_NULL);
7272 }
7273
7274 if (!entry->is_sub_map &&
7275 !entry->needs_copy &&
7276 *upl_size != 0 &&
7277 local_object->vo_size > *upl_size && /* partial UPL */
7278 entry->wired_count == 0 && /* No COW for entries that are wired */
7279 (map->pmap != kernel_pmap) && /* alias checks */
7280 (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */
7281 ||
7282 ( /* case 2 */
7283 local_object->internal &&
7284 (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
7285 os_ref_get_count_raw(&local_object->ref_count) > 1))) {
7286 vm_prot_t prot;
7287
7288 /*
7289 * Case 1:
7290 * Set up the targeted range for copy-on-write to avoid
7291 * applying true_share/copy_delay to the entire object.
7292 *
7293 * Case 2:
7294 * This map entry covers only part of an internal
7295 * object. There could be other map entries covering
7296 * other areas of this object and some of these map
7297 * entries could be marked as "needs_copy", which
7298 * assumes that the object is COPY_SYMMETRIC.
7299 * To avoid marking this object as COPY_DELAY and
7300 * "true_share", let's shadow it and mark the new
7301 * (smaller) object as "true_share" and COPY_DELAY.
7302 */
7303
7304 if (vm_map_lock_read_to_write(map)) {
7305 goto REDISCOVER_ENTRY;
7306 }
7307 vm_map_lock_assert_exclusive(map);
7308 assert(VME_OBJECT(entry) == local_object);
7309
7310 vm_map_clip_start(map,
7311 entry,
7312 vm_map_trunc_page(offset,
7313 VM_MAP_PAGE_MASK(map)));
7314 vm_map_clip_end(map,
7315 entry,
7316 vm_map_round_page(offset + *upl_size,
7317 VM_MAP_PAGE_MASK(map)));
7318 if ((entry->vme_end - offset) < *upl_size) {
7319 *upl_size = (upl_size_t) (entry->vme_end - offset);
7320 assert(*upl_size == entry->vme_end - offset);
7321 }
7322
7323 prot = entry->protection & ~VM_PROT_WRITE;
7324 if (override_nx(map, VME_ALIAS(entry)) && prot) {
7325 prot |= VM_PROT_EXECUTE;
7326 }
7327 vm_object_pmap_protect(local_object,
7328 VME_OFFSET(entry),
7329 entry->vme_end - entry->vme_start,
7330 ((entry->is_shared ||
7331 map->mapped_in_other_pmaps)
7332 ? PMAP_NULL
7333 : map->pmap),
7334 VM_MAP_PAGE_SIZE(map),
7335 entry->vme_start,
7336 prot);
7337
7338 assert(entry->wired_count == 0);
7339
7340 /*
7341 * Lock the VM object and re-check its status: if it's mapped
7342 * in another address space, we could still be racing with
7343 * another thread holding that other VM map exclusively.
7344 */
7345 vm_object_lock(local_object);
7346 if (local_object->true_share) {
7347 /* object is already in proper state: no COW needed */
7348 assert(local_object->copy_strategy !=
7349 MEMORY_OBJECT_COPY_SYMMETRIC);
7350 } else {
7351 /* not true_share: ask for copy-on-write below */
7352 assert(local_object->copy_strategy ==
7353 MEMORY_OBJECT_COPY_SYMMETRIC);
7354 entry->needs_copy = TRUE;
7355 }
7356 vm_object_unlock(local_object);
7357
7358 vm_map_lock_write_to_read(map);
7359 }
7360
7361 if (entry->needs_copy) {
7362 /*
7363 * Honor copy-on-write for COPY_SYMMETRIC
7364 * strategy.
7365 */
7366 vm_map_t local_map;
7367 vm_object_t object;
7368 vm_object_offset_t new_offset;
7369 vm_prot_t prot;
7370 boolean_t wired;
7371 vm_map_version_t version;
7372 vm_map_t real_map;
7373 vm_prot_t fault_type;
7374
7375 local_map = map;
7376
7377 if (caller_flags & UPL_COPYOUT_FROM) {
7378 fault_type = VM_PROT_READ | VM_PROT_COPY;
7379 vm_counters.create_upl_extra_cow++;
7380 vm_counters.create_upl_extra_cow_pages +=
7381 (entry->vme_end - entry->vme_start) / PAGE_SIZE;
7382 } else {
7383 fault_type = VM_PROT_WRITE;
7384 }
7385 if (vm_map_lookup_and_lock_object(&local_map,
7386 offset, fault_type,
7387 OBJECT_LOCK_EXCLUSIVE,
7388 &version, &object,
7389 &new_offset, &prot, &wired,
7390 NULL,
7391 &real_map, NULL) != KERN_SUCCESS) {
7392 if (fault_type == VM_PROT_WRITE) {
7393 vm_counters.create_upl_lookup_failure_write++;
7394 } else {
7395 vm_counters.create_upl_lookup_failure_copy++;
7396 }
7397 vm_map_unlock_read(local_map);
7398 ret = KERN_FAILURE;
7399 goto done;
7400 }
7401 if (real_map != local_map) {
7402 vm_map_unlock(real_map);
7403 }
7404 vm_map_unlock_read(local_map);
7405
7406 vm_object_unlock(object);
7407
7408 goto REDISCOVER_ENTRY;
7409 }
7410
7411 if (entry->is_sub_map) {
7412 vm_map_t submap;
7413
7414 submap = VME_SUBMAP(entry);
7415 local_start = entry->vme_start;
7416 local_offset = (vm_map_offset_t)VME_OFFSET(entry);
7417
7418 vm_map_reference(submap);
7419 vm_map_unlock_read(map);
7420
7421 DEBUG4K_UPL("map %p offset 0x%llx (0x%llx) size 0x%x (adjusted 0x%llx original 0x%llx) submap %p\n", map, (uint64_t)offset, (uint64_t)original_offset, *upl_size, (uint64_t)adjusted_size, (uint64_t)original_size, submap);
7422
7423 if (release_map) {
7424 vm_map_deallocate(map);
7425 }
7426 map = submap;
7427 release_map = TRUE;
7428 offset = local_offset + (offset - local_start);
7429 goto start_with_map;
7430 }
7431
7432 if (sync_cow_data &&
7433 (VME_OBJECT(entry)->shadow ||
7434 VME_OBJECT(entry)->vo_copy)) {
7435 local_object = VME_OBJECT(entry);
7436 local_start = entry->vme_start;
7437 local_offset = (vm_map_offset_t)VME_OFFSET(entry);
7438
7439 vm_object_reference(local_object);
7440 vm_map_unlock_read(map);
7441
7442 if (local_object->shadow && local_object->vo_copy) {
7443 vm_object_lock_request(local_object->shadow,
7444 ((vm_object_offset_t)
7445 ((offset - local_start) +
7446 local_offset) +
7447 local_object->vo_shadow_offset),
7448 *upl_size, FALSE,
7449 MEMORY_OBJECT_DATA_SYNC,
7450 VM_PROT_NO_CHANGE);
7451 }
7452 sync_cow_data = FALSE;
7453 vm_object_deallocate(local_object);
7454
7455 goto REDISCOVER_ENTRY;
7456 }
7457 if (force_data_sync) {
7458 local_object = VME_OBJECT(entry);
7459 local_start = entry->vme_start;
7460 local_offset = (vm_map_offset_t)VME_OFFSET(entry);
7461
7462 vm_object_reference(local_object);
7463 vm_map_unlock_read(map);
7464
7465 vm_object_lock_request(local_object,
7466 ((vm_object_offset_t)
7467 ((offset - local_start) +
7468 local_offset)),
7469 (vm_object_size_t)*upl_size,
7470 FALSE,
7471 MEMORY_OBJECT_DATA_SYNC,
7472 VM_PROT_NO_CHANGE);
7473
7474 force_data_sync = FALSE;
7475 vm_object_deallocate(local_object);
7476
7477 goto REDISCOVER_ENTRY;
7478 }
7479 if (VME_OBJECT(entry)->private) {
7480 *flags = UPL_DEV_MEMORY;
7481 } else {
7482 *flags = 0;
7483 }
7484
7485 if (VME_OBJECT(entry)->phys_contiguous) {
7486 *flags |= UPL_PHYS_CONTIG;
7487 }
7488
7489 local_object = VME_OBJECT(entry);
7490 local_offset = (vm_map_offset_t)VME_OFFSET(entry);
7491 local_start = entry->vme_start;
7492
7493 #if HAS_MTE
7494 if (local_object && vm_object_is_mte_mappable(local_object)) {
7495 vm_size_t size = entry->vme_end - entry->vme_start;
7496 if (!vm_map_allow_mte_operation(map, local_start, size, VM_MTE_OPERATION_TYPE_CREATE_UPL, optional_vm_object_none() /* irrelevant here */)) {
7497 vm_map_unlock(map);
7498 ret = KERN_NOT_SUPPORTED;
7499 goto done;
7500 }
7501 }
7502 #endif /* HAS_MTE */
7503
7504 /*
7505 * Wiring will copy the pages to the shadow object.
7506 * The shadow object will not be code-signed so
7507 * attempting to execute code from these copied pages
7508 * would trigger a code-signing violation.
7509 */
7510 if (entry->protection & VM_PROT_EXECUTE) {
7511 #if MACH_ASSERT
7512 printf("pid %d[%s] create_upl out of executable range from "
7513 "0x%llx to 0x%llx: side effects may include "
7514 "code-signing violations later on\n",
7515 proc_selfpid(),
7516 (get_bsdtask_info(current_task())
7517 ? proc_name_address(get_bsdtask_info(current_task()))
7518 : "?"),
7519 (uint64_t) entry->vme_start,
7520 (uint64_t) entry->vme_end);
7521 #endif /* MACH_ASSERT */
7522 DTRACE_VM2(cs_executable_create_upl,
7523 uint64_t, (uint64_t)entry->vme_start,
7524 uint64_t, (uint64_t)entry->vme_end);
7525 cs_executable_create_upl++;
7526 }
7527
7528 vm_object_lock(local_object);
7529
7530 /*
7531 * Ensure that this object is "true_share" and "copy_delay" now,
7532 * while we're still holding the VM map lock. After we unlock the map,
7533 * anything could happen to that mapping, including some copy-on-write
7534 * activity. We need to make sure that the IOPL will point at the
7535 * same memory as the mapping.
7536 */
7537 if (local_object->true_share) {
7538 assert(local_object->copy_strategy !=
7539 MEMORY_OBJECT_COPY_SYMMETRIC);
7540 } else if (!is_kernel_object(local_object) &&
7541 local_object != compressor_object &&
7542 !local_object->phys_contiguous) {
7543 #if VM_OBJECT_TRACKING_OP_TRUESHARE
7544 if (!local_object->true_share &&
7545 vm_object_tracking_btlog) {
7546 btlog_record(vm_object_tracking_btlog, local_object,
7547 VM_OBJECT_TRACKING_OP_TRUESHARE,
7548 btref_get(__builtin_frame_address(0), 0));
7549 }
7550 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
7551 VM_OBJECT_SET_TRUE_SHARE(local_object, TRUE);
7552 if (local_object->copy_strategy ==
7553 MEMORY_OBJECT_COPY_SYMMETRIC) {
7554 local_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
7555 }
7556 }
7557
7558 vm_object_reference_locked(local_object);
7559 vm_object_unlock(local_object);
7560
7561 vm_map_unlock_read(map);
7562
7563 ret = vm_object_iopl_request(local_object,
7564 ((vm_object_offset_t)
7565 ((offset - local_start) + local_offset)),
7566 *upl_size,
7567 upl,
7568 page_list,
7569 count,
7570 caller_flags,
7571 tag);
7572 vm_object_deallocate(local_object);
7573
7574 done:
7575 if (release_map) {
7576 vm_map_deallocate(map);
7577 }
7578
7579 vmlp_api_end(VM_MAP_CREATE_UPL, ret);
7580 return ret;
7581 }
7582
7583 /*
7584 * Internal routine to enter a UPL into a VM map.
7585 *
7586 * JMM - This should just be doable through the standard
7587 * vm_map_enter() API.
7588 */
7589 kern_return_t
7590 vm_map_enter_upl_range(
7591 vm_map_t map,
7592 upl_t upl,
7593 vm_object_offset_t offset_to_map,
7594 vm_size_t size_to_map,
7595 vm_prot_t prot_to_map,
7596 vm_map_offset_t *dst_addr)
7597 {
7598 vm_map_size_t size;
7599 vm_object_offset_t offset;
7600 vm_map_offset_t addr;
7601 vm_page_t m;
7602 kern_return_t kr;
7603 int isVectorUPL = 0, curr_upl = 0;
7604 upl_t vector_upl = NULL;
7605 mach_vm_offset_t vector_upl_dst_addr = 0;
7606 upl_offset_t subupl_offset = 0;
7607 upl_size_t subupl_size = 0;
7608
7609 if (upl == UPL_NULL) {
7610 return KERN_INVALID_ARGUMENT;
7611 }
7612
7613 DEBUG4K_UPL("map %p upl %p flags 0x%x object %p offset 0x%llx (uploff: 0x%llx) size 0x%lx (uplsz: 0x%x) \n", map, upl, upl->flags, upl->map_object, offset_to_map, upl->u_offset, size_to_map, upl->u_size);
7614 assert(map == kernel_map);
7615
7616 if ((isVectorUPL = vector_upl_is_valid(upl))) {
7617 int mapped = 0, valid_upls = 0;
7618 vector_upl = upl;
7619
7620 upl_lock(vector_upl);
7621 for (curr_upl = 0; curr_upl < vector_upl_max_upls(vector_upl); curr_upl++) {
7622 upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
7623 if (upl == NULL) {
7624 continue;
7625 }
7626 valid_upls++;
7627 if (UPL_PAGE_LIST_MAPPED & upl->flags) {
7628 mapped++;
7629 }
7630 }
7631
7632 if (mapped) {
7633 if (mapped != valid_upls) {
7634 panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped", mapped, valid_upls);
7635 } else {
7636 upl_unlock(vector_upl);
7637 return KERN_FAILURE;
7638 }
7639 }
7640
7641 if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) {
7642 panic("TODO4K: vector UPL not implemented");
7643 }
7644
7645 kern_return_t kr2;
7646 vm_offset_t alloc_addr = 0;
7647 kr2 = vm_allocate(map, &alloc_addr, vector_upl->u_size, VM_FLAGS_ANYWHERE);
7648 if (kr2 != KERN_SUCCESS) {
7649 os_log(OS_LOG_DEFAULT, "%s: vm_allocate(0x%x) -> %d",
7650 __func__, vector_upl->u_size, kr2);
7651 upl_unlock(vector_upl);
7652 return kr2;
7653 }
7654 vector_upl_dst_addr = alloc_addr;
7655 vector_upl_set_addr(vector_upl, vector_upl_dst_addr);
7656 curr_upl = 0;
7657 } else {
7658 upl_lock(upl);
7659 }
7660
7661 process_upl_to_enter:
7662 if (isVectorUPL) {
7663 if (curr_upl == vector_upl_max_upls(vector_upl)) {
7664 *dst_addr = vector_upl_dst_addr;
7665 upl_unlock(vector_upl);
7666 return KERN_SUCCESS;
7667 }
7668 upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
7669 if (upl == NULL) {
7670 goto process_upl_to_enter;
7671 }
7672
7673 vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
7674 *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
7675 } else {
7676 /*
7677 * check to see if already mapped
7678 */
7679 if (UPL_PAGE_LIST_MAPPED & upl->flags) {
7680 upl_unlock(upl);
7681 return KERN_FAILURE;
7682 }
7683 }
7684
7685 if ((!(upl->flags & UPL_SHADOWED)) &&
7686 ((upl->flags & UPL_HAS_BUSY) ||
7687 !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
7688 vm_object_t object;
7689 vm_page_t alias_page;
7690 vm_object_offset_t new_offset;
7691 unsigned int pg_num;
7692
7693 size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
7694 object = upl->map_object;
7695 upl->map_object = vm_object_allocate(
7696 vm_object_round_page(size),
7697 /* Provenance is copied from the object we're shadowing */
7698 object->vmo_provenance);
7699
7700 vm_object_lock(upl->map_object);
7701
7702 upl->map_object->shadow = object;
7703 VM_OBJECT_SET_PAGEOUT(upl->map_object, TRUE);
7704 VM_OBJECT_SET_CAN_PERSIST(upl->map_object, FALSE);
7705 upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
7706 upl->map_object->vo_shadow_offset = upl_adjusted_offset(upl, PAGE_MASK) - object->paging_offset;
7707 assertf(page_aligned(upl->map_object->vo_shadow_offset),
7708 "object %p shadow_offset 0x%llx",
7709 upl->map_object,
7710 (uint64_t)upl->map_object->vo_shadow_offset);
7711 upl->map_object->wimg_bits = object->wimg_bits;
7712 offset = upl->map_object->vo_shadow_offset;
7713 new_offset = 0;
7714
7715 upl->flags |= UPL_SHADOWED;
7716
7717 while (size) {
7718 pg_num = (unsigned int) (new_offset / PAGE_SIZE);
7719 assert(pg_num == new_offset / PAGE_SIZE);
7720
7721 if (bitmap_test(upl->lite_list, pg_num)) {
7722 alias_page = vm_page_create_fictitious();
7723
7724 vm_object_lock(object);
7725
7726 m = vm_page_lookup(object, offset);
7727 if (m == VM_PAGE_NULL) {
7728 panic("vm_upl_map: page missing");
7729 }
7730
7731 /*
7732 * Convert the fictitious page to a private
7733 * shadow of the real page.
7734 */
7735 alias_page->vmp_free_when_done = TRUE;
7736 /*
7737 * since m is a page in the upl it must
7738 * already be wired or BUSY, so it's
7739 * safe to assign the underlying physical
7740 * page to the alias
7741 */
7742
7743 vm_object_unlock(object);
7744
7745 vm_page_lockspin_queues();
7746 vm_page_make_private(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
7747 vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
7748 vm_page_unlock_queues();
7749
7750 vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
7751
7752 assert(!alias_page->vmp_wanted);
7753 alias_page->vmp_busy = FALSE;
7754 alias_page->vmp_absent = FALSE;
7755 }
7756 size -= PAGE_SIZE;
7757 offset += PAGE_SIZE_64;
7758 new_offset += PAGE_SIZE_64;
7759 }
7760 vm_object_unlock(upl->map_object);
7761 }
7762 if (upl->flags & UPL_SHADOWED) {
7763 if (isVectorUPL) {
7764 offset = 0;
7765 } else {
7766 offset = offset_to_map;
7767 }
7768 } else {
7769 offset = upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)) - upl->map_object->paging_offset;
7770 if (!isVectorUPL) {
7771 offset += offset_to_map;
7772 }
7773 }
7774
7775 if (isVectorUPL) {
7776 size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
7777 } else {
7778 size = MIN(upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map)), size_to_map);
7779 }
7780
7781 vm_object_reference(upl->map_object);
7782
7783 if (!isVectorUPL) {
7784 *dst_addr = 0;
7785 /*
7786 * NEED A UPL_MAP ALIAS
7787 */
7788 kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
7789 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(.vm_tag = VM_KERN_MEMORY_OSFMK),
7790 upl->map_object, offset, FALSE,
7791 prot_to_map, VM_PROT_ALL, VM_INHERIT_DEFAULT);
7792
7793 if (kr != KERN_SUCCESS) {
7794 vm_object_deallocate(upl->map_object);
7795 upl_unlock(upl);
7796 return kr;
7797 }
7798 } else {
7799 kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
7800 VM_MAP_KERNEL_FLAGS_FIXED(
7801 .vm_tag = VM_KERN_MEMORY_OSFMK,
7802 .vmf_overwrite = true),
7803 upl->map_object, offset, FALSE,
7804 prot_to_map, VM_PROT_ALL, VM_INHERIT_DEFAULT);
7805 if (kr) {
7806 panic("vm_map_enter failed for a Vector UPL");
7807 }
7808 }
7809 upl->u_mapped_size = (upl_size_t) size; /* When we allow multiple submappings of the UPL */
7810 /* this will have to be an increment rather than */
7811 /* an assignment. */
7812 vm_object_lock(upl->map_object);
7813
7814 for (addr = *dst_addr; size > 0; size -= PAGE_SIZE, addr += PAGE_SIZE) {
7815 m = vm_page_lookup(upl->map_object, offset);
7816
7817 if (m) {
7818 m->vmp_pmapped = TRUE;
7819
7820 /*
7821 * CODE SIGNING ENFORCEMENT: page has been wpmapped,
7822 * but only in kernel space. If this was on a user map,
7823 * we'd have to set the wpmapped bit.
7824 */
7825 /* m->vmp_wpmapped = TRUE; */
7826 assert(map->pmap == kernel_pmap);
7827
7828 kr = pmap_enter_check(map->pmap, addr, m, prot_to_map, VM_PROT_NONE, TRUE);
7829
7830 assert(kr == KERN_SUCCESS);
7831 #if KASAN
7832 kasan_notify_address(addr, PAGE_SIZE_64);
7833 #endif
7834 }
7835 offset += PAGE_SIZE_64;
7836 }
7837 vm_object_unlock(upl->map_object);
7838
7839 /*
7840 * hold a reference for the mapping
7841 */
7842 upl->ref_count++;
7843 upl->flags |= UPL_PAGE_LIST_MAPPED;
7844 upl->kaddr = (vm_offset_t) *dst_addr;
7845 assert(upl->kaddr == *dst_addr);
7846
7847 if (isVectorUPL) {
7848 goto process_upl_to_enter;
7849 }
7850
7851 if (!isVectorUPL) {
7852 vm_map_offset_t addr_adjustment;
7853
7854 addr_adjustment = (vm_map_offset_t)(upl->u_offset - upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)));
7855 if (addr_adjustment) {
7856 DEBUG4K_UPL("dst_addr 0x%llx (+ 0x%llx) -> 0x%llx\n", (uint64_t)*dst_addr, (uint64_t)addr_adjustment, (uint64_t)(*dst_addr + addr_adjustment));
7857 *dst_addr += addr_adjustment;
7858 }
7859 }
7860
7861 upl_unlock(upl);
7862
7863 return KERN_SUCCESS;
7864 }
7865
7866 kern_return_t
7867 vm_map_enter_upl(
7868 vm_map_t map,
7869 upl_t upl,
7870 vm_map_offset_t *dst_addr)
7871 {
7872 upl_size_t upl_size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
7873 return vm_map_enter_upl_range(map, upl, 0, upl_size, VM_PROT_DEFAULT, dst_addr);
7874 }
7875
7876 /*
7877 * Internal routine to remove a UPL mapping from a VM map.
7878 *
7879 * XXX - This should just be doable through a standard
7880 * vm_map_remove() operation. Otherwise, implicit clean-up
7881 * of the target map won't be able to correctly remove
7882 * these (and release the reference on the UPL). Having
7883 * to do this means we can't map these into user-space
7884 * maps yet.
7885 */
7886 kern_return_t
7887 vm_map_remove_upl_range(
7888 vm_map_t map,
7889 upl_t upl,
7890 __unused vm_object_offset_t offset_to_unmap,
7891 __unused vm_size_t size_to_unmap)
7892 {
7893 vm_address_t addr;
7894 upl_size_t size;
7895 int isVectorUPL = 0, curr_upl = 0;
7896 upl_t vector_upl = NULL;
7897
7898 if (upl == UPL_NULL) {
7899 return KERN_INVALID_ARGUMENT;
7900 }
7901
7902 if ((isVectorUPL = vector_upl_is_valid(upl))) {
7903 int unmapped = 0, valid_upls = 0;
7904 vector_upl = upl;
7905 upl_lock(vector_upl);
7906 for (curr_upl = 0; curr_upl < vector_upl_max_upls(vector_upl); curr_upl++) {
7907 upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
7908 if (upl == NULL) {
7909 continue;
7910 }
7911 valid_upls++;
7912 if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) {
7913 unmapped++;
7914 }
7915 }
7916
7917 if (unmapped) {
7918 if (unmapped != valid_upls) {
7919 panic("%d of the %d sub-upls within the Vector UPL is/are not mapped", unmapped, valid_upls);
7920 } else {
7921 upl_unlock(vector_upl);
7922 return KERN_FAILURE;
7923 }
7924 }
7925 curr_upl = 0;
7926 } else {
7927 upl_lock(upl);
7928 }
7929
7930 process_upl_to_remove:
7931 if (isVectorUPL) {
7932 if (curr_upl == vector_upl_max_upls(vector_upl)) {
7933 vm_offset_t v_upl_dst_addr;
7934 kern_return_t kr;
7935 vector_upl_get_addr(vector_upl, &v_upl_dst_addr);
7936
7937 kr = vm_deallocate(map, v_upl_dst_addr, vector_upl->u_size);
7938 if (kr != KERN_SUCCESS) {
7939 os_log(OS_LOG_DEFAULT, "%s: vm_deallocate(0x%llx, 0x%x) -> %d",
7940 __func__, (uint64_t)v_upl_dst_addr,
7941 vector_upl->u_size, kr);
7942 }
7943 v_upl_dst_addr = 0;
7944 vector_upl_set_addr(vector_upl, v_upl_dst_addr);
7945 upl_unlock(vector_upl);
7946 return KERN_SUCCESS;
7947 }
7948
7949 upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
7950 if (upl == NULL) {
7951 goto process_upl_to_remove;
7952 }
7953 }
7954
7955 if (upl->flags & UPL_PAGE_LIST_MAPPED) {
7956 addr = upl->kaddr;
7957 size = upl->u_mapped_size;
7958
7959 assert(upl->ref_count > 1);
7960 upl->ref_count--; /* removing mapping ref */
7961
7962 upl->flags &= ~UPL_PAGE_LIST_MAPPED;
7963 upl->kaddr = (vm_offset_t) 0;
7964 upl->u_mapped_size = 0;
7965
7966 if (isVectorUPL) {
7967 /*
7968 * If it's a Vectored UPL, we'll be removing the entire
7969 * address range anyway, so no need to remove individual UPL
7970 * element mappings from within the range
7971 */
7972 goto process_upl_to_remove;
7973 }
7974
7975 upl_unlock(upl);
7976
7977 vm_map_remove(map,
7978 vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(map)),
7979 vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(map)));
7980 return KERN_SUCCESS;
7981 }
7982 upl_unlock(upl);
7983
7984 return KERN_FAILURE;
7985 }
7986
7987 kern_return_t
7988 vm_map_remove_upl(
7989 vm_map_t map,
7990 upl_t upl)
7991 {
7992 upl_size_t upl_size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
7993 return vm_map_remove_upl_range(map, upl, 0, upl_size);
7994 }
7995
7996 void
7997 iopl_valid_data(
7998 upl_t upl,
7999 vm_tag_t tag)
8000 {
8001 vm_object_t object;
8002 vm_offset_t offset;
8003 vm_page_t m, nxt_page = VM_PAGE_NULL;
8004 upl_size_t size;
8005 int wired_count = 0;
8006
8007 if (upl == NULL) {
8008 panic("iopl_valid_data: NULL upl");
8009 }
8010 if (vector_upl_is_valid(upl)) {
8011 panic("iopl_valid_data: vector upl");
8012 }
8013 if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_SHADOWED | UPL_ACCESS_BLOCKED | UPL_IO_WIRE | UPL_INTERNAL)) != UPL_IO_WIRE) {
8014 panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags);
8015 }
8016
8017 object = upl->map_object;
8018
8019 if (is_kernel_object(object) || object == compressor_object) {
8020 panic("iopl_valid_data: object == kernel or compressor");
8021 }
8022
8023 if (object->purgable == VM_PURGABLE_VOLATILE ||
8024 object->purgable == VM_PURGABLE_EMPTY) {
8025 panic("iopl_valid_data: object %p purgable %d",
8026 object, object->purgable);
8027 }
8028
8029 size = upl_adjusted_size(upl, PAGE_MASK);
8030
8031 vm_object_lock(object);
8032 VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
8033
8034 bool whole_object;
8035
8036 if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) {
8037 nxt_page = (vm_page_t)vm_page_queue_first(&object->memq);
8038 whole_object = true;
8039 } else {
8040 offset = (vm_offset_t)(upl_adjusted_offset(upl, PAGE_MASK) - object->paging_offset);
8041 whole_object = false;
8042 }
8043
8044 while (size) {
8045 if (whole_object) {
8046 if (nxt_page != VM_PAGE_NULL) {
8047 m = nxt_page;
8048 nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
8049 }
8050 } else {
8051 m = vm_page_lookup(object, offset);
8052 offset += PAGE_SIZE;
8053
8054 if (m == VM_PAGE_NULL) {
8055 panic("iopl_valid_data: missing expected page at offset %lx", (long)offset);
8056 }
8057 }
8058 if (m->vmp_busy) {
8059 if (!m->vmp_absent) {
8060 panic("iopl_valid_data: busy page w/o absent");
8061 }
8062
8063 if (m->vmp_pageq.next || m->vmp_pageq.prev) {
8064 panic("iopl_valid_data: busy+absent page on page queue");
8065 }
8066 if (m->vmp_reusable) {
8067 panic("iopl_valid_data: %p is reusable", m);
8068 }
8069
8070 m->vmp_absent = FALSE;
8071 m->vmp_dirty = TRUE;
8072 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
8073 assert(m->vmp_wire_count == 0);
8074 m->vmp_wire_count++;
8075 m->vmp_iopl_wired = true;
8076 assert(m->vmp_wire_count);
8077 if (m->vmp_wire_count == 1) {
8078 m->vmp_q_state = VM_PAGE_IS_WIRED;
8079 wired_count++;
8080 } else {
8081 panic("iopl_valid_data: %p already wired", m);
8082 }
8083
8084 #if HAS_MTE
8085 mteinfo_increment_wire_count(m);
8086 #endif /* HAS_MTE */
8087
8088 vm_page_wakeup_done(object, m);
8089 }
8090 size -= PAGE_SIZE;
8091 }
8092 if (wired_count) {
8093 VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count);
8094 assert(object->resident_page_count >= object->wired_page_count);
8095
8096 /* no need to adjust purgeable accounting for this object: */
8097 assert(object->purgable != VM_PURGABLE_VOLATILE);
8098 assert(object->purgable != VM_PURGABLE_EMPTY);
8099
8100 vm_page_lockspin_queues();
8101 vm_page_wire_count += wired_count;
8102 vm_page_unlock_queues();
8103 }
8104 VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
8105 vm_object_unlock(object);
8106 }
8107
8108
8109 void
8110 vm_object_set_pmap_cache_attr(
8111 vm_object_t object,
8112 upl_page_info_array_t user_page_list,
8113 unsigned int num_pages,
8114 boolean_t batch_pmap_op)
8115 {
8116 unsigned int cache_attr = 0;
8117
8118 cache_attr = object->wimg_bits & VM_WIMG_MASK;
8119 assert(user_page_list);
8120 if (!HAS_DEFAULT_CACHEABILITY(cache_attr)) {
8121 PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, cache_attr, num_pages, batch_pmap_op);
8122 }
8123 }
8124
8125
8126 static bool
8127 vm_object_iopl_wire_full(
8128 vm_object_t object,
8129 upl_t upl,
8130 upl_page_info_array_t user_page_list,
8131 upl_control_flags_t cntrl_flags,
8132 vm_tag_t tag)
8133 {
8134 vm_page_t dst_page;
8135 unsigned int entry;
8136 int page_count;
8137 int delayed_unlock = 0;
8138 boolean_t retval = TRUE;
8139 ppnum_t phys_page;
8140
8141 vm_object_lock_assert_exclusive(object);
8142 assert(object->purgable != VM_PURGABLE_VOLATILE);
8143 assert(object->purgable != VM_PURGABLE_EMPTY);
8144 assert(object->pager == NULL);
8145 assert(object->vo_copy == NULL);
8146 assert(object->shadow == NULL);
8147
8148 page_count = object->resident_page_count;
8149 dst_page = (vm_page_t)vm_page_queue_first(&object->memq);
8150
8151 vm_page_lock_queues();
8152
8153 while (page_count--) {
8154 if (dst_page->vmp_busy ||
8155 #if CONFIG_SPTM
8156 PMAP_PAGE_IS_USER_EXECUTABLE(dst_page) ||
8157 #endif
8158 vm_page_is_fictitious(dst_page) ||
8159 dst_page->vmp_absent ||
8160 VMP_ERROR_GET(dst_page) ||
8161 dst_page->vmp_cleaning ||
8162 dst_page->vmp_restart ||
8163 dst_page->vmp_laundry) {
8164 retval = FALSE;
8165 goto done;
8166 }
8167 if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {
8168 retval = FALSE;
8169 goto done;
8170 }
8171 dst_page->vmp_reference = TRUE;
8172
8173 vm_page_wire(dst_page, tag, FALSE);
8174 dst_page->vmp_iopl_wired = true;
8175
8176 if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
8177 SET_PAGE_DIRTY(dst_page, FALSE);
8178 }
8179 entry = (unsigned int)(dst_page->vmp_offset / PAGE_SIZE);
8180 assert(entry >= 0 && entry < object->resident_page_count);
8181 bitmap_set(upl->lite_list, entry);
8182
8183 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
8184
8185 if (phys_page > upl->highest_page) {
8186 upl->highest_page = phys_page;
8187 }
8188
8189 if (user_page_list) {
8190 user_page_list[entry].phys_addr = phys_page;
8191 user_page_list[entry].absent = dst_page->vmp_absent;
8192 user_page_list[entry].dirty = dst_page->vmp_dirty;
8193 user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
8194 user_page_list[entry].precious = dst_page->vmp_precious;
8195 user_page_list[entry].device = FALSE;
8196 user_page_list[entry].speculative = FALSE;
8197 user_page_list[entry].cs_validated = FALSE;
8198 user_page_list[entry].cs_tainted = FALSE;
8199 user_page_list[entry].cs_nx = FALSE;
8200 user_page_list[entry].needed = FALSE;
8201 user_page_list[entry].mark = FALSE;
8202 }
8203 if (delayed_unlock++ > 256) {
8204 delayed_unlock = 0;
8205 lck_mtx_yield(&vm_page_queue_lock);
8206
8207 VM_CHECK_MEMORYSTATUS;
8208 }
8209 dst_page = (vm_page_t)vm_page_queue_next(&dst_page->vmp_listq);
8210 }
8211 done:
8212 vm_page_unlock_queues();
8213
8214 VM_CHECK_MEMORYSTATUS;
8215
8216 return retval;
8217 }
8218
8219
8220 static kern_return_t
8221 vm_object_iopl_wire_empty(
8222 vm_object_t object,
8223 upl_t upl,
8224 upl_page_info_array_t user_page_list,
8225 upl_control_flags_t cntrl_flags,
8226 vm_tag_t tag,
8227 vm_object_offset_t *dst_offset,
8228 int page_count,
8229 int *page_grab_count)
8230 {
8231 vm_page_t dst_page;
8232 boolean_t no_zero_fill = FALSE;
8233 int interruptible;
8234 int pages_wired = 0;
8235 int pages_inserted = 0;
8236 int entry = 0;
8237 uint64_t delayed_ledger_update = 0;
8238 kern_return_t ret = KERN_SUCCESS;
8239 vm_grab_options_t grab_options;
8240 ppnum_t phys_page;
8241
8242 vm_object_lock_assert_exclusive(object);
8243 assert(object->purgable != VM_PURGABLE_VOLATILE);
8244 assert(object->purgable != VM_PURGABLE_EMPTY);
8245 assert(object->pager == NULL);
8246 assert(object->vo_copy == NULL);
8247 assert(object->shadow == NULL);
8248
8249 if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {
8250 interruptible = THREAD_ABORTSAFE;
8251 } else {
8252 interruptible = THREAD_UNINT;
8253 }
8254
8255 if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
8256 no_zero_fill = TRUE;
8257 }
8258
8259 grab_options = VM_PAGE_GRAB_OPTIONS_NONE;
8260 #if CONFIG_SECLUDED_MEMORY
8261 if (object->can_grab_secluded) {
8262 grab_options |= VM_PAGE_GRAB_SECLUDED;
8263 }
8264 #endif /* CONFIG_SECLUDED_MEMORY */
8265
8266 while (page_count--) {
8267 while ((dst_page = vm_page_grab_options(grab_options))
8268 == VM_PAGE_NULL) {
8269 OSAddAtomic(page_count, &vm_upl_wait_for_pages);
8270
8271 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
8272
8273 if (vm_page_wait(interruptible) == FALSE) {
8274 /*
8275 * interrupted case
8276 */
8277 OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
8278
8279 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
8280
8281 ret = MACH_SEND_INTERRUPTED;
8282 goto done;
8283 }
8284 OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
8285
8286 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
8287 }
8288
8289 dst_page->vmp_absent = no_zero_fill;
8290 dst_page->vmp_reference = TRUE;
8291
8292 if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
8293 SET_PAGE_DIRTY(dst_page, FALSE);
8294 }
8295 if (dst_page->vmp_absent == FALSE) {
8296 assert(dst_page->vmp_q_state == VM_PAGE_NOT_ON_Q);
8297 assert(dst_page->vmp_wire_count == 0);
8298 dst_page->vmp_wire_count++;
8299 dst_page->vmp_iopl_wired = true;
8300 dst_page->vmp_q_state = VM_PAGE_IS_WIRED;
8301 assert(dst_page->vmp_wire_count);
8302 pages_wired++;
8303
8304 #if HAS_MTE
8305 mteinfo_increment_wire_count(dst_page);
8306 #endif /* HAS_MTE */
8307
8308 vm_page_wakeup_done(object, dst_page);
8309 }
8310 pages_inserted++;
8311
8312 vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update);
8313
8314 if (no_zero_fill == FALSE) {
8315 vm_page_zero_fill(
8316 dst_page
8317 #if HAS_MTE
8318 , true /* zero_tags */
8319 #endif /* HAS_MTE */
8320 );
8321 }
8322
8323 bitmap_set(upl->lite_list, entry);
8324
8325 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
8326
8327 if (phys_page > upl->highest_page) {
8328 upl->highest_page = phys_page;
8329 }
8330
8331 if (user_page_list) {
8332 user_page_list[entry].phys_addr = phys_page;
8333 user_page_list[entry].absent = dst_page->vmp_absent;
8334 user_page_list[entry].dirty = dst_page->vmp_dirty;
8335 user_page_list[entry].free_when_done = FALSE;
8336 user_page_list[entry].precious = FALSE;
8337 user_page_list[entry].device = FALSE;
8338 user_page_list[entry].speculative = FALSE;
8339 user_page_list[entry].cs_validated = FALSE;
8340 user_page_list[entry].cs_tainted = FALSE;
8341 user_page_list[entry].cs_nx = FALSE;
8342 user_page_list[entry].needed = FALSE;
8343 user_page_list[entry].mark = FALSE;
8344 }
8345 entry++;
8346 *dst_offset += PAGE_SIZE_64;
8347 }
8348 done:
8349 if (pages_wired) {
8350 vm_page_lockspin_queues();
8351 vm_page_wire_count += pages_wired;
8352 vm_page_unlock_queues();
8353 }
8354 if (pages_inserted) {
8355 if (object->internal) {
8356 OSAddAtomic(pages_inserted, &vm_page_internal_count);
8357 } else {
8358 OSAddAtomic(pages_inserted, &vm_page_external_count);
8359 }
8360 }
8361 if (delayed_ledger_update) {
8362 task_t owner;
8363 int ledger_idx_volatile;
8364 int ledger_idx_nonvolatile;
8365 int ledger_idx_volatile_compressed;
8366 int ledger_idx_nonvolatile_compressed;
8367 int ledger_idx_composite;
8368 int ledger_idx_external_wired;
8369 boolean_t do_footprint;
8370
8371 owner = VM_OBJECT_OWNER(object);
8372 assert(owner);
8373
8374 vm_object_ledger_tag_ledgers(object,
8375 &ledger_idx_volatile,
8376 &ledger_idx_nonvolatile,
8377 &ledger_idx_volatile_compressed,
8378 &ledger_idx_nonvolatile_compressed,
8379 &ledger_idx_composite,
8380 &ledger_idx_external_wired,
8381 &do_footprint);
8382
8383 if (object->internal) {
8384 /* more non-volatile bytes */
8385 ledger_credit(owner->ledger,
8386 ledger_idx_nonvolatile,
8387 delayed_ledger_update);
8388 if (do_footprint) {
8389 /* more footprint */
8390 ledger_credit(owner->ledger,
8391 task_ledgers.phys_footprint,
8392 delayed_ledger_update);
8393 } else if (ledger_idx_composite != -1) {
8394 ledger_credit(owner->ledger,
8395 ledger_idx_composite,
8396 delayed_ledger_update);
8397 }
8398 } else {
8399 /* more external wired bytes */
8400 ledger_credit(owner->ledger,
8401 ledger_idx_external_wired,
8402 delayed_ledger_update);
8403 if (do_footprint) {
8404 /* more footprint */
8405 ledger_credit(owner->ledger,
8406 task_ledgers.phys_footprint,
8407 delayed_ledger_update);
8408 } else if (ledger_idx_composite != -1) {
8409 ledger_credit(owner->ledger,
8410 ledger_idx_composite,
8411 delayed_ledger_update);
8412 }
8413 }
8414 }
8415
8416 assert(page_grab_count);
8417 *page_grab_count = pages_inserted;
8418
8419 return ret;
8420 }
8421
8422
8423 kern_return_t
8424 vm_object_iopl_request(
8425 vm_object_t object,
8426 vm_object_offset_t offset,
8427 upl_size_t size,
8428 upl_t *upl_ptr,
8429 upl_page_info_array_t user_page_list,
8430 unsigned int *page_list_count,
8431 upl_control_flags_t cntrl_flags,
8432 vm_tag_t tag)
8433 {
8434 vm_page_t dst_page;
8435 vm_object_offset_t dst_offset;
8436 upl_size_t xfer_size;
8437 upl_t upl = NULL;
8438 unsigned int entry;
8439 int no_zero_fill = FALSE;
8440 unsigned int size_in_pages;
8441 int page_grab_count = 0;
8442 u_int32_t psize;
8443 kern_return_t ret;
8444 vm_prot_t prot;
8445 struct vm_object_fault_info fault_info = {};
8446 struct vm_page_delayed_work dw_array;
8447 struct vm_page_delayed_work *dwp, *dwp_start;
8448 bool dwp_finish_ctx = TRUE;
8449 int dw_count;
8450 int dw_limit;
8451 int dw_index;
8452 boolean_t caller_lookup;
8453 int io_tracking_flag = 0;
8454 int interruptible;
8455 ppnum_t phys_page;
8456 bool need_pl_req_end = false;
8457 thread_pri_floor_t token;
8458
8459 boolean_t set_cache_attr_needed = FALSE;
8460 boolean_t free_wired_pages = FALSE;
8461 boolean_t fast_path_empty_req = FALSE;
8462 boolean_t fast_path_full_req = FALSE;
8463
8464 task_t task = current_task();
8465
8466 dwp_start = dwp = NULL;
8467 *upl_ptr = NULL;
8468
8469 vm_object_offset_t original_offset = offset;
8470 upl_size_t original_size = size;
8471
8472 // DEBUG4K_UPL("object %p offset 0x%llx size 0x%llx cntrl_flags 0x%llx\n", object, (uint64_t)offset, (uint64_t)size, cntrl_flags);
8473
8474 size = (upl_size_t)(vm_object_round_page(offset + size) - vm_object_trunc_page(offset));
8475 offset = vm_object_trunc_page(offset);
8476 if (size != original_size || offset != original_offset) {
8477 DEBUG4K_IOKIT("flags 0x%llx object %p offset 0x%llx size 0x%x -> offset 0x%llx size 0x%x\n", cntrl_flags, object, original_offset, original_size, offset, size);
8478 }
8479
8480 if (cntrl_flags & ~UPL_VALID_FLAGS) {
8481 /*
8482 * For forward compatibility's sake,
8483 * reject any unknown flag.
8484 */
8485 return KERN_INVALID_VALUE;
8486 }
8487 if (!vm_lopage_needed) {
8488 cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
8489 }
8490
8491 if (cntrl_flags & UPL_NEED_32BIT_ADDR) {
8492 if ((cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE)) {
8493 return KERN_INVALID_VALUE;
8494 }
8495
8496 if (object->phys_contiguous) {
8497 if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) {
8498 return KERN_INVALID_ADDRESS;
8499 }
8500
8501 if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) {
8502 return KERN_INVALID_ADDRESS;
8503 }
8504 }
8505 }
8506 if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
8507 no_zero_fill = TRUE;
8508 }
8509
8510 if (cntrl_flags & UPL_COPYOUT_FROM) {
8511 prot = VM_PROT_READ;
8512 } else {
8513 prot = VM_PROT_READ | VM_PROT_WRITE;
8514 }
8515
8516 if ((!object->internal) && (object->paging_offset != 0)) {
8517 panic("vm_object_iopl_request: external object with non-zero paging offset");
8518 }
8519
8520 VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, DBG_VM_IOPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, prot, 0);
8521
8522 #if CONFIG_IOSCHED || UPL_DEBUG
8523 if ((object->io_tracking && !is_kernel_object(object)) || upl_debug_enabled) {
8524 io_tracking_flag |= UPL_CREATE_IO_TRACKING;
8525 }
8526 #endif
8527
8528 #if CONFIG_IOSCHED
8529 if (object->io_tracking) {
8530 /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */
8531 if (!is_kernel_object(object)) {
8532 io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
8533 }
8534 }
8535 #endif
8536
8537 if (object->phys_contiguous) {
8538 psize = PAGE_SIZE;
8539 } else {
8540 psize = size;
8541
8542 dw_count = 0;
8543 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
8544 dwp_start = vm_page_delayed_work_get_ctx();
8545 if (dwp_start == NULL) {
8546 dwp_start = &dw_array;
8547 dw_limit = 1;
8548 dwp_finish_ctx = FALSE;
8549 }
8550
8551 dwp = dwp_start;
8552 }
8553
8554 if (cntrl_flags & UPL_SET_INTERNAL) {
8555 upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
8556 user_page_list = size ? upl->page_list : NULL;
8557 } else {
8558 upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
8559 }
8560 if (user_page_list) {
8561 user_page_list[0].device = FALSE;
8562 }
8563 *upl_ptr = upl;
8564
8565 if (cntrl_flags & UPL_NOZEROFILLIO) {
8566 DTRACE_VM4(upl_nozerofillio,
8567 vm_object_t, object,
8568 vm_object_offset_t, offset,
8569 upl_size_t, size,
8570 upl_t, upl);
8571 }
8572
8573 upl->map_object = object;
8574 upl->u_offset = original_offset;
8575 upl->u_size = original_size;
8576
8577 size_in_pages = size / PAGE_SIZE;
8578
8579 if (is_kernel_object(object) &&
8580 !(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS))) {
8581 upl->flags |= UPL_KERNEL_OBJECT;
8582 #if UPL_DEBUG
8583 vm_object_lock(object);
8584 #else
8585 vm_object_lock_shared(object);
8586 #endif
8587 } else {
8588 vm_object_lock(object);
8589 if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
8590 token = thread_priority_floor_start();
8591 vm_object_pl_req_begin(object);
8592 need_pl_req_end = true;
8593 }
8594 vm_object_activity_begin(object);
8595 }
8596 /*
8597 * paging in progress also protects the paging_offset
8598 */
8599 upl->u_offset = original_offset + object->paging_offset;
8600
8601 if (cntrl_flags & UPL_BLOCK_ACCESS) {
8602 /*
8603 * The user requested that access to the pages in this UPL
8604 * be blocked until the UPL is commited or aborted.
8605 */
8606 upl->flags |= UPL_ACCESS_BLOCKED;
8607 }
8608
8609 #if CONFIG_IOSCHED || UPL_DEBUG
8610 if ((upl->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
8611 vm_object_activity_begin(object);
8612 queue_enter(&object->uplq, upl, upl_t, uplq);
8613 }
8614 #endif
8615
8616 if (object->phys_contiguous) {
8617 if (upl->flags & UPL_ACCESS_BLOCKED) {
8618 assert(!object->blocked_access);
8619 object->blocked_access = TRUE;
8620 }
8621 if (need_pl_req_end) {
8622 vm_object_pl_req_end(object);
8623 thread_priority_floor_end(&token);
8624 }
8625 vm_object_unlock(object);
8626
8627 /*
8628 * don't need any shadow mappings for this one
8629 * since it is already I/O memory
8630 */
8631 upl->flags |= UPL_DEVICE_MEMORY;
8632
8633 upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1) >> PAGE_SHIFT);
8634
8635 if (user_page_list) {
8636 user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset) >> PAGE_SHIFT);
8637 user_page_list[0].device = TRUE;
8638 }
8639 if (page_list_count != NULL) {
8640 if (upl->flags & UPL_INTERNAL) {
8641 *page_list_count = 0;
8642 } else {
8643 *page_list_count = 1;
8644 }
8645 }
8646
8647 VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, DBG_VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
8648 if (task != NULL) {
8649 ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
8650 }
8651 counter_add(&vm_page_grab_count_iopl, page_grab_count);
8652 return KERN_SUCCESS;
8653 }
8654 if (!is_kernel_object(object) && object != compressor_object) {
8655 /*
8656 * Protect user space from future COW operations
8657 */
8658 #if VM_OBJECT_TRACKING_OP_TRUESHARE
8659 if (!object->true_share &&
8660 vm_object_tracking_btlog) {
8661 btlog_record(vm_object_tracking_btlog, object,
8662 VM_OBJECT_TRACKING_OP_TRUESHARE,
8663 btref_get(__builtin_frame_address(0), 0));
8664 }
8665 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
8666
8667 vm_object_lock_assert_exclusive(object);
8668 VM_OBJECT_SET_TRUE_SHARE(object, TRUE);
8669
8670 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
8671 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
8672 }
8673 }
8674
8675 if (!(cntrl_flags & UPL_COPYOUT_FROM) &&
8676 object->vo_copy != VM_OBJECT_NULL) {
8677 /*
8678 * Honor copy-on-write obligations
8679 *
8680 * The caller is gathering these pages and
8681 * might modify their contents. We need to
8682 * make sure that the copy object has its own
8683 * private copies of these pages before we let
8684 * the caller modify them.
8685 *
8686 * NOTE: someone else could map the original object
8687 * after we've done this copy-on-write here, and they
8688 * could then see an inconsistent picture of the memory
8689 * while it's being modified via the UPL. To prevent this,
8690 * we would have to block access to these pages until the
8691 * UPL is released. We could use the UPL_BLOCK_ACCESS
8692 * code path for that...
8693 */
8694 vm_object_update(object,
8695 offset,
8696 size,
8697 NULL,
8698 NULL,
8699 FALSE, /* should_return */
8700 MEMORY_OBJECT_COPY_SYNC,
8701 VM_PROT_NO_CHANGE);
8702 VM_PAGEOUT_DEBUG(iopl_cow, 1);
8703 VM_PAGEOUT_DEBUG(iopl_cow_pages, (size >> PAGE_SHIFT));
8704 }
8705 if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
8706 object->purgable != VM_PURGABLE_VOLATILE &&
8707 object->purgable != VM_PURGABLE_EMPTY &&
8708 object->vo_copy == NULL &&
8709 size == object->vo_size &&
8710 offset == 0 &&
8711 object->shadow == NULL &&
8712 object->pager == NULL) {
8713 if (object->resident_page_count == size_in_pages) {
8714 assert(object != compressor_object);
8715 assert(!is_kernel_object(object));
8716 fast_path_full_req = TRUE;
8717 } else if (object->resident_page_count == 0) {
8718 assert(object != compressor_object);
8719 assert(!is_kernel_object(object));
8720 fast_path_empty_req = TRUE;
8721 set_cache_attr_needed = TRUE;
8722 }
8723 }
8724
8725 if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {
8726 interruptible = THREAD_ABORTSAFE;
8727 } else {
8728 interruptible = THREAD_UNINT;
8729 }
8730
8731 entry = 0;
8732
8733 xfer_size = size;
8734 dst_offset = offset;
8735
8736 if (fast_path_full_req) {
8737 if (vm_object_iopl_wire_full(object, upl, user_page_list, cntrl_flags, tag) == TRUE) {
8738 goto finish;
8739 }
8740 /*
8741 * we couldn't complete the processing of this request on the fast path
8742 * so fall through to the slow path and finish up
8743 */
8744 } else if (fast_path_empty_req) {
8745 if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
8746 ret = KERN_MEMORY_ERROR;
8747 goto return_err;
8748 }
8749 ret = vm_object_iopl_wire_empty(object, upl, user_page_list,
8750 cntrl_flags, tag, &dst_offset, size_in_pages, &page_grab_count);
8751
8752 if (ret) {
8753 free_wired_pages = TRUE;
8754 goto return_err;
8755 }
8756 goto finish;
8757 }
8758
8759 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
8760 fault_info.lo_offset = offset;
8761 fault_info.hi_offset = offset + xfer_size;
8762 fault_info.mark_zf_absent = TRUE;
8763 fault_info.interruptible = interruptible;
8764 fault_info.batch_pmap_op = TRUE;
8765
8766 while (xfer_size) {
8767 vm_fault_return_t result;
8768
8769 dwp->dw_mask = 0;
8770
8771 if (fast_path_full_req) {
8772 /*
8773 * if we get here, it means that we ran into a page
8774 * state we couldn't handle in the fast path and
8775 * bailed out to the slow path... since the order
8776 * we look at pages is different between the 2 paths,
8777 * the following check is needed to determine whether
8778 * this page was already processed in the fast path
8779 */
8780 if (bitmap_test(upl->lite_list, entry)) {
8781 goto skip_page;
8782 }
8783 }
8784 dst_page = vm_page_lookup(object, dst_offset);
8785
8786 if (dst_page == VM_PAGE_NULL ||
8787 dst_page->vmp_busy ||
8788 VMP_ERROR_GET(dst_page) ||
8789 dst_page->vmp_restart ||
8790 dst_page->vmp_absent ||
8791 vm_page_is_fictitious(dst_page)) {
8792 if (is_kernel_object(object)) {
8793 panic("vm_object_iopl_request: missing/bad page in kernel object");
8794 }
8795 if (object == compressor_object) {
8796 panic("vm_object_iopl_request: missing/bad page in compressor object");
8797 }
8798
8799 if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
8800 ret = KERN_MEMORY_ERROR;
8801 goto return_err;
8802 }
8803
8804 if (dst_page != VM_PAGE_NULL &&
8805 dst_page->vmp_busy) {
8806 wait_result_t wait_result;
8807 vm_object_lock_assert_exclusive(object);
8808 wait_result = vm_page_sleep(object, dst_page,
8809 interruptible, LCK_SLEEP_DEFAULT);
8810 if (wait_result == THREAD_AWAKENED ||
8811 wait_result == THREAD_RESTART) {
8812 continue;
8813 }
8814 ret = MACH_SEND_INTERRUPTED;
8815 goto return_err;
8816 }
8817
8818 set_cache_attr_needed = TRUE;
8819
8820 /*
8821 * We just looked up the page and the result remains valid
8822 * until the object lock is release, so send it to
8823 * vm_fault_page() (as "dst_page"), to avoid having to
8824 * look it up again there.
8825 */
8826 caller_lookup = TRUE;
8827
8828 do {
8829 vm_page_t top_page;
8830 kern_return_t error_code;
8831
8832 fault_info.cluster_size = xfer_size;
8833 vm_object_paging_begin(object);
8834
8835 result = vm_fault_page(object, dst_offset,
8836 prot | VM_PROT_WRITE, FALSE,
8837 caller_lookup,
8838 &prot, &dst_page, &top_page,
8839 (int *)0,
8840 &error_code, no_zero_fill,
8841 &fault_info);
8842
8843 /* our lookup is no longer valid at this point */
8844 caller_lookup = FALSE;
8845
8846 switch (result) {
8847 case VM_FAULT_SUCCESS:
8848 page_grab_count++;
8849
8850 if (!dst_page->vmp_absent) {
8851 vm_page_wakeup_done(object, dst_page);
8852 } else {
8853 /*
8854 * we only get back an absent page if we
8855 * requested that it not be zero-filled
8856 * because we are about to fill it via I/O
8857 *
8858 * absent pages should be left BUSY
8859 * to prevent them from being faulted
8860 * into an address space before we've
8861 * had a chance to complete the I/O on
8862 * them since they may contain info that
8863 * shouldn't be seen by the faulting task
8864 */
8865 }
8866 /*
8867 * Release paging references and
8868 * top-level placeholder page, if any.
8869 */
8870 if (top_page != VM_PAGE_NULL) {
8871 vm_object_t local_object;
8872
8873 local_object = VM_PAGE_OBJECT(top_page);
8874
8875 /*
8876 * comparing 2 packed pointers
8877 */
8878 if (top_page->vmp_object != dst_page->vmp_object) {
8879 vm_object_lock(local_object);
8880 VM_PAGE_FREE(top_page);
8881 vm_object_paging_end(local_object);
8882 vm_object_unlock(local_object);
8883 } else {
8884 VM_PAGE_FREE(top_page);
8885 vm_object_paging_end(local_object);
8886 }
8887 }
8888 vm_object_paging_end(object);
8889 break;
8890
8891 case VM_FAULT_RETRY:
8892 vm_object_lock(object);
8893 break;
8894
8895 case VM_FAULT_MEMORY_SHORTAGE:
8896 OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages);
8897
8898 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
8899
8900 if (vm_page_wait(interruptible)) {
8901 OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
8902
8903 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
8904 vm_object_lock(object);
8905
8906 break;
8907 }
8908 OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
8909
8910 VM_DEBUG_EVENT(vm_iopl_page_wait, DBG_VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
8911 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_FAULT_OBJIOPLREQ_MEMORY_SHORTAGE), 0 /* arg */);
8912 OS_FALLTHROUGH;
8913
8914 case VM_FAULT_INTERRUPTED:
8915 error_code = MACH_SEND_INTERRUPTED;
8916 OS_FALLTHROUGH;
8917 case VM_FAULT_MEMORY_ERROR:
8918 memory_error:
8919 ret = (error_code ? error_code: KERN_MEMORY_ERROR);
8920
8921 vm_object_lock(object);
8922 goto return_err;
8923
8924 case VM_FAULT_SUCCESS_NO_VM_PAGE:
8925 /* success but no page: fail */
8926 vm_object_paging_end(object);
8927 vm_object_unlock(object);
8928 goto memory_error;
8929
8930 default:
8931 panic("vm_object_iopl_request: unexpected error"
8932 " 0x%x from vm_fault_page()\n", result);
8933 }
8934 } while (result != VM_FAULT_SUCCESS);
8935 }
8936 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
8937
8938 if (upl->flags & UPL_KERNEL_OBJECT) {
8939 goto record_phys_addr;
8940 }
8941
8942 if (dst_page->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
8943 dst_page->vmp_busy = TRUE;
8944 goto record_phys_addr;
8945 }
8946
8947 if (dst_page->vmp_cleaning) {
8948 /*
8949 * Someone else is cleaning this page in place.
8950 * In theory, we should be able to proceed and use this
8951 * page but they'll probably end up clearing the "busy"
8952 * bit on it in upl_commit_range() but they didn't set
8953 * it, so they would clear our "busy" bit and open
8954 * us to race conditions.
8955 * We'd better wait for the cleaning to complete and
8956 * then try again.
8957 */
8958 VM_PAGEOUT_DEBUG(vm_object_iopl_request_sleep_for_cleaning, 1);
8959 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_EXCLUSIVE);
8960 continue;
8961 }
8962 if (dst_page->vmp_laundry) {
8963 vm_pageout_steal_laundry(dst_page, FALSE);
8964 }
8965
8966 if ((cntrl_flags & UPL_NEED_32BIT_ADDR) &&
8967 phys_page >= (max_valid_dma_address >> PAGE_SHIFT)) {
8968 vm_page_t new_page;
8969 int refmod;
8970
8971 /*
8972 * support devices that can't DMA above 32 bits
8973 * by substituting pages from a pool of low address
8974 * memory for any pages we find above the 4G mark
8975 * can't substitute if the page is already wired because
8976 * we don't know whether that physical address has been
8977 * handed out to some other 64 bit capable DMA device to use
8978 */
8979 if (VM_PAGE_WIRED(dst_page)) {
8980 ret = KERN_PROTECTION_FAILURE;
8981 goto return_err;
8982 }
8983
8984 new_page = vm_page_grablo(VM_PAGE_GRAB_OPTIONS_NONE);
8985
8986 if (new_page == VM_PAGE_NULL) {
8987 ret = KERN_RESOURCE_SHORTAGE;
8988 goto return_err;
8989 }
8990 /*
8991 * from here until the vm_page_replace completes
8992 * we musn't drop the object lock... we don't
8993 * want anyone refaulting this page in and using
8994 * it after we disconnect it... we want the fault
8995 * to find the new page being substituted.
8996 */
8997 if (dst_page->vmp_pmapped) {
8998 refmod = pmap_disconnect(phys_page);
8999 } else {
9000 refmod = 0;
9001 }
9002
9003 if (!dst_page->vmp_absent) {
9004 vm_page_copy(dst_page, new_page);
9005 }
9006
9007 new_page->vmp_reference = dst_page->vmp_reference;
9008 new_page->vmp_dirty = dst_page->vmp_dirty;
9009 new_page->vmp_absent = dst_page->vmp_absent;
9010
9011 if (refmod & VM_MEM_REFERENCED) {
9012 new_page->vmp_reference = TRUE;
9013 }
9014 if (refmod & VM_MEM_MODIFIED) {
9015 SET_PAGE_DIRTY(new_page, FALSE);
9016 }
9017
9018 vm_page_replace(new_page, object, dst_offset);
9019
9020 dst_page = new_page;
9021 /*
9022 * vm_page_grablo returned the page marked
9023 * BUSY... we don't need a PAGE_WAKEUP_DONE
9024 * here, because we've never dropped the object lock
9025 */
9026 if (!dst_page->vmp_absent) {
9027 dst_page->vmp_busy = FALSE;
9028 }
9029
9030 phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
9031 }
9032 if (!dst_page->vmp_busy) {
9033 /*
9034 * Specify that we're wiring the page for I/O, which also means
9035 * that the delayed work handler may return KERN_PROTECTION_FAILURE
9036 * on certain configs if a page's mapping state doesn't allow I/O
9037 * wiring. For the specifc case in which we're creating an IOPL
9038 * against an executable mapping, the buffer copy performed by
9039 * vm_map_create_upl() should prevent failure here, but we still
9040 * want to gracefully fail here if someone attempts to I/O-wire
9041 * an executable page through a named entry or non-executable
9042 * alias mapping.
9043 */
9044 dwp->dw_mask |= (DW_vm_page_wire | DW_vm_page_iopl_wire);
9045 if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
9046 dwp->dw_mask |= DW_vm_page_iopl_wire_write;
9047 }
9048 }
9049
9050 if (cntrl_flags & UPL_BLOCK_ACCESS) {
9051 /*
9052 * Mark the page "busy" to block any future page fault
9053 * on this page in addition to wiring it.
9054 * We'll also remove the mapping
9055 * of all these pages before leaving this routine.
9056 */
9057 assert(!vm_page_is_fictitious(dst_page));
9058 dst_page->vmp_busy = TRUE;
9059 }
9060 /*
9061 * expect the page to be used
9062 * page queues lock must be held to set 'reference'
9063 */
9064 dwp->dw_mask |= DW_set_reference;
9065
9066 if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
9067 SET_PAGE_DIRTY(dst_page, TRUE);
9068 /*
9069 * Page belonging to a code-signed object is about to
9070 * be written. Mark it tainted and disconnect it from
9071 * all pmaps so processes have to fault it back in and
9072 * deal with the tainted bit.
9073 */
9074 if (object->code_signed && dst_page->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
9075 dst_page->vmp_cs_tainted = VMP_CS_ALL_TRUE;
9076 vm_page_iopl_tainted++;
9077 if (dst_page->vmp_pmapped) {
9078 int refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
9079 if (refmod & VM_MEM_REFERENCED) {
9080 dst_page->vmp_reference = TRUE;
9081 }
9082 }
9083 }
9084 }
9085 if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {
9086 pmap_sync_page_attributes_phys(phys_page);
9087 dst_page->vmp_written_by_kernel = FALSE;
9088 }
9089
9090 record_phys_addr:
9091 if (dst_page->vmp_busy) {
9092 upl->flags |= UPL_HAS_BUSY;
9093 }
9094
9095 bitmap_set(upl->lite_list, entry);
9096
9097 if (phys_page > upl->highest_page) {
9098 upl->highest_page = phys_page;
9099 }
9100
9101 if (user_page_list) {
9102 user_page_list[entry].phys_addr = phys_page;
9103 user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
9104 user_page_list[entry].absent = dst_page->vmp_absent;
9105 user_page_list[entry].dirty = dst_page->vmp_dirty;
9106 user_page_list[entry].precious = dst_page->vmp_precious;
9107 user_page_list[entry].device = FALSE;
9108 user_page_list[entry].needed = FALSE;
9109 if (dst_page->vmp_clustered == TRUE) {
9110 user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
9111 } else {
9112 user_page_list[entry].speculative = FALSE;
9113 }
9114 user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
9115 user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
9116 user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;
9117 user_page_list[entry].mark = FALSE;
9118 }
9119 if (!is_kernel_object(object) && object != compressor_object) {
9120 /*
9121 * someone is explicitly grabbing this page...
9122 * update clustered and speculative state
9123 *
9124 */
9125 if (dst_page->vmp_clustered) {
9126 VM_PAGE_CONSUME_CLUSTERED(dst_page);
9127 }
9128 }
9129 skip_page:
9130 entry++;
9131 dst_offset += PAGE_SIZE_64;
9132 xfer_size -= PAGE_SIZE;
9133
9134 if (dwp->dw_mask) {
9135 VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
9136
9137 if (dw_count >= dw_limit) {
9138 ret = vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
9139
9140 dwp = dwp_start;
9141 dw_count = 0;
9142 if (ret != KERN_SUCCESS) {
9143 goto return_err;
9144 }
9145 }
9146 }
9147 }
9148 assert(entry == size_in_pages);
9149
9150 if (dw_count) {
9151 ret = vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
9152 dwp = dwp_start;
9153 dw_count = 0;
9154 if (ret != KERN_SUCCESS) {
9155 goto return_err;
9156 }
9157 }
9158 finish:
9159 if (user_page_list && set_cache_attr_needed == TRUE) {
9160 vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE);
9161 }
9162
9163 if (page_list_count != NULL) {
9164 if (upl->flags & UPL_INTERNAL) {
9165 *page_list_count = 0;
9166 } else if (*page_list_count > size_in_pages) {
9167 *page_list_count = size_in_pages;
9168 }
9169 }
9170 vm_object_unlock(object);
9171
9172 if (cntrl_flags & UPL_BLOCK_ACCESS) {
9173 /*
9174 * We've marked all the pages "busy" so that future
9175 * page faults will block.
9176 * Now remove the mapping for these pages, so that they
9177 * can't be accessed without causing a page fault.
9178 */
9179 vm_object_pmap_protect(object, offset, (vm_object_size_t)size,
9180 PMAP_NULL,
9181 PAGE_SIZE,
9182 0, VM_PROT_NONE);
9183 vm_object_lock(object);
9184 assert(!object->blocked_access);
9185 object->blocked_access = TRUE;
9186 vm_object_unlock(object);
9187 }
9188
9189 VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, DBG_VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
9190 if (task != NULL) {
9191 ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
9192 }
9193 counter_add(&vm_page_grab_count_iopl, page_grab_count);
9194
9195 if (dwp_start && dwp_finish_ctx) {
9196 vm_page_delayed_work_finish_ctx(dwp_start);
9197 dwp_start = dwp = NULL;
9198 }
9199
9200 if (need_pl_req_end) {
9201 /* object should still be alive due to its "pl_req_in_progress" */
9202 vm_object_lock(object);
9203 vm_object_pl_req_end(object);
9204 vm_object_unlock(object);
9205 object = VM_OBJECT_NULL; /* object might no longer be valid */
9206 thread_priority_floor_end(&token);
9207 }
9208
9209 return KERN_SUCCESS;
9210
9211 return_err:
9212 dw_index = 0;
9213
9214 for (; offset < dst_offset; offset += PAGE_SIZE) {
9215 boolean_t need_unwire;
9216 bool need_wakeup;
9217
9218 dst_page = vm_page_lookup(object, offset);
9219
9220 if (dst_page == VM_PAGE_NULL) {
9221 panic("vm_object_iopl_request: Wired page missing.");
9222 }
9223
9224 /*
9225 * if we've already processed this page in an earlier
9226 * dw_do_work, we need to undo the wiring... we will
9227 * leave the dirty and reference bits on if they
9228 * were set, since we don't have a good way of knowing
9229 * what the previous state was and we won't get here
9230 * under any normal circumstances... we will always
9231 * clear BUSY and wakeup any waiters via vm_page_free
9232 * or PAGE_WAKEUP_DONE
9233 */
9234 need_unwire = TRUE;
9235
9236 need_wakeup = false;
9237 if (dw_count) {
9238 if ((dwp_start)[dw_index].dw_m == dst_page) {
9239 /*
9240 * still in the deferred work list
9241 * which means we haven't yet called
9242 * vm_page_wire on this page
9243 */
9244 need_unwire = FALSE;
9245
9246 if (dst_page->vmp_busy &&
9247 ((dwp_start)[dw_index].dw_mask & DW_clear_busy)) {
9248 /*
9249 * It's our own "busy" bit, so we need to clear it
9250 * now and wake up waiters below.
9251 */
9252 dst_page->vmp_busy = false;
9253 need_wakeup = true;
9254 }
9255
9256 dw_index++;
9257 dw_count--;
9258 }
9259 }
9260 vm_page_lock_queues();
9261
9262 if (dst_page->vmp_absent || free_wired_pages == TRUE) {
9263 vm_page_free(dst_page);
9264
9265 need_unwire = FALSE;
9266 } else {
9267 if (need_unwire == TRUE) {
9268 vm_page_unwire(dst_page, TRUE);
9269 }
9270 if (dst_page->vmp_busy) {
9271 /* not our "busy" or we would have cleared it above */
9272 assert(!need_wakeup);
9273 }
9274 if (need_wakeup) {
9275 assert(!dst_page->vmp_busy);
9276 vm_page_wakeup(object, dst_page);
9277 }
9278 }
9279 vm_page_unlock_queues();
9280
9281 if (need_unwire == TRUE) {
9282 counter_inc(&vm_statistics_reactivations);
9283 }
9284 }
9285 #if UPL_DEBUG
9286 upl->upl_state = 2;
9287 #endif
9288 if (!(upl->flags & UPL_KERNEL_OBJECT)) {
9289 vm_object_activity_end(object);
9290 vm_object_collapse(object, 0, TRUE);
9291 }
9292 vm_object_unlock(object);
9293 upl_destroy(upl);
9294 *upl_ptr = NULL;
9295
9296 VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, DBG_VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, ret, 0, 0);
9297 if (task != NULL) {
9298 ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
9299 }
9300 counter_add(&vm_page_grab_count_iopl, page_grab_count);
9301
9302 if (dwp_start && dwp_finish_ctx) {
9303 vm_page_delayed_work_finish_ctx(dwp_start);
9304 dwp_start = dwp = NULL;
9305 }
9306
9307 if (need_pl_req_end) {
9308 /* object should still be alive due to its "pl_req_in_progress" */
9309 vm_object_lock(object);
9310 vm_object_pl_req_end(object);
9311 vm_object_unlock(object);
9312 thread_priority_floor_end(&token);
9313 object = VM_OBJECT_NULL; /* object might no longer be valid */
9314 }
9315
9316 return ret;
9317 }
9318
9319 kern_return_t
9320 upl_transpose(
9321 upl_t upl1,
9322 upl_t upl2)
9323 {
9324 kern_return_t retval;
9325 boolean_t upls_locked;
9326 vm_object_t object1, object2;
9327
9328 /* LD: Should mapped UPLs be eligible for a transpose? */
9329 if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR) == UPL_VECTOR) || ((upl2->flags & UPL_VECTOR) == UPL_VECTOR)) {
9330 return KERN_INVALID_ARGUMENT;
9331 }
9332
9333 upls_locked = FALSE;
9334
9335 /*
9336 * Since we need to lock both UPLs at the same time,
9337 * avoid deadlocks by always taking locks in the same order.
9338 */
9339 if (upl1 < upl2) {
9340 upl_lock(upl1);
9341 upl_lock(upl2);
9342 } else {
9343 upl_lock(upl2);
9344 upl_lock(upl1);
9345 }
9346 upls_locked = TRUE; /* the UPLs will need to be unlocked */
9347
9348 object1 = upl1->map_object;
9349 object2 = upl2->map_object;
9350
9351 if (upl1->u_offset != 0 || upl2->u_offset != 0 ||
9352 upl1->u_size != upl2->u_size) {
9353 /*
9354 * We deal only with full objects, not subsets.
9355 * That's because we exchange the entire backing store info
9356 * for the objects: pager, resident pages, etc... We can't do
9357 * only part of it.
9358 */
9359 retval = KERN_INVALID_VALUE;
9360 goto done;
9361 }
9362
9363 /*
9364 * Tranpose the VM objects' backing store.
9365 */
9366 retval = vm_object_transpose(object1, object2,
9367 upl_adjusted_size(upl1, PAGE_MASK));
9368
9369 if (retval == KERN_SUCCESS) {
9370 /*
9371 * Make each UPL point to the correct VM object, i.e. the
9372 * object holding the pages that the UPL refers to...
9373 */
9374 #if CONFIG_IOSCHED || UPL_DEBUG
9375 if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
9376 vm_object_lock(object1);
9377 vm_object_lock(object2);
9378 }
9379 if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
9380 queue_remove(&object1->uplq, upl1, upl_t, uplq);
9381 }
9382 if ((upl2->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
9383 queue_remove(&object2->uplq, upl2, upl_t, uplq);
9384 }
9385 #endif
9386 upl1->map_object = object2;
9387 upl2->map_object = object1;
9388
9389 #if CONFIG_IOSCHED || UPL_DEBUG
9390 if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
9391 queue_enter(&object2->uplq, upl1, upl_t, uplq);
9392 }
9393 if ((upl2->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
9394 queue_enter(&object1->uplq, upl2, upl_t, uplq);
9395 }
9396 if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
9397 vm_object_unlock(object2);
9398 vm_object_unlock(object1);
9399 }
9400 #endif
9401 }
9402
9403 done:
9404 /*
9405 * Cleanup.
9406 */
9407 if (upls_locked) {
9408 upl_unlock(upl1);
9409 upl_unlock(upl2);
9410 upls_locked = FALSE;
9411 }
9412
9413 return retval;
9414 }
9415
9416 void
9417 upl_range_needed(
9418 upl_t upl,
9419 int index,
9420 int count)
9421 {
9422 int size_in_pages;
9423
9424 if (!(upl->flags & UPL_INTERNAL) || count <= 0) {
9425 return;
9426 }
9427
9428 size_in_pages = upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE;
9429
9430 while (count-- && index < size_in_pages) {
9431 upl->page_list[index++].needed = TRUE;
9432 }
9433 }
9434
9435
9436 /*
9437 * Reserve of virtual addresses in the kernel address space.
9438 * We need to map the physical pages in the kernel, so that we
9439 * can call the code-signing or slide routines with a kernel
9440 * virtual address. We keep this pool of pre-allocated kernel
9441 * virtual addresses so that we don't have to scan the kernel's
9442 * virtaul address space each time we need to work with
9443 * a physical page.
9444 */
9445 SIMPLE_LOCK_DECLARE(vm_paging_lock, 0);
9446 #define VM_PAGING_NUM_PAGES 64
9447 SECURITY_READ_ONLY_LATE(vm_offset_t) vm_paging_base_address = 0;
9448 bool vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
9449 int vm_paging_max_index = 0;
9450 int vm_paging_page_waiter = 0;
9451 int vm_paging_page_waiter_total = 0;
9452
9453 unsigned long vm_paging_no_kernel_page = 0;
9454 unsigned long vm_paging_objects_mapped = 0;
9455 unsigned long vm_paging_pages_mapped = 0;
9456 unsigned long vm_paging_objects_mapped_slow = 0;
9457 unsigned long vm_paging_pages_mapped_slow = 0;
9458
9459 __startup_func
9460 static void
9461 vm_paging_map_init(void)
9462 {
9463 kmem_alloc(kernel_map, &vm_paging_base_address,
9464 ptoa(VM_PAGING_NUM_PAGES),
9465 KMA_DATA_SHARED | KMA_NOFAIL | KMA_KOBJECT | KMA_PERMANENT | KMA_PAGEABLE,
9466 VM_KERN_MEMORY_NONE);
9467 }
9468 STARTUP(ZALLOC, STARTUP_RANK_LAST, vm_paging_map_init);
9469
9470 /*
9471 * vm_paging_map_object:
9472 * Maps part of a VM object's pages in the kernel
9473 * virtual address space, using the pre-allocated
9474 * kernel virtual addresses, if possible.
9475 * Context:
9476 * The VM object is locked. This lock will get
9477 * dropped and re-acquired though, so the caller
9478 * must make sure the VM object is kept alive
9479 * (by holding a VM map that has a reference
9480 * on it, for example, or taking an extra reference).
9481 * The page should also be kept busy to prevent
9482 * it from being reclaimed.
9483 */
9484 kern_return_t
9485 vm_paging_map_object(
9486 vm_page_t page,
9487 vm_object_t object,
9488 vm_object_offset_t offset,
9489 vm_prot_t protection,
9490 boolean_t can_unlock_object,
9491 vm_map_size_t *size, /* IN/OUT */
9492 vm_map_offset_t *address, /* OUT */
9493 boolean_t *need_unmap) /* OUT */
9494 {
9495 kern_return_t kr;
9496 vm_map_offset_t page_map_offset;
9497 vm_map_size_t map_size;
9498 vm_object_offset_t object_offset;
9499 int i;
9500
9501 if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
9502 /* use permanent 1-to-1 kernel mapping of physical memory ? */
9503 *address = (vm_map_offset_t)
9504 phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT);
9505 *need_unmap = FALSE;
9506 return KERN_SUCCESS;
9507
9508 assert(page->vmp_busy);
9509 /*
9510 * Use one of the pre-allocated kernel virtual addresses
9511 * and just enter the VM page in the kernel address space
9512 * at that virtual address.
9513 */
9514 simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
9515
9516 /*
9517 * Try and find an available kernel virtual address
9518 * from our pre-allocated pool.
9519 */
9520 page_map_offset = 0;
9521 for (;;) {
9522 for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
9523 if (vm_paging_page_inuse[i] == FALSE) {
9524 page_map_offset =
9525 vm_paging_base_address +
9526 (i * PAGE_SIZE);
9527 break;
9528 }
9529 }
9530 if (page_map_offset != 0) {
9531 /* found a space to map our page ! */
9532 break;
9533 }
9534
9535 if (can_unlock_object) {
9536 /*
9537 * If we can afford to unlock the VM object,
9538 * let's take the slow path now...
9539 */
9540 break;
9541 }
9542 /*
9543 * We can't afford to unlock the VM object, so
9544 * let's wait for a space to become available...
9545 */
9546 vm_paging_page_waiter_total++;
9547 vm_paging_page_waiter++;
9548 kr = assert_wait((event_t)&vm_paging_page_waiter, THREAD_UNINT);
9549 if (kr == THREAD_WAITING) {
9550 simple_unlock(&vm_paging_lock);
9551 kr = thread_block(THREAD_CONTINUE_NULL);
9552 simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
9553 }
9554 vm_paging_page_waiter--;
9555 /* ... and try again */
9556 }
9557
9558 if (page_map_offset != 0) {
9559 /*
9560 * We found a kernel virtual address;
9561 * map the physical page to that virtual address.
9562 */
9563 if (i > vm_paging_max_index) {
9564 vm_paging_max_index = i;
9565 }
9566 vm_paging_page_inuse[i] = TRUE;
9567 simple_unlock(&vm_paging_lock);
9568
9569 page->vmp_pmapped = TRUE;
9570
9571 /*
9572 * Keep the VM object locked over the PMAP_ENTER
9573 * and the actual use of the page by the kernel,
9574 * or this pmap mapping might get undone by a
9575 * vm_object_pmap_protect() call...
9576 */
9577 kr = pmap_enter_check(kernel_pmap,
9578 page_map_offset,
9579 page,
9580 protection,
9581 VM_PROT_NONE,
9582 TRUE);
9583 assert(kr == KERN_SUCCESS);
9584 vm_paging_objects_mapped++;
9585 vm_paging_pages_mapped++;
9586 *address = page_map_offset;
9587 *need_unmap = TRUE;
9588
9589 #if KASAN
9590 kasan_notify_address(page_map_offset, PAGE_SIZE);
9591 #endif
9592
9593 /* all done and mapped, ready to use ! */
9594 return KERN_SUCCESS;
9595 }
9596
9597 /*
9598 * We ran out of pre-allocated kernel virtual
9599 * addresses. Just map the page in the kernel
9600 * the slow and regular way.
9601 */
9602 vm_paging_no_kernel_page++;
9603 simple_unlock(&vm_paging_lock);
9604 }
9605
9606 if (!can_unlock_object) {
9607 *address = 0;
9608 *size = 0;
9609 *need_unmap = FALSE;
9610 return KERN_NOT_SUPPORTED;
9611 }
9612
9613 object_offset = vm_object_trunc_page(offset);
9614 map_size = vm_map_round_page(*size,
9615 VM_MAP_PAGE_MASK(kernel_map));
9616
9617 /*
9618 * Try and map the required range of the object
9619 * in the kernel_map. Given that allocation is
9620 * for pageable memory, it shouldn't contain
9621 * pointers and is mapped into the data range.
9622 */
9623
9624 vm_object_reference_locked(object); /* for the map entry */
9625 vm_object_unlock(object);
9626
9627 kr = vm_map_enter(kernel_map,
9628 address,
9629 map_size,
9630 0,
9631 VM_MAP_KERNEL_FLAGS_DATA_SHARED_ANYWHERE(),
9632 object,
9633 object_offset,
9634 FALSE,
9635 protection,
9636 VM_PROT_ALL,
9637 VM_INHERIT_NONE);
9638 if (kr != KERN_SUCCESS) {
9639 *address = 0;
9640 *size = 0;
9641 *need_unmap = FALSE;
9642 vm_object_deallocate(object); /* for the map entry */
9643 vm_object_lock(object);
9644 return kr;
9645 }
9646
9647 *size = map_size;
9648
9649 /*
9650 * Enter the mapped pages in the page table now.
9651 */
9652 vm_object_lock(object);
9653 /*
9654 * VM object must be kept locked from before PMAP_ENTER()
9655 * until after the kernel is done accessing the page(s).
9656 * Otherwise, the pmap mappings in the kernel could be
9657 * undone by a call to vm_object_pmap_protect().
9658 */
9659
9660 for (page_map_offset = 0;
9661 map_size != 0;
9662 map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
9663 page = vm_page_lookup(object, offset + page_map_offset);
9664 if (page == VM_PAGE_NULL) {
9665 printf("vm_paging_map_object: no page !?");
9666 vm_object_unlock(object);
9667 vm_map_remove(kernel_map, *address, *size);
9668 *address = 0;
9669 *size = 0;
9670 *need_unmap = FALSE;
9671 vm_object_lock(object);
9672 return KERN_MEMORY_ERROR;
9673 }
9674 page->vmp_pmapped = TRUE;
9675
9676 kr = pmap_enter_check(kernel_pmap,
9677 *address + page_map_offset,
9678 page,
9679 protection,
9680 VM_PROT_NONE,
9681 TRUE);
9682 assert(kr == KERN_SUCCESS);
9683 #if KASAN
9684 kasan_notify_address(*address + page_map_offset, PAGE_SIZE);
9685 #endif
9686 }
9687
9688 vm_paging_objects_mapped_slow++;
9689 vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
9690
9691 *need_unmap = TRUE;
9692
9693 return KERN_SUCCESS;
9694 }
9695
9696 /*
9697 * vm_paging_unmap_object:
9698 * Unmaps part of a VM object's pages from the kernel
9699 * virtual address space.
9700 * Context:
9701 * The VM object is locked. This lock will get
9702 * dropped and re-acquired though.
9703 */
9704 void
9705 vm_paging_unmap_object(
9706 vm_object_t object,
9707 vm_map_offset_t start,
9708 vm_map_offset_t end)
9709 {
9710 int i;
9711
9712 if ((vm_paging_base_address == 0) ||
9713 (start < vm_paging_base_address) ||
9714 (end > (vm_paging_base_address
9715 + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
9716 /*
9717 * We didn't use our pre-allocated pool of
9718 * kernel virtual address. Deallocate the
9719 * virtual memory.
9720 */
9721 if (object != VM_OBJECT_NULL) {
9722 vm_object_unlock(object);
9723 }
9724 vm_map_remove(kernel_map, start, end);
9725 if (object != VM_OBJECT_NULL) {
9726 vm_object_lock(object);
9727 }
9728 } else {
9729 /*
9730 * We used a kernel virtual address from our
9731 * pre-allocated pool. Put it back in the pool
9732 * for next time.
9733 */
9734 assert(end - start == PAGE_SIZE);
9735 i = (int) ((start - vm_paging_base_address) >> PAGE_SHIFT);
9736 assert(i >= 0 && i < VM_PAGING_NUM_PAGES);
9737
9738 /* undo the pmap mapping */
9739 pmap_remove(kernel_pmap, start, end);
9740
9741 simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
9742 vm_paging_page_inuse[i] = FALSE;
9743 if (vm_paging_page_waiter) {
9744 thread_wakeup(&vm_paging_page_waiter);
9745 }
9746 simple_unlock(&vm_paging_lock);
9747 }
9748 }
9749
9750
9751 /*
9752 * page->vmp_object must be locked
9753 */
9754 void
9755 vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
9756 {
9757 if (!queues_locked) {
9758 vm_page_lockspin_queues();
9759 }
9760
9761 page->vmp_free_when_done = FALSE;
9762 /*
9763 * need to drop the laundry count...
9764 * we may also need to remove it
9765 * from the I/O paging queue...
9766 * vm_pageout_throttle_up handles both cases
9767 *
9768 * the laundry and pageout_queue flags are cleared...
9769 */
9770 vm_pageout_throttle_up(page);
9771
9772 if (!queues_locked) {
9773 vm_page_unlock_queues();
9774 }
9775 }
9776
9777 #define VECTOR_UPL_ELEMENTS_UPPER_LIMIT 64
9778
9779 upl_t
9780 vector_upl_create(vm_offset_t upl_offset, uint32_t max_upls)
9781 {
9782 upl_t upl;
9783
9784 assert(max_upls > 0);
9785 if (max_upls == 0) {
9786 return NULL;
9787 }
9788
9789 if (max_upls > VECTOR_UPL_ELEMENTS_UPPER_LIMIT) {
9790 max_upls = VECTOR_UPL_ELEMENTS_UPPER_LIMIT;
9791 }
9792 vector_upl_t vector_upl = kalloc_type(struct _vector_upl, typeof(vector_upl->upls[0]), max_upls, Z_WAITOK | Z_NOFAIL | Z_ZERO);
9793
9794 upl = upl_create(0, UPL_VECTOR, 0);
9795 upl->vector_upl = vector_upl;
9796 upl->u_offset = upl_offset;
9797 vector_upl->offset = upl_offset;
9798 vector_upl->max_upls = max_upls;
9799
9800 return upl;
9801 }
9802
9803 upl_size_t
9804 vector_upl_get_size(const upl_t upl)
9805 {
9806 if (!vector_upl_is_valid(upl)) {
9807 return upl_get_size(upl);
9808 } else {
9809 return round_page_32(upl->vector_upl->size);
9810 }
9811 }
9812
9813 uint32_t
9814 vector_upl_max_upls(const upl_t upl)
9815 {
9816 if (!vector_upl_is_valid(upl)) {
9817 return 0;
9818 }
9819 return ((vector_upl_t)(upl->vector_upl))->max_upls;
9820 }
9821
9822 void
9823 vector_upl_deallocate(upl_t upl)
9824 {
9825 vector_upl_t vector_upl = upl->vector_upl;
9826
9827 assert(vector_upl_is_valid(upl));
9828
9829 if (vector_upl->invalid_upls != vector_upl->num_upls) {
9830 panic("Deallocating non-empty Vectored UPL");
9831 }
9832 uint32_t max_upls = vector_upl->max_upls;
9833 kfree_type(struct upl_page_info, atop(vector_upl->size), vector_upl->pagelist);
9834 kfree_type(struct _vector_upl, typeof(vector_upl->upls[0]), max_upls, vector_upl);
9835 upl->vector_upl = NULL;
9836 }
9837
9838 boolean_t
9839 vector_upl_is_valid(upl_t upl)
9840 {
9841 return upl && (upl->flags & UPL_VECTOR) && upl->vector_upl;
9842 }
9843
9844 boolean_t
9845 vector_upl_set_subupl(upl_t upl, upl_t subupl, uint32_t io_size)
9846 {
9847 if (vector_upl_is_valid(upl)) {
9848 vector_upl_t vector_upl = upl->vector_upl;
9849
9850 if (vector_upl) {
9851 if (subupl) {
9852 if (io_size) {
9853 if (io_size < PAGE_SIZE) {
9854 io_size = PAGE_SIZE;
9855 }
9856 subupl->vector_upl = (void*)vector_upl;
9857 vector_upl->upls[vector_upl->num_upls++].elem = subupl;
9858 vector_upl->size += io_size;
9859 upl->u_size += io_size;
9860 } else {
9861 uint32_t i = 0, invalid_upls = 0;
9862 for (i = 0; i < vector_upl->num_upls; i++) {
9863 if (vector_upl->upls[i].elem == subupl) {
9864 break;
9865 }
9866 }
9867 if (i == vector_upl->num_upls) {
9868 panic("Trying to remove sub-upl when none exists");
9869 }
9870
9871 vector_upl->upls[i].elem = NULL;
9872 invalid_upls = os_atomic_inc(&(vector_upl)->invalid_upls,
9873 relaxed);
9874 if (invalid_upls == vector_upl->num_upls) {
9875 return TRUE;
9876 } else {
9877 return FALSE;
9878 }
9879 }
9880 } else {
9881 panic("vector_upl_set_subupl was passed a NULL upl element");
9882 }
9883 } else {
9884 panic("vector_upl_set_subupl was passed a non-vectored upl");
9885 }
9886 } else {
9887 panic("vector_upl_set_subupl was passed a NULL upl");
9888 }
9889
9890 return FALSE;
9891 }
9892
9893 void
9894 vector_upl_set_pagelist(upl_t upl)
9895 {
9896 if (vector_upl_is_valid(upl)) {
9897 uint32_t i = 0;
9898 vector_upl_t vector_upl = upl->vector_upl;
9899
9900 if (vector_upl) {
9901 vm_offset_t pagelist_size = 0, cur_upl_pagelist_size = 0;
9902
9903 vector_upl->pagelist = kalloc_type(struct upl_page_info,
9904 atop(vector_upl->size), Z_WAITOK);
9905
9906 for (i = 0; i < vector_upl->num_upls; i++) {
9907 cur_upl_pagelist_size = sizeof(struct upl_page_info) * upl_adjusted_size(vector_upl->upls[i].elem, PAGE_MASK) / PAGE_SIZE;
9908 bcopy(vector_upl->upls[i].elem->page_list, (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
9909 pagelist_size += cur_upl_pagelist_size;
9910 if (vector_upl->upls[i].elem->highest_page > upl->highest_page) {
9911 upl->highest_page = vector_upl->upls[i].elem->highest_page;
9912 }
9913 }
9914 assert( pagelist_size == (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)));
9915 } else {
9916 panic("vector_upl_set_pagelist was passed a non-vectored upl");
9917 }
9918 } else {
9919 panic("vector_upl_set_pagelist was passed a NULL upl");
9920 }
9921 }
9922
9923 upl_t
9924 vector_upl_subupl_byindex(upl_t upl, uint32_t index)
9925 {
9926 if (vector_upl_is_valid(upl)) {
9927 vector_upl_t vector_upl = upl->vector_upl;
9928 if (vector_upl) {
9929 if (index < vector_upl->num_upls) {
9930 return vector_upl->upls[index].elem;
9931 }
9932 } else {
9933 panic("vector_upl_subupl_byindex was passed a non-vectored upl");
9934 }
9935 }
9936 return NULL;
9937 }
9938
9939 upl_t
9940 vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
9941 {
9942 if (vector_upl_is_valid(upl)) {
9943 uint32_t i = 0;
9944 vector_upl_t vector_upl = upl->vector_upl;
9945
9946 if (vector_upl) {
9947 upl_t subupl = NULL;
9948 vector_upl_iostates_t subupl_state;
9949
9950 for (i = 0; i < vector_upl->num_upls; i++) {
9951 subupl = vector_upl->upls[i].elem;
9952 subupl_state = vector_upl->upls[i].iostate;
9953 if (*upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {
9954 /* We could have been passed an offset/size pair that belongs
9955 * to an UPL element that has already been committed/aborted.
9956 * If so, return NULL.
9957 */
9958 if (subupl == NULL) {
9959 return NULL;
9960 }
9961 if ((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {
9962 *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;
9963 if (*upl_size > subupl_state.size) {
9964 *upl_size = subupl_state.size;
9965 }
9966 }
9967 if (*upl_offset >= subupl_state.offset) {
9968 *upl_offset -= subupl_state.offset;
9969 } else if (i) {
9970 panic("Vector UPL offset miscalculation");
9971 }
9972 return subupl;
9973 }
9974 }
9975 } else {
9976 panic("vector_upl_subupl_byoffset was passed a non-vectored UPL");
9977 }
9978 }
9979 return NULL;
9980 }
9981
9982 void
9983 vector_upl_get_addr(upl_t upl, vm_offset_t *dst_addr)
9984 {
9985 if (vector_upl_is_valid(upl)) {
9986 vector_upl_t vector_upl = upl->vector_upl;
9987 if (vector_upl) {
9988 assert(vector_upl->dst_addr != 0);
9989 *dst_addr = vector_upl->dst_addr;
9990 } else {
9991 panic("%s was passed a non-vectored UPL", __func__);
9992 }
9993 } else {
9994 panic("%s was passed a null UPL", __func__);
9995 }
9996 }
9997
9998 void
9999 vector_upl_set_addr(upl_t upl, vm_offset_t dst_addr)
10000 {
10001 if (vector_upl_is_valid(upl)) {
10002 vector_upl_t vector_upl = upl->vector_upl;
10003 if (vector_upl) {
10004 if (dst_addr) {
10005 /* setting a new value: do not overwrite an old one */
10006 assert(vector_upl->dst_addr == 0);
10007 } else {
10008 /* resetting: make sure there was an old value */
10009 assert(vector_upl->dst_addr != 0);
10010 }
10011 vector_upl->dst_addr = dst_addr;
10012 } else {
10013 panic("%s was passed a non-vectored UPL", __func__);
10014 }
10015 } else {
10016 panic("%s was passed a NULL UPL", __func__);
10017 }
10018 }
10019
10020 void
10021 vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
10022 {
10023 if (vector_upl_is_valid(upl)) {
10024 uint32_t i = 0;
10025 vector_upl_t vector_upl = upl->vector_upl;
10026
10027 if (vector_upl) {
10028 for (i = 0; i < vector_upl->num_upls; i++) {
10029 if (vector_upl->upls[i].elem == subupl) {
10030 break;
10031 }
10032 }
10033
10034 if (i == vector_upl->num_upls) {
10035 panic("setting sub-upl iostate when none exists");
10036 }
10037
10038 vector_upl->upls[i].iostate.offset = offset;
10039 if (size < PAGE_SIZE) {
10040 size = PAGE_SIZE;
10041 }
10042 vector_upl->upls[i].iostate.size = size;
10043 } else {
10044 panic("vector_upl_set_iostate was passed a non-vectored UPL");
10045 }
10046 } else {
10047 panic("vector_upl_set_iostate was passed a NULL UPL");
10048 }
10049 }
10050
10051 void
10052 vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
10053 {
10054 if (vector_upl_is_valid(upl)) {
10055 uint32_t i = 0;
10056 vector_upl_t vector_upl = upl->vector_upl;
10057
10058 if (vector_upl) {
10059 for (i = 0; i < vector_upl->num_upls; i++) {
10060 if (vector_upl->upls[i].elem == subupl) {
10061 break;
10062 }
10063 }
10064
10065 if (i == vector_upl->num_upls) {
10066 panic("getting sub-upl iostate when none exists");
10067 }
10068
10069 *offset = vector_upl->upls[i].iostate.offset;
10070 *size = vector_upl->upls[i].iostate.size;
10071 } else {
10072 panic("vector_upl_get_iostate was passed a non-vectored UPL");
10073 }
10074 } else {
10075 panic("vector_upl_get_iostate was passed a NULL UPL");
10076 }
10077 }
10078
10079 void
10080 vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
10081 {
10082 if (vector_upl_is_valid(upl)) {
10083 vector_upl_t vector_upl = upl->vector_upl;
10084 if (vector_upl) {
10085 if (index < vector_upl->num_upls) {
10086 *offset = vector_upl->upls[index].iostate.offset;
10087 *size = vector_upl->upls[index].iostate.size;
10088 } else {
10089 *offset = *size = 0;
10090 }
10091 } else {
10092 panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL");
10093 }
10094 } else {
10095 panic("vector_upl_get_iostate_byindex was passed a NULL UPL");
10096 }
10097 }
10098
10099 void *
10100 upl_get_internal_vectorupl(upl_t upl)
10101 {
10102 return upl->vector_upl;
10103 }
10104
10105 upl_page_info_t *
10106 upl_get_internal_vectorupl_pagelist(upl_t upl)
10107 {
10108 return upl->vector_upl->pagelist;
10109 }
10110
10111 upl_page_info_t *
10112 upl_get_internal_page_list(upl_t upl)
10113 {
10114 return upl->vector_upl ? upl->vector_upl->pagelist : upl->page_list;
10115 }
10116
10117 void
10118 upl_clear_dirty(
10119 upl_t upl,
10120 boolean_t value)
10121 {
10122 if (value) {
10123 upl->flags |= UPL_CLEAR_DIRTY;
10124 } else {
10125 upl->flags &= ~UPL_CLEAR_DIRTY;
10126 }
10127 }
10128
10129 void
10130 upl_set_referenced(
10131 upl_t upl,
10132 boolean_t value)
10133 {
10134 upl_lock(upl);
10135 if (value) {
10136 upl->ext_ref_count++;
10137 } else {
10138 if (!upl->ext_ref_count) {
10139 panic("upl_set_referenced not %p", upl);
10140 }
10141 upl->ext_ref_count--;
10142 }
10143 upl_unlock(upl);
10144 }
10145
10146 void
10147 upl_set_map_exclusive(upl_t upl)
10148 {
10149 upl_lock(upl);
10150 while (upl->map_addr_owner) {
10151 upl->flags |= UPL_MAP_EXCLUSIVE_WAIT;
10152 upl_lock_sleep(upl, &upl->map_addr_owner, ctid_get_thread(upl->map_addr_owner));
10153 }
10154 upl->map_addr_owner = thread_get_ctid(current_thread());
10155 upl_unlock(upl);
10156 }
10157
10158 void
10159 upl_clear_map_exclusive(upl_t upl)
10160 {
10161 assert(upl->map_addr_owner == thread_get_ctid(current_thread()));
10162 upl_lock(upl);
10163 if (upl->flags & UPL_MAP_EXCLUSIVE_WAIT) {
10164 upl->flags &= ~UPL_MAP_EXCLUSIVE_WAIT;
10165 upl_wakeup(&upl->map_addr_owner);
10166 }
10167 upl->map_addr_owner = 0;
10168 upl_unlock(upl);
10169 }
10170
10171 #if CONFIG_IOSCHED
10172 void
10173 upl_set_blkno(
10174 upl_t upl,
10175 vm_offset_t upl_offset,
10176 int io_size,
10177 int64_t blkno)
10178 {
10179 int i, j;
10180 if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) {
10181 return;
10182 }
10183
10184 assert(upl->upl_reprio_info != 0);
10185 for (i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) {
10186 UPL_SET_REPRIO_INFO(upl, i, blkno, io_size);
10187 }
10188 }
10189 #endif
10190
10191 void inline
10192 memoryshot(unsigned int event, unsigned int control)
10193 {
10194 if (vm_debug_events) {
10195 KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control,
10196 vm_page_active_count, vm_page_inactive_count,
10197 vm_page_free_count, vm_page_speculative_count,
10198 vm_page_throttled_count);
10199 } else {
10200 (void) event;
10201 (void) control;
10202 }
10203 }
10204
10205 #ifdef MACH_BSD
10206
10207 boolean_t
10208 upl_device_page(upl_page_info_t *upl)
10209 {
10210 return UPL_DEVICE_PAGE(upl);
10211 }
10212 boolean_t
10213 upl_page_present(upl_page_info_t *upl, int index)
10214 {
10215 return UPL_PAGE_PRESENT(upl, index);
10216 }
10217 boolean_t
10218 upl_speculative_page(upl_page_info_t *upl, int index)
10219 {
10220 return UPL_SPECULATIVE_PAGE(upl, index);
10221 }
10222 boolean_t
10223 upl_dirty_page(upl_page_info_t *upl, int index)
10224 {
10225 return UPL_DIRTY_PAGE(upl, index);
10226 }
10227 boolean_t
10228 upl_valid_page(upl_page_info_t *upl, int index)
10229 {
10230 return UPL_VALID_PAGE(upl, index);
10231 }
10232 ppnum_t
10233 upl_phys_page(upl_page_info_t *upl, int index)
10234 {
10235 return UPL_PHYS_PAGE(upl, index);
10236 }
10237
10238 void
10239 upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v)
10240 {
10241 upl[index].mark = v;
10242 }
10243
10244 boolean_t
10245 upl_page_get_mark(upl_page_info_t *upl, int index)
10246 {
10247 return upl[index].mark;
10248 }
10249
10250 boolean_t
10251 upl_page_is_needed(upl_page_info_t *upl, int index)
10252 {
10253 return upl[index].needed;
10254 }
10255
10256 void
10257 vm_countdirtypages(void)
10258 {
10259 vm_page_t m;
10260 int dpages;
10261 int pgopages;
10262 int precpages;
10263
10264
10265 dpages = 0;
10266 pgopages = 0;
10267 precpages = 0;
10268
10269 vm_page_lock_queues();
10270 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
10271 do {
10272 if (m == (vm_page_t)0) {
10273 break;
10274 }
10275
10276 if (m->vmp_dirty) {
10277 dpages++;
10278 }
10279 if (m->vmp_free_when_done) {
10280 pgopages++;
10281 }
10282 if (m->vmp_precious) {
10283 precpages++;
10284 }
10285
10286 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
10287 m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
10288 if (m == (vm_page_t)0) {
10289 break;
10290 }
10291 } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m));
10292 vm_page_unlock_queues();
10293
10294 vm_page_lock_queues();
10295 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
10296 do {
10297 if (m == (vm_page_t)0) {
10298 break;
10299 }
10300
10301 dpages++;
10302 assert(m->vmp_dirty);
10303 assert(!m->vmp_free_when_done);
10304 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
10305 m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
10306 if (m == (vm_page_t)0) {
10307 break;
10308 }
10309 } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m));
10310 vm_page_unlock_queues();
10311
10312 vm_page_lock_queues();
10313 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
10314 do {
10315 if (m == (vm_page_t)0) {
10316 break;
10317 }
10318
10319 if (m->vmp_dirty) {
10320 dpages++;
10321 }
10322 if (m->vmp_free_when_done) {
10323 pgopages++;
10324 }
10325 if (m->vmp_precious) {
10326 precpages++;
10327 }
10328
10329 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
10330 m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
10331 if (m == (vm_page_t)0) {
10332 break;
10333 }
10334 } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m));
10335 vm_page_unlock_queues();
10336
10337 printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
10338
10339 dpages = 0;
10340 pgopages = 0;
10341 precpages = 0;
10342
10343 vm_page_lock_queues();
10344 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
10345
10346 do {
10347 if (m == (vm_page_t)0) {
10348 break;
10349 }
10350 if (m->vmp_dirty) {
10351 dpages++;
10352 }
10353 if (m->vmp_free_when_done) {
10354 pgopages++;
10355 }
10356 if (m->vmp_precious) {
10357 precpages++;
10358 }
10359
10360 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
10361 m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
10362 if (m == (vm_page_t)0) {
10363 break;
10364 }
10365 } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m));
10366 vm_page_unlock_queues();
10367
10368 printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
10369 }
10370 #endif /* MACH_BSD */
10371
10372
10373 #if CONFIG_IOSCHED
10374 int
10375 upl_get_cached_tier(upl_t upl)
10376 {
10377 assert(upl);
10378 if (upl->flags & UPL_TRACKED_BY_OBJECT) {
10379 return upl->upl_priority;
10380 }
10381 return -1;
10382 }
10383 #endif /* CONFIG_IOSCHED */
10384
10385
10386 void
10387 upl_callout_iodone(upl_t upl)
10388 {
10389 struct upl_io_completion *upl_ctx = upl->upl_iodone;
10390
10391 if (upl_ctx) {
10392 void (*iodone_func)(void *, int) = upl_ctx->io_done;
10393
10394 assert(upl_ctx->io_done);
10395
10396 (*iodone_func)(upl_ctx->io_context, upl_ctx->io_error);
10397 }
10398 }
10399
10400 void
10401 upl_set_iodone(upl_t upl, void *upl_iodone)
10402 {
10403 upl->upl_iodone = (struct upl_io_completion *)upl_iodone;
10404 }
10405
10406 void
10407 upl_set_iodone_error(upl_t upl, int error)
10408 {
10409 struct upl_io_completion *upl_ctx = upl->upl_iodone;
10410
10411 if (upl_ctx) {
10412 upl_ctx->io_error = error;
10413 }
10414 }
10415
10416
10417 ppnum_t
10418 upl_get_highest_page(
10419 upl_t upl)
10420 {
10421 return upl->highest_page;
10422 }
10423
10424 upl_size_t
10425 upl_get_size(
10426 upl_t upl)
10427 {
10428 return upl_adjusted_size(upl, PAGE_MASK);
10429 }
10430
10431 upl_size_t
10432 upl_adjusted_size(
10433 upl_t upl,
10434 vm_map_offset_t pgmask)
10435 {
10436 vm_object_offset_t start_offset, end_offset;
10437
10438 start_offset = trunc_page_mask_64(upl->u_offset, pgmask);
10439 end_offset = round_page_mask_64(upl->u_offset + upl->u_size, pgmask);
10440
10441 return (upl_size_t)(end_offset - start_offset);
10442 }
10443
10444 vm_object_offset_t
10445 upl_adjusted_offset(
10446 upl_t upl,
10447 vm_map_offset_t pgmask)
10448 {
10449 return trunc_page_mask_64(upl->u_offset, pgmask);
10450 }
10451
10452 vm_object_offset_t
10453 upl_get_data_offset(
10454 upl_t upl)
10455 {
10456 return upl->u_offset - upl_adjusted_offset(upl, PAGE_MASK);
10457 }
10458
10459 upl_t
10460 upl_associated_upl(upl_t upl)
10461 {
10462 if (!(upl->flags & UPL_HAS_FS_VERIFY_INFO)) {
10463 return upl->u_fs_un.associated_upl;
10464 }
10465 return NULL;
10466 }
10467
10468 void
10469 upl_set_associated_upl(upl_t upl, upl_t associated_upl)
10470 {
10471 assert(!(upl->flags & UPL_HAS_FS_VERIFY_INFO));
10472 upl->u_fs_un.associated_upl = associated_upl;
10473 }
10474
10475 bool
10476 upl_has_fs_verify_info(upl_t upl)
10477 {
10478 return upl->flags & UPL_HAS_FS_VERIFY_INFO;
10479 }
10480
10481 void
10482 upl_set_fs_verify_info(upl_t upl, uint32_t size)
10483 {
10484 struct upl_fs_verify_info *fs_verify_infop;
10485
10486 if (upl->flags & UPL_HAS_FS_VERIFY_INFO || !size) {
10487 return;
10488 }
10489
10490 fs_verify_infop = kalloc_type(struct upl_fs_verify_info, Z_WAITOK);
10491 fs_verify_infop->verify_data_ptr = kalloc_data(size, Z_WAITOK);
10492 fs_verify_infop->verify_data_len = size;
10493
10494 upl_lock(upl);
10495 if (upl->flags & UPL_HAS_FS_VERIFY_INFO) {
10496 upl_unlock(upl);
10497
10498 assert(upl->u_fs_un.verify_info &&
10499 upl->u_fs_un.verify_info->verify_data_len > 0 &&
10500 upl->u_fs_un.verify_info->verify_data_len <= upl_adjusted_size(upl, PAGE_MASK));
10501
10502 kfree_data(fs_verify_infop->verify_data_ptr, size);
10503 kfree_type(struct upl_fs_verify_info, fs_verify_infop);
10504 } else {
10505 upl->flags |= UPL_HAS_FS_VERIFY_INFO;
10506 upl->u_fs_un.verify_info = fs_verify_infop;
10507
10508 upl_unlock(upl);
10509 }
10510 }
10511
10512 uint8_t *
10513 upl_fs_verify_buf(upl_t upl, uint32_t *size)
10514 {
10515 assert(size);
10516
10517 if (!(upl->flags & UPL_HAS_FS_VERIFY_INFO)) {
10518 *size = 0;
10519 return NULL;
10520 }
10521
10522 *size = upl->u_fs_un.verify_info->verify_data_len;
10523 return upl->u_fs_un.verify_info->verify_data_ptr;
10524 }
10525
10526 struct vnode *
10527 upl_lookup_vnode(upl_t upl)
10528 {
10529 if (!upl->map_object->internal) {
10530 return vnode_pager_lookup_vnode(upl->map_object->pager);
10531 } else {
10532 return NULL;
10533 }
10534 }
10535
10536 boolean_t
10537 upl_has_wired_pages(upl_t upl)
10538 {
10539 return (upl->flags & UPL_HAS_WIRED) ? TRUE : FALSE;
10540 }
10541
10542 #if UPL_DEBUG
10543 kern_return_t
10544 upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
10545 {
10546 upl->ubc_alias1 = alias1;
10547 upl->ubc_alias2 = alias2;
10548 return KERN_SUCCESS;
10549 }
10550 int
10551 upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
10552 {
10553 if (al) {
10554 *al = upl->ubc_alias1;
10555 }
10556 if (al2) {
10557 *al2 = upl->ubc_alias2;
10558 }
10559 return KERN_SUCCESS;
10560 }
10561 #endif /* UPL_DEBUG */
10562
10563 #if VM_PRESSURE_EVENTS
10564 /*
10565 * Upward trajectory.
10566 */
10567
10568 boolean_t
10569 VM_PRESSURE_NORMAL_TO_WARNING(void)
10570 {
10571 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
10572 /* Available pages below our threshold */
10573 uint32_t available_pages = memorystatus_get_available_page_count();
10574 if (available_pages < memorystatus_get_soft_memlimit_page_shortage_threshold()) {
10575 #if CONFIG_FREEZE
10576 /* No frozen processes to kill */
10577 if (memorystatus_frozen_count == 0) {
10578 /* Not enough suspended processes available. */
10579 if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) {
10580 return TRUE;
10581 }
10582 }
10583 #else /* CONFIG_FREEZE */
10584 return TRUE;
10585 #endif /* CONFIG_FREEZE */
10586 }
10587 return FALSE;
10588 } else {
10589 return (AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0;
10590 }
10591 }
10592
10593 boolean_t
10594 VM_PRESSURE_WARNING_TO_CRITICAL(void)
10595 {
10596 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
10597 /* Available pages below our threshold */
10598 uint32_t available_pages = memorystatus_get_available_page_count();
10599 return available_pages < memorystatus_get_critical_page_shortage_threshold();
10600 } else {
10601 return vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;
10602 }
10603 }
10604
10605 /*
10606 * Downward trajectory.
10607 */
10608 boolean_t
10609 VM_PRESSURE_WARNING_TO_NORMAL(void)
10610 {
10611 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
10612 /* Available pages above our threshold */
10613 uint32_t available_pages = memorystatus_get_available_page_count();
10614 uint32_t target_threshold = (((115 * memorystatus_get_soft_memlimit_page_shortage_threshold()) / 100));
10615 return available_pages > target_threshold;
10616 } else {
10617 return (AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0;
10618 }
10619 }
10620
10621 boolean_t
10622 VM_PRESSURE_CRITICAL_TO_WARNING(void)
10623 {
10624 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
10625 uint32_t available_pages = memorystatus_get_available_page_count();
10626 uint32_t target_threshold = (((115 * memorystatus_get_critical_page_shortage_threshold()) / 100));
10627 return available_pages > target_threshold;
10628 } else {
10629 return (AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;
10630 }
10631 }
10632 #endif /* VM_PRESSURE_EVENTS */
10633
10634 #if DEVELOPMENT || DEBUG
10635 bool compressor_running_perf_test;
10636 uint64_t compressor_perf_test_pages_processed;
10637
10638 static kern_return_t
10639 move_pages_to_queue(
10640 vm_map_t map,
10641 user_addr_t start_addr,
10642 size_t buffer_size,
10643 vm_page_queue_head_t *queue,
10644 size_t *pages_moved)
10645 {
10646 kern_return_t err = KERN_SUCCESS;
10647 vm_map_entry_t curr_entry = VM_MAP_ENTRY_NULL;
10648 boolean_t addr_in_map = FALSE;
10649 user_addr_t end_addr = USER_ADDR_NULL, curr_addr = USER_ADDR_NULL;
10650 vm_object_t curr_object = VM_OBJECT_NULL;
10651 *pages_moved = 0;
10652
10653 vmlp_api_start(MOVE_PAGES_TO_QUEUE);
10654
10655 if (VM_MAP_PAGE_SIZE(map) != PAGE_SIZE_64) {
10656 /*
10657 * We don't currently support benchmarking maps with a different page size
10658 * than the kernel.
10659 */
10660 vmlp_api_end(MOVE_PAGES_TO_QUEUE, KERN_INVALID_ARGUMENT);
10661 return KERN_INVALID_ARGUMENT;
10662 }
10663
10664 if (os_add_overflow(start_addr, buffer_size, &end_addr)) {
10665 vmlp_api_end(MOVE_PAGES_TO_QUEUE, KERN_INVALID_ARGUMENT);
10666 return KERN_INVALID_ARGUMENT;
10667 }
10668
10669 vm_map_lock_read(map);
10670 curr_addr = vm_map_trunc_page_mask(start_addr, VM_MAP_PAGE_MASK(map));
10671 end_addr = vm_map_round_page_mask(start_addr + buffer_size, VM_MAP_PAGE_MASK(map));
10672
10673
10674 while (curr_addr < end_addr) {
10675 addr_in_map = vm_map_lookup_entry(map, curr_addr, &curr_entry);
10676 if (!addr_in_map) {
10677 err = KERN_INVALID_ARGUMENT;
10678 break;
10679 }
10680
10681 vmlp_range_event_entry(map, curr_entry);
10682
10683 curr_object = VME_OBJECT(curr_entry);
10684 if (curr_object) {
10685 vm_object_lock(curr_object);
10686 /* We really only want anonymous memory that's in the top level map and object here. */
10687 if (curr_entry->is_sub_map || curr_entry->wired_count != 0 ||
10688 curr_object->shadow != VM_OBJECT_NULL || !curr_object->internal) {
10689 err = KERN_INVALID_ARGUMENT;
10690 vm_object_unlock(curr_object);
10691 break;
10692 }
10693 vm_map_offset_t start_offset = (curr_addr - curr_entry->vme_start) + VME_OFFSET(curr_entry);
10694 vm_map_offset_t end_offset = MIN(curr_entry->vme_end, end_addr) -
10695 (curr_entry->vme_start + VME_OFFSET(curr_entry));
10696 vm_map_offset_t curr_offset = start_offset;
10697 vm_page_t curr_page;
10698 while (curr_offset < end_offset) {
10699 curr_page = vm_page_lookup(curr_object, vm_object_trunc_page(curr_offset));
10700 if (curr_page != VM_PAGE_NULL) {
10701 vm_page_lock_queues();
10702 if (curr_page->vmp_laundry) {
10703 vm_pageout_steal_laundry(curr_page, TRUE);
10704 }
10705 /*
10706 * we've already factored out pages in the laundry which
10707 * means this page can't be on the pageout queue so it's
10708 * safe to do the vm_page_queues_remove
10709 */
10710 bool donate = (curr_page->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE);
10711 vm_page_queues_remove(curr_page, TRUE);
10712 if (donate) {
10713 /*
10714 * The compressor needs to see this bit to know
10715 * where this page needs to land. Also if stolen,
10716 * this bit helps put the page back in the right
10717 * special queue where it belongs.
10718 */
10719 curr_page->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
10720 }
10721 // Clear the referenced bit so we ensure this gets paged out
10722 curr_page->vmp_reference = false;
10723 if (curr_page->vmp_pmapped) {
10724 pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(curr_page),
10725 VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void*)NULL);
10726 }
10727 vm_page_queue_enter(queue, curr_page, vmp_pageq);
10728 vm_page_unlock_queues();
10729 *pages_moved += 1;
10730 }
10731 curr_offset += PAGE_SIZE_64;
10732 curr_addr += PAGE_SIZE_64;
10733 }
10734 }
10735 vm_object_unlock(curr_object);
10736 }
10737 vm_map_unlock_read(map);
10738 vmlp_api_end(MOVE_PAGES_TO_QUEUE, err);
10739 return err;
10740 }
10741
10742 /*
10743 * Local queue for processing benchmark pages.
10744 * Can't be allocated on the stack because the pointer has to
10745 * be packable.
10746 */
10747 vm_page_queue_head_t compressor_perf_test_queue VM_PAGE_PACKED_ALIGNED;
10748 kern_return_t
10749 run_compressor_perf_test(
10750 user_addr_t buf,
10751 size_t buffer_size,
10752 uint64_t *time,
10753 uint64_t *bytes_compressed,
10754 uint64_t *compressor_growth)
10755 {
10756 kern_return_t err = KERN_SUCCESS;
10757 if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
10758 return KERN_NOT_SUPPORTED;
10759 }
10760 if (current_task() == kernel_task) {
10761 return KERN_INVALID_ARGUMENT;
10762 }
10763 vm_page_lock_queues();
10764 if (compressor_running_perf_test) {
10765 /* Only run one instance of the benchmark at a time. */
10766 vm_page_unlock_queues();
10767 return KERN_RESOURCE_SHORTAGE;
10768 }
10769 vm_page_unlock_queues();
10770 size_t page_count = 0;
10771 vm_map_t map;
10772 vm_page_t p, next;
10773 uint64_t compressor_perf_test_start = 0, compressor_perf_test_end = 0;
10774 uint64_t compressed_bytes_start = 0, compressed_bytes_end = 0;
10775 *bytes_compressed = *compressor_growth = 0;
10776
10777 vm_page_queue_init(&compressor_perf_test_queue);
10778 map = current_task()->map;
10779 err = move_pages_to_queue(map, buf, buffer_size, &compressor_perf_test_queue, &page_count);
10780 if (err != KERN_SUCCESS) {
10781 goto out;
10782 }
10783
10784 vm_page_lock_queues();
10785 compressor_running_perf_test = true;
10786 compressor_perf_test_pages_processed = 0;
10787 /*
10788 * At this point the compressor threads should only process the benchmark queue
10789 * so we can look at the difference in c_segment_compressed_bytes while the perf test is running
10790 * to determine how many compressed bytes we ended up using.
10791 */
10792 compressed_bytes_start = os_atomic_load(&c_segment_compressed_bytes, relaxed);
10793 vm_page_unlock_queues();
10794
10795 page_count = vm_pageout_page_queue(&compressor_perf_test_queue, page_count, true);
10796
10797 vm_page_lock_queues();
10798 compressor_perf_test_start = mach_absolute_time();
10799
10800 // Wake up the compressor thread(s)
10801 sched_cond_signal(&pgo_iothread_internal_state[0].pgo_wakeup,
10802 pgo_iothread_internal_state[0].pgo_iothread);
10803
10804 /*
10805 * Depending on when this test is run we could overshoot or be right on the mark
10806 * with our page_count. So the comparison is of the _less than_ variety.
10807 */
10808 while (compressor_perf_test_pages_processed < page_count) {
10809 assert_wait((event_t) &compressor_perf_test_pages_processed, THREAD_UNINT);
10810 vm_page_unlock_queues();
10811 thread_block(THREAD_CONTINUE_NULL);
10812 vm_page_lock_queues();
10813 }
10814 compressor_perf_test_end = mach_absolute_time();
10815 compressed_bytes_end = os_atomic_load(&c_segment_compressed_bytes, relaxed);
10816 vm_page_unlock_queues();
10817
10818
10819 out:
10820 /*
10821 * If we errored out above, then we could still have some pages
10822 * on the local queue. Make sure to put them back on the active queue before
10823 * returning so they're not orphaned.
10824 */
10825 vm_page_lock_queues();
10826 absolutetime_to_nanoseconds(compressor_perf_test_end - compressor_perf_test_start, time);
10827 p = (vm_page_t) vm_page_queue_first(&compressor_perf_test_queue);
10828 while (p && !vm_page_queue_end(&compressor_perf_test_queue, (vm_page_queue_entry_t)p)) {
10829 next = (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.next);
10830
10831 vm_page_enqueue_active(p, FALSE);
10832 p = next;
10833 }
10834
10835 compressor_running_perf_test = false;
10836 vm_page_unlock_queues();
10837 if (err == KERN_SUCCESS) {
10838 *bytes_compressed = page_count * PAGE_SIZE_64;
10839 *compressor_growth = compressed_bytes_end - compressed_bytes_start;
10840 }
10841
10842 /*
10843 * pageout_scan will consider waking the compactor swapper
10844 * before it blocks. Do the same thing here before we return
10845 * to ensure that back to back benchmark runs can't overly fragment the
10846 * compressor pool.
10847 */
10848 vm_consider_waking_compactor_swapper();
10849 return err;
10850 }
10851 #endif /* DEVELOPMENT || DEBUG */
10852