1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * Resident memory management module.
63 */
64 #include <AppleFeatures.h>
65 #include <debug.h>
66 #include <libkern/OSDebug.h>
67
68 #include <mach/clock_types.h>
69 #include <mach/vm_prot.h>
70 #include <mach/vm_statistics.h>
71 #include <mach/sdt.h>
72 #include <kern/counter.h>
73 #include <kern/host_statistics.h>
74 #include <kern/sched_prim.h>
75 #include <kern/policy_internal.h>
76 #include <kern/task.h>
77 #include <kern/thread.h>
78 #include <kern/kalloc.h>
79 #include <kern/zalloc_internal.h>
80 #include <kern/ledger.h>
81 #include <kern/ecc.h>
82 #include <vm/pmap.h>
83 #include <vm/vm_init_xnu.h>
84 #include <vm/vm_map_internal.h>
85 #include <vm/vm_page_internal.h>
86 #include <vm/vm_pageout_internal.h>
87 #include <vm/vm_kern_xnu.h> /* kmem_alloc() */
88 #include <vm/vm_compressor_pager_internal.h>
89 #include <kern/misc_protos.h>
90 #include <mach_debug/zone_info.h>
91 #include <vm/cpm_internal.h>
92 #include <pexpert/pexpert.h>
93 #include <pexpert/device_tree.h>
94 #include <san/kasan.h>
95 #include <vm/vm_log.h>
96
97 #include <libkern/coreanalytics/coreanalytics.h>
98 #include <kern/backtrace.h>
99 #include <kern/telemetry.h>
100
101 #include <vm/vm_protos_internal.h>
102 #include <vm/memory_object.h>
103 #include <vm/vm_purgeable_internal.h>
104 #include <vm/vm_compressor_internal.h>
105 #include <vm/vm_iokit.h>
106 #include <vm/vm_object_internal.h>
107
108
109 #if defined (__x86_64__)
110 #include <i386/misc_protos.h>
111 #endif
112
113 #if CONFIG_SPTM
114 #include <arm64/sptm/sptm.h>
115 #endif
116
117 #if CONFIG_PHANTOM_CACHE
118 #include <vm/vm_phantom_cache_internal.h>
119 #endif
120
121 #if HIBERNATION
122 #include <IOKit/IOHibernatePrivate.h>
123 #include <machine/pal_hibernate.h>
124 #endif /* HIBERNATION */
125
126 #if CONFIG_SECLUDED_MEMORY
127 static_assert(!XNU_VM_HAS_LOPAGE,
128 "VM_PAGE_ON_SECLUDED_Q and VM_PAGE_ON_FREE_LOPAGE_Q alias");
129 #endif
130
131 #include <sys/kdebug.h>
132
133 #if defined(HAS_APPLE_PAC)
134 #include <ptrauth.h>
135 #endif
136 #if defined(__arm64__)
137 #include <arm/cpu_internal.h>
138 #endif /* defined(__arm64__) */
139
140 /*
141 * During single threaded early boot we don't initialize all pages.
142 * This avoids some delay during boot. They'll be initialized and
143 * added to the free list as needed or after we are multithreaded by
144 * what becomes the pageout thread.
145 *
146 * This slows down booting the DEBUG kernel, particularly on
147 * large memory systems, but is worthwhile in deterministically
148 * trapping uninitialized memory usage.
149 */
150 #if DEBUG
151 static TUNABLE(uint32_t, fillval, "fill", 0xDEB8F177);
152 #else
153 static TUNABLE(uint32_t, fillval, "fill", 0);
154 #endif
155
156 #if MACH_ASSERT
157
158 TUNABLE(bool, vm_check_refs_on_alloc, "vm_check_refs_on_alloc", false);
159
160 #endif /* MACH_ASSERT */
161
162 extern boolean_t vm_pageout_running;
163 extern thread_t vm_pageout_scan_thread;
164 extern bool vps_dynamic_priority_enabled;
165
166 const uint16_t vm_page_inactive_states =
167 BIT(VM_PAGE_ON_INACTIVE_INTERNAL_Q) |
168 BIT(VM_PAGE_ON_INACTIVE_EXTERNAL_Q) |
169 BIT(VM_PAGE_ON_INACTIVE_CLEANED_Q);
170
171 const uint16_t vm_page_active_or_inactive_states =
172 vm_page_inactive_states |
173 #if CONFIG_SECLUDED_MEMORY
174 BIT(VM_PAGE_ON_SECLUDED_Q) |
175 #endif /* CONFIG_SECLUDED_MEMORY */
176 BIT(VM_PAGE_ON_ACTIVE_Q);
177
178 const uint16_t vm_page_non_speculative_pageable_states =
179 vm_page_active_or_inactive_states |
180 BIT(VM_PAGE_ON_THROTTLED_Q);
181
182 const uint16_t vm_page_pageable_states =
183 vm_page_non_speculative_pageable_states |
184 BIT(VM_PAGE_ON_SPECULATIVE_Q);
185
186 #if CONFIG_SECLUDED_MEMORY
187 struct vm_page_secluded_data vm_page_secluded;
188 #endif /* CONFIG_SECLUDED_MEMORY */
189 #if HIBERNATION
190 static bool hibernate_rebuild_needed = false;
191 #endif /* HIBERNATION */
192
193 #if DEVELOPMENT || DEBUG
194 extern struct memory_object_pager_ops shared_region_pager_ops;
195 unsigned int shared_region_pagers_resident_count = 0;
196 unsigned int shared_region_pagers_resident_peak = 0;
197 #endif /* DEVELOPMENT || DEBUG */
198
199
200
201 unsigned int PERCPU_DATA(start_color);
202 vm_page_t PERCPU_DATA(free_pages);
203 SCALABLE_COUNTER_DEFINE(vm_cpu_free_count);
204 boolean_t hibernate_cleaning_in_progress = FALSE;
205
206 atomic_counter_t vm_guard_count;
207
208 #if XNU_VM_HAS_LOPAGE
209 /*
210 * this interface exists to support hardware controllers
211 * incapable of generating DMAs with more than 32 bits
212 * of address on platforms with physical memory > 4G...
213 */
214 vm_page_queue_head_t vm_lopage_queue_free VM_PAGE_PACKED_ALIGNED;
215 uint32_t vm_lopage_free_count = 0;
216 uint32_t vm_lopage_free_limit = 0;
217 uint32_t vm_lopage_lowater = 0;
218 bool vm_lopage_refill = false;
219 bool vm_lopage_needed = false;
220 unsigned int vm_lopages_allocated_q = 0;
221 unsigned int vm_lopages_allocated_cpm_success = 0;
222 unsigned int vm_lopages_allocated_cpm_failed = 0;
223 #endif /* XNU_VM_HAS_LOPAGE */
224
225
226 int speculative_age_index = 0;
227 int speculative_steal_index = 0;
228 struct vm_speculative_age_q vm_page_queue_speculative[VM_PAGE_RESERVED_SPECULATIVE_AGE_Q + 1];
229
230 boolean_t hibernation_vmqueues_inspection = FALSE; /* Tracks if the hibernation code is looking at the VM queues.
231 * Updated and checked behind the vm_page_queues_lock. */
232
233 static void vm_page_free_prepare(vm_page_t page);
234
235
236 static void vm_tag_init(void);
237
238 /* for debugging purposes */
239 SECURITY_READ_ONLY_EARLY(uint32_t) vm_packed_from_vm_pages_array_mask =
240 VM_PAGE_PACKED_FROM_ARRAY;
241 #ifndef __BUILDING_XNU_LIB_UNITTEST__ /* This is not a compile-time constant when building unit-test */
242 SECURITY_READ_ONLY_EARLY(vm_packing_params_t) vm_page_packing_params =
243 VM_PACKING_PARAMS(VM_PAGE_PACKED_PTR);
244 #endif /* __BUILDING_XNU_LIB_UNITTEST__ */
245
246 /*
247 * Associated with page of user-allocatable memory is a
248 * page structure.
249 */
250
251 /*
252 * These variables record the values returned by vm_page_bootstrap,
253 * for debugging purposes. The implementation of pmap_steal_memory
254 * and pmap_startup here also uses them internally.
255 */
256
257 vm_offset_t virtual_space_start;
258 vm_offset_t virtual_space_end;
259 uint32_t vm_page_pages;
260
261 /*
262 * The vm_page_lookup() routine, which provides for fast
263 * (virtual memory object, offset) to page lookup, employs
264 * the following hash table. The vm_page_{insert,remove}
265 * routines install and remove associations in the table.
266 * [This table is often called the virtual-to-physical,
267 * or VP, table.]
268 */
269 typedef struct {
270 vm_page_packed_t page_list;
271 #if MACH_PAGE_HASH_STATS
272 int cur_count; /* current count */
273 int hi_count; /* high water mark */
274 #endif /* MACH_PAGE_HASH_STATS */
275 } vm_page_bucket_t;
276
277
278 #define BUCKETS_PER_LOCK 16
279
280 SECURITY_READ_ONLY_LATE(vm_page_bucket_t *) vm_page_buckets; /* Array of buckets */
281 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_bucket_count = 0; /* How big is array? */
282 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_hash_mask; /* Mask for hash function */
283 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_hash_shift; /* Shift for hash function */
284 SECURITY_READ_ONLY_LATE(uint32_t) vm_page_bucket_hash; /* Basic bucket hash */
285 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_bucket_lock_count = 0; /* How big is array of locks? */
286
287 #ifndef VM_TAG_ACTIVE_UPDATE
288 #error VM_TAG_ACTIVE_UPDATE
289 #endif
290 #ifndef VM_TAG_SIZECLASSES
291 #error VM_TAG_SIZECLASSES
292 #endif
293
294 /* for debugging */
295 SECURITY_READ_ONLY_LATE(bool) vm_tag_active_update = VM_TAG_ACTIVE_UPDATE;
296 SECURITY_READ_ONLY_LATE(lck_ticket_t *) vm_page_bucket_locks;
297
298 vm_allocation_site_t vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC + 1];
299 vm_allocation_site_t * vm_allocation_sites[VM_MAX_TAG_VALUE];
300 #if VM_TAG_SIZECLASSES
301 static vm_allocation_zone_total_t **vm_allocation_zone_totals;
302 #endif /* VM_TAG_SIZECLASSES */
303
304 vm_tag_t vm_allocation_tag_highest;
305
306 #if VM_PAGE_BUCKETS_CHECK
307 boolean_t vm_page_buckets_check_ready = FALSE;
308 #if VM_PAGE_FAKE_BUCKETS
309 vm_page_bucket_t *vm_page_fake_buckets; /* decoy buckets */
310 vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
311 #endif /* VM_PAGE_FAKE_BUCKETS */
312 #endif /* VM_PAGE_BUCKETS_CHECK */
313
314 #if MACH_PAGE_HASH_STATS
315 /* This routine is only for debug. It is intended to be called by
316 * hand by a developer using a kernel debugger. This routine prints
317 * out vm_page_hash table statistics to the kernel debug console.
318 */
319 void
hash_debug(void)320 hash_debug(void)
321 {
322 int i;
323 int numbuckets = 0;
324 int highsum = 0;
325 int maxdepth = 0;
326
327 for (i = 0; i < vm_page_bucket_count; i++) {
328 if (vm_page_buckets[i].hi_count) {
329 numbuckets++;
330 highsum += vm_page_buckets[i].hi_count;
331 if (vm_page_buckets[i].hi_count > maxdepth) {
332 maxdepth = vm_page_buckets[i].hi_count;
333 }
334 }
335 }
336 printf("Total number of buckets: %d\n", vm_page_bucket_count);
337 printf("Number used buckets: %d = %d%%\n",
338 numbuckets, 100 * numbuckets / vm_page_bucket_count);
339 printf("Number unused buckets: %d = %d%%\n",
340 vm_page_bucket_count - numbuckets,
341 100 * (vm_page_bucket_count - numbuckets) / vm_page_bucket_count);
342 printf("Sum of bucket max depth: %d\n", highsum);
343 printf("Average bucket depth: %d.%2d\n",
344 highsum / vm_page_bucket_count,
345 highsum % vm_page_bucket_count);
346 printf("Maximum bucket depth: %d\n", maxdepth);
347 }
348 #endif /* MACH_PAGE_HASH_STATS */
349
350 /*
351 * The virtual page size is currently implemented as a runtime
352 * variable, but is constant once initialized using vm_set_page_size.
353 * This initialization must be done in the machine-dependent
354 * bootstrap sequence, before calling other machine-independent
355 * initializations.
356 *
357 * All references to the virtual page size outside this
358 * module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT
359 * constants.
360 */
361 #if defined(__arm64__)
362 vm_size_t page_size;
363 vm_size_t page_mask;
364 int page_shift;
365 #else
366 vm_size_t page_size = PAGE_SIZE;
367 vm_size_t page_mask = PAGE_MASK;
368 int page_shift = PAGE_SHIFT;
369 #endif
370
371 SECURITY_READ_ONLY_LATE(vm_page_t) vm_pages;
372 #if XNU_VM_HAS_DELAYED_PAGES
373 vm_page_t vm_pages_end;
374 uint32_t vm_pages_count;
375 #else
376 SECURITY_READ_ONLY_LATE(vm_page_t) vm_pages_end;
377 SECURITY_READ_ONLY_LATE(uint32_t) vm_pages_count;
378 #endif /* XNU_VM_HAS_DELAYED_PAGES */
379 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
380 SECURITY_READ_ONLY_LATE(ppnum_t) vm_pages_first_pnum;
381 #endif /* XNU_VM_HAS_LINEAR_PAGES_ARRAY */
382 #if CONFIG_SPTM
383 /*
384 * When used, these 128bit (MAX_COLORS bits) masks represent a "cluster"
385 * of contiguous free physical pages.
386 *
387 * For each cluster, there is an enqueue "index", which is -1 when there is no
388 * free page in the cluster, or the index in [0, 128) of the page that is
389 * enqueued on the vm_page_free_queue to represent the entire cluster.
390 *
391 * Grouping pages this way has the double nice effect to reduce doubly linked
392 * list (the worst data structure known to man when considering cache misses)
393 * manipulations, and also to mechanically make the VM serve more "contiguous"
394 * pages naturally.
395 */
396 static_assert(XNU_VM_HAS_LINEAR_PAGES_ARRAY);
397 SECURITY_READ_ONLY_LATE(__uint128_t *) _vm_pages_free_masks;
398 SECURITY_READ_ONLY_LATE(int8_t *) _vm_pages_free_enqueue_idx;
399 #endif /* CONFIG_SPTM */
400
401
402 /*
403 * Resident pages that represent real memory
404 * are allocated from a set of free lists,
405 * one per color.
406 */
407 SECURITY_READ_ONLY_LATE(unsigned int) vm_colors;
408 SECURITY_READ_ONLY_LATE(unsigned int) vm_color_mask; /* mask is == (vm_colors-1) */
409 unsigned int vm_cache_geometry_colors = 0; /* set by hw dependent code during startup */
410 unsigned int vm_free_magazine_refill_limit = 0;
411
412 struct vm_page_free_queue vm_page_queue_free;
413
414 unsigned int vm_page_free_wanted;
415 unsigned int vm_page_free_wanted_privileged;
416 #if CONFIG_SECLUDED_MEMORY
417 unsigned int vm_page_free_wanted_secluded;
418 #endif /* CONFIG_SECLUDED_MEMORY */
419 unsigned int vm_page_free_count;
420
421 unsigned int vm_page_realtime_count;
422
423 /*
424 * Occasionally, the virtual memory system uses
425 * resident page structures that do not refer to
426 * real pages, for example to leave a page with
427 * important state information in the VP table.
428 *
429 * These page structures are allocated the way
430 * most other kernel structures are.
431 */
432 SECURITY_READ_ONLY_LATE(zone_t) vm_page_zone;
433 vm_locks_array_t vm_page_locks;
434
435 LCK_ATTR_DECLARE(vm_page_lck_attr, 0, 0);
436 LCK_GRP_DECLARE(vm_page_lck_grp_free, "vm_page_free");
437 LCK_GRP_DECLARE(vm_page_lck_grp_queue, "vm_page_queue");
438 LCK_GRP_DECLARE(vm_page_lck_grp_local, "vm_page_queue_local");
439 LCK_GRP_DECLARE(vm_page_lck_grp_purge, "vm_page_purge");
440 LCK_GRP_DECLARE(vm_page_lck_grp_alloc, "vm_page_alloc");
441 LCK_GRP_DECLARE(vm_page_lck_grp_bucket, "vm_page_bucket");
442 LCK_SPIN_DECLARE_ATTR(vm_objects_wired_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
443 LCK_TICKET_DECLARE(vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
444
445 unsigned int vm_page_local_q_soft_limit = 250;
446 unsigned int vm_page_local_q_hard_limit = 500;
447 struct vpl *__zpercpu vm_page_local_q;
448
449 /* N.B. Guard and fictitious pages must not
450 * be assigned a zero phys_page value.
451 */
452 /*
453 * Fictitious pages don't have a physical address,
454 * but we must initialize phys_page to something.
455 * For debugging, this should be a strange value
456 * that the pmap module can recognize in assertions.
457 */
458 const ppnum_t vm_page_fictitious_addr = (ppnum_t) -1;
459
460 /*
461 * Guard pages are not accessible so they don't
462 * need a physical address, but we need to enter
463 * one in the pmap.
464 * Let's make it recognizable and make sure that
465 * we don't use a real physical page with that
466 * physical address.
467 */
468 const ppnum_t vm_page_guard_addr = (ppnum_t) -2;
469
470 /*
471 * Resident page structures are also chained on
472 * queues that are used by the page replacement
473 * system (pageout daemon). These queues are
474 * defined here, but are shared by the pageout
475 * module. The inactive queue is broken into
476 * file backed and anonymous for convenience as the
477 * pageout daemon often assignes a higher
478 * importance to anonymous pages (less likely to pick)
479 */
480 vm_page_queue_head_t vm_page_queue_active VM_PAGE_PACKED_ALIGNED;
481 vm_page_queue_head_t vm_page_queue_inactive VM_PAGE_PACKED_ALIGNED;
482 #if CONFIG_SECLUDED_MEMORY
483 vm_page_queue_head_t vm_page_queue_secluded VM_PAGE_PACKED_ALIGNED;
484 #endif /* CONFIG_SECLUDED_MEMORY */
485 vm_page_queue_head_t vm_page_queue_anonymous VM_PAGE_PACKED_ALIGNED; /* inactive memory queue for anonymous pages */
486 vm_page_queue_head_t vm_page_queue_throttled VM_PAGE_PACKED_ALIGNED;
487
488 queue_head_t vm_objects_wired;
489
490 vm_page_queue_head_t vm_page_queue_donate VM_PAGE_PACKED_ALIGNED;
491 uint32_t vm_page_donate_mode;
492 uint32_t vm_page_donate_target, vm_page_donate_target_high, vm_page_donate_target_low;
493 uint32_t vm_page_donate_count;
494 bool vm_page_donate_queue_ripe;
495
496
497 vm_page_queue_head_t vm_page_queue_background VM_PAGE_PACKED_ALIGNED;
498 uint32_t vm_page_background_target;
499 uint32_t vm_page_background_target_snapshot;
500 uint32_t vm_page_background_count;
501 uint64_t vm_page_background_promoted_count;
502
503 uint32_t vm_page_background_internal_count;
504 uint32_t vm_page_background_external_count;
505
506 uint32_t vm_page_background_mode;
507 uint32_t vm_page_background_exclude_external;
508
509 unsigned int vm_page_active_count;
510 unsigned int vm_page_inactive_count;
511 unsigned int vm_page_kernelcache_count;
512 #if CONFIG_SECLUDED_MEMORY
513 unsigned int vm_page_secluded_count;
514 unsigned int vm_page_secluded_count_free;
515 unsigned int vm_page_secluded_count_inuse;
516 unsigned int vm_page_secluded_count_over_target;
517 #endif /* CONFIG_SECLUDED_MEMORY */
518 unsigned int vm_page_anonymous_count;
519 unsigned int vm_page_throttled_count;
520 unsigned int vm_page_speculative_count;
521
522 unsigned int vm_page_wire_count;
523 unsigned int vm_page_wire_count_on_boot = 0;
524 unsigned int vm_page_stolen_count = 0;
525 unsigned int vm_page_wire_count_initial;
526 unsigned int vm_page_gobble_count = 0;
527 unsigned int vm_page_kern_lpage_count = 0;
528
529 uint64_t booter_size; /* external so it can be found in core dumps */
530
531 #define VM_PAGE_WIRE_COUNT_WARNING 0
532 #define VM_PAGE_GOBBLE_COUNT_WARNING 0
533
534 unsigned int vm_page_purgeable_count = 0; /* # of pages purgeable now */
535 unsigned int vm_page_purgeable_wired_count = 0; /* # of purgeable pages that are wired now */
536 uint64_t vm_page_purged_count = 0; /* total count of purged pages */
537
538 unsigned int vm_page_xpmapped_external_count = 0;
539 unsigned int vm_page_external_count = 0;
540 unsigned int vm_page_internal_count = 0;
541 unsigned int vm_page_pageable_external_count = 0;
542 unsigned int vm_page_pageable_internal_count = 0;
543
544 #if DEVELOPMENT || DEBUG
545 unsigned int vm_page_speculative_recreated = 0;
546 unsigned int vm_page_speculative_created = 0;
547 unsigned int vm_page_speculative_used = 0;
548 #endif
549
550 _Atomic unsigned int vm_page_swapped_count = 0;
551
552 vm_page_queue_head_t vm_page_queue_cleaned VM_PAGE_PACKED_ALIGNED;
553
554 unsigned int vm_page_cleaned_count = 0;
555
556 uint64_t max_valid_dma_address = 0xffffffffffffffffULL;
557 ppnum_t max_valid_low_ppnum = PPNUM_MAX;
558
559
560 /*
561 * Several page replacement parameters are also
562 * shared with this module, so that page allocation
563 * (done here in vm_page_alloc) can trigger the
564 * pageout daemon.
565 */
566 unsigned int vm_page_free_target = 0;
567 unsigned int vm_page_free_min = 0;
568 unsigned int vm_page_throttle_limit = 0;
569 unsigned int vm_page_inactive_target = 0;
570 #if CONFIG_SECLUDED_MEMORY
571 unsigned int vm_page_secluded_target = 0;
572 #endif /* CONFIG_SECLUDED_MEMORY */
573 unsigned int vm_page_anonymous_min = 0;
574 unsigned int vm_page_free_reserved = 0;
575
576
577 /*
578 * The VM system has a couple of heuristics for deciding
579 * that pages are "uninteresting" and should be placed
580 * on the inactive queue as likely candidates for replacement.
581 * These variables let the heuristics be controlled at run-time
582 * to make experimentation easier.
583 */
584
585 boolean_t vm_page_deactivate_hint = TRUE;
586
587 struct vm_page_stats_reusable vm_page_stats_reusable;
588
589 /*
590 * vm_set_page_size:
591 *
592 * Sets the page size, perhaps based upon the memory
593 * size. Must be called before any use of page-size
594 * dependent functions.
595 *
596 * Sets page_shift and page_mask from page_size.
597 */
598 void
vm_set_page_size(void)599 vm_set_page_size(void)
600 {
601 page_size = PAGE_SIZE;
602 page_mask = PAGE_MASK;
603 page_shift = PAGE_SHIFT;
604
605 if ((page_mask & page_size) != 0) {
606 panic("vm_set_page_size: page size not a power of two");
607 }
608
609 for (page_shift = 0;; page_shift++) {
610 if ((1U << page_shift) == page_size) {
611 break;
612 }
613 }
614 }
615
616
617 /*
618 * @abstract
619 * Given a page, returns the memory class of that page.
620 */
621 static vm_memory_class_t
vm_page_get_memory_class(vm_page_t mem __unused,ppnum_t pnum __unused)622 vm_page_get_memory_class(vm_page_t mem __unused, ppnum_t pnum __unused)
623 {
624 assert(!vm_page_is_fictitious(mem));
625
626 #if XNU_VM_HAS_LOPAGE
627 if (mem->vmp_lopage) {
628 return VM_MEMORY_CLASS_LOPAGE;
629 }
630 #endif /* XNU_VM_HAS_LOPAGE */
631 return VM_MEMORY_CLASS_REGULAR;
632 }
633
634 /*
635 * vm_page_is_restricted:
636 *
637 * Checks if a given vm_page_t is a restricted page.
638 */
639 inline bool
vm_page_is_restricted(vm_page_t mem)640 vm_page_is_restricted(vm_page_t mem)
641 {
642 ppnum_t pn = VM_PAGE_GET_PHYS_PAGE(mem);
643 return pmap_is_page_restricted(pn);
644 }
645
646 #ifdef __x86_64__
647
648 #define MAX_CLUMP_SIZE 16
649 #define DEFAULT_CLUMP_SIZE 4
650
651 unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
652
653 #if DEVELOPMENT || DEBUG
654 unsigned long vm_clump_stats[MAX_CLUMP_SIZE + 1];
655 unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
656
657 static inline void
vm_clump_update_stats(unsigned int c)658 vm_clump_update_stats(unsigned int c)
659 {
660 assert(c <= vm_clump_size);
661 if (c > 0 && c <= vm_clump_size) {
662 vm_clump_stats[c] += c;
663 }
664 vm_clump_allocs += c;
665 }
666 #endif /* if DEVELOPMENT || DEBUG */
667
668 /* Called once to setup the VM clump knobs */
669 static void
vm_page_setup_clump(void)670 vm_page_setup_clump( void )
671 {
672 unsigned int override, n;
673
674 vm_clump_size = DEFAULT_CLUMP_SIZE;
675 if (PE_parse_boot_argn("clump_size", &override, sizeof(override))) {
676 vm_clump_size = override;
677 }
678
679 if (vm_clump_size > MAX_CLUMP_SIZE) {
680 panic("vm_page_setup_clump:: clump_size is too large!");
681 }
682 if (vm_clump_size < 1) {
683 panic("vm_page_setup_clump:: clump_size must be >= 1");
684 }
685 if ((vm_clump_size & (vm_clump_size - 1)) != 0) {
686 panic("vm_page_setup_clump:: clump_size must be a power of 2");
687 }
688
689 vm_clump_promote_threshold = vm_clump_size;
690 vm_clump_mask = vm_clump_size - 1;
691 for (vm_clump_shift = 0, n = vm_clump_size; n > 1; n >>= 1, vm_clump_shift++) {
692 ;
693 }
694
695 #if DEVELOPMENT || DEBUG
696 bzero(vm_clump_stats, sizeof(vm_clump_stats));
697 vm_clump_allocs = vm_clump_inserts = vm_clump_inrange = vm_clump_promotes = 0;
698 #endif /* if DEVELOPMENT || DEBUG */
699 }
700
701 #endif /* __x86_64__ */
702
703 void
vm_page_free_queue_init(vm_page_free_queue_t free_queue)704 vm_page_free_queue_init(vm_page_free_queue_t free_queue)
705 {
706 for (unsigned int color = 0; color < MAX_COLORS; color++) {
707 vm_page_queue_init(&free_queue->vmpfq_queues[color].qhead);
708 }
709 }
710
711 /*!
712 * @function vm_page_free_queue_for_class()
713 *
714 * @abstract
715 * Returns the appropriate free queue for the given class and page color.
716 */
717 __pure2
718 static vm_page_queue_t
vm_page_free_queue_for_class(vm_memory_class_t mem_class,unsigned int color)719 vm_page_free_queue_for_class(vm_memory_class_t mem_class, unsigned int color)
720 {
721 switch (mem_class) {
722 case VM_MEMORY_CLASS_REGULAR:
723 return &vm_page_queue_free.vmpfq_queues[color].qhead;
724 #if XNU_VM_HAS_LOPAGE
725 case VM_MEMORY_CLASS_LOPAGE:
726 return &vm_lopage_queue_free;
727 #endif /* XNU_VM_HAS_LOPAGE */
728 #if CONFIG_SECLUDED_MEMORY
729 case VM_MEMORY_CLASS_SECLUDED:
730 return &vm_page_queue_secluded;
731 #endif
732 }
733 }
734
735 __pure2
736 static bool
vm_page_free_queue_has_colors(vm_memory_class_t mem_class)737 vm_page_free_queue_has_colors(vm_memory_class_t mem_class)
738 {
739 switch (mem_class) {
740 case VM_MEMORY_CLASS_REGULAR:
741 return true;
742 #if XNU_VM_HAS_LOPAGE
743 case VM_MEMORY_CLASS_LOPAGE:
744 return false;
745 #endif /* XNU_VM_HAS_LOPAGE */
746 #if CONFIG_SECLUDED_MEMORY
747 case VM_MEMORY_CLASS_SECLUDED:
748 return false;
749 #endif
750 }
751 }
752
753
754 #if CONFIG_SECLUDED_MEMORY
755
756 static bool
vm_page_secluded_pool_eligible(vm_memory_class_t class)757 vm_page_secluded_pool_eligible(vm_memory_class_t class)
758 {
759 switch (class) {
760 #if XNU_VM_HAS_LOPAGE
761 case VM_MEMORY_CLASS_LOPAGE:
762 return false;
763 #endif /* XNU_VM_HAS_LOPAGE */
764 default:
765 return true;
766 }
767 }
768
769 static bool
vm_page_secluded_pool_depleted(void)770 vm_page_secluded_pool_depleted(void)
771 {
772 if (vm_page_free_count <= vm_page_free_reserved) {
773 return false;
774 }
775 if (num_tasks_can_use_secluded_mem) {
776 return false;
777 }
778 return vm_page_secluded_count < vm_page_secluded_target;
779 }
780
781 #endif /* CONFIG_SECLUDED_MEMORY */
782 #if HIBERNATION
783
784 __attribute__((overloadable))
785 static void
786 vm_page_free_queue_foreach(vm_page_queue_t queue, void (^block)(vm_page_t))
787 {
788 vm_page_t page;
789
vm_page_queue_iterate(queue,page,vmp_pageq)790 vm_page_queue_iterate(queue, page, vmp_pageq) {
791 block(page);
792 }
793 }
794
795 __attribute__((overloadable))
796 static void
797 vm_page_free_queue_foreach(vm_page_free_queue_t queue, void (^block)(vm_page_t))
798 {
799 for (unsigned int color = 0; color < vm_colors; color++) {
800 vm_page_free_queue_foreach(&queue->vmpfq_queues[color].qhead, block);
801 }
802 }
803
804 #endif /* HIBERNATION */
805 #if CONFIG_SPTM
806
807 static inline uint32_t
vm_pages_free_mask_len(void)808 vm_pages_free_mask_len(void)
809 {
810 extern pmap_paddr_t real_avail_end;
811
812 uint64_t pnums = atop(real_avail_end) - pmap_first_pnum;
813 static_assert(8 * sizeof(__uint128_t) == MAX_COLORS);
814 return (uint32_t)((pnums + MAX_COLORS - 1) / MAX_COLORS);
815 }
816
817 static inline int8_t
vm_pages_free_mask_bit(ppnum_t pnum)818 vm_pages_free_mask_bit(ppnum_t pnum)
819 {
820 return (int8_t)(pnum & (MAX_COLORS - 1));
821 }
822
823 static inline uint32_t
vm_pages_free_mask_index(ppnum_t pnum)824 vm_pages_free_mask_index(ppnum_t pnum)
825 {
826 return (pnum - pmap_first_pnum) / MAX_COLORS;
827 }
828
829 __pure2
830 static inline __uint128_t *
vm_pages_free_masks(void)831 vm_pages_free_masks(void)
832 {
833 return _vm_pages_free_masks;
834 }
835
836 __pure2
837 static inline bitmap_t *
vm_pages_free_masks_as_bitmap(uint32_t index)838 vm_pages_free_masks_as_bitmap(uint32_t index)
839 {
840 /*
841 * this conversion is gross but helps with codegen for bit-wise
842 * accesses where the __uint128_t type is really yielding poor code.
843 *
844 * This conversion is only legal on little endian architectures.
845 */
846 #ifndef __LITTLE_ENDIAN__
847 #error unsupported configuration
848 #endif
849 return (bitmap_t *)(_vm_pages_free_masks + index);
850 }
851
852 __pure2
853 static inline int8_t *
vm_pages_free_enqueue_idx(uint32_t index)854 vm_pages_free_enqueue_idx(uint32_t index)
855 {
856 return &_vm_pages_free_enqueue_idx[index];
857 }
858
859 /*!
860 * @brief
861 * Return the position of the next bit in "circular" order for a given cluster
862 * of pages, starting at and including @c bit.
863 */
864 static inline int8_t
vm_pages_free_mask_next_bit(uint32_t index,int8_t bit)865 vm_pages_free_mask_next_bit(uint32_t index, int8_t bit)
866 {
867 __uint128_t value = vm_pages_free_masks()[index];
868 __uint128_t mask = ((__uint128_t)1 << bit) - 1;
869
870 if (value == 0) {
871 return -1;
872 }
873
874 if (value & ~mask) {
875 value &= ~mask;
876 }
877 if ((uint64_t)value) {
878 return (int8_t)__builtin_ctzll((uint64_t)value);
879 }
880 return 64 + (int8_t)__builtin_ctzll((uint64_t)(value >> 64));
881 }
882
883 static inline bool
vm_pages_free_mask_test(uint32_t index,int8_t bit)884 vm_pages_free_mask_test(uint32_t index, int8_t bit)
885 {
886 return bitmap_test(vm_pages_free_masks_as_bitmap(index), bit);
887 }
888
889 static inline void
vm_pages_free_mask_set(uint32_t index,int8_t bit)890 vm_pages_free_mask_set(uint32_t index, int8_t bit)
891 {
892 assert(!vm_pages_free_mask_test(index, bit));
893 bitmap_set(vm_pages_free_masks_as_bitmap(index), bit);
894 }
895
896 static inline void
vm_pages_free_mask_clear(uint32_t index,int8_t bit)897 vm_pages_free_mask_clear(uint32_t index, int8_t bit)
898 {
899 assert(vm_pages_free_mask_test(index, bit));
900 bitmap_clear(vm_pages_free_masks_as_bitmap(index), bit);
901 }
902
903 #endif /* CONFIG_SPTM */
904
905 __attribute__((always_inline))
906 void
vm_page_free_queue_enter(vm_memory_class_t class,vm_page_t mem,ppnum_t pnum)907 vm_page_free_queue_enter(vm_memory_class_t class, vm_page_t mem, ppnum_t pnum)
908 {
909 bool enter_first;
910 unsigned int color;
911 vm_page_queue_t queue;
912
913 if (startup_phase >= STARTUP_SUB_KMEM) {
914 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
915 }
916
917 assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)));
918 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0 &&
919 mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0 &&
920 mem->vmp_specialq.next == 0 && mem->vmp_specialq.prev == 0 &&
921 mem->vmp_next_m == 0 &&
922 mem->vmp_object == 0 &&
923 mem->vmp_wire_count == 0 &&
924 mem->vmp_busy &&
925 !mem->vmp_tabled &&
926 !mem->vmp_laundry &&
927 !mem->vmp_pmapped &&
928 !mem->vmp_wpmapped &&
929 !mem->vmp_realtime);
930
931 switch (class) {
932 #if XNU_VM_HAS_LOPAGE
933 case VM_MEMORY_CLASS_LOPAGE:
934 mem->vmp_q_state = VM_PAGE_ON_FREE_LOPAGE_Q;
935 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
936 mem->vmp_lopage = true;
937 mem->vmp_canonical = true;
938 enter_first = true;
939 break;
940 #endif /* XNU_VM_HAS_LOPAGE */
941 #if CONFIG_SECLUDED_MEMORY
942 case VM_MEMORY_CLASS_SECLUDED:
943 if (startup_phase >= STARTUP_SUB_KMEM) {
944 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
945 }
946 mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
947 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
948 mem->vmp_lopage = false;
949 mem->vmp_canonical = true;
950 enter_first = true;
951 break;
952 #endif
953 default:
954 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
955 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
956 mem->vmp_lopage = false;
957 mem->vmp_canonical = true;
958 enter_first = false;
959 break;
960 }
961
962
963 color = VM_PAGE_GET_COLOR_PNUM(pnum);
964 queue = vm_page_free_queue_for_class(class, color);
965 #if CONFIG_SPTM
966 if (class == VM_MEMORY_CLASS_REGULAR && vm_pages_free_masks()) {
967 uint32_t index = vm_pages_free_mask_index(pnum);
968 int8_t bit = vm_pages_free_mask_bit(pnum);
969
970 if (vm_pages_free_masks()[index] == 0) {
971 vm_page_queue_enter(queue, mem, vmp_pageq);
972 *vm_pages_free_enqueue_idx(index) = bit;
973 }
974 vm_pages_free_mask_set(index, bit);
975 } else
976 #endif /* CONFIG_SPTM */
977 if (enter_first) {
978 vm_page_queue_enter_first(queue, mem, vmp_pageq);
979 } else {
980 #if defined(__x86_64__)
981 vm_page_queue_enter_clump(queue, mem);
982 #else
983 vm_page_queue_enter(queue, mem, vmp_pageq);
984 #endif
985 }
986
987 switch (class) {
988 case VM_MEMORY_CLASS_REGULAR:
989 VM_COUNTER_INC(&vm_page_queue_free.vmpfq_count);
990 VM_COUNTER_INC(&vm_page_free_count);
991 break;
992 #if XNU_VM_HAS_LOPAGE
993 case VM_MEMORY_CLASS_LOPAGE:
994 VM_COUNTER_INC(&vm_lopage_free_count);
995 if (vm_lopage_free_count >= vm_lopage_free_limit) {
996 vm_lopage_refill = false;
997 }
998 break;
999 #endif /* XNU_VM_HAS_LOPAGE */
1000 #if CONFIG_SECLUDED_MEMORY
1001 case VM_MEMORY_CLASS_SECLUDED:
1002 vm_page_secluded_count++;
1003 vm_page_secluded_count_free++;
1004 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
1005 break;
1006 #endif /* CONFIG_SECLUDED_MEMORY */
1007 default:
1008 __builtin_unreachable();
1009 }
1010 }
1011
1012 /*!
1013 * @typedef vmp_free_list_result_t
1014 *
1015 * @discussion
1016 * This data structure is used by vm_page_free_queue_add_list to track
1017 * how many pages were freed to which free lists, so that it can then drive
1018 * which waiters we are going to wake up.
1019 *
1020 * uint8_t counters are enough because we never free more than 64 pages at
1021 * a time, and this allows for the data structure to be passed by register.
1022 */
1023 typedef struct {
1024 uint8_t vmpr_regular;
1025 uint8_t vmpr_lopage;
1026 #if CONFIG_SECLUDED_MEMORY
1027 uint8_t vmpr_secluded;
1028 #endif /* CONFIG_SECLUDED_MEMORY */
1029 } vmp_free_list_result_t;
1030
1031 /*!
1032 * @abstract
1033 * Returns whether there are any threads blocked in VM_PAGE_WAIT().
1034 *
1035 * @discussion
1036 * The page free queue lock must be held.
1037 */
1038 static bool
vm_page_free_queue_has_any_waiters(void)1039 vm_page_free_queue_has_any_waiters(void)
1040 {
1041 uint32_t result = 0;
1042
1043 result |= vm_page_free_wanted;
1044 result |= vm_page_free_wanted_privileged;
1045 #if CONFIG_SECLUDED_MEMORY
1046 result |= vm_page_free_wanted_secluded;
1047 #endif /* CONFIG_SECLUDED_MEMORY */
1048
1049 return result != 0;
1050 }
1051
1052 void
vm_page_free_wakeup(event_t event,uint32_t n)1053 vm_page_free_wakeup(event_t event, uint32_t n)
1054 {
1055 if (vps_dynamic_priority_enabled) {
1056 if (n == UINT32_MAX) {
1057 wakeup_all_with_inheritor(event, THREAD_AWAKENED);
1058 } else {
1059 while (n-- > 0) {
1060 wakeup_one_with_inheritor(event, THREAD_AWAKENED,
1061 LCK_WAKE_DO_NOT_TRANSFER_PUSH, NULL);
1062 }
1063 }
1064 } else {
1065 thread_wakeup_nthreads(event, n);
1066 }
1067 }
1068
1069 /*!
1070 * @abstract
1071 * Helper to wakeup threads in VM_PAGE_WAIT() given
1072 * a vm_page_free_queue_enter_list() result.
1073 *
1074 * @discussion
1075 * The page free queue lock must be held, and is unlocked on return.
1076 *
1077 * @param vmpr The result of a vm_page_free_queue_enter_list() call.
1078 */
1079 __attribute__((noinline))
1080 static void
vm_page_free_queue_handle_wakeups_and_unlock(vmp_free_list_result_t vmpr)1081 vm_page_free_queue_handle_wakeups_and_unlock(vmp_free_list_result_t vmpr)
1082 {
1083 unsigned int need_wakeup = 0;
1084 unsigned int need_priv_wakeup = 0;
1085 #if CONFIG_SECLUDED_MEMORY
1086 unsigned int need_wakeup_secluded = 0;
1087 #endif /* CONFIG_SECLUDED_MEMORY */
1088 unsigned int unpriv_limit;
1089
1090 #define DONATE_TO_WAITERS(wake, count, waiters_count, limit) ({ \
1091 uint32_t __n = MIN(MIN(waiters_count, vmpr.count), limit); \
1092 waiters_count -= __n; \
1093 vmpr.count -= __n; \
1094 wake += __n; \
1095 __n; \
1096 })
1097
1098 /*
1099 * Step 1: privileged waiters get to be satisfied first
1100 */
1101 if (vm_page_free_wanted_privileged) {
1102 DONATE_TO_WAITERS(need_priv_wakeup,
1103 vmpr_regular, vm_page_free_wanted_privileged,
1104 UINT32_MAX);
1105 }
1106
1107
1108 /*
1109 * Step 2: the privileged reserve needs to be replenished
1110 *
1111 * Let's make sure that we only wake up regular threads
1112 * for free pages above the reserve threshold.
1113 */
1114 if (vm_page_free_count <= vm_page_free_reserved) {
1115 unpriv_limit = 0;
1116 } else {
1117 unpriv_limit = vm_page_free_count - vm_page_free_reserved;
1118 }
1119
1120 /*
1121 * Step 3: satisfy secluded waiters, using the secluded pool first,
1122 * regular pages second.
1123 */
1124 #if CONFIG_SECLUDED_MEMORY
1125 if (vm_page_free_wanted_secluded) {
1126 DONATE_TO_WAITERS(need_wakeup_secluded,
1127 vmpr_secluded, vm_page_free_wanted_secluded,
1128 UINT32_MAX);
1129 unpriv_limit -= DONATE_TO_WAITERS(need_wakeup_secluded,
1130 vmpr_regular, vm_page_free_wanted_secluded,
1131 unpriv_limit);
1132
1133 if (vm_page_free_wanted_secluded == 0) {
1134 need_wakeup_secluded = UINT32_MAX;
1135 }
1136 }
1137 #endif /* CONFIG_SECLUDED_MEMORY */
1138
1139 /*
1140 * Step 4: satisfy regular demand last.
1141 */
1142 if (vm_page_free_wanted) {
1143 unpriv_limit -= DONATE_TO_WAITERS(need_wakeup,
1144 vmpr_regular, vm_page_free_wanted,
1145 unpriv_limit);
1146 if (vm_page_free_wanted == 0) {
1147 need_wakeup = UINT32_MAX;
1148 }
1149 }
1150
1151 /*
1152 * We have updated waiter counts, and if that release page happens
1153 * from the context of a thread that's super low priority we might
1154 * starve waking up privileged threads.
1155 *
1156 * While we hold the free page lock, such threads would wake us up via
1157 * the mutex priority inheritance mechanism, but as soon as we drop the
1158 * lock all bets are off.
1159 *
1160 * To avoid this priority inversion that could really hurt the VM,
1161 * disable preemption until we've woken up everyone.
1162 */
1163 disable_preemption();
1164 vm_free_page_unlock();
1165
1166 /*
1167 * Dispatch privileged wakeups
1168 *
1169 * There shouldn't be that many VM-privileged threads,
1170 * so let's wake them all up, even if we don't quite
1171 * have enough pages to satisfy them all.
1172 */
1173 if (need_priv_wakeup) {
1174 vm_page_free_wakeup(&vm_page_free_wanted_privileged,
1175 UINT32_MAX);
1176 }
1177 if (need_wakeup) {
1178 vm_page_free_wakeup(&vm_page_free_count, need_wakeup);
1179 }
1180 #if CONFIG_SECLUDED_MEMORY
1181 if (need_wakeup_secluded) {
1182 vm_page_free_wakeup(&vm_page_free_wanted_secluded,
1183 need_wakeup_secluded);
1184 }
1185 #endif /* CONFIG_SECLUDED_MEMORY */
1186
1187 enable_preemption();
1188
1189 #undef DONATE_TO_WAITERS
1190 }
1191
1192 /*
1193 * @abstract
1194 * Given a list of pages, put each page on whichever global free queue is
1195 * appropriate.
1196 *
1197 * @discussion
1198 * Must be called with the VM free page lock unlocked.
1199 *
1200 * The list must contain less than 255 elements.
1201 */
1202 static void
vm_page_free_queue_enter_list(vm_page_list_t list,vmp_release_options_t opts)1203 vm_page_free_queue_enter_list(vm_page_list_t list, vmp_release_options_t opts)
1204 {
1205 bool page_queues_unlock = false;
1206 bool page_queues_locked = false;
1207 bool do_secluded = false;
1208 vmp_free_list_result_t result = { };
1209 vm_page_t mem;
1210
1211 LCK_MTX_ASSERT(&vm_page_queue_lock,
1212 (opts & VMP_RELEASE_Q_LOCKED)
1213 ? LCK_MTX_ASSERT_OWNED
1214 : LCK_MTX_ASSERT_NOTOWNED);
1215
1216 /*
1217 * Hibernation and startup do not really need the lock because
1218 * these are single threaded paths, so from the PoV of that function,
1219 * it's as if VMP_RELEASE_Q_LOCKED was passed.
1220 */
1221 page_queues_locked = (opts & (VMP_RELEASE_STARTUP |
1222 VMP_RELEASE_HIBERNATE |
1223 VMP_RELEASE_Q_LOCKED));
1224
1225 #if CONFIG_SECLUDED_MEMORY
1226 do_secluded = vm_page_secluded_pool_depleted();
1227 #endif /* CONFIG_SECLUDED_MEMORY */
1228
1229 if (!page_queues_locked && (list.vmpl_has_realtime || do_secluded)) {
1230 vm_page_lock_queues();
1231 page_queues_locked = true;
1232 page_queues_unlock = true;
1233 }
1234
1235 if (opts & VMP_RELEASE_STARTUP) {
1236 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
1237 } else {
1238 vm_free_page_lock_spin();
1239 }
1240
1241 vm_page_list_foreach_consume(mem, &list) {
1242 ppnum_t pnum = VM_PAGE_GET_PHYS_PAGE(mem);
1243 vm_memory_class_t class = vm_page_get_memory_class(mem, pnum);
1244
1245 if (mem->vmp_realtime) {
1246 mem->vmp_realtime = false;
1247 VM_COUNTER_DEC(&vm_page_realtime_count);
1248 }
1249
1250 #if XNU_VM_HAS_LOPAGE
1251 if ((class == VM_MEMORY_CLASS_REGULAR ||
1252 class == VM_MEMORY_CLASS_LOPAGE) &&
1253 vm_lopage_refill &&
1254 vm_lopage_free_count < vm_lopage_free_limit &&
1255 pnum < max_valid_low_ppnum) {
1256 class = VM_MEMORY_CLASS_LOPAGE;
1257 } else {
1258 class = VM_MEMORY_CLASS_REGULAR;
1259 }
1260 #endif /* XNU_VM_HAS_LOPAGE */
1261
1262 #if CONFIG_SECLUDED_MEMORY
1263 /*
1264 * XXX FBDP TODO: also avoid refilling secluded queue
1265 * when some IOKit objects are already grabbing from it...
1266 */
1267 if (page_queues_locked &&
1268 vm_page_secluded_pool_eligible(class) &&
1269 vm_page_secluded_pool_depleted()) {
1270 class = VM_MEMORY_CLASS_SECLUDED;
1271 }
1272 #endif /* CONFIG_SECLUDED_MEMORY */
1273
1274 vm_page_free_queue_enter(class, mem, pnum);
1275
1276 switch (class) {
1277 case VM_MEMORY_CLASS_REGULAR:
1278 result.vmpr_regular++;
1279 break;
1280 #if XNU_VM_HAS_LOPAGE
1281 case VM_MEMORY_CLASS_LOPAGE:
1282 result.vmpr_lopage++;
1283 break;
1284 #endif /* XNU_VM_HAS_LOPAGE */
1285 #if CONFIG_SECLUDED_MEMORY
1286 case VM_MEMORY_CLASS_SECLUDED:
1287 result.vmpr_secluded++;
1288 continue;
1289 #endif /* CONFIG_SECLUDED_MEMORY */
1290 }
1291 }
1292
1293 if (page_queues_unlock) {
1294 vm_page_unlock_queues();
1295 }
1296
1297 vm_pageout_vminfo.vm_page_pages_freed += list.vmpl_count;
1298 VM_DEBUG_CONSTANT_EVENT(vm_page_release, DBG_VM_PAGE_RELEASE,
1299 DBG_FUNC_NONE, list.vmpl_count, 0, 0, 0);
1300
1301 if (opts & VMP_RELEASE_STARTUP) {
1302 /*
1303 * On purpose skip the VM_CHECK_MEMORYSTATUS,
1304 * pmap_startup() will do it,
1305 * and the caller holds the free queue lock the whole time.
1306 */
1307 return;
1308 }
1309
1310 if (vm_page_free_queue_has_any_waiters()) {
1311 vm_page_free_queue_handle_wakeups_and_unlock(result);
1312 } else {
1313 vm_free_page_unlock();
1314 }
1315
1316 if ((opts & VMP_RELEASE_HIBERNATE) == 0) {
1317 /*
1318 * Skip VM_CHECK_MEMORYSTATUS here as
1319 * hibernate_rebuild_vm_structs() will run it after the last flush.
1320 */
1321 VM_CHECK_MEMORYSTATUS;
1322 }
1323 }
1324
1325 __attribute__((always_inline))
1326 void
vm_page_free_queue_remove(vm_memory_class_t class,vm_page_t mem,ppnum_t pnum,vm_page_q_state_t q_state)1327 vm_page_free_queue_remove(
1328 vm_memory_class_t class,
1329 vm_page_t mem,
1330 ppnum_t pnum,
1331 vm_page_q_state_t q_state)
1332 {
1333 unsigned int color;
1334 vm_page_queue_t queue;
1335
1336 if (startup_phase >= STARTUP_SUB_KMEM) {
1337 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
1338 }
1339
1340 mem->vmp_q_state = q_state;
1341
1342
1343 color = VM_PAGE_GET_COLOR_PNUM(pnum);
1344 queue = vm_page_free_queue_for_class(class, color);
1345 #if CONFIG_SPTM
1346 if (class == VM_MEMORY_CLASS_REGULAR && vm_pages_free_masks()) {
1347 uint32_t index = vm_pages_free_mask_index(pnum);
1348 int8_t bit = vm_pages_free_mask_bit(pnum);
1349
1350 vm_pages_free_mask_clear(index, bit);
1351 if (*vm_pages_free_enqueue_idx(index) == bit) {
1352 vm_page_queue_remove(queue, mem, vmp_pageq);
1353 bit = vm_pages_free_mask_next_bit(index, bit);
1354 *vm_pages_free_enqueue_idx(index) = bit;
1355
1356 if (bit != -1) {
1357 assert(vm_pages_free_mask_test(index, bit));
1358 pnum = (pnum & -MAX_COLORS) + bit;
1359 mem = vm_page_find_canonical(pnum);
1360 color = VM_PAGE_GET_COLOR_PNUM(pnum);
1361 queue = vm_page_free_queue_for_class(class, color);
1362 vm_page_queue_enter(queue, mem, vmp_pageq);
1363 }
1364 }
1365 } else
1366 #endif /* CONFIG_SPTM */
1367 {
1368 vm_page_queue_remove(queue, mem, vmp_pageq);
1369 }
1370
1371 switch (class) {
1372 case VM_MEMORY_CLASS_REGULAR:
1373 VM_COUNTER_DEC(&vm_page_queue_free.vmpfq_count);
1374 VM_COUNTER_DEC(&vm_page_free_count);
1375 break;
1376 #if XNU_VM_HAS_LOPAGE
1377 case VM_MEMORY_CLASS_LOPAGE:
1378 VM_COUNTER_DEC(&vm_lopage_free_count);
1379 vm_lopages_allocated_q += 1;
1380 if (vm_lopage_free_count < vm_lopage_lowater) {
1381 vm_lopage_refill = true;
1382 }
1383 break;
1384 #endif /* XNU_VM_HAS_LOPAGE */
1385 default:
1386 __builtin_unreachable();
1387 }
1388 }
1389
1390 vm_page_list_t
vm_page_free_queue_grab(vm_grab_options_t options __unused,vm_memory_class_t class,unsigned int num_pages,vm_page_q_state_t q_state)1391 vm_page_free_queue_grab(
1392 vm_grab_options_t options __unused,
1393 vm_memory_class_t class,
1394 unsigned int num_pages,
1395 vm_page_q_state_t q_state)
1396 {
1397 unsigned int *colorp;
1398 unsigned int color;
1399 #if defined(__x86_64__)
1400 unsigned int clump_end = 1;
1401 unsigned int sub_count = 0;
1402 #endif /* __x86_64__ */
1403 vm_page_list_t list = { };
1404
1405 if (startup_phase >= STARTUP_SUB_KMEM) {
1406 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
1407 }
1408 assert(get_preemption_level() != 0);
1409 assert(q_state <= VM_PAGE_Q_STATE_LAST_VALID_VALUE);
1410
1411
1412 colorp = PERCPU_GET(start_color);
1413 color = *colorp;
1414
1415 /* Get the pages. */
1416 while (list.vmpl_count < num_pages) {
1417 uint32_t color_offset = 1;
1418 vm_page_queue_t queue;
1419 vm_page_t mem;
1420
1421 queue = vm_page_free_queue_for_class(class, color);
1422 if (!vm_page_free_queue_has_colors(class)) {
1423 assert(!vm_page_queue_empty(queue));
1424 color_offset = 0;
1425 }
1426 while (vm_page_queue_empty(queue)) {
1427 color = (color + 1) & vm_color_mask;
1428 queue = vm_page_free_queue_for_class(class, color);
1429 }
1430
1431 #if defined(__x86_64__)
1432 if (class == VM_MEMORY_CLASS_REGULAR) {
1433 /*
1434 * x86_64 uses a bespoke free queue scheme, where the free path
1435 * tries to cluster clumps of contiguous pages together on
1436 * the free queue to optimize for the platform's memory
1437 * controller.
1438 */
1439 vm_page_queue_remove_first_with_clump(queue, mem, clump_end);
1440 sub_count++;
1441 if (clump_end) {
1442 #if DEVELOPMENT || DEBUG
1443 vm_clump_update_stats(sub_count);
1444 #endif /* !DEVELOPMENT && !DEBUG */
1445 sub_count = 0;
1446 } else {
1447 /* Only change colors at the end of a clump. */
1448 color_offset = 0;
1449 }
1450 } else
1451 #endif /* !defined(__x86_64__) */
1452 {
1453 /* Other targets default to rotating colors after each pop. */
1454 vm_page_queue_remove_first(queue, mem, vmp_pageq);
1455 }
1456
1457 #if CONFIG_SPTM
1458 if (vm_pages_free_masks()) {
1459 ppnum_t pnum = VM_PAGE_GET_PHYS_PAGE(mem);
1460 ppnum_t first_pnum = pnum & -MAX_COLORS;
1461 uint32_t index = vm_pages_free_mask_index(pnum);
1462 int8_t bit = vm_pages_free_mask_bit(pnum);
1463
1464 for (;;) {
1465 vm_pages_free_mask_clear(index, bit);
1466 mem->vmp_q_state = q_state;
1467 vm_page_list_push(&list, mem);
1468
1469 bit = (bit + 1) & (MAX_COLORS - 1);
1470
1471 if (!vm_pages_free_mask_test(index, bit) ||
1472 num_pages <= list.vmpl_count) {
1473 break;
1474 }
1475 mem = vm_page_find_canonical(first_pnum + bit);
1476 }
1477
1478 color = bit & vm_color_mask;
1479
1480 bit = vm_pages_free_mask_next_bit(index, bit);
1481 *vm_pages_free_enqueue_idx(index) = bit;
1482
1483 if (bit != -1) {
1484 assert(vm_pages_free_mask_test(index, bit));
1485 mem = vm_page_find_canonical(first_pnum + bit);
1486 queue = vm_page_free_queue_for_class(class,
1487 bit & vm_color_mask);
1488 vm_page_queue_enter_first(queue, mem, vmp_pageq);
1489 }
1490 } else
1491 #endif /* CONFIG_SPTM */
1492 {
1493 /* Set the page to the client's desired queue state. */
1494 mem->vmp_q_state = q_state;
1495 vm_page_list_push(&list, mem);
1496
1497 color = (color + color_offset) & vm_color_mask;
1498 }
1499 }
1500
1501 switch (class) {
1502 case VM_MEMORY_CLASS_REGULAR:
1503 VM_COUNTER_SUB(&vm_page_queue_free.vmpfq_count, list.vmpl_count);
1504 VM_COUNTER_SUB(&vm_page_free_count, list.vmpl_count);
1505 break;
1506 #if XNU_VM_HAS_LOPAGE
1507 case VM_MEMORY_CLASS_LOPAGE:
1508 VM_COUNTER_SUB(&vm_lopage_free_count, list.vmpl_count);
1509 vm_lopages_allocated_q += list.vmpl_count;
1510 if (vm_lopage_free_count < vm_lopage_lowater) {
1511 vm_lopage_refill = true;
1512 }
1513 break;
1514 #endif /* XNU_VM_HAS_LOPAGE */
1515 default:
1516 __builtin_unreachable();
1517 }
1518
1519 /* Record the next page color the CPU should try to get. */
1520 *colorp = color;
1521 #if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
1522 vm_clump_update_stats(sub_count);
1523 #endif /* defined(__x86_64__) && (DEVELOPMENT || DEBUG) */
1524
1525 return list;
1526 }
1527
1528
1529 #define COLOR_GROUPS_TO_STEAL 4
1530
1531 /* Called once during statup, once the cache geometry is known.
1532 */
1533 static void
vm_page_set_colors(void)1534 vm_page_set_colors( void )
1535 {
1536 unsigned int n, override;
1537
1538 #if defined (__x86_64__)
1539 /* adjust #colors because we need to color outside the clump boundary */
1540 vm_cache_geometry_colors >>= vm_clump_shift;
1541 #endif
1542 if (PE_parse_boot_argn("colors", &override, sizeof(override))) { /* colors specified as a boot-arg? */
1543 n = override;
1544 } else if (vm_cache_geometry_colors) { /* do we know what the cache geometry is? */
1545 n = vm_cache_geometry_colors;
1546 } else {
1547 n = DEFAULT_COLORS; /* use default if all else fails */
1548 }
1549 if (n == 0) {
1550 n = 1;
1551 }
1552 if (n > MAX_COLORS) {
1553 n = MAX_COLORS;
1554 }
1555
1556 /* the count must be a power of 2 */
1557 if ((n & (n - 1)) != 0) {
1558 n = DEFAULT_COLORS; /* use default if all else fails */
1559 }
1560 vm_colors = n;
1561 vm_color_mask = n - 1;
1562
1563 vm_free_magazine_refill_limit = vm_colors * COLOR_GROUPS_TO_STEAL;
1564
1565 #if defined (__x86_64__)
1566 /* adjust for reduction in colors due to clumping and multiple cores */
1567 if (real_ncpus) {
1568 vm_free_magazine_refill_limit *= (vm_clump_size * real_ncpus);
1569 }
1570 #endif
1571 }
1572
1573 #if XNU_VM_HAS_DELAYED_PAGES
1574
1575 static uint32_t vm_delayed_count = 0; /* when non-zero, indicates we may have more pages to init */
1576 static ppnum_t delay_above_pnum = PPNUM_MAX;
1577
1578 /*
1579 * For x86 first 8 Gig initializes quickly and gives us lots of lowmem + mem above to start off with.
1580 * If ARM ever uses delayed page initialization, this value may need to be quite different.
1581 */
1582 #define DEFAULT_DELAY_ABOVE_PHYS_GB (8)
1583
1584 /*
1585 * When we have to dip into more delayed pages due to low memory, free up
1586 * a large chunk to get things back to normal. This avoids contention on the
1587 * delayed code allocating page by page.
1588 */
1589 #define VM_DELAY_PAGE_CHUNK ((1024 * 1024 * 1024) / PAGE_SIZE)
1590
1591 /*
1592 * Get and initialize the next delayed page.
1593 */
1594 __attribute__((noinline))
1595 static vm_page_t
vm_get_delayed_page(vm_grab_options_t grab_options)1596 vm_get_delayed_page(vm_grab_options_t grab_options)
1597 {
1598 vm_page_t p;
1599 ppnum_t pnum;
1600
1601 /*
1602 * Get a new page if we have one.
1603 */
1604 vm_free_page_lock();
1605 if (vm_delayed_count == 0) {
1606 vm_free_page_unlock();
1607 return NULL;
1608 }
1609
1610 if (!pmap_next_page(&pnum)) {
1611 vm_delayed_count = 0;
1612 vm_free_page_unlock();
1613 return NULL;
1614 }
1615
1616
1617 assert(vm_delayed_count > 0);
1618 --vm_delayed_count;
1619
1620 #if defined(__x86_64__)
1621 /* x86 cluster code requires increasing phys_page in vm_pages[] */
1622 if (vm_pages_count > 0) {
1623 assert(pnum > vm_page_get(vm_pages_count - 1)->vmp_phys_page);
1624 }
1625 #endif
1626 p = vm_page_get(vm_pages_count);
1627 assert(p < vm_pages_end);
1628 vm_page_init(p, pnum);
1629 ++vm_pages_count;
1630 ++vm_page_pages;
1631 vm_free_page_unlock();
1632
1633 /*
1634 * These pages were initially counted as wired, undo that now.
1635 */
1636 if (grab_options & VM_PAGE_GRAB_Q_LOCK_HELD) {
1637 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
1638 } else {
1639 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
1640 vm_page_lockspin_queues();
1641 }
1642 --vm_page_wire_count;
1643 --vm_page_wire_count_initial;
1644 if (vm_page_wire_count_on_boot != 0) {
1645 --vm_page_wire_count_on_boot;
1646 }
1647 if (!(grab_options & VM_PAGE_GRAB_Q_LOCK_HELD)) {
1648 vm_page_unlock_queues();
1649 }
1650
1651
1652 if (fillval) {
1653 fillPage(pnum, fillval);
1654 }
1655 return p;
1656 }
1657
1658 /*
1659 * Free all remaining delayed pages to the free lists.
1660 */
1661 void
vm_free_delayed_pages(void)1662 vm_free_delayed_pages(void)
1663 {
1664 vm_page_t p;
1665 vm_page_t list = NULL;
1666 uint_t cnt = 0;
1667 vm_offset_t start_free_va;
1668 int64_t free_size;
1669
1670 while ((p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE)) != NULL) {
1671 if (vm_himemory_mode) {
1672 vm_page_release(p, VMP_RELEASE_NONE);
1673 } else {
1674 p->vmp_snext = list;
1675 list = p;
1676 }
1677 ++cnt;
1678 }
1679
1680 /*
1681 * Free the pages in reverse order if not himemory mode.
1682 * Hence the low memory pages will be first on free lists. (LIFO)
1683 */
1684 while (list != NULL) {
1685 p = list;
1686 list = p->vmp_snext;
1687 p->vmp_snext = NULL;
1688 vm_page_release(p, VMP_RELEASE_NONE);
1689 }
1690 #if DEVELOPMENT || DEBUG
1691 kprintf("vm_free_delayed_pages: initialized %d free pages\n", cnt);
1692 #endif
1693
1694 /*
1695 * Free up any unused full pages at the end of the vm_pages[] array
1696 */
1697 start_free_va = round_page((vm_offset_t)vm_page_get(vm_pages_count));
1698
1699 #if defined(__x86_64__)
1700 /*
1701 * Since x86 might have used large pages for vm_pages[], we can't
1702 * free starting in the middle of a partially used large page.
1703 */
1704 if (pmap_query_pagesize(kernel_pmap, start_free_va) == I386_LPGBYTES) {
1705 start_free_va = ((start_free_va + I386_LPGMASK) & ~I386_LPGMASK);
1706 }
1707 #endif
1708 if (start_free_va < (vm_offset_t)vm_pages_end) {
1709 free_size = trunc_page((vm_offset_t)vm_pages_end - start_free_va);
1710 if (free_size > 0) {
1711 ml_static_mfree(start_free_va, (vm_offset_t)free_size);
1712 vm_pages_end = (void *)start_free_va;
1713
1714 /*
1715 * Note there's no locking here, as only this thread will ever change this value.
1716 * The reader, vm_page_diagnose, doesn't grab any locks for the counts it looks at.
1717 */
1718 vm_page_stolen_count -= (free_size >> PAGE_SHIFT);
1719
1720 #if DEVELOPMENT || DEBUG
1721 kprintf("Freeing final unused %ld bytes from vm_pages[] at 0x%lx\n",
1722 (long)free_size, (long)start_free_va);
1723 #endif
1724 }
1725 }
1726 }
1727
1728 /*
1729 * Try and free up enough delayed pages to match a contig memory allocation.
1730 */
1731 static void
vm_free_delayed_pages_contig(uint_t npages,ppnum_t max_pnum,ppnum_t pnum_mask)1732 vm_free_delayed_pages_contig(
1733 uint_t npages,
1734 ppnum_t max_pnum,
1735 ppnum_t pnum_mask)
1736 {
1737 vm_page_t p;
1738 ppnum_t pnum;
1739 uint_t cnt = 0;
1740
1741 /*
1742 * Treat 0 as the absolute max page number.
1743 */
1744 if (max_pnum == 0) {
1745 max_pnum = PPNUM_MAX;
1746 }
1747
1748 /*
1749 * Free till we get a properly aligned start page
1750 */
1751 for (;;) {
1752 p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
1753 if (p == NULL) {
1754 return;
1755 }
1756 pnum = VM_PAGE_GET_PHYS_PAGE(p);
1757 vm_page_release(p, VMP_RELEASE_NONE);
1758 if (pnum >= max_pnum) {
1759 return;
1760 }
1761 if ((pnum & pnum_mask) == 0) {
1762 break;
1763 }
1764 }
1765
1766 /*
1767 * Having a healthy pool of free pages will help performance. We don't
1768 * want to fall back to the delayed code for every page allocation.
1769 */
1770 if (vm_page_free_count < VM_DELAY_PAGE_CHUNK) {
1771 npages += VM_DELAY_PAGE_CHUNK;
1772 }
1773
1774 /*
1775 * Now free up the pages
1776 */
1777 for (cnt = 1; cnt < npages; ++cnt) {
1778 p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
1779 if (p == NULL) {
1780 return;
1781 }
1782 vm_page_release(p, VMP_RELEASE_NONE);
1783 }
1784 }
1785
1786 #endif /* XNU_VM_HAS_DELAYED_PAGES */
1787
1788 #define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1)))
1789
1790 void
vm_page_init_local_q(unsigned int num_cpus)1791 vm_page_init_local_q(unsigned int num_cpus)
1792 {
1793 struct vpl *t_local_q;
1794
1795 /*
1796 * no point in this for a uni-processor system
1797 */
1798 if (num_cpus >= 2) {
1799 ml_cpu_info_t cpu_info;
1800
1801 /*
1802 * Force the allocation alignment to a cacheline,
1803 * because the `vpl` struct has a lock and will be taken
1804 * cross CPU so we want to isolate the rest of the per-CPU
1805 * data to avoid false sharing due to this lock being taken.
1806 */
1807
1808 ml_cpu_get_info(&cpu_info);
1809
1810 t_local_q = zalloc_percpu_permanent(sizeof(struct vpl),
1811 cpu_info.cache_line_size - 1);
1812
1813 zpercpu_foreach(lq, t_local_q) {
1814 VPL_LOCK_INIT(lq, &vm_page_lck_grp_local, &vm_page_lck_attr);
1815 vm_page_queue_init(&lq->vpl_queue);
1816 }
1817
1818 /* make the initialization visible to all cores */
1819 os_atomic_store(&vm_page_local_q, t_local_q, release);
1820 }
1821 }
1822
1823 /*
1824 * vm_init_before_launchd
1825 *
1826 * This should be called right before launchd is loaded.
1827 */
1828 void
vm_init_before_launchd(void)1829 vm_init_before_launchd(void)
1830 {
1831 vm_page_lockspin_queues();
1832 vm_page_wire_count_on_boot = vm_page_wire_count;
1833 vm_page_unlock_queues();
1834 }
1835
1836
1837 /*
1838 * vm_page_bootstrap:
1839 *
1840 * Initializes the resident memory module.
1841 *
1842 * Allocates memory for the page cells, and
1843 * for the object/offset-to-page hash table headers.
1844 * Each page cell is initialized and placed on the free list.
1845 * Returns the range of available kernel virtual memory.
1846 */
1847 __startup_func
1848 void
vm_page_bootstrap(vm_offset_t * startp,vm_offset_t * endp)1849 vm_page_bootstrap(
1850 vm_offset_t *startp,
1851 vm_offset_t *endp)
1852 {
1853 unsigned int i;
1854 unsigned int log1;
1855 unsigned int log2;
1856 unsigned int size;
1857
1858 /*
1859 * Initialize the page queues.
1860 */
1861
1862 lck_mtx_init(&vm_page_queue_free_lock, &vm_page_lck_grp_free, &vm_page_lck_attr);
1863 lck_mtx_init(&vm_page_queue_lock, &vm_page_lck_grp_queue, &vm_page_lck_attr);
1864 lck_mtx_init(&vm_purgeable_queue_lock, &vm_page_lck_grp_purge, &vm_page_lck_attr);
1865
1866 for (i = 0; i < PURGEABLE_Q_TYPE_MAX; i++) {
1867 int group;
1868
1869 purgeable_queues[i].token_q_head = 0;
1870 purgeable_queues[i].token_q_tail = 0;
1871 for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
1872 queue_init(&purgeable_queues[i].objq[group]);
1873 }
1874
1875 purgeable_queues[i].type = i;
1876 purgeable_queues[i].new_pages = 0;
1877 #if MACH_ASSERT
1878 purgeable_queues[i].debug_count_tokens = 0;
1879 purgeable_queues[i].debug_count_objects = 0;
1880 #endif
1881 }
1882 ;
1883 purgeable_nonvolatile_count = 0;
1884 queue_init(&purgeable_nonvolatile_queue);
1885
1886 vm_page_free_queue_init(&vm_page_queue_free);
1887 #if XNU_VM_HAS_LOPAGE
1888 vm_page_queue_init(&vm_lopage_queue_free);
1889 #endif /* XNU_VM_HAS_LOPAGE */
1890 vm_page_queue_init(&vm_page_queue_active);
1891 vm_page_queue_init(&vm_page_queue_inactive);
1892 #if CONFIG_SECLUDED_MEMORY
1893 vm_page_queue_init(&vm_page_queue_secluded);
1894 #endif /* CONFIG_SECLUDED_MEMORY */
1895 vm_page_queue_init(&vm_page_queue_cleaned);
1896 vm_page_queue_init(&vm_page_queue_throttled);
1897 vm_page_queue_init(&vm_page_queue_anonymous);
1898 queue_init(&vm_objects_wired);
1899
1900 for (i = 0; i <= vm_page_max_speculative_age_q; i++) {
1901 vm_page_queue_init(&vm_page_queue_speculative[i].age_q);
1902
1903 vm_page_queue_speculative[i].age_ts.tv_sec = 0;
1904 vm_page_queue_speculative[i].age_ts.tv_nsec = 0;
1905 }
1906
1907 vm_page_queue_init(&vm_page_queue_donate);
1908 vm_page_queue_init(&vm_page_queue_background);
1909
1910 vm_page_background_count = 0;
1911 vm_page_background_internal_count = 0;
1912 vm_page_background_external_count = 0;
1913 vm_page_background_promoted_count = 0;
1914
1915 vm_page_background_target = (unsigned int)(atop_64(max_mem) / 25);
1916
1917 if (vm_page_background_target > VM_PAGE_BACKGROUND_TARGET_MAX) {
1918 vm_page_background_target = VM_PAGE_BACKGROUND_TARGET_MAX;
1919 }
1920
1921 #if defined(__LP64__)
1922 vm_page_background_mode = VM_PAGE_BG_ENABLED;
1923 vm_page_donate_mode = VM_PAGE_DONATE_ENABLED;
1924 #else
1925 vm_page_background_mode = VM_PAGE_BG_DISABLED;
1926 vm_page_donate_mode = VM_PAGE_DONATE_DISABLED;
1927 #endif
1928 vm_page_background_exclude_external = 0;
1929
1930 PE_parse_boot_argn("vm_page_bg_mode", &vm_page_background_mode, sizeof(vm_page_background_mode));
1931 PE_parse_boot_argn("vm_page_bg_exclude_external", &vm_page_background_exclude_external, sizeof(vm_page_background_exclude_external));
1932 PE_parse_boot_argn("vm_page_bg_target", &vm_page_background_target, sizeof(vm_page_background_target));
1933
1934 if (vm_page_background_mode != VM_PAGE_BG_DISABLED && vm_page_background_mode != VM_PAGE_BG_ENABLED) {
1935 vm_page_background_mode = VM_PAGE_BG_DISABLED;
1936 }
1937
1938 PE_parse_boot_argn("vm_page_donate_mode", &vm_page_donate_mode, sizeof(vm_page_donate_mode));
1939 if (vm_page_donate_mode != VM_PAGE_DONATE_DISABLED && vm_page_donate_mode != VM_PAGE_DONATE_ENABLED) {
1940 vm_page_donate_mode = VM_PAGE_DONATE_DISABLED;
1941 }
1942
1943 vm_page_donate_target_high = VM_PAGE_DONATE_TARGET_HIGHWATER;
1944 vm_page_donate_target_low = VM_PAGE_DONATE_TARGET_LOWWATER;
1945 vm_page_donate_target = vm_page_donate_target_high;
1946 vm_page_donate_count = 0;
1947
1948 vm_page_free_wanted = 0;
1949 vm_page_free_wanted_privileged = 0;
1950 #if CONFIG_SECLUDED_MEMORY
1951 vm_page_free_wanted_secluded = 0;
1952 #endif /* CONFIG_SECLUDED_MEMORY */
1953
1954 #if defined (__x86_64__)
1955 /* this must be called before vm_page_set_colors() */
1956 vm_page_setup_clump();
1957 #endif
1958
1959 vm_page_set_colors();
1960
1961 for (vm_tag_t t = 0; t < VM_KERN_MEMORY_FIRST_DYNAMIC; t++) {
1962 vm_allocation_sites_static[t].refcount = 2;
1963 vm_allocation_sites_static[t].tag = t;
1964 vm_allocation_sites[t] = &vm_allocation_sites_static[t];
1965 }
1966 vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].refcount = 2;
1967 vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].tag = VM_KERN_MEMORY_ANY;
1968 vm_allocation_sites[VM_KERN_MEMORY_ANY] = &vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC];
1969
1970 /*
1971 * Steal memory for the map and zone subsystems.
1972 *
1973 * make sure initialize_ram_ranges() has run before we steal pages for the first time on arm
1974 */
1975 (void)pmap_free_pages();
1976
1977 kernel_startup_initialize_upto(STARTUP_SUB_PMAP_STEAL);
1978
1979 /*
1980 * Allocate (and initialize) the virtual-to-physical
1981 * table hash buckets.
1982 *
1983 * The number of buckets should be a power of two to
1984 * get a good hash function. The following computation
1985 * chooses the first power of two that is greater
1986 * than the number of physical pages in the system.
1987 */
1988
1989 if (vm_page_bucket_count == 0) {
1990 unsigned int npages = pmap_free_pages();
1991
1992 vm_page_bucket_count = 1;
1993 while (vm_page_bucket_count < npages) {
1994 vm_page_bucket_count <<= 1;
1995 }
1996 }
1997 vm_page_bucket_lock_count = (vm_page_bucket_count + BUCKETS_PER_LOCK - 1) / BUCKETS_PER_LOCK;
1998
1999 vm_page_hash_mask = vm_page_bucket_count - 1;
2000
2001 /*
2002 * Calculate object shift value for hashing algorithm:
2003 * O = log2(sizeof(struct vm_object))
2004 * B = log2(vm_page_bucket_count)
2005 * hash shifts the object left by
2006 * B/2 - O
2007 */
2008 size = vm_page_bucket_count;
2009 for (log1 = 0; size > 1; log1++) {
2010 size /= 2;
2011 }
2012 size = sizeof(struct vm_object);
2013 for (log2 = 0; size > 1; log2++) {
2014 size /= 2;
2015 }
2016 vm_page_hash_shift = log1 / 2 - log2 + 1;
2017
2018 vm_page_bucket_hash = 1 << ((log1 + 1) >> 1); /* Get (ceiling of sqrt of table size) */
2019 vm_page_bucket_hash |= 1 << ((log1 + 1) >> 2); /* Get (ceiling of quadroot of table size) */
2020 vm_page_bucket_hash |= 1; /* Set bit and add 1 - always must be 1 to insure unique series */
2021
2022 if (vm_page_hash_mask & vm_page_bucket_count) {
2023 printf("vm_page_bootstrap: WARNING -- strange page hash\n");
2024 }
2025
2026 #if VM_PAGE_BUCKETS_CHECK
2027 #if VM_PAGE_FAKE_BUCKETS
2028 /*
2029 * Allocate a decoy set of page buckets, to detect
2030 * any stomping there.
2031 */
2032 vm_page_fake_buckets = (vm_page_bucket_t *)
2033 pmap_steal_memory(vm_page_bucket_count *
2034 sizeof(vm_page_bucket_t), 0);
2035 vm_page_fake_buckets_start = (vm_map_offset_t) vm_page_fake_buckets;
2036 vm_page_fake_buckets_end =
2037 vm_map_round_page((vm_page_fake_buckets_start +
2038 (vm_page_bucket_count *
2039 sizeof(vm_page_bucket_t))),
2040 PAGE_MASK);
2041 char *cp;
2042 for (cp = (char *)vm_page_fake_buckets_start;
2043 cp < (char *)vm_page_fake_buckets_end;
2044 cp++) {
2045 *cp = 0x5a;
2046 }
2047 #endif /* VM_PAGE_FAKE_BUCKETS */
2048 #endif /* VM_PAGE_BUCKETS_CHECK */
2049
2050 kernel_debug_string_early("vm_page_buckets");
2051 vm_page_buckets = (vm_page_bucket_t *)
2052 pmap_steal_memory(vm_page_bucket_count *
2053 sizeof(vm_page_bucket_t), 0);
2054
2055 kernel_debug_string_early("vm_page_bucket_locks");
2056 vm_page_bucket_locks = (lck_ticket_t *)
2057 pmap_steal_memory(vm_page_bucket_lock_count *
2058 sizeof(lck_ticket_t), 0);
2059
2060 for (i = 0; i < vm_page_bucket_count; i++) {
2061 vm_page_bucket_t *bucket = &vm_page_buckets[i];
2062
2063 bucket->page_list = VM_PAGE_PACK_PTR(VM_PAGE_NULL);
2064 #if MACH_PAGE_HASH_STATS
2065 bucket->cur_count = 0;
2066 bucket->hi_count = 0;
2067 #endif /* MACH_PAGE_HASH_STATS */
2068 }
2069
2070 for (i = 0; i < vm_page_bucket_lock_count; i++) {
2071 lck_ticket_init(&vm_page_bucket_locks[i], &vm_page_lck_grp_bucket);
2072 }
2073
2074 vm_tag_init();
2075
2076 #if VM_PAGE_BUCKETS_CHECK
2077 vm_page_buckets_check_ready = TRUE;
2078 #endif /* VM_PAGE_BUCKETS_CHECK */
2079
2080 /*
2081 * Machine-dependent code allocates the resident page table.
2082 * It uses vm_page_init to initialize the page frames.
2083 * The code also returns to us the virtual space available
2084 * to the kernel. We don't trust the pmap module
2085 * to get the alignment right.
2086 */
2087
2088 kernel_debug_string_early("pmap_startup");
2089 pmap_startup(&virtual_space_start, &virtual_space_end);
2090 virtual_space_start = round_page(virtual_space_start);
2091 virtual_space_end = trunc_page(virtual_space_end);
2092
2093 *startp = virtual_space_start;
2094 *endp = virtual_space_end;
2095
2096 /*
2097 * Compute the initial "wire" count.
2098 * Up until now, the pages which have been set aside are not under
2099 * the VM system's control, so although they aren't explicitly
2100 * wired, they nonetheless can't be moved. At this moment,
2101 * all VM managed pages are "free", courtesy of pmap_startup.
2102 */
2103 assert((unsigned int) atop_64(max_mem) == atop_64(max_mem));
2104 vm_page_wire_count = ((unsigned int) atop_64(max_mem)) -
2105 vm_page_free_count - vm_lopage_free_count;
2106 #if CONFIG_SECLUDED_MEMORY
2107 vm_page_wire_count -= vm_page_secluded_count;
2108 #endif
2109 vm_page_wire_count_initial = vm_page_wire_count;
2110
2111 /* capture this for later use */
2112 booter_size = ml_get_booter_memory_size();
2113
2114 printf("vm_page_bootstrap: %d free pages, %d wired pages"
2115 #if XNU_VM_HAS_DELAYED_PAGES
2116 ", (up to %d of which are delayed free)"
2117 #endif /* XNU_VM_HAS_DELAYED_PAGES */
2118 "%c",
2119 vm_page_free_count,
2120 vm_page_wire_count,
2121 #if XNU_VM_HAS_DELAYED_PAGES
2122 vm_delayed_count,
2123 #endif /* XNU_VM_HAS_DELAYED_PAGES */
2124 '\n');
2125
2126 kernel_debug_string_early("vm_page_bootstrap complete");
2127 }
2128
2129 #ifndef MACHINE_PAGES
2130 /*
2131 * This is the early boot time allocator for data structures needed to bootstrap the VM system.
2132 * On x86 it will allocate large pages if size is sufficiently large. We don't need to do this
2133 * on ARM yet, due to the combination of a large base page size and smaller RAM devices.
2134 */
2135 __static_testable void *
pmap_steal_memory_internal(vm_size_t size,vm_size_t alignment,boolean_t might_free,unsigned int flags,pmap_mapping_type_t mapping_type)2136 pmap_steal_memory_internal(
2137 vm_size_t size,
2138 vm_size_t alignment,
2139 boolean_t might_free,
2140 unsigned int flags,
2141 pmap_mapping_type_t mapping_type)
2142 {
2143 kern_return_t kr;
2144 vm_offset_t addr;
2145 vm_offset_t end = 0;
2146 vm_offset_t map_addr;
2147 ppnum_t phys_page;
2148 unsigned int pmap_flags;
2149
2150 if (size > UINT64_MAX - sizeof(void *)) {
2151 panic("pmap_steal_memory(): size: 0x%lx", size);
2152 }
2153 /*
2154 * Size needs to be aligned to word size.
2155 */
2156 size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
2157
2158 /*
2159 * Alignment defaults to word size if not specified.
2160 */
2161 if (alignment == 0) {
2162 alignment = sizeof(void*);
2163 }
2164
2165 /*
2166 * Alignment must be no greater than a page and must be a power of two.
2167 */
2168 assert(alignment <= PAGE_SIZE);
2169 assert((alignment & (alignment - 1)) == 0);
2170
2171 /*
2172 * On the first call, get the initial values for virtual address space
2173 * and page align them.
2174 */
2175 if (virtual_space_start == virtual_space_end) {
2176 pmap_virtual_space(&virtual_space_start, &virtual_space_end);
2177 virtual_space_start = round_page(virtual_space_start);
2178 virtual_space_end = trunc_page(virtual_space_end);
2179
2180 #if defined(__x86_64__)
2181 /*
2182 * Release remaining unused section of preallocated KVA and the 4K page tables
2183 * that map it. This makes the VA available for large page mappings.
2184 */
2185 Idle_PTs_release(virtual_space_start, virtual_space_end);
2186 #endif
2187 }
2188
2189 /*
2190 * Allocate the virtual space for this request. On x86, we'll align to a large page
2191 * address if the size is big enough to back with at least 1 large page.
2192 */
2193 #if defined(__x86_64__)
2194 if (size >= I386_LPGBYTES) {
2195 virtual_space_start = ((virtual_space_start + I386_LPGMASK) & ~I386_LPGMASK);
2196 }
2197 #endif
2198 virtual_space_start = (virtual_space_start + (alignment - 1)) & ~(alignment - 1);
2199 addr = virtual_space_start;
2200 virtual_space_start += size;
2201
2202 //kprintf("pmap_steal_memory: %08lX - %08lX; size=%08lX\n", (long)addr, (long)virtual_space_start, (long)size); /* (TEST/DEBUG) */
2203
2204 /*
2205 * Allocate and map physical pages to back the new virtual space.
2206 */
2207 map_addr = round_page(addr);
2208 if (os_add_overflow(addr, size, &end)) {
2209 panic("pmap_steal_memory() overflow, addr: %lx, size: 0x%lx", addr, size);
2210 }
2211 while (map_addr < end) {
2212 #if defined(__x86_64__)
2213 /*
2214 * Back with a large page if properly aligned on x86
2215 */
2216 if ((map_addr & I386_LPGMASK) == 0 &&
2217 map_addr + I386_LPGBYTES <= addr + size &&
2218 pmap_pre_expand_large(kernel_pmap, map_addr) == KERN_SUCCESS &&
2219 pmap_next_page_large(&phys_page) == KERN_SUCCESS) {
2220 kr = pmap_enter(kernel_pmap, map_addr, phys_page,
2221 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
2222 VM_WIMG_USE_DEFAULT | VM_MEM_SUPERPAGE, FALSE, mapping_type);
2223
2224 if (kr != KERN_SUCCESS) {
2225 panic("pmap_steal_memory: pmap_enter() large failed, new_addr=%#lx, phys_page=%u",
2226 (unsigned long)map_addr, phys_page);
2227 }
2228 map_addr += I386_LPGBYTES;
2229 vm_page_wire_count += I386_LPGBYTES >> PAGE_SHIFT;
2230 vm_page_stolen_count += I386_LPGBYTES >> PAGE_SHIFT;
2231 vm_page_kern_lpage_count++;
2232 continue;
2233 }
2234 #endif
2235
2236 if (!pmap_next_page_hi(&phys_page, might_free)) {
2237 panic("pmap_steal_memory() size: 0x%llx", (uint64_t)size);
2238 }
2239
2240 #if defined(__x86_64__)
2241 pmap_pre_expand(kernel_pmap, map_addr);
2242 #endif
2243 pmap_flags = flags ? flags : VM_WIMG_USE_DEFAULT;
2244
2245
2246 kr = pmap_enter(kernel_pmap, map_addr, phys_page,
2247 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
2248 pmap_flags, FALSE, mapping_type);
2249
2250 if (kr != KERN_SUCCESS) {
2251 panic("pmap_steal_memory() pmap_enter failed, map_addr=%#lx, phys_page=%u",
2252 (unsigned long)map_addr, phys_page);
2253 }
2254 map_addr += PAGE_SIZE;
2255
2256 /*
2257 * Account for newly stolen memory
2258 */
2259 vm_page_wire_count++;
2260 vm_page_stolen_count++;
2261 }
2262
2263 #if defined(__x86_64__)
2264 /*
2265 * The call with might_free is currently the last use of pmap_steal_memory*().
2266 * Notify the pmap layer to record which high pages were allocated so far.
2267 */
2268 if (might_free) {
2269 pmap_hi_pages_done();
2270 }
2271 #endif
2272 #if KASAN
2273 kasan_notify_address(round_page(addr), size);
2274 #endif
2275 return (void *) addr;
2276 }
2277
2278 __mockable void *
pmap_steal_memory(vm_size_t size,vm_size_t alignment)2279 pmap_steal_memory(
2280 vm_size_t size,
2281 vm_size_t alignment)
2282 {
2283 return pmap_steal_memory_internal(size, alignment, FALSE, 0, PMAP_MAPPING_TYPE_RESTRICTED);
2284 }
2285
2286 void *
pmap_steal_freeable_memory(vm_size_t size)2287 pmap_steal_freeable_memory(
2288 vm_size_t size)
2289 {
2290 return pmap_steal_memory_internal(size, 0, TRUE, 0, PMAP_MAPPING_TYPE_RESTRICTED);
2291 }
2292
2293
2294
2295 #if CONFIG_SECLUDED_MEMORY
2296 /* boot-args to control secluded memory */
2297 TUNABLE_DT(unsigned int, secluded_mem_mb, "/defaults", "kern.secluded_mem_mb", "secluded_mem_mb", 0, TUNABLE_DT_NONE);
2298 /* IOKit can use secluded memory */
2299 TUNABLE(bool, secluded_for_iokit, "secluded_for_iokit", true);
2300 /* apps can use secluded memory */
2301 TUNABLE(bool, secluded_for_apps, "secluded_for_apps", true);
2302 /* filecache can use seclude memory */
2303 TUNABLE(secluded_filecache_mode_t, secluded_for_filecache, "secluded_for_filecache", SECLUDED_FILECACHE_RDONLY);
2304 uint64_t secluded_shutoff_trigger = 0;
2305 uint64_t secluded_shutoff_headroom = 150 * 1024 * 1024; /* original value from N56 */
2306 #endif /* CONFIG_SECLUDED_MEMORY */
2307
2308
2309 #if defined(__arm64__)
2310 extern void patch_low_glo_vm_page_info(void *, void *, uint32_t);
2311 #endif
2312
2313 void vm_page_release_startup(vm_page_t mem);
2314 __mockable void
pmap_startup(vm_offset_t * startp,vm_offset_t * endp)2315 pmap_startup(
2316 vm_offset_t *startp,
2317 vm_offset_t *endp)
2318 {
2319 unsigned int npages;
2320 ppnum_t phys_page;
2321 uint64_t mem_sz;
2322 uint64_t start_ns;
2323 uint64_t now_ns;
2324 uint32_t divisor;
2325 #if XNU_VM_HAS_DELAYED_PAGES
2326 uint_t low_page_count = 0;
2327 #endif /* XNU_VM_HAS_DELAYED_PAGES */
2328
2329 /*
2330 * make sure we are aligned on a 64 byte boundary
2331 * for VM_PAGE_PACK_PTR (it clips off the low-order
2332 * 6 bits of the pointer)
2333 */
2334 if (virtual_space_start != virtual_space_end) {
2335 virtual_space_start = round_page(virtual_space_start);
2336 }
2337
2338 /*
2339 * We calculate how many page frames we will have
2340 * and then allocate the page structures in one chunk.
2341 *
2342 * Note that the calculation here doesn't take into account
2343 * the memory needed to map what's being allocated, i.e. the page
2344 * table entries. So the actual number of pages we get will be
2345 * less than this. To do someday: include that in the computation.
2346 *
2347 * Also for ARM, we don't use the count of free_pages, but rather the
2348 * range from last page to first page (ignore holes due to retired pages).
2349 */
2350
2351 /*
2352 * Initialize and release the page frames.
2353 */
2354 kernel_debug_string_early("page_frame_init");
2355 absolutetime_to_nanoseconds(mach_absolute_time(), &start_ns);
2356 if (fillval) {
2357 kprintf("Filling vm_pages with pattern: 0x%x\n", fillval);
2358 }
2359
2360 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
2361 mem_sz = ptoa(pmap_free_pages_span());
2362 #if CONFIG_SPTM
2363 {
2364 uint32_t count = vm_pages_free_mask_len();
2365
2366 _vm_pages_free_masks = pmap_steal_memory(count *
2367 sizeof(__uint128_t), sizeof(__uint128_t));
2368 _vm_pages_free_enqueue_idx = pmap_steal_memory(count, sizeof(uint8_t));
2369 bzero(_vm_pages_free_masks, count * sizeof(__uint128_t));
2370 memset(_vm_pages_free_enqueue_idx, 0xff, count);
2371 }
2372 #endif /* CONFIG_SPTM */
2373 #else
2374 mem_sz = ptoa(pmap_free_pages());
2375 #endif
2376 mem_sz += round_page(virtual_space_start) - virtual_space_start; /* Account for any slop */
2377 divisor = PAGE_SIZE + sizeof(struct vm_page);
2378 npages = (uint32_t)((mem_sz + divisor - 1) / divisor); /* scaled to include the vm_page_ts */
2379
2380
2381 vm_pages = pmap_steal_freeable_memory(npages * sizeof(struct vm_page));
2382 vm_pages_end = vm_page_get(npages);
2383
2384 #if CONFIG_SECLUDED_MEMORY
2385 /*
2386 * Figure out how much secluded memory to have before we start
2387 * release pages to free lists.
2388 * The default, if specified nowhere else, is no secluded mem.
2389 */
2390 vm_page_secluded_target = (unsigned int)atop_64(secluded_mem_mb * 1024ULL * 1024ULL);
2391
2392 /*
2393 * Allow a really large app to effectively use secluded memory until it exits.
2394 */
2395 if (vm_page_secluded_target != 0) {
2396 /*
2397 * Get an amount from boot-args, else use 1/2 of max_mem.
2398 * 1/2 max_mem was chosen from a Peace daemon tentpole test which
2399 * used munch to induce jetsam thrashing of false idle daemons on N56.
2400 */
2401 int secluded_shutoff_mb;
2402 if (PE_parse_boot_argn("secluded_shutoff_mb", &secluded_shutoff_mb,
2403 sizeof(secluded_shutoff_mb))) {
2404 secluded_shutoff_trigger = (uint64_t)secluded_shutoff_mb * 1024 * 1024;
2405 } else {
2406 secluded_shutoff_trigger = max_mem / 2;
2407 }
2408
2409 /* ensure the headroom value is sensible and avoid underflows */
2410 assert(secluded_shutoff_trigger == 0 || secluded_shutoff_trigger > secluded_shutoff_headroom);
2411 }
2412 #endif /* CONFIG_SECLUDED_MEMORY */
2413
2414 #if defined(__x86_64__)
2415
2416 /*
2417 * Decide how much memory we delay freeing at boot time.
2418 */
2419 uint32_t delay_above_gb;
2420 if (!PE_parse_boot_argn("delay_above_gb", &delay_above_gb, sizeof(delay_above_gb))) {
2421 delay_above_gb = DEFAULT_DELAY_ABOVE_PHYS_GB;
2422 }
2423
2424 if (delay_above_gb == 0) {
2425 delay_above_pnum = PPNUM_MAX;
2426 } else {
2427 delay_above_pnum = delay_above_gb * (1024 * 1024 * 1024 / PAGE_SIZE);
2428 }
2429
2430 /* make sure we have sane breathing room: 1G above low memory */
2431 if (delay_above_pnum <= max_valid_low_ppnum) {
2432 delay_above_pnum = max_valid_low_ppnum + ((1024 * 1024 * 1024) >> PAGE_SHIFT);
2433 }
2434
2435 if (delay_above_pnum < PPNUM_MAX) {
2436 printf("pmap_startup() delaying init/free of page nums > 0x%x\n", delay_above_pnum);
2437 }
2438
2439 #endif /* defined(__x86_64__) */
2440
2441
2442 vm_free_page_lock();
2443
2444 for (uint32_t i = 0; i < npages && pmap_next_page(&phys_page); i++) {
2445 #if XNU_VM_HAS_DELAYED_PAGES
2446 if (phys_page < max_valid_low_ppnum) {
2447 ++low_page_count;
2448 }
2449
2450 /* Are we at high enough pages to delay the rest? */
2451 if (low_page_count > vm_lopage_free_limit &&
2452 phys_page > delay_above_pnum) {
2453 vm_delayed_count = pmap_free_pages();
2454 assert3u(vm_pages_count + vm_delayed_count, <=, npages);
2455 break;
2456 }
2457 #endif /* XNU_VM_HAS_DELAYED_PAGES */
2458
2459 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
2460 if (i == 0) {
2461 vm_pages_first_pnum = phys_page;
2462 patch_low_glo_vm_page_info(vm_pages, vm_pages_end,
2463 vm_pages_first_pnum);
2464 }
2465 #else
2466 /* The x86 clump freeing code requires increasing ppn's to work correctly */
2467 if (i > 0) {
2468 assert(phys_page > vm_page_get(i - 1)->vmp_phys_page);
2469 }
2470 #endif /* !XNU_VM_HAS_LINEAR_PAGES_ARRAY */
2471
2472 ++vm_pages_count;
2473 vm_page_init(vm_page_get(i), phys_page);
2474 if (fillval) {
2475 fillPage(phys_page, fillval);
2476 }
2477 if (vm_himemory_mode) {
2478 vm_page_release_startup(vm_page_get(i));
2479 }
2480 }
2481
2482 vm_page_pages = vm_pages_count; /* used to report to user space */
2483
2484 if (!vm_himemory_mode) {
2485 for (uint32_t i = npages; i-- > 0;) {
2486 /* skip retired pages */
2487 if (!VMP_ERROR_GET(vm_page_get(i))) {
2488 vm_page_release_startup(vm_page_get(i));
2489 }
2490 }
2491 }
2492
2493 vm_free_page_unlock();
2494
2495 absolutetime_to_nanoseconds(mach_absolute_time(), &now_ns);
2496 printf("pmap_startup() init/release time: %lld microsec\n",
2497 (now_ns - start_ns) / NSEC_PER_USEC);
2498 #if XNU_VM_HAS_DELAYED_PAGES
2499 printf("pmap_startup() delayed init/release of %d pages\n",
2500 vm_delayed_count);
2501 #endif /* XNU_VM_HAS_DELAYED_PAGES */
2502
2503 /*
2504 * Validate packing will work properly. This needs to be done last
2505 * after vm_pages_count has been computed.
2506 */
2507 if (npages >= VM_PAGE_PACKED_FROM_ARRAY) {
2508 panic("pmap_startup(): too many pages to support vm_page packing");
2509 }
2510 if ((vm_page_t)VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(vm_pages)) != vm_pages) {
2511 panic("VM_PAGE_PACK_PTR failed on vm_pages - %p", vm_pages);
2512 }
2513 if ((vm_page_t)VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(vm_page_get(vm_pages_count - 1))) !=
2514 vm_page_get(vm_pages_count - 1)) {
2515 panic("VM_PAGE_PACK_PTR failed on vm_pages_end - %p",
2516 vm_page_get(vm_pages_count - 1));
2517 }
2518
2519 VM_CHECK_MEMORYSTATUS;
2520
2521 /*
2522 * We have to re-align virtual_space_start,
2523 * because pmap_steal_memory has been using it.
2524 */
2525 virtual_space_start = round_page(virtual_space_start);
2526 *startp = virtual_space_start;
2527 *endp = virtual_space_end;
2528 }
2529 #endif /* MACHINE_PAGES */
2530
2531 /*
2532 * Create the zone that represents the vm_pages[] array. Nothing ever allocates
2533 * or frees to this zone. It's just here for reporting purposes via zprint command.
2534 * This needs to be done after all initially delayed pages are put on the free lists.
2535 */
2536 void
vm_pages_array_finalize(void)2537 vm_pages_array_finalize(void)
2538 {
2539 (void)zone_create_ext("vm pages array", sizeof(struct vm_page),
2540 ZC_KASAN_NOREDZONE | ZC_KASAN_NOQUARANTINE, ZONE_ID_VM_PAGES, ^(zone_t z) {
2541 uint64_t vm_page_zone_pages, vm_page_array_zone_data_size;
2542
2543 zone_set_exhaustible(z, 0, true);
2544 /*
2545 * Reflect size and usage information for vm_pages[].
2546 */
2547
2548 z->z_elems_avail = (uint32_t)(vm_pages_end - vm_pages);
2549 z->z_elems_free = z->z_elems_avail - vm_pages_count;
2550 zpercpu_get_cpu(z->z_stats, 0)->zs_mem_allocated =
2551 vm_pages_count * sizeof(struct vm_page);
2552 vm_page_array_zone_data_size = (uint64_t)vm_pages_end - (uint64_t)vm_pages;
2553 vm_page_zone_pages = atop(round_page((vm_offset_t)vm_page_array_zone_data_size));
2554 z->z_wired_cur += vm_page_zone_pages;
2555 z->z_wired_hwm = z->z_wired_cur;
2556 z->z_va_cur = z->z_wired_cur;
2557 /* since zone accounts for these, take them out of stolen */
2558 VM_PAGE_MOVE_STOLEN(vm_page_zone_pages);
2559 });
2560 }
2561
2562 /*
2563 * Create the vm_pages zone. This is used for the vm_page structures for the pages
2564 * that are scavanged from other boot time usages by ml_static_mfree(). As such,
2565 * this needs to happen in early VM bootstrap.
2566 */
2567
2568 __startup_func
2569 static void
vm_page_module_init(void)2570 vm_page_module_init(void)
2571 {
2572 vm_size_t vm_page_with_ppnum_size;
2573
2574 /*
2575 * Since the pointers to elements in this zone will be packed, they
2576 * must have appropriate size. Not strictly what sizeof() reports.
2577 */
2578 vm_page_with_ppnum_size =
2579 (sizeof(struct vm_page_with_ppnum) + (VM_PAGE_PACKED_PTR_ALIGNMENT - 1)) &
2580 ~(VM_PAGE_PACKED_PTR_ALIGNMENT - 1);
2581
2582 vm_page_zone = zone_create_ext("vm pages", vm_page_with_ppnum_size,
2583 ZC_ALIGNMENT_REQUIRED | ZC_VM,
2584 ZONE_ID_ANY, ^(zone_t z) {
2585 /*
2586 * The number "10" is a small number that is larger than the number
2587 * of fictitious pages that any single caller will attempt to allocate
2588 * without blocking.
2589 *
2590 * The largest such number at the moment is kmem_alloc()
2591 * when 2 guard pages are asked. 10 is simply a somewhat larger number,
2592 * taking into account the 50% hysteresis the zone allocator uses.
2593 *
2594 * Note: this works at all because the zone allocator
2595 * doesn't ever allocate fictitious pages.
2596 */
2597 zone_raise_reserve(z, 10);
2598 });
2599 }
2600 STARTUP(ZALLOC, STARTUP_RANK_SECOND, vm_page_module_init);
2601
2602 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
2603 /*
2604 * Radix tree of pages within the [pmap_first_pnum, vm_pages_first_pnum) range,
2605 * in order to support page lookup by pnum (@see vm_page_find_canonical()),
2606 * which corresponds to pages returned to the VM via @c ml_static_mfree().
2607 *
2608 * Kernel vm pages are never freed, which means that this data structure
2609 * is insert only.
2610 *
2611 * Empirically we have about 4-5k such pages, typically in only few rather dense
2612 * contiguous spans, inside a range of roughly 32k pnums.
2613 *
2614 * A radix tree works well with the distribution of keys, but also allows for
2615 * a straightforward lockless lookup path.
2616 */
2617
2618 #define VM_PAGE_RADIX_FANOUT_SHIFT 8
2619 #define VM_PAGE_RADIX_FANOUT (1u << VM_PAGE_RADIX_FANOUT_SHIFT)
2620
2621 typedef uint32_t vm_page_radix_ptr_t;
2622
2623 typedef struct vm_page_radix_node {
2624 vm_page_radix_ptr_t vmpr_array[VM_PAGE_RADIX_FANOUT];
2625 } *vm_page_radix_node_t;
2626
2627 static LCK_GRP_DECLARE(vm_pages_radix_lock_grp, "VM pages radix");
2628 static LCK_MTX_DECLARE(vm_pages_radix_lock, &vm_pages_radix_lock_grp);
2629
2630 static SECURITY_READ_ONLY_LATE(uintptr_t) vm_pages_radix_root;
2631 static uint32_t vm_pages_radix_count;
2632
2633 static vm_page_radix_node_t
vm_page_radix_node_unpack(vm_page_radix_ptr_t ptr)2634 vm_page_radix_node_unpack(vm_page_radix_ptr_t ptr)
2635 {
2636 return (vm_page_radix_node_t)VM_UNPACK_POINTER(ptr, VM_PAGE_PACKED_PTR);
2637 }
2638
2639 static vm_page_radix_ptr_t
vm_page_radix_node_pack(vm_page_radix_node_t node)2640 vm_page_radix_node_pack(vm_page_radix_node_t node)
2641 {
2642 vm_offset_t ptr = (vm_offset_t)node;
2643
2644 VM_ASSERT_POINTER_PACKABLE(ptr, VM_PAGE_PACKED_PTR);
2645 return (vm_page_radix_ptr_t)VM_PACK_POINTER(ptr, VM_PAGE_PACKED_PTR);
2646 }
2647
2648 static uint32_t
vm_page_radix_key(uint32_t level,uint32_t index)2649 vm_page_radix_key(uint32_t level, uint32_t index)
2650 {
2651 uint32_t key = index >> (VM_PAGE_RADIX_FANOUT_SHIFT * level);
2652
2653 return key & (VM_PAGE_RADIX_FANOUT - 1);
2654 }
2655
2656 static vm_page_radix_ptr_t *
vm_page_radix_slot(vm_page_radix_node_t node,uint32_t level,uint32_t index)2657 vm_page_radix_slot(vm_page_radix_node_t node, uint32_t level, uint32_t index)
2658 {
2659 return node->vmpr_array + vm_page_radix_key(level, index);
2660 }
2661
2662 __startup_func
2663 __attribute__((noinline))
2664 static vm_page_radix_node_t
vm_pages_radix_init_root(uint32_t * levelp)2665 vm_pages_radix_init_root(uint32_t *levelp)
2666 {
2667 uint32_t max_index = vm_pages_first_pnum - pmap_first_pnum - 1;
2668 vm_page_radix_node_t root;
2669 uint32_t level;
2670 vm_size_t size;
2671
2672 /*
2673 * Init a top-level node right away, to cover any index within
2674 * [0, vm_pages_first_pnum - pmap_first_pnum)
2675 */
2676 level = (fls(max_index | 1) - 1) / VM_PAGE_RADIX_FANOUT_SHIFT;
2677 size = (vm_page_radix_key(level, max_index) + 1) *
2678 sizeof(vm_page_radix_ptr_t);
2679
2680 root = zalloc_permanent(size, ZALIGN_64);
2681
2682 /*
2683 * Pack the level into the root pointer low bits,
2684 * so that pointer and level can be read atomically.
2685 *
2686 * See vm_pages_radix_load_root().
2687 */
2688 os_atomic_store(&vm_pages_radix_root, (uintptr_t)root | level, release);
2689
2690 *levelp = level;
2691 return root;
2692 }
2693
2694 static vm_page_radix_node_t
vm_pages_radix_node_alloc(vm_page_radix_ptr_t * slot)2695 vm_pages_radix_node_alloc(vm_page_radix_ptr_t *slot)
2696 {
2697 vm_page_radix_node_t node;
2698
2699 node = zalloc_permanent(sizeof(struct vm_page_radix_node),
2700 VM_PAGE_PACKED_PTR_ALIGNMENT - 1);
2701 os_atomic_store(slot, vm_page_radix_node_pack(node), release);
2702 return node;
2703 }
2704
2705 static vm_page_radix_node_t
vm_pages_radix_load_root(uint32_t * level)2706 vm_pages_radix_load_root(uint32_t *level)
2707 {
2708 const uintptr_t VM_PAGE_RADIX_LEVEL_MASK = 0x7ul;
2709
2710 uintptr_t root = os_atomic_load(&vm_pages_radix_root, dependency);
2711
2712 *level = root & VM_PAGE_RADIX_LEVEL_MASK;
2713 root &= ~VM_PAGE_RADIX_LEVEL_MASK;
2714 return (vm_page_radix_node_t)root;
2715 }
2716
2717 vm_page_t
vm_pages_radix_next(uint32_t * cursor,ppnum_t * pnum)2718 vm_pages_radix_next(uint32_t *cursor, ppnum_t *pnum)
2719 {
2720 const uint32_t max_index = vm_pages_first_pnum - pmap_first_pnum;
2721 vm_page_radix_node_t node;
2722 uint32_t level, index;
2723
2724 index = *cursor;
2725 node = vm_pages_radix_load_root(&level);
2726
2727 while (index < max_index) {
2728 vm_page_radix_ptr_t *slot = vm_page_radix_slot(node, level, index);
2729 vm_page_radix_ptr_t ptr = os_atomic_load(slot, dependency);
2730
2731 if (ptr == 0) {
2732 uint32_t stride = 1 << (VM_PAGE_RADIX_FANOUT_SHIFT * level);
2733
2734 index = (index + stride) & -stride;
2735 if (vm_page_radix_key(level, index) == 0) {
2736 /* restart lookup at the top */
2737 node = vm_pages_radix_load_root(&level);
2738 }
2739 } else if (level > 0) {
2740 node = vm_page_radix_node_unpack(ptr);
2741 level -= 1;
2742 } else {
2743 *cursor = index + 1;
2744 if (pnum) {
2745 *pnum = pmap_first_pnum + index;
2746 }
2747 return (vm_page_t)VM_PAGE_UNPACK_PTR(ptr);
2748 }
2749 }
2750
2751 if (pnum) {
2752 *pnum = 0;
2753 }
2754 return VM_PAGE_NULL;
2755 }
2756
2757 #if DEBUG || DEVELOPMENT
2758
2759 static int
vm_page_radix_verify_test(int64_t in __unused,int64_t * out)2760 vm_page_radix_verify_test(int64_t in __unused, int64_t *out)
2761 {
2762 uint32_t count = 0;
2763 vm_page_t mem;
2764
2765 lck_mtx_lock(&vm_pages_radix_lock);
2766
2767 vm_pages_radix_for_each(mem) {
2768 count++;
2769 assert(mem == vm_page_find_canonical(VM_PAGE_GET_PHYS_PAGE(mem)));
2770 }
2771
2772 assert(count == vm_pages_radix_count);
2773
2774 lck_mtx_unlock(&vm_pages_radix_lock);
2775
2776 *out = 1;
2777 return 0;
2778 }
2779 SYSCTL_TEST_REGISTER(vm_page_radix_verify, vm_page_radix_verify_test);
2780
2781 #endif /* DEBUG || DEVELOPMENT */
2782
2783 __attribute__((noinline))
2784 static void
vm_pages_radix_insert(ppnum_t pnum,vm_page_t page)2785 vm_pages_radix_insert(ppnum_t pnum, vm_page_t page)
2786 {
2787 vm_page_radix_ptr_t *slot;
2788 vm_page_radix_node_t node;
2789 uint32_t level, index;
2790
2791 assert(!vm_page_in_array(page));
2792 index = pnum - pmap_first_pnum;
2793
2794 lck_mtx_lock(&vm_pages_radix_lock);
2795
2796 node = vm_pages_radix_load_root(&level);
2797 if (node == NULL) {
2798 node = vm_pages_radix_init_root(&level);
2799 }
2800
2801 for (; level > 0; level--) {
2802 slot = vm_page_radix_slot(node, level, index);
2803 if (*slot == 0) {
2804 node = vm_pages_radix_node_alloc(slot);
2805 } else {
2806 node = vm_page_radix_node_unpack(*slot);
2807 }
2808 }
2809
2810 slot = vm_page_radix_slot(node, 0, index);
2811 assert(*slot == 0);
2812 os_atomic_store(slot, VM_PAGE_PACK_PTR(page), release);
2813 vm_pages_radix_count++;
2814
2815 lck_mtx_unlock(&vm_pages_radix_lock);
2816 }
2817
2818 __abortlike
2819 static void
vm_page_for_ppnum_panic(ppnum_t pnum)2820 vm_page_for_ppnum_panic(ppnum_t pnum)
2821 {
2822 if (pnum < pmap_first_pnum) {
2823 panic("physical page is before the start of DRAM: %#x < %#x)",
2824 pnum, pmap_first_pnum);
2825 }
2826 panic("physical page is beyond the end of managed DRAM: %#x >= %#x)",
2827 pnum, vm_pages_first_pnum + vm_pages_count);
2828 }
2829
2830 vm_page_t
vm_page_find_canonical(ppnum_t pnum)2831 vm_page_find_canonical(ppnum_t pnum)
2832 {
2833 vm_page_radix_ptr_t *slot;
2834 vm_page_radix_node_t node;
2835 vm_page_radix_ptr_t ptr;
2836 uint32_t level, index;
2837
2838 if (pnum < pmap_first_pnum) {
2839 vm_page_for_ppnum_panic(pnum);
2840 }
2841
2842 if (pnum >= vm_pages_first_pnum + vm_pages_count) {
2843 /*
2844 * We could receive requests for pages which are beyond the xnu's managed space. (eg: ECC errors)
2845 * These need to be handled gracefully, so we return VM_PAGE_NULL here.
2846 */
2847 return VM_PAGE_NULL;
2848 }
2849
2850 if (__probable(pnum >= vm_pages_first_pnum)) {
2851 return vm_page_get(pnum - vm_pages_first_pnum);
2852 }
2853
2854 index = pnum - pmap_first_pnum;
2855 node = vm_pages_radix_load_root(&level);
2856
2857 for (; node && level > 0; level--) {
2858 slot = vm_page_radix_slot(node, level, index);
2859 ptr = os_atomic_load(slot, dependency);
2860 node = vm_page_radix_node_unpack(ptr);
2861 }
2862
2863 if (__probable(node)) {
2864 slot = vm_page_radix_slot(node, 0, index);
2865 ptr = os_atomic_load(slot, dependency);
2866 return (vm_page_t)VM_PAGE_UNPACK_PTR(ptr);
2867 }
2868
2869 return VM_PAGE_NULL;
2870 }
2871
2872 #endif /* XNU_VM_HAS_LINEAR_PAGES_ARRAY */
2873
2874 /*!
2875 * @function vm_page_create()
2876 *
2877 * @brief
2878 * Common helper for all vm_page_create* functions.
2879 */
2880 vm_page_t
vm_page_create(ppnum_t phys_page,bool canonical,zalloc_flags_t flags)2881 vm_page_create(ppnum_t phys_page, bool canonical, zalloc_flags_t flags)
2882 {
2883 vm_page_t m;
2884
2885 m = zalloc_flags(vm_page_zone, flags);
2886 if (m) {
2887 vm_page_init(m, phys_page);
2888 if (phys_page == vm_page_guard_addr) {
2889 counter_inc(&vm_guard_count);
2890 }
2891 }
2892 if (canonical) {
2893 assert((flags & (Z_NOWAIT | Z_NOPAGEWAIT)) == 0);
2894 m->vmp_canonical = true;
2895 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
2896 vm_pages_radix_insert(phys_page, m);
2897 #endif /* XNU_VM_HAS_LINEAR_PAGES_ARRAY */
2898 vm_free_page_lock();
2899 vm_page_pages++;
2900 vm_free_page_unlock();
2901 }
2902 return m;
2903 }
2904
2905 /*
2906 * Routine: vm_page_create_canonical
2907 * Purpose:
2908 * After the VM system is up, machine-dependent code
2909 * may stumble across more physical memory. For example,
2910 * memory that it was reserving for a frame buffer.
2911 * vm_page_create_canonical turns this memory into available pages.
2912 */
2913
2914 void
vm_page_create_canonical(ppnum_t phys_page)2915 vm_page_create_canonical(ppnum_t phys_page)
2916 {
2917 vm_page_t m;
2918
2919 m = vm_page_create(phys_page, true, Z_WAITOK);
2920 vm_page_release(m, VMP_RELEASE_NONE);
2921 }
2922
2923
2924 /*
2925 * vm_page_hash:
2926 *
2927 * Distributes the object/offset key pair among hash buckets.
2928 *
2929 * NOTE: The bucket count must be a power of 2
2930 */
2931 #define vm_page_hash(object, offset) (\
2932 ( (natural_t)((uintptr_t)object * vm_page_bucket_hash) + ((uint32_t)atop_64(offset) ^ vm_page_bucket_hash))\
2933 & vm_page_hash_mask)
2934
2935
2936 /*
2937 * vm_page_insert: [ internal use only ]
2938 *
2939 * Inserts the given mem entry into the object/object-page
2940 * table and object list.
2941 *
2942 * The object must be locked.
2943 */
2944 void
vm_page_insert(vm_page_t mem,vm_object_t object,vm_object_offset_t offset)2945 vm_page_insert(
2946 vm_page_t mem,
2947 vm_object_t object,
2948 vm_object_offset_t offset)
2949 {
2950 vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, FALSE, FALSE, NULL);
2951 }
2952
2953 void
vm_page_insert_wired(vm_page_t mem,vm_object_t object,vm_object_offset_t offset,vm_tag_t tag)2954 vm_page_insert_wired(
2955 vm_page_t mem,
2956 vm_object_t object,
2957 vm_object_offset_t offset,
2958 vm_tag_t tag)
2959 {
2960 vm_page_insert_internal(mem, object, offset, tag, FALSE, TRUE, FALSE, FALSE, NULL);
2961 }
2962
2963 void
vm_page_insert_internal(vm_page_t mem,vm_object_t object,vm_object_offset_t offset,vm_tag_t tag,boolean_t queues_lock_held,boolean_t insert_in_hash,boolean_t batch_pmap_op,boolean_t batch_accounting,uint64_t * delayed_ledger_update)2964 vm_page_insert_internal(
2965 vm_page_t mem,
2966 vm_object_t object,
2967 vm_object_offset_t offset,
2968 vm_tag_t tag,
2969 boolean_t queues_lock_held,
2970 boolean_t insert_in_hash,
2971 boolean_t batch_pmap_op,
2972 boolean_t batch_accounting,
2973 uint64_t *delayed_ledger_update)
2974 {
2975 vm_page_bucket_t *bucket;
2976 lck_ticket_t *bucket_lock;
2977 int hash_id;
2978 task_t owner;
2979 int ledger_idx_volatile;
2980 int ledger_idx_nonvolatile;
2981 int ledger_idx_volatile_compressed;
2982 int ledger_idx_nonvolatile_compressed;
2983 int ledger_idx_composite;
2984 int ledger_idx_external_wired;
2985 boolean_t do_footprint;
2986
2987 #if 0
2988 /*
2989 * we may not hold the page queue lock
2990 * so this check isn't safe to make
2991 */
2992 VM_PAGE_CHECK(mem);
2993 #endif
2994
2995 assertf(page_aligned(offset), "0x%llx\n", offset);
2996
2997 assert(!VM_PAGE_WIRED(mem) || !vm_page_is_canonical(mem) ||
2998 (tag != VM_KERN_MEMORY_NONE));
2999
3000 vm_object_lock_assert_exclusive(object);
3001 LCK_MTX_ASSERT(&vm_page_queue_lock,
3002 queues_lock_held ? LCK_MTX_ASSERT_OWNED
3003 : LCK_MTX_ASSERT_NOTOWNED);
3004
3005 if (queues_lock_held == FALSE) {
3006 assert(!VM_PAGE_PAGEABLE(mem));
3007 }
3008
3009 if (insert_in_hash == TRUE) {
3010 #if DEBUG || VM_PAGE_BUCKETS_CHECK
3011 if (mem->vmp_tabled || mem->vmp_object) {
3012 panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) "
3013 "already in (obj=%p,off=0x%llx)",
3014 mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
3015 }
3016 #endif
3017 if (object->internal && (offset >= object->vo_size)) {
3018 panic("vm_page_insert_internal: (page=%p,obj=%p,off=0x%llx,size=0x%llx) inserted at offset past object bounds",
3019 mem, object, offset, object->vo_size);
3020 }
3021
3022 assert(vm_page_lookup(object, offset) == VM_PAGE_NULL);
3023
3024 /*
3025 * Record the object/offset pair in this page
3026 */
3027
3028 mem->vmp_object = VM_PAGE_PACK_OBJECT(object);
3029 mem->vmp_offset = offset;
3030
3031 #if CONFIG_SECLUDED_MEMORY
3032 if (object->eligible_for_secluded) {
3033 vm_page_secluded.eligible_for_secluded++;
3034 }
3035 #endif /* CONFIG_SECLUDED_MEMORY */
3036
3037 /*
3038 * Insert it into the object_object/offset hash table
3039 */
3040 hash_id = vm_page_hash(object, offset);
3041 bucket = &vm_page_buckets[hash_id];
3042 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
3043
3044 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
3045
3046 mem->vmp_next_m = bucket->page_list;
3047 bucket->page_list = VM_PAGE_PACK_PTR(mem);
3048 assert(mem == (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)));
3049
3050 #if MACH_PAGE_HASH_STATS
3051 if (++bucket->cur_count > bucket->hi_count) {
3052 bucket->hi_count = bucket->cur_count;
3053 }
3054 #endif /* MACH_PAGE_HASH_STATS */
3055 mem->vmp_hashed = TRUE;
3056 lck_ticket_unlock(bucket_lock);
3057 }
3058
3059 {
3060 unsigned int cache_attr;
3061
3062 cache_attr = object->wimg_bits & VM_WIMG_MASK;
3063
3064
3065 if (cache_attr != VM_WIMG_USE_DEFAULT) {
3066 PMAP_SET_CACHE_ATTR(mem, object, cache_attr, batch_pmap_op);
3067 }
3068
3069 }
3070
3071 /*
3072 * Now link into the object's list of backed pages.
3073 */
3074 vm_page_queue_enter(&object->memq, mem, vmp_listq);
3075 object->memq_hint = mem;
3076 mem->vmp_tabled = TRUE;
3077
3078 /*
3079 * Show that the object has one more resident page.
3080 */
3081
3082 object->resident_page_count++;
3083 if (VM_PAGE_WIRED(mem)) {
3084 assert(mem->vmp_wire_count > 0);
3085 VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
3086 VM_OBJECT_WIRED_PAGE_ADD(object, mem);
3087 VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
3088 }
3089 assert(object->resident_page_count >= object->wired_page_count);
3090
3091 #if COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT > 1
3092 vm_object_set_chead_hint(object);
3093 #endif
3094
3095 #if DEVELOPMENT || DEBUG
3096 if (object->object_is_shared_cache &&
3097 object->pager != NULL &&
3098 object->pager->mo_pager_ops == &shared_region_pager_ops) {
3099 int new, old;
3100 assert(!object->internal);
3101 new = OSAddAtomic(+1, &shared_region_pagers_resident_count);
3102 do {
3103 old = shared_region_pagers_resident_peak;
3104 } while (old < new &&
3105 !OSCompareAndSwap(old, new, &shared_region_pagers_resident_peak));
3106 }
3107 #endif /* DEVELOPMENT || DEBUG */
3108
3109 if (batch_accounting == FALSE) {
3110 if (object->internal) {
3111 OSAddAtomic(1, &vm_page_internal_count);
3112 } else {
3113 OSAddAtomic(1, &vm_page_external_count);
3114 }
3115 }
3116
3117 /*
3118 * It wouldn't make sense to insert a "reusable" page in
3119 * an object (the page would have been marked "reusable" only
3120 * at the time of a madvise(MADV_FREE_REUSABLE) if it was already
3121 * in the object at that time).
3122 * But a page could be inserted in a "all_reusable" object, if
3123 * something faults it in (a vm_read() from another task or a
3124 * "use-after-free" issue in user space, for example). It can
3125 * also happen if we're relocating a page from that object to
3126 * a different physical page during a physically-contiguous
3127 * allocation.
3128 */
3129 assert(!mem->vmp_reusable);
3130 if (object->all_reusable) {
3131 OSAddAtomic(+1, &vm_page_stats_reusable.reusable_count);
3132 }
3133
3134 if (object->purgable == VM_PURGABLE_DENY &&
3135 !object->vo_ledger_tag) {
3136 owner = TASK_NULL;
3137 } else {
3138 owner = VM_OBJECT_OWNER(object);
3139 vm_object_ledger_tag_ledgers(object,
3140 &ledger_idx_volatile,
3141 &ledger_idx_nonvolatile,
3142 &ledger_idx_volatile_compressed,
3143 &ledger_idx_nonvolatile_compressed,
3144 &ledger_idx_composite,
3145 &ledger_idx_external_wired,
3146 &do_footprint);
3147 }
3148 if (owner &&
3149 object->internal &&
3150 (object->purgable == VM_PURGABLE_NONVOLATILE ||
3151 object->purgable == VM_PURGABLE_DENY ||
3152 VM_PAGE_WIRED(mem))) {
3153 if (delayed_ledger_update) {
3154 *delayed_ledger_update += PAGE_SIZE;
3155 } else {
3156 /* more non-volatile bytes */
3157 ledger_credit(owner->ledger,
3158 ledger_idx_nonvolatile,
3159 PAGE_SIZE);
3160 if (do_footprint) {
3161 /* more footprint */
3162 ledger_credit(owner->ledger,
3163 task_ledgers.phys_footprint,
3164 PAGE_SIZE);
3165 } else if (ledger_idx_composite != -1) {
3166 ledger_credit(owner->ledger,
3167 ledger_idx_composite,
3168 PAGE_SIZE);
3169 }
3170 }
3171 } else if (owner &&
3172 object->internal &&
3173 (object->purgable == VM_PURGABLE_VOLATILE ||
3174 object->purgable == VM_PURGABLE_EMPTY)) {
3175 assert(!VM_PAGE_WIRED(mem));
3176 /* more volatile bytes */
3177 ledger_credit(owner->ledger,
3178 ledger_idx_volatile,
3179 PAGE_SIZE);
3180 }
3181
3182 if (object->purgable == VM_PURGABLE_VOLATILE) {
3183 if (VM_PAGE_WIRED(mem)) {
3184 OSAddAtomic(+1, &vm_page_purgeable_wired_count);
3185 } else {
3186 OSAddAtomic(+1, &vm_page_purgeable_count);
3187 }
3188 } else if (object->purgable == VM_PURGABLE_EMPTY &&
3189 mem->vmp_q_state == VM_PAGE_ON_THROTTLED_Q) {
3190 /*
3191 * This page belongs to a purged VM object but hasn't
3192 * been purged (because it was "busy").
3193 * It's in the "throttled" queue and hence not
3194 * visible to vm_pageout_scan(). Move it to a pageable
3195 * queue, so that it can eventually be reclaimed, instead
3196 * of lingering in the "empty" object.
3197 */
3198 if (queues_lock_held == FALSE) {
3199 vm_page_lockspin_queues();
3200 }
3201 vm_page_deactivate(mem);
3202 if (queues_lock_held == FALSE) {
3203 vm_page_unlock_queues();
3204 }
3205 }
3206
3207
3208 #if VM_OBJECT_TRACKING_OP_MODIFIED
3209 if (vm_object_tracking_btlog &&
3210 object->internal &&
3211 object->resident_page_count == 0 &&
3212 object->pager == NULL &&
3213 object->shadow != NULL &&
3214 object->shadow->vo_copy == object) {
3215 btlog_record(vm_object_tracking_btlog, object,
3216 VM_OBJECT_TRACKING_OP_MODIFIED,
3217 btref_get(__builtin_frame_address(0), 0));
3218 }
3219 #endif /* VM_OBJECT_TRACKING_OP_MODIFIED */
3220 }
3221
3222 /*
3223 * vm_page_replace:
3224 *
3225 * Exactly like vm_page_insert, except that we first
3226 * remove any existing page at the given offset in object.
3227 *
3228 * The object must be locked.
3229 */
3230 void
vm_page_replace(vm_page_t mem,vm_object_t object,vm_object_offset_t offset)3231 vm_page_replace(
3232 vm_page_t mem,
3233 vm_object_t object,
3234 vm_object_offset_t offset)
3235 {
3236 vm_page_bucket_t *bucket;
3237 vm_page_t found_m = VM_PAGE_NULL;
3238 lck_ticket_t *bucket_lock;
3239 int hash_id;
3240
3241 #if 0
3242 /*
3243 * we don't hold the page queue lock
3244 * so this check isn't safe to make
3245 */
3246 VM_PAGE_CHECK(mem);
3247 #endif
3248 vm_object_lock_assert_exclusive(object);
3249 #if DEBUG || VM_PAGE_BUCKETS_CHECK
3250 if (mem->vmp_tabled || mem->vmp_object) {
3251 panic("vm_page_replace: page %p for (obj=%p,off=0x%llx) "
3252 "already in (obj=%p,off=0x%llx)",
3253 mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
3254 }
3255 #endif
3256 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
3257
3258 assert(!VM_PAGE_PAGEABLE(mem));
3259
3260 /*
3261 * Record the object/offset pair in this page
3262 */
3263 mem->vmp_object = VM_PAGE_PACK_OBJECT(object);
3264 mem->vmp_offset = offset;
3265
3266 /*
3267 * Insert it into the object_object/offset hash table,
3268 * replacing any page that might have been there.
3269 */
3270
3271 hash_id = vm_page_hash(object, offset);
3272 bucket = &vm_page_buckets[hash_id];
3273 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
3274
3275 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
3276
3277 if (bucket->page_list) {
3278 vm_page_packed_t *mp = &bucket->page_list;
3279 vm_page_t m = (vm_page_t)(VM_PAGE_UNPACK_PTR(*mp));
3280
3281 do {
3282 /*
3283 * compare packed object pointers
3284 */
3285 if (m->vmp_object == mem->vmp_object && m->vmp_offset == offset) {
3286 /*
3287 * Remove old page from hash list
3288 */
3289 *mp = m->vmp_next_m;
3290 m->vmp_hashed = FALSE;
3291 m->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
3292
3293 found_m = m;
3294 break;
3295 }
3296 mp = &m->vmp_next_m;
3297 } while ((m = (vm_page_t)(VM_PAGE_UNPACK_PTR(*mp))));
3298
3299 mem->vmp_next_m = bucket->page_list;
3300 } else {
3301 mem->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
3302 }
3303 /*
3304 * insert new page at head of hash list
3305 */
3306 bucket->page_list = VM_PAGE_PACK_PTR(mem);
3307 mem->vmp_hashed = TRUE;
3308
3309 lck_ticket_unlock(bucket_lock);
3310
3311 if (found_m) {
3312 /*
3313 * there was already a page at the specified
3314 * offset for this object... remove it from
3315 * the object and free it back to the free list
3316 */
3317 vm_page_free_unlocked(found_m, FALSE);
3318 }
3319 vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, FALSE, FALSE, FALSE, NULL);
3320 }
3321
3322 /*
3323 * vm_page_remove: [ internal use only ]
3324 *
3325 * Removes the given mem entry from the object/offset-page
3326 * table and the object page list.
3327 *
3328 * The object must be locked.
3329 */
3330
3331 void
vm_page_remove(vm_page_t mem,boolean_t remove_from_hash)3332 vm_page_remove(
3333 vm_page_t mem,
3334 boolean_t remove_from_hash)
3335 {
3336 vm_page_bucket_t *bucket;
3337 vm_page_t this;
3338 lck_ticket_t *bucket_lock;
3339 int hash_id;
3340 task_t owner;
3341 vm_object_t m_object;
3342 int ledger_idx_volatile;
3343 int ledger_idx_nonvolatile;
3344 int ledger_idx_volatile_compressed;
3345 int ledger_idx_nonvolatile_compressed;
3346 int ledger_idx_composite;
3347 int ledger_idx_external_wired;
3348 int do_footprint;
3349
3350 m_object = VM_PAGE_OBJECT(mem);
3351
3352 vm_object_lock_assert_exclusive(m_object);
3353 assert(mem->vmp_tabled);
3354 assert(!mem->vmp_cleaning);
3355 assert(!mem->vmp_laundry);
3356
3357 if (VM_PAGE_PAGEABLE(mem)) {
3358 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3359 }
3360 #if 0
3361 /*
3362 * we don't hold the page queue lock
3363 * so this check isn't safe to make
3364 */
3365 VM_PAGE_CHECK(mem);
3366 #endif
3367 if (remove_from_hash == TRUE) {
3368 /*
3369 * Remove from the object_object/offset hash table
3370 */
3371 hash_id = vm_page_hash(m_object, mem->vmp_offset);
3372 bucket = &vm_page_buckets[hash_id];
3373 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
3374
3375 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
3376
3377 if ((this = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list))) == mem) {
3378 /* optimize for common case */
3379
3380 bucket->page_list = mem->vmp_next_m;
3381 } else {
3382 vm_page_packed_t *prev;
3383
3384 for (prev = &this->vmp_next_m;
3385 (this = (vm_page_t)(VM_PAGE_UNPACK_PTR(*prev))) != mem;
3386 prev = &this->vmp_next_m) {
3387 continue;
3388 }
3389 *prev = this->vmp_next_m;
3390 }
3391 #if MACH_PAGE_HASH_STATS
3392 bucket->cur_count--;
3393 #endif /* MACH_PAGE_HASH_STATS */
3394 mem->vmp_hashed = FALSE;
3395 this->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
3396 lck_ticket_unlock(bucket_lock);
3397 }
3398 /*
3399 * Now remove from the object's list of backed pages.
3400 */
3401
3402 vm_page_remove_internal(mem);
3403
3404 /*
3405 * And show that the object has one fewer resident
3406 * page.
3407 */
3408
3409 assert(m_object->resident_page_count > 0);
3410 m_object->resident_page_count--;
3411
3412 #if DEVELOPMENT || DEBUG
3413 if (m_object->object_is_shared_cache &&
3414 m_object->pager != NULL &&
3415 m_object->pager->mo_pager_ops == &shared_region_pager_ops) {
3416 assert(!m_object->internal);
3417 OSAddAtomic(-1, &shared_region_pagers_resident_count);
3418 }
3419 #endif /* DEVELOPMENT || DEBUG */
3420
3421 if (m_object->internal) {
3422 #if DEBUG
3423 assert(vm_page_internal_count);
3424 #endif /* DEBUG */
3425
3426 OSAddAtomic(-1, &vm_page_internal_count);
3427 } else {
3428 assert(vm_page_external_count);
3429 OSAddAtomic(-1, &vm_page_external_count);
3430
3431 if (mem->vmp_xpmapped) {
3432 assert(vm_page_xpmapped_external_count);
3433 OSAddAtomic(-1, &vm_page_xpmapped_external_count);
3434 }
3435 }
3436 if (!m_object->internal &&
3437 m_object->cached_list.next &&
3438 m_object->cached_list.prev) {
3439 if (m_object->resident_page_count == 0) {
3440 vm_object_cache_remove(m_object);
3441 }
3442 }
3443
3444 if (VM_PAGE_WIRED(mem)) {
3445 assert(mem->vmp_wire_count > 0);
3446 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
3447 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
3448 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
3449 }
3450 assert(m_object->resident_page_count >=
3451 m_object->wired_page_count);
3452 if (mem->vmp_reusable) {
3453 assert(m_object->reusable_page_count > 0);
3454 m_object->reusable_page_count--;
3455 assert(m_object->reusable_page_count <=
3456 m_object->resident_page_count);
3457 mem->vmp_reusable = FALSE;
3458 OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
3459 vm_page_stats_reusable.reused_remove++;
3460 } else if (m_object->all_reusable) {
3461 OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
3462 vm_page_stats_reusable.reused_remove++;
3463 }
3464
3465 if (m_object->purgable == VM_PURGABLE_DENY &&
3466 !m_object->vo_ledger_tag) {
3467 owner = TASK_NULL;
3468 } else {
3469 owner = VM_OBJECT_OWNER(m_object);
3470 vm_object_ledger_tag_ledgers(m_object,
3471 &ledger_idx_volatile,
3472 &ledger_idx_nonvolatile,
3473 &ledger_idx_volatile_compressed,
3474 &ledger_idx_nonvolatile_compressed,
3475 &ledger_idx_composite,
3476 &ledger_idx_external_wired,
3477 &do_footprint);
3478 }
3479 if (owner &&
3480 m_object->internal &&
3481 (m_object->purgable == VM_PURGABLE_NONVOLATILE ||
3482 m_object->purgable == VM_PURGABLE_DENY ||
3483 VM_PAGE_WIRED(mem))) {
3484 /* less non-volatile bytes */
3485 ledger_debit(owner->ledger,
3486 ledger_idx_nonvolatile,
3487 PAGE_SIZE);
3488 if (do_footprint) {
3489 /* less footprint */
3490 ledger_debit(owner->ledger,
3491 task_ledgers.phys_footprint,
3492 PAGE_SIZE);
3493 } else if (ledger_idx_composite != -1) {
3494 ledger_debit(owner->ledger,
3495 ledger_idx_composite,
3496 PAGE_SIZE);
3497 }
3498 } else if (owner &&
3499 m_object->internal &&
3500 (m_object->purgable == VM_PURGABLE_VOLATILE ||
3501 m_object->purgable == VM_PURGABLE_EMPTY)) {
3502 assert(!VM_PAGE_WIRED(mem));
3503 /* less volatile bytes */
3504 ledger_debit(owner->ledger,
3505 ledger_idx_volatile,
3506 PAGE_SIZE);
3507 }
3508
3509 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
3510 if (VM_PAGE_WIRED(mem)) {
3511 assert(vm_page_purgeable_wired_count > 0);
3512 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
3513 } else {
3514 assert(vm_page_purgeable_count > 0);
3515 OSAddAtomic(-1, &vm_page_purgeable_count);
3516 }
3517 }
3518
3519 if (m_object->set_cache_attr == TRUE) {
3520 pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), 0);
3521 }
3522
3523 mem->vmp_tabled = FALSE;
3524 mem->vmp_object = 0;
3525 mem->vmp_offset = (vm_object_offset_t) -1;
3526 }
3527
3528
3529 /*
3530 * vm_page_lookup:
3531 *
3532 * Returns the page associated with the object/offset
3533 * pair specified; if none is found, VM_PAGE_NULL is returned.
3534 *
3535 * The object must be locked. No side effects.
3536 */
3537
3538 #define VM_PAGE_HASH_LOOKUP_THRESHOLD 10
3539
3540 #if DEBUG_VM_PAGE_LOOKUP
3541
3542 struct {
3543 uint64_t vpl_total;
3544 uint64_t vpl_empty_obj;
3545 uint64_t vpl_bucket_NULL;
3546 uint64_t vpl_hit_hint;
3547 uint64_t vpl_hit_hint_next;
3548 uint64_t vpl_hit_hint_prev;
3549 uint64_t vpl_fast;
3550 uint64_t vpl_slow;
3551 uint64_t vpl_hit;
3552 uint64_t vpl_miss;
3553
3554 uint64_t vpl_fast_elapsed;
3555 uint64_t vpl_slow_elapsed;
3556 } vm_page_lookup_stats __attribute__((aligned(8)));
3557
3558 #endif
3559
3560 #define KDP_VM_PAGE_WALK_MAX 1000
3561
3562 vm_page_t
kdp_vm_page_lookup(vm_object_t object,vm_object_offset_t offset)3563 kdp_vm_page_lookup(
3564 vm_object_t object,
3565 vm_object_offset_t offset)
3566 {
3567 vm_page_t cur_page;
3568 int num_traversed = 0;
3569
3570 if (not_in_kdp) {
3571 panic("panic: kdp_vm_page_lookup done outside of kernel debugger");
3572 }
3573
3574 vm_page_queue_iterate(&object->memq, cur_page, vmp_listq) {
3575 if (cur_page->vmp_offset == offset) {
3576 return cur_page;
3577 }
3578 num_traversed++;
3579
3580 if (num_traversed >= KDP_VM_PAGE_WALK_MAX) {
3581 return VM_PAGE_NULL;
3582 }
3583 }
3584
3585 return VM_PAGE_NULL;
3586 }
3587
3588 vm_page_t
vm_page_lookup(vm_object_t object,vm_object_offset_t offset)3589 vm_page_lookup(
3590 vm_object_t object,
3591 vm_object_offset_t offset)
3592 {
3593 vm_page_t mem;
3594 vm_page_bucket_t *bucket;
3595 vm_page_queue_entry_t qe;
3596 lck_ticket_t *bucket_lock = NULL;
3597 int hash_id;
3598 #if DEBUG_VM_PAGE_LOOKUP
3599 uint64_t start, elapsed;
3600
3601 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_total);
3602 #endif
3603
3604 #if KASAN_TBI
3605 if (is_kernel_object(object)) {
3606 offset = vm_memtag_canonicalize_kernel(offset);
3607 }
3608 #endif /* KASAN_TBI */
3609
3610 vm_object_lock_assert_held(object);
3611 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
3612
3613 if (object->resident_page_count == 0) {
3614 #if DEBUG_VM_PAGE_LOOKUP
3615 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_empty_obj);
3616 #endif
3617 return VM_PAGE_NULL;
3618 }
3619
3620 mem = object->memq_hint;
3621
3622 if (mem != VM_PAGE_NULL) {
3623 assert(VM_PAGE_OBJECT(mem) == object);
3624
3625 if (mem->vmp_offset == offset) {
3626 #if DEBUG_VM_PAGE_LOOKUP
3627 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint);
3628 #endif
3629 return mem;
3630 }
3631 qe = (vm_page_queue_entry_t)vm_page_queue_next(&mem->vmp_listq);
3632
3633 if (!vm_page_queue_end(&object->memq, qe)) {
3634 vm_page_t next_page;
3635
3636 next_page = (vm_page_t)((uintptr_t)qe);
3637 assert(VM_PAGE_OBJECT(next_page) == object);
3638
3639 if (next_page->vmp_offset == offset) {
3640 object->memq_hint = next_page; /* new hint */
3641 #if DEBUG_VM_PAGE_LOOKUP
3642 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_next);
3643 #endif
3644 return next_page;
3645 }
3646 }
3647 qe = (vm_page_queue_entry_t)vm_page_queue_prev(&mem->vmp_listq);
3648
3649 if (!vm_page_queue_end(&object->memq, qe)) {
3650 vm_page_t prev_page;
3651
3652 prev_page = (vm_page_t)((uintptr_t)qe);
3653 assert(VM_PAGE_OBJECT(prev_page) == object);
3654
3655 if (prev_page->vmp_offset == offset) {
3656 object->memq_hint = prev_page; /* new hint */
3657 #if DEBUG_VM_PAGE_LOOKUP
3658 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_prev);
3659 #endif
3660 return prev_page;
3661 }
3662 }
3663 }
3664 /*
3665 * Search the hash table for this object/offset pair
3666 */
3667 hash_id = vm_page_hash(object, offset);
3668 bucket = &vm_page_buckets[hash_id];
3669
3670 /*
3671 * since we hold the object lock, we are guaranteed that no
3672 * new pages can be inserted into this object... this in turn
3673 * guarantess that the page we're looking for can't exist
3674 * if the bucket it hashes to is currently NULL even when looked
3675 * at outside the scope of the hash bucket lock... this is a
3676 * really cheap optimiztion to avoid taking the lock
3677 */
3678 if (!bucket->page_list) {
3679 #if DEBUG_VM_PAGE_LOOKUP
3680 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_bucket_NULL);
3681 #endif
3682 return VM_PAGE_NULL;
3683 }
3684
3685 #if DEBUG_VM_PAGE_LOOKUP
3686 start = mach_absolute_time();
3687 #endif
3688 if (object->resident_page_count <= VM_PAGE_HASH_LOOKUP_THRESHOLD) {
3689 /*
3690 * on average, it's roughly 3 times faster to run a short memq list
3691 * than to take the spin lock and go through the hash list
3692 */
3693 mem = (vm_page_t)vm_page_queue_first(&object->memq);
3694
3695 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {
3696 if (mem->vmp_offset == offset) {
3697 break;
3698 }
3699
3700 mem = (vm_page_t)vm_page_queue_next(&mem->vmp_listq);
3701 }
3702 if (vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {
3703 mem = NULL;
3704 }
3705 } else {
3706 vm_page_object_t packed_object;
3707
3708 packed_object = VM_PAGE_PACK_OBJECT(object);
3709
3710 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
3711
3712 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
3713
3714 for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
3715 mem != VM_PAGE_NULL;
3716 mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m))) {
3717 #if 0
3718 /*
3719 * we don't hold the page queue lock
3720 * so this check isn't safe to make
3721 */
3722 VM_PAGE_CHECK(mem);
3723 #endif
3724 if ((mem->vmp_object == packed_object) && (mem->vmp_offset == offset)) {
3725 break;
3726 }
3727 }
3728 lck_ticket_unlock(bucket_lock);
3729 }
3730
3731 #if DEBUG_VM_PAGE_LOOKUP
3732 elapsed = mach_absolute_time() - start;
3733
3734 if (bucket_lock) {
3735 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_slow);
3736 OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_slow_elapsed);
3737 } else {
3738 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_fast);
3739 OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_fast_elapsed);
3740 }
3741 if (mem != VM_PAGE_NULL) {
3742 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit);
3743 } else {
3744 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_miss);
3745 }
3746 #endif
3747 if (mem != VM_PAGE_NULL) {
3748 assert(VM_PAGE_OBJECT(mem) == object);
3749
3750 object->memq_hint = mem;
3751 }
3752 return mem;
3753 }
3754
3755
3756 /*
3757 * vm_page_rename:
3758 *
3759 * Move the given memory entry from its
3760 * current object to the specified target object/offset.
3761 *
3762 * The object must be locked.
3763 */
3764 void
vm_page_rename(vm_page_t mem,vm_object_t new_object,vm_object_offset_t new_offset)3765 vm_page_rename(
3766 vm_page_t mem,
3767 vm_object_t new_object,
3768 vm_object_offset_t new_offset)
3769 {
3770 boolean_t internal_to_external, external_to_internal;
3771 vm_tag_t tag;
3772 vm_object_t m_object;
3773
3774 m_object = VM_PAGE_OBJECT(mem);
3775
3776 assert(m_object != new_object);
3777 assert(m_object);
3778
3779 /*
3780 * Changes to mem->vmp_object require the page lock because
3781 * the pageout daemon uses that lock to get the object.
3782 */
3783 vm_page_lockspin_queues();
3784
3785 internal_to_external = FALSE;
3786 external_to_internal = FALSE;
3787
3788 if (mem->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q) {
3789 /*
3790 * it's much easier to get the vm_page_pageable_xxx accounting correct
3791 * if we first move the page to the active queue... it's going to end
3792 * up there anyway, and we don't do vm_page_rename's frequently enough
3793 * for this to matter.
3794 */
3795 vm_page_queues_remove(mem, FALSE);
3796 vm_page_activate(mem);
3797 }
3798 if (VM_PAGE_PAGEABLE(mem)) {
3799 if (m_object->internal && !new_object->internal) {
3800 internal_to_external = TRUE;
3801 }
3802 if (!m_object->internal && new_object->internal) {
3803 external_to_internal = TRUE;
3804 }
3805 }
3806
3807 tag = m_object->wire_tag;
3808 vm_page_remove(mem, TRUE);
3809 vm_page_insert_internal(mem, new_object, new_offset, tag, TRUE, TRUE, FALSE, FALSE, NULL);
3810
3811 if (internal_to_external) {
3812 vm_page_pageable_internal_count--;
3813 vm_page_pageable_external_count++;
3814 } else if (external_to_internal) {
3815 vm_page_pageable_external_count--;
3816 vm_page_pageable_internal_count++;
3817 }
3818
3819 vm_page_unlock_queues();
3820 }
3821
3822 /*
3823 * vm_page_init:
3824 *
3825 * Initialize the fields in a new page.
3826 * This takes a structure with random values and initializes it
3827 * so that it can be given to vm_page_release or vm_page_insert.
3828 */
3829 void
vm_page_init(vm_page_t mem,ppnum_t phys_page)3830 vm_page_init(vm_page_t mem, ppnum_t phys_page)
3831 {
3832 assert(phys_page);
3833
3834 #if DEBUG
3835 if ((phys_page != vm_page_fictitious_addr) && (phys_page != vm_page_guard_addr)) {
3836 if (!(pmap_valid_page(phys_page))) {
3837 panic("vm_page_init: non-DRAM phys_page 0x%x", phys_page);
3838 }
3839 }
3840 #endif /* DEBUG */
3841
3842 /*
3843 * Initialize the fields of the vm_page. If adding any new fields to vm_page,
3844 * try to use initial values which match 0. This minimizes the number of writes
3845 * needed for boot-time initialization.
3846 */
3847 assert(VM_PAGE_NOT_ON_Q == 0);
3848 assert(sizeof(*mem) % sizeof(uintptr_t) == 0);
3849 *mem = (struct vm_page) {
3850 .vmp_offset = (vm_object_offset_t)-1,
3851 .vmp_q_state = VM_PAGE_NOT_ON_Q,
3852 .vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY,
3853 .vmp_canonical = vm_page_in_array(mem),
3854 .vmp_busy = true,
3855 };
3856
3857 VM_PAGE_INIT_PHYS_PAGE(mem, phys_page);
3858
3859 #if 0
3860 /*
3861 * we're leaving this turned off for now... currently pages
3862 * come off the free list and are either immediately dirtied/referenced
3863 * due to zero-fill or COW faults, or are used to read or write files...
3864 * in the file I/O case, the UPL mechanism takes care of clearing
3865 * the state of the HW ref/mod bits in a somewhat fragile way.
3866 * Since we may change the way this works in the future (to toughen it up),
3867 * I'm leaving this as a reminder of where these bits could get cleared
3868 */
3869
3870 /*
3871 * make sure both the h/w referenced and modified bits are
3872 * clear at this point... we are especially dependent on
3873 * not finding a 'stale' h/w modified in a number of spots
3874 * once this page goes back into use
3875 */
3876 pmap_clear_refmod(phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
3877 #endif
3878 }
3879
3880 vm_page_t
vm_page_create_fictitious(void)3881 vm_page_create_fictitious(void)
3882 {
3883 return vm_page_create(vm_page_fictitious_addr, false, Z_WAITOK);
3884 }
3885
3886 vm_page_t
vm_page_create_guard(bool canwait)3887 vm_page_create_guard(bool canwait)
3888 {
3889 return vm_page_create(vm_page_guard_addr, false, canwait ? Z_WAITOK : Z_NOWAIT);
3890 }
3891
3892 vm_page_t
vm_page_create_private(ppnum_t base_page)3893 vm_page_create_private(ppnum_t base_page)
3894 {
3895 assert(base_page != vm_page_fictitious_addr &&
3896 base_page != vm_page_guard_addr);
3897 return vm_page_create(base_page, false, Z_WAITOK);
3898 }
3899
3900 bool
vm_page_is_canonical(const struct vm_page * m)3901 vm_page_is_canonical(const struct vm_page *m)
3902 {
3903 return m->vmp_canonical;
3904 }
3905
3906 bool
vm_page_is_fictitious(const struct vm_page * m)3907 vm_page_is_fictitious(const struct vm_page *m)
3908 {
3909 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
3910 if (vm_page_in_array(m)) {
3911 return false;
3912 }
3913 #endif /* XNU_VM_HAS_LINEAR_PAGES_ARRAY */
3914 switch (VM_PAGE_GET_PHYS_PAGE(m)) {
3915 case vm_page_guard_addr:
3916 case vm_page_fictitious_addr:
3917 return true;
3918 default:
3919 return false;
3920 }
3921 }
3922
3923 bool
vm_page_is_guard(const struct vm_page * m)3924 vm_page_is_guard(const struct vm_page *m)
3925 {
3926 #if XNU_VM_HAS_LINEAR_PAGES_ARRAY
3927 if (vm_page_in_array(m)) {
3928 return false;
3929 }
3930 #endif /* XNU_VM_HAS_LINEAR_PAGES_ARRAY */
3931 return VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr;
3932 }
3933
3934 bool
vm_page_is_private(const struct vm_page * m)3935 vm_page_is_private(const struct vm_page *m)
3936 {
3937 return !vm_page_is_canonical(m) && !vm_page_is_fictitious(m);
3938 }
3939
3940 void
vm_page_make_private(vm_page_t m,ppnum_t base_page)3941 vm_page_make_private(vm_page_t m, ppnum_t base_page)
3942 {
3943 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3944 assert(VM_PAGE_GET_PHYS_PAGE(m) == vm_page_fictitious_addr);
3945
3946 VM_PAGE_SET_PHYS_PAGE(m, base_page);
3947 }
3948
3949 void
vm_page_reset_private(vm_page_t m)3950 vm_page_reset_private(vm_page_t m)
3951 {
3952 assert(vm_page_is_private(m));
3953
3954 VM_PAGE_SET_PHYS_PAGE(m, vm_page_fictitious_addr);
3955 }
3956
3957 /*
3958 * vm_page_release_fictitious:
3959 *
3960 * Release a fictitious page to the zone pool
3961 */
3962 static void
vm_page_release_fictitious(vm_page_t m)3963 vm_page_release_fictitious(vm_page_t m)
3964 {
3965 assert((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
3966 (m->vmp_q_state == VM_PAGE_IS_WIRED));
3967 assert(vm_page_is_fictitious(m));
3968 assert(!m->vmp_realtime);
3969
3970 if (vm_page_is_guard(m)) {
3971 counter_dec(&vm_guard_count);
3972 }
3973 zfree(vm_page_zone, m);
3974 }
3975
3976 /*
3977 * vm_pool_low():
3978 *
3979 * Return true if it is not likely that a non-vm_privileged thread
3980 * can get memory without blocking. Advisory only, since the
3981 * situation may change under us.
3982 */
3983 bool
vm_pool_low(void)3984 vm_pool_low(void)
3985 {
3986 /* No locking, at worst we will fib. */
3987 return vm_page_free_count <= vm_page_free_reserved;
3988 }
3989
3990 boolean_t vm_darkwake_mode = FALSE;
3991
3992 /*
3993 * vm_update_darkwake_mode():
3994 *
3995 * Tells the VM that the system is in / out of darkwake.
3996 *
3997 * Today, the VM only lowers/raises the background queue target
3998 * so as to favor consuming more/less background pages when
3999 * darwake is ON/OFF.
4000 *
4001 * We might need to do more things in the future.
4002 */
4003
4004 void
vm_update_darkwake_mode(boolean_t darkwake_mode)4005 vm_update_darkwake_mode(boolean_t darkwake_mode)
4006 {
4007 #if XNU_TARGET_OS_OSX && defined(__arm64__)
4008 #pragma unused(darkwake_mode)
4009 assert(vm_darkwake_mode == FALSE);
4010 /*
4011 * Darkwake mode isn't supported for AS macOS.
4012 */
4013 return;
4014 #else /* XNU_TARGET_OS_OSX && __arm64__ */
4015 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
4016
4017 vm_page_lockspin_queues();
4018
4019 if (vm_darkwake_mode == darkwake_mode) {
4020 /*
4021 * No change.
4022 */
4023 vm_page_unlock_queues();
4024 return;
4025 }
4026
4027 vm_darkwake_mode = darkwake_mode;
4028
4029 if (vm_darkwake_mode == TRUE) {
4030 /* save background target to restore later */
4031 vm_page_background_target_snapshot = vm_page_background_target;
4032
4033 /* target is set to 0...no protection for background pages */
4034 vm_page_background_target = 0;
4035 } else if (vm_darkwake_mode == FALSE) {
4036 if (vm_page_background_target_snapshot) {
4037 vm_page_background_target = vm_page_background_target_snapshot;
4038 }
4039 }
4040 vm_page_unlock_queues();
4041 #endif
4042 }
4043
4044 void
vm_page_update_special_state(vm_page_t mem)4045 vm_page_update_special_state(vm_page_t mem)
4046 {
4047 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR || mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY) {
4048 return;
4049 }
4050
4051 switch (mem->vmp_on_specialq) {
4052 case VM_PAGE_SPECIAL_Q_BG:
4053 {
4054 task_t my_task = current_task_early();
4055
4056 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
4057 return;
4058 }
4059
4060 if (my_task) {
4061 if (task_get_darkwake_mode(my_task)) {
4062 return;
4063 }
4064 }
4065
4066 if (my_task) {
4067 if (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG)) {
4068 return;
4069 }
4070 }
4071 vm_page_lockspin_queues();
4072
4073 vm_page_background_promoted_count++;
4074
4075 vm_page_remove_from_specialq(mem);
4076 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
4077
4078 vm_page_unlock_queues();
4079 break;
4080 }
4081
4082 case VM_PAGE_SPECIAL_Q_DONATE:
4083 {
4084 task_t my_task = current_task_early();
4085
4086 if (vm_page_donate_mode == VM_PAGE_DONATE_DISABLED) {
4087 return;
4088 }
4089
4090 if (my_task->donates_own_pages == false) {
4091 vm_page_lockspin_queues();
4092
4093 vm_page_remove_from_specialq(mem);
4094 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
4095
4096 vm_page_unlock_queues();
4097 }
4098 break;
4099 }
4100
4101 default:
4102 {
4103 assert(VM_PAGE_UNPACK_PTR(mem->vmp_specialq.next) == (uintptr_t)NULL &&
4104 VM_PAGE_UNPACK_PTR(mem->vmp_specialq.prev) == (uintptr_t)NULL);
4105 break;
4106 }
4107 }
4108 }
4109
4110
4111 void
vm_page_assign_special_state(vm_page_t mem,vm_page_specialq_t mode)4112 vm_page_assign_special_state(vm_page_t mem, vm_page_specialq_t mode)
4113 {
4114 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
4115 return;
4116 }
4117
4118 switch (mode) {
4119 case VM_PAGE_SPECIAL_Q_BG:
4120 {
4121 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
4122 return;
4123 }
4124
4125 task_t my_task = current_task_early();
4126
4127 if (my_task) {
4128 if (task_get_darkwake_mode(my_task)) {
4129 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_BG;
4130 return;
4131 }
4132 }
4133
4134 if (my_task) {
4135 mem->vmp_on_specialq = (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG) ? VM_PAGE_SPECIAL_Q_BG : VM_PAGE_SPECIAL_Q_EMPTY);
4136 }
4137 break;
4138 }
4139
4140 case VM_PAGE_SPECIAL_Q_DONATE:
4141 {
4142 if (vm_page_donate_mode == VM_PAGE_DONATE_DISABLED) {
4143 return;
4144 }
4145 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
4146 break;
4147 }
4148
4149 default:
4150 break;
4151 }
4152 }
4153
4154
4155 void
vm_page_remove_from_specialq(vm_page_t mem)4156 vm_page_remove_from_specialq(vm_page_t mem)
4157 {
4158 vm_object_t m_object;
4159
4160 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4161
4162 switch (mem->vmp_on_specialq) {
4163 case VM_PAGE_SPECIAL_Q_BG:
4164 {
4165 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
4166 vm_page_queue_remove(&vm_page_queue_background, mem, vmp_specialq);
4167
4168 mem->vmp_specialq.next = 0;
4169 mem->vmp_specialq.prev = 0;
4170
4171 vm_page_background_count--;
4172
4173 m_object = VM_PAGE_OBJECT(mem);
4174
4175 if (m_object->internal) {
4176 vm_page_background_internal_count--;
4177 } else {
4178 vm_page_background_external_count--;
4179 }
4180 }
4181 break;
4182 }
4183
4184 case VM_PAGE_SPECIAL_Q_DONATE:
4185 {
4186 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
4187 vm_page_queue_remove((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
4188 mem->vmp_specialq.next = 0;
4189 mem->vmp_specialq.prev = 0;
4190 vm_page_donate_count--;
4191 if (vm_page_donate_queue_ripe && (vm_page_donate_count < vm_page_donate_target)) {
4192 assert(vm_page_donate_target == vm_page_donate_target_low);
4193 vm_page_donate_target = vm_page_donate_target_high;
4194 vm_page_donate_queue_ripe = false;
4195 }
4196 }
4197
4198 break;
4199 }
4200
4201 default:
4202 {
4203 assert(VM_PAGE_UNPACK_PTR(mem->vmp_specialq.next) == (uintptr_t)NULL &&
4204 VM_PAGE_UNPACK_PTR(mem->vmp_specialq.prev) == (uintptr_t)NULL);
4205 break;
4206 }
4207 }
4208 }
4209
4210
4211 void
vm_page_add_to_specialq(vm_page_t mem,boolean_t first)4212 vm_page_add_to_specialq(vm_page_t mem, boolean_t first)
4213 {
4214 vm_object_t m_object;
4215
4216 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4217
4218 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
4219 return;
4220 }
4221
4222 switch (mem->vmp_on_specialq) {
4223 case VM_PAGE_SPECIAL_Q_BG:
4224 {
4225 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
4226 return;
4227 }
4228
4229 m_object = VM_PAGE_OBJECT(mem);
4230
4231 if (vm_page_background_exclude_external && !m_object->internal) {
4232 return;
4233 }
4234
4235 if (first == TRUE) {
4236 vm_page_queue_enter_first(&vm_page_queue_background, mem, vmp_specialq);
4237 } else {
4238 vm_page_queue_enter(&vm_page_queue_background, mem, vmp_specialq);
4239 }
4240 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_BG;
4241
4242 vm_page_background_count++;
4243
4244 if (m_object->internal) {
4245 vm_page_background_internal_count++;
4246 } else {
4247 vm_page_background_external_count++;
4248 }
4249 break;
4250 }
4251
4252 case VM_PAGE_SPECIAL_Q_DONATE:
4253 {
4254 if (first == TRUE) {
4255 vm_page_queue_enter_first((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
4256 } else {
4257 vm_page_queue_enter((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
4258 }
4259 vm_page_donate_count++;
4260 if (!vm_page_donate_queue_ripe && (vm_page_donate_count > vm_page_donate_target)) {
4261 assert(vm_page_donate_target == vm_page_donate_target_high);
4262 vm_page_donate_target = vm_page_donate_target_low;
4263 vm_page_donate_queue_ripe = true;
4264 }
4265 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
4266 break;
4267 }
4268
4269 default:
4270 break;
4271 }
4272 }
4273
4274 /*!
4275 * @brief
4276 * Prepares a page that has been successfully grabbed for the caller.
4277 *
4278 * @discussion
4279 * This function will update accounting, emit tracements, ...
4280 */
4281 static vm_page_t
vm_page_grab_finalize(vm_grab_options_t grab_options __unused,vm_page_t mem)4282 vm_page_grab_finalize(vm_grab_options_t grab_options __unused, vm_page_t mem)
4283 {
4284 task_t task;
4285
4286 #if MACH_ASSERT
4287 /*
4288 * For all free pages, no matter their provenance...
4289 * ensure they are not referenced anywhere,
4290 * and their state is clean.
4291 */
4292 if (vm_check_refs_on_alloc) {
4293 pmap_recycle_page(VM_PAGE_GET_PHYS_PAGE(mem));
4294 }
4295 assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)));
4296 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0 &&
4297 mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0 &&
4298 mem->vmp_specialq.next == 0 && mem->vmp_specialq.prev == 0 &&
4299 mem->vmp_next_m == 0 &&
4300 mem->vmp_object == 0 &&
4301 mem->vmp_wire_count == 0 &&
4302 mem->vmp_busy &&
4303 !mem->vmp_tabled &&
4304 !mem->vmp_laundry &&
4305 !mem->vmp_pmapped &&
4306 !mem->vmp_wpmapped &&
4307 !mem->vmp_realtime);
4308 #endif /* MACH_ASSERT */
4309
4310 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
4311 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
4312
4313 {
4314 VM_DEBUG_EVENT(vm_page_grab, DBG_VM_PAGE_GRAB,
4315 DBG_FUNC_NONE, grab_options, 0, 0, 0);
4316 }
4317
4318 counter_inc(&vm_page_grab_count);
4319
4320 task = current_task_early();
4321 if (task != TASK_NULL) {
4322 ledger_credit(task->ledger, task_ledgers.pages_grabbed, 1);
4323 }
4324 if (task != TASK_NULL && task != kernel_task) {
4325 /*
4326 * tag:DONATE this is where the donate state of the page
4327 * is decided according to what task grabs it
4328 */
4329 if (task->donates_own_pages) {
4330 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_DONATE);
4331 } else {
4332 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_BG);
4333 }
4334 }
4335
4336 return mem;
4337 }
4338
4339 #if __x86_64__
4340 /*
4341 * This can be switched to FALSE to help debug drivers
4342 * that are having problems with memory > 4G.
4343 */
4344 boolean_t vm_himemory_mode = TRUE;
4345 #endif /* __x86_64__ */
4346
4347 #if XNU_VM_HAS_LOPAGE
4348
4349 vm_page_t
vm_page_grablo(vm_grab_options_t grab_options)4350 vm_page_grablo(vm_grab_options_t grab_options)
4351 {
4352 vm_page_t mem = VM_PAGE_NULL;
4353
4354 if (!vm_lopage_needed) {
4355 return vm_page_grab_options(grab_options);
4356 }
4357
4358 vm_free_page_lock_spin();
4359 if (vm_lopage_free_count) {
4360 #if LCK_MTX_USE_ARCH
4361 /*
4362 * Intel locks do not really always disable preemption
4363 * for lck_mtx_lock_spin(), and vm_page_free_queue_grab()
4364 * really want that.
4365 */
4366 disable_preemption();
4367 #endif
4368 mem = vm_page_free_queue_grab(grab_options,
4369 VM_MEMORY_CLASS_LOPAGE, 1, VM_PAGE_NOT_ON_Q).vmpl_head;
4370 #if LCK_MTX_USE_ARCH
4371 enable_preemption();
4372 #endif
4373 }
4374 vm_free_page_unlock();
4375
4376 if (mem == VM_PAGE_NULL) {
4377 if (cpm_allocate(PAGE_SIZE, &mem, atop(PPNUM_MAX), 0, FALSE, KMA_LOMEM) != KERN_SUCCESS) {
4378 vm_free_page_lock_spin();
4379 vm_lopages_allocated_cpm_failed++;
4380 vm_free_page_unlock();
4381
4382 return VM_PAGE_NULL;
4383 }
4384 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
4385
4386 mem->vmp_busy = TRUE;
4387
4388 vm_page_lockspin_queues();
4389
4390 mem->vmp_gobbled = FALSE;
4391 vm_page_gobble_count--;
4392 vm_page_wire_count--;
4393
4394 vm_lopages_allocated_cpm_success++;
4395 vm_page_unlock_queues();
4396 }
4397
4398 return vm_page_grab_finalize(grab_options, mem);
4399 }
4400
4401 #endif /* XNU_VM_HAS_LOPAGE */
4402 #if CONFIG_SECLUDED_MEMORY
4403
4404 /*!
4405 * @brief
4406 * Attempt to allocate a page from the secluded queue
4407 *
4408 * @discussion
4409 * This function will check that the caller is eligible
4410 * for the secluded pool, and if not, return VM_PAGE_NULL.
4411 */
4412 __attribute__((noinline))
4413 static vm_page_t
vm_page_grab_secluded(vm_grab_options_t grab_options)4414 vm_page_grab_secluded(vm_grab_options_t grab_options)
4415 {
4416 vm_page_t mem;
4417 vm_object_t object;
4418 int refmod_state;
4419
4420 if (vm_page_secluded_count == 0) {
4421 return VM_PAGE_NULL;
4422 }
4423
4424 if (grab_options & VM_PAGE_GRAB_SECLUDED) {
4425 vm_page_secluded.grab_for_iokit++;
4426 } else if (!task_can_use_secluded_mem(current_task(), TRUE)) {
4427 return VM_PAGE_NULL;
4428 }
4429
4430
4431 /* secluded queue is protected by the VM page queue lock */
4432 vm_page_lock_queues();
4433
4434 if (vm_page_secluded_count == 0) {
4435 /* no secluded pages to grab... */
4436 vm_page_unlock_queues();
4437 return VM_PAGE_NULL;
4438 }
4439
4440 #if 00
4441 /* can we grab from the secluded queue? */
4442 if (vm_page_secluded_count > vm_page_secluded_target ||
4443 (vm_page_secluded_count > 0 &&
4444 task_can_use_secluded_mem(current_task(), TRUE))) {
4445 /* OK */
4446 } else {
4447 /* can't grab from secluded queue... */
4448 vm_page_unlock_queues();
4449 return VM_PAGE_NULL;
4450 }
4451 #endif
4452
4453 /* we can grab a page from secluded queue! */
4454 assert((vm_page_secluded_count_free +
4455 vm_page_secluded_count_inuse) ==
4456 vm_page_secluded_count);
4457 if (current_task()->task_can_use_secluded_mem) {
4458 assert(num_tasks_can_use_secluded_mem > 0);
4459 }
4460 assert(!vm_page_queue_empty(&vm_page_queue_secluded));
4461 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4462 mem = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
4463 assert(mem->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
4464 vm_page_queues_remove(mem, TRUE);
4465
4466 object = VM_PAGE_OBJECT(mem);
4467
4468 assert(!vm_page_is_fictitious(mem));
4469 assert(!VM_PAGE_WIRED(mem));
4470 if (object == VM_OBJECT_NULL) {
4471 /* free for grab! */
4472 vm_page_unlock_queues();
4473 vm_page_secluded.grab_success_free++;
4474 goto out_success;
4475 }
4476
4477 assert(!object->internal);
4478 // vm_page_pageable_external_count--;
4479
4480 if (!vm_object_lock_try(object)) {
4481 // printf("SECLUDED: page %p: object %p locked\n", mem, object);
4482 vm_page_secluded.grab_failure_locked++;
4483 reactivate_secluded_page:
4484 vm_page_activate(mem);
4485 vm_page_unlock_queues();
4486 return VM_PAGE_NULL;
4487 }
4488 if (mem->vmp_busy ||
4489 mem->vmp_cleaning ||
4490 mem->vmp_laundry) {
4491 /* can't steal page in this state... */
4492 vm_object_unlock(object);
4493 vm_page_secluded.grab_failure_state++;
4494 goto reactivate_secluded_page;
4495 }
4496 if (mem->vmp_realtime) {
4497 /* don't steal pages used by realtime threads... */
4498 vm_object_unlock(object);
4499 vm_page_secluded.grab_failure_realtime++;
4500 goto reactivate_secluded_page;
4501 }
4502
4503 mem->vmp_busy = TRUE;
4504 refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(mem));
4505 if (refmod_state & VM_MEM_REFERENCED) {
4506 mem->vmp_reference = TRUE;
4507 }
4508 if (refmod_state & VM_MEM_MODIFIED) {
4509 SET_PAGE_DIRTY(mem, FALSE);
4510 }
4511 if (mem->vmp_dirty || mem->vmp_precious) {
4512 /* can't grab a dirty page; re-activate */
4513 // printf("SECLUDED: dirty page %p\n", mem);
4514 vm_page_wakeup_done(object, mem);
4515 vm_page_secluded.grab_failure_dirty++;
4516 vm_object_unlock(object);
4517 goto reactivate_secluded_page;
4518 }
4519 if (mem->vmp_reference) {
4520 /* it's been used but we do need to grab a page... */
4521 }
4522
4523 vm_page_unlock_queues();
4524
4525 /* finish what vm_page_free() would have done... */
4526 vm_page_free_prepare_object(mem, TRUE);
4527 vm_object_unlock(object);
4528 object = VM_OBJECT_NULL;
4529
4530 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
4531 vm_page_secluded.grab_success_other++;
4532
4533 out_success:
4534
4535 if (grab_options & VM_PAGE_GRAB_SECLUDED) {
4536 vm_page_secluded.grab_for_iokit_success++;
4537 }
4538 return mem;
4539 }
4540
4541 uint64_t
vm_page_secluded_drain(void)4542 vm_page_secluded_drain(void)
4543 {
4544 vm_page_t local_freeq;
4545 int local_freed;
4546 uint64_t num_reclaimed;
4547 unsigned int saved_secluded_count, saved_secluded_target;
4548
4549 num_reclaimed = 0;
4550 local_freeq = NULL;
4551 local_freed = 0;
4552
4553 vm_page_lock_queues();
4554
4555 saved_secluded_count = vm_page_secluded_count;
4556 saved_secluded_target = vm_page_secluded_target;
4557 vm_page_secluded_target = 0;
4558 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
4559 while (vm_page_secluded_count) {
4560 vm_page_t secluded_page;
4561
4562 assert((vm_page_secluded_count_free +
4563 vm_page_secluded_count_inuse) ==
4564 vm_page_secluded_count);
4565 secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
4566 assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
4567
4568 vm_page_queues_remove(secluded_page, FALSE);
4569 assert(!vm_page_is_fictitious(secluded_page));
4570 assert(!VM_PAGE_WIRED(secluded_page));
4571
4572 if (secluded_page->vmp_object == 0) {
4573 /* transfer to free queue */
4574 assert(secluded_page->vmp_busy);
4575 secluded_page->vmp_snext = local_freeq;
4576 local_freeq = secluded_page;
4577 local_freed += 1;
4578 } else {
4579 /* transfer to head of active queue */
4580 vm_page_enqueue_active(secluded_page, FALSE);
4581 secluded_page = VM_PAGE_NULL;
4582 }
4583 num_reclaimed++;
4584 }
4585 vm_page_secluded_target = saved_secluded_target;
4586 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
4587
4588 // printf("FBDP %s:%d secluded_count %d->%d, target %d, reclaimed %lld\n", __FUNCTION__, __LINE__, saved_secluded_count, vm_page_secluded_count, vm_page_secluded_target, num_reclaimed);
4589
4590 vm_page_unlock_queues();
4591
4592 if (local_freed) {
4593 vm_page_free_list(local_freeq, TRUE);
4594 local_freeq = NULL;
4595 local_freed = 0;
4596 }
4597
4598 return num_reclaimed;
4599 }
4600
4601 #endif /* CONFIG_SECLUDED_MEMORY */
4602
4603 /*!
4604 * @brief
4605 * Attempts to allocate a page from the specified per-cpu page queue.
4606 */
4607 static vm_page_t
vm_page_grab_from_cpu(vm_page_t * cpu_list,scalable_counter_t * counter)4608 vm_page_grab_from_cpu(vm_page_t *cpu_list, scalable_counter_t *counter)
4609 {
4610 vm_page_t mem = _vm_page_list_pop(cpu_list);
4611
4612 if (mem != VM_PAGE_NULL) {
4613 #if HIBERNATION
4614 if (hibernate_rebuild_needed) {
4615 panic("should not modify cpu->free_pages while hibernating");
4616 }
4617 #endif /* HIBERNATION */
4618 counter_dec_preemption_disabled(counter);
4619 }
4620 return mem;
4621 }
4622
4623
4624 /*!
4625 * @brief
4626 * Attempts to allocate pages from free queues, and to populate the per-cpu
4627 * queue as a side effect.
4628 *
4629 * @discussion
4630 * This function will take the properties of the allocating thread into account
4631 * to decide how many pages it can allocate.
4632 *
4633 * If the free queues are depleted, then it will return VM_PAGE_NULL.
4634 */
4635 __attribute__((noinline))
4636 static vm_page_t
vm_page_grab_slow(vm_grab_options_t grab_options)4637 vm_page_grab_slow(vm_grab_options_t grab_options)
4638 {
4639 unsigned int target = vm_free_magazine_refill_limit;
4640 vm_memory_class_t class = VM_MEMORY_CLASS_REGULAR;
4641 vm_page_t mem = VM_PAGE_NULL;
4642 vm_page_list_t list = { };
4643 vm_page_t *cpu_list = NULL;
4644 scalable_counter_t *counter = NULL;
4645
4646 vm_free_page_lock_spin();
4647 #if LCK_MTX_USE_ARCH
4648 /* Intel does't disable preemption with vm_free_page_lock_spin() */
4649 disable_preemption();
4650 #endif /* LCK_MTX_USE_ARCH */
4651 cpu_list = PERCPU_GET(free_pages);
4652 counter = &vm_cpu_free_count;
4653 {
4654 mem = vm_page_grab_from_cpu(cpu_list, counter);
4655 }
4656 if (mem != VM_PAGE_NULL) {
4657 #if LCK_MTX_USE_ARCH
4658 enable_preemption();
4659 #endif /* LCK_MTX_USE_ARCH */
4660 vm_free_page_unlock();
4661 return mem;
4662 }
4663
4664 if (vm_page_free_count <= vm_page_free_reserved) {
4665 if ((current_thread()->options & TH_OPT_VMPRIV) == 0) {
4666 target = 0;
4667 } else if (vm_page_free_count == 0) {
4668 target = 0;
4669 } else {
4670 target = 1;
4671 }
4672 } else {
4673 target = MIN(target, vm_page_free_count - vm_page_free_reserved);
4674 }
4675
4676 #if HIBERNATION
4677 if (target > 0 && hibernate_rebuild_needed) {
4678 panic("should not modify CPU free_pages while hibernating");
4679 }
4680 #endif /* HIBERNATION */
4681
4682 /*
4683 * Convert the lock hold into a mutex, to signal to waiters that the
4684 * lock may be held for longer.
4685 */
4686 #if !LCK_MTX_USE_ARCH
4687 disable_preemption();
4688 #endif /* !LCK_MTX_USE_ARCH */
4689 vm_free_page_lock_convert();
4690
4691 if (target != 0) {
4692 list = vm_page_free_queue_grab(grab_options, class, target,
4693 VM_PAGE_ON_FREE_LOCAL_Q);
4694 }
4695
4696 #if VM_PAGE_WIRE_COUNT_WARNING
4697 if (vm_page_wire_count >= VM_PAGE_WIRE_COUNT_WARNING) {
4698 printf("mk: vm_page_grab(): high wired page count of %d\n",
4699 vm_page_wire_count);
4700 }
4701 #endif
4702 #if VM_PAGE_GOBBLE_COUNT_WARNING
4703 if (vm_page_gobble_count >= VM_PAGE_GOBBLE_COUNT_WARNING) {
4704 printf("mk: vm_page_grab(): high gobbled page count of %d\n",
4705 vm_page_gobble_count);
4706 }
4707 #endif
4708
4709 if (vm_page_free_count < vm_page_free_min && !vm_pageout_running) {
4710 thread_wakeup(&vm_page_free_wanted);
4711 }
4712
4713 vm_free_page_unlock();
4714
4715 VM_CHECK_MEMORYSTATUS;
4716
4717 if (list.vmpl_head) {
4718 /* Steal a page off the list for the caller. */
4719 mem = vm_page_list_pop(&list);
4720
4721 /* Add the remaining pages to the CPU's free list. */
4722 assert(*cpu_list == VM_PAGE_NULL);
4723 *cpu_list = list.vmpl_head;
4724 counter_add_preemption_disabled(counter, list.vmpl_count);
4725 }
4726
4727 enable_preemption();
4728
4729 return mem;
4730 }
4731
4732 vm_page_t
vm_page_grab_options(vm_grab_options_t options)4733 vm_page_grab_options(vm_grab_options_t options)
4734 {
4735 vm_page_t mem;
4736
4737 restart:
4738
4739 /*
4740 * Step 1: look at the CPU magazines.
4741 */
4742
4743 disable_preemption();
4744 mem = vm_page_grab_from_cpu(PERCPU_GET(free_pages), &vm_cpu_free_count);
4745 enable_preemption();
4746
4747 if (mem != VM_PAGE_NULL) {
4748 return vm_page_grab_finalize(options, mem);
4749 }
4750
4751 #if XNU_VM_HAS_DELAYED_PAGES
4752 /*
4753 * If free count is low and we have delayed pages from early boot,
4754 * get one of those instead.
4755 */
4756 if (__improbable(vm_delayed_count > 0 &&
4757 vm_page_free_count <= vm_page_free_target)) {
4758 mem = vm_get_delayed_page(options);
4759 if (mem != VM_PAGE_NULL) {
4760 return vm_page_grab_finalize(options, mem);
4761 }
4762 }
4763 #endif /* XNU_VM_HAS_DELAYED_PAGES */
4764
4765
4766 /*
4767 * Step 2: Try to promote pages from the free queues,
4768 * or the secluded queue if appropriate.
4769 */
4770
4771 mem = vm_page_grab_slow(options);
4772 if (mem != VM_PAGE_NULL) {
4773 return vm_page_grab_finalize(options, mem);
4774 }
4775
4776 #if CONFIG_SECLUDED_MEMORY
4777 mem = vm_page_grab_secluded(options);
4778 if (mem != VM_PAGE_NULL) {
4779 return vm_page_grab_finalize(options, mem);
4780 }
4781 #endif /* CONFIG_SECLUDED_MEMORY */
4782
4783
4784 /*
4785 * Step 3: Privileged threads block and retry, others fail.
4786 */
4787
4788 if ((options & VM_PAGE_GRAB_NOPAGEWAIT) == 0 &&
4789 (current_thread()->options & TH_OPT_VMPRIV) != 0) {
4790 VM_PAGE_WAIT();
4791 goto restart;
4792 }
4793
4794 return VM_PAGE_NULL;
4795 }
4796
4797 vm_grab_options_t
vm_page_grab_options_for_object(vm_object_t object __unused)4798 vm_page_grab_options_for_object(vm_object_t object __unused)
4799 {
4800 vm_grab_options_t options = VM_PAGE_GRAB_OPTIONS_NONE;
4801
4802 #if CONFIG_SECLUDED_MEMORY
4803 if (object->can_grab_secluded) {
4804 options |= VM_PAGE_GRAB_SECLUDED;
4805 }
4806 #endif /* CONFIG_SECLUDED_MEMORY */
4807
4808 return options;
4809 }
4810
4811 /*!
4812 * @function vm_page_free_queue_steal()
4813 *
4814 * @abstract
4815 * Steal a given page from the free queues.
4816 *
4817 * @discussion
4818 * The given page must be in the given free queue, or state may be corrupted.
4819 *
4820 * Internally, the free queue is not synchronized, so any locking must be done
4821 * outside of this function.
4822 *
4823 * This function, like vm_page_grab(), takes care of waking up
4824 * page out scan as needed.
4825 */
4826 static void
vm_page_free_queue_steal(vm_grab_options_t options,vm_page_t mem)4827 vm_page_free_queue_steal(vm_grab_options_t options, vm_page_t mem)
4828 {
4829 ppnum_t pnum = VM_PAGE_GET_PHYS_PAGE(mem);
4830 vm_memory_class_t class = vm_page_get_memory_class(mem, pnum);
4831
4832 assert(mem->vmp_q_state == VM_PAGE_ON_FREE_Q);
4833 assert(!mem->vmp_lopage && mem->vmp_busy);
4834
4835 vm_page_free_queue_remove(class, mem, pnum, VM_PAGE_NOT_ON_Q);
4836 vm_page_grab_finalize(options, mem);
4837
4838 if (vm_page_free_count < vm_page_free_min && !vm_pageout_running) {
4839 thread_wakeup(&vm_page_free_wanted);
4840 }
4841 }
4842
4843
4844 /*
4845 * vm_page_wait:
4846 *
4847 * Wait for a page to become available.
4848 * If there are plenty of free pages, then we don't sleep.
4849 *
4850 * Returns:
4851 * TRUE: There may be another page, try again
4852 * FALSE: We were interrupted out of our wait, don't try again
4853 */
4854
4855 boolean_t
vm_page_wait(int interruptible)4856 vm_page_wait(int interruptible)
4857 {
4858 /*
4859 * We can't use vm_page_free_reserved to make this
4860 * determination. Consider: some thread might
4861 * need to allocate two pages. The first allocation
4862 * succeeds, the second fails. After the first page is freed,
4863 * a call to vm_page_wait must really block.
4864 */
4865 kern_return_t wait_result = THREAD_NOT_WAITING;
4866 thread_t cur_thread = current_thread();
4867 bool is_privileged = cur_thread->options & TH_OPT_VMPRIV;
4868 bool need_wakeup = false;
4869 event_t wait_event = NULL;
4870
4871 vm_free_page_lock_spin();
4872
4873 if (is_privileged) {
4874 if (vm_page_free_count) {
4875 vm_free_page_unlock();
4876 goto out;
4877 }
4878
4879 if (vm_page_free_wanted_privileged++ == 0) {
4880 need_wakeup = true;
4881 }
4882
4883 wait_event = (event_t)&vm_page_free_wanted_privileged;
4884 } else if (vm_page_free_count >= vm_page_free_target) {
4885 vm_free_page_unlock();
4886 goto out;
4887 #if CONFIG_SECLUDED_MEMORY
4888 } else if (secluded_for_apps &&
4889 task_can_use_secluded_mem(current_task(), FALSE)) {
4890 #if 00
4891 /* XXX FBDP: need pageq lock for this... */
4892 /* XXX FBDP: might wait even if pages available, */
4893 /* XXX FBDP: hopefully not for too long... */
4894 if (vm_page_secluded_count > 0) {
4895 vm_free_page_unlock();
4896 goto out;
4897 }
4898 #endif
4899 if (vm_page_free_wanted_secluded++ == 0) {
4900 need_wakeup = true;
4901 }
4902
4903 wait_event = (event_t)&vm_page_free_wanted_secluded;
4904 #endif /* CONFIG_SECLUDED_MEMORY */
4905 } else {
4906 if (vm_page_free_wanted++ == 0) {
4907 need_wakeup = true;
4908 }
4909
4910 wait_event = (event_t)&vm_page_free_count;
4911 }
4912
4913 if (vm_pageout_running) {
4914 need_wakeup = false;
4915 }
4916
4917 /*
4918 * We don't do a vm_pageout_scan wakeup if we already have
4919 * some waiters because vm_pageout_scan checks for waiters
4920 * before it returns and does so behind the vm_page_queue_free_lock,
4921 * which we own when we bump the waiter counts.
4922 */
4923
4924 if (vps_dynamic_priority_enabled) {
4925 /*
4926 * We are waking up vm_pageout_scan here. If it needs
4927 * the vm_page_queue_free_lock before we unlock it
4928 * we'll end up just blocking and incur an extra
4929 * context switch. Could be a perf. issue.
4930 */
4931
4932 if (need_wakeup) {
4933 thread_wakeup((event_t)&vm_page_free_wanted);
4934 }
4935
4936 /*
4937 * LD: This event is going to get recorded every time because
4938 * we don't get back THREAD_WAITING from lck_mtx_sleep_with_inheritor.
4939 * We just block in that routine.
4940 */
4941 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, DBG_VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,
4942 vm_page_free_wanted_privileged,
4943 vm_page_free_wanted,
4944 #if CONFIG_SECLUDED_MEMORY
4945 vm_page_free_wanted_secluded,
4946 #else /* CONFIG_SECLUDED_MEMORY */
4947 0,
4948 #endif /* CONFIG_SECLUDED_MEMORY */
4949 0);
4950 wait_result = lck_mtx_sleep_with_inheritor(&vm_page_queue_free_lock,
4951 LCK_SLEEP_UNLOCK,
4952 wait_event,
4953 vm_pageout_scan_thread,
4954 interruptible,
4955 0);
4956 } else {
4957 wait_result = assert_wait(wait_event, interruptible);
4958
4959 vm_free_page_unlock();
4960
4961 if (need_wakeup) {
4962 thread_wakeup((event_t)&vm_page_free_wanted);
4963 }
4964
4965 if (wait_result != THREAD_WAITING) {
4966 goto out;
4967 }
4968
4969
4970 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block,
4971 DBG_VM_PAGE_WAIT_BLOCK,
4972 DBG_FUNC_START,
4973 vm_page_free_wanted_privileged,
4974 vm_page_free_wanted,
4975 #if CONFIG_SECLUDED_MEMORY
4976 vm_page_free_wanted_secluded,
4977 #else /* CONFIG_SECLUDED_MEMORY */
4978 0,
4979 #endif /* CONFIG_SECLUDED_MEMORY */
4980 0);
4981 wait_result = thread_block(THREAD_CONTINUE_NULL);
4982 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block,
4983 DBG_VM_PAGE_WAIT_BLOCK, DBG_FUNC_END, 0, 0, 0, 0);
4984 }
4985
4986 out:
4987 return (wait_result == THREAD_AWAKENED) || (wait_result == THREAD_NOT_WAITING);
4988 }
4989
4990 /*
4991 * vm_page_free_prepare:
4992 *
4993 * Removes page from any queue it may be on
4994 * and disassociates it from its VM object.
4995 *
4996 * Object and page queues must be locked prior to entry.
4997 */
4998 static void
vm_page_free_prepare(vm_page_t mem)4999 vm_page_free_prepare(
5000 vm_page_t mem)
5001 {
5002 vm_page_free_prepare_queues(mem);
5003 vm_page_free_prepare_object(mem, TRUE);
5004 #if CONFIG_SPTM
5005 /**
5006 * The pmap should retype frames as necessary when pmap_recycle_page()
5007 * is called. In order to catch potential cases where this does not
5008 * happen, add an appropriate assert here. This code should be
5009 * executed on every frame that is about to be released to the VM.
5010 */
5011 const sptm_paddr_t paddr = ((uint64_t)VM_PAGE_GET_PHYS_PAGE(mem)) << PAGE_SHIFT;
5012 __unused const sptm_frame_type_t frame_type = sptm_get_frame_type(paddr);
5013
5014 assert(frame_type == XNU_DEFAULT);
5015 #endif /* CONFIG_SPTM */
5016
5017 }
5018
5019
5020 void
vm_page_free_prepare_queues(vm_page_t mem)5021 vm_page_free_prepare_queues(
5022 vm_page_t mem)
5023 {
5024 vm_object_t m_object;
5025
5026 VM_PAGE_CHECK(mem);
5027
5028 assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
5029 assert(!mem->vmp_cleaning);
5030 m_object = VM_PAGE_OBJECT(mem);
5031
5032 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5033 if (m_object) {
5034 vm_object_lock_assert_exclusive(m_object);
5035 }
5036 if (mem->vmp_laundry) {
5037 /*
5038 * We may have to free a page while it's being laundered
5039 * if we lost its pager (due to a forced unmount, for example).
5040 * We need to call vm_pageout_steal_laundry() before removing
5041 * the page from its VM object, so that we can remove it
5042 * from its pageout queue and adjust the laundry accounting
5043 */
5044 vm_pageout_steal_laundry(mem, TRUE);
5045 }
5046
5047 vm_page_queues_remove(mem, TRUE);
5048
5049 if (mem->vmp_realtime) {
5050 mem->vmp_realtime = false;
5051 VM_COUNTER_DEC(&vm_page_realtime_count);
5052 }
5053
5054 if (VM_PAGE_WIRED(mem)) {
5055 assert(mem->vmp_wire_count > 0);
5056
5057 if (m_object) {
5058 task_t owner;
5059 int ledger_idx_volatile;
5060 int ledger_idx_nonvolatile;
5061 int ledger_idx_volatile_compressed;
5062 int ledger_idx_nonvolatile_compressed;
5063 int ledger_idx_composite;
5064 int ledger_idx_external_wired;
5065 boolean_t do_footprint;
5066
5067 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
5068 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
5069 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
5070
5071 assert(m_object->resident_page_count >=
5072 m_object->wired_page_count);
5073
5074 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
5075 OSAddAtomic(+1, &vm_page_purgeable_count);
5076 assert(vm_page_purgeable_wired_count > 0);
5077 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
5078 }
5079 if (m_object->internal &&
5080 m_object->vo_owner != TASK_NULL &&
5081 (m_object->purgable == VM_PURGABLE_VOLATILE ||
5082 m_object->purgable == VM_PURGABLE_EMPTY)) {
5083 owner = VM_OBJECT_OWNER(m_object);
5084 vm_object_ledger_tag_ledgers(
5085 m_object,
5086 &ledger_idx_volatile,
5087 &ledger_idx_nonvolatile,
5088 &ledger_idx_volatile_compressed,
5089 &ledger_idx_nonvolatile_compressed,
5090 &ledger_idx_composite,
5091 &ledger_idx_external_wired,
5092 &do_footprint);
5093 /*
5094 * While wired, this page was accounted
5095 * as "non-volatile" but it should now
5096 * be accounted as "volatile".
5097 */
5098 /* one less "non-volatile"... */
5099 ledger_debit(owner->ledger,
5100 ledger_idx_nonvolatile,
5101 PAGE_SIZE);
5102 if (do_footprint) {
5103 /* ... and "phys_footprint" */
5104 ledger_debit(owner->ledger,
5105 task_ledgers.phys_footprint,
5106 PAGE_SIZE);
5107 } else if (ledger_idx_composite != -1) {
5108 ledger_debit(owner->ledger,
5109 ledger_idx_composite,
5110 PAGE_SIZE);
5111 }
5112 /* one more "volatile" */
5113 ledger_credit(owner->ledger,
5114 ledger_idx_volatile,
5115 PAGE_SIZE);
5116 }
5117 }
5118 if (vm_page_is_canonical(mem)) {
5119 vm_page_wire_count--;
5120 }
5121
5122
5123 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
5124 mem->vmp_iopl_wired = false;
5125 mem->vmp_wire_count = 0;
5126 assert(!mem->vmp_gobbled);
5127 } else if (mem->vmp_gobbled) {
5128 if (vm_page_is_canonical(mem)) {
5129 vm_page_wire_count--;
5130 }
5131 vm_page_gobble_count--;
5132 }
5133 }
5134
5135 /*
5136 * like vm_page_init, but we have to preserve fields related to phys page
5137 */
5138 inline static void
vm_page_reset_canonical(vm_page_t mem)5139 vm_page_reset_canonical(vm_page_t mem)
5140 {
5141 *mem = (struct vm_page){
5142 .vmp_offset = (vm_object_offset_t)-1,
5143 .vmp_q_state = VM_PAGE_NOT_ON_Q,
5144 .vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY,
5145 #if XNU_VM_HAS_LOPAGE
5146 .vmp_lopage = mem->vmp_lopage,
5147 #endif /* XNU_VM_HAS_LOPAGE */
5148 .vmp_canonical = true,
5149 .vmp_busy = true,
5150 .vmp_realtime = mem->vmp_realtime,
5151 #if !XNU_VM_HAS_LINEAR_PAGES_ARRAY
5152 .vmp_phys_page = mem->vmp_phys_page,
5153 #endif /* !XNU_VM_HAS_LINEAR_PAGES_ARRAY */
5154 };
5155 /* ECC information is out of `struct vm_page` and preserved */
5156 }
5157
5158 void
vm_page_free_prepare_object(vm_page_t mem,boolean_t remove_from_hash)5159 vm_page_free_prepare_object(vm_page_t mem, boolean_t remove_from_hash)
5160 {
5161 if (mem->vmp_tabled) {
5162 vm_page_remove(mem, remove_from_hash); /* clears tabled, object, offset */
5163 }
5164 vm_page_wakeup(VM_OBJECT_NULL, mem); /* clears wanted */
5165
5166 if (vm_page_is_private(mem)) {
5167 vm_page_reset_private(mem);
5168 }
5169 if (vm_page_is_canonical(mem)) {
5170 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0 &&
5171 mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0 &&
5172 mem->vmp_specialq.next == 0 && mem->vmp_specialq.prev == 0 &&
5173 mem->vmp_next_m == 0);
5174
5175 pmap_recycle_page(VM_PAGE_GET_PHYS_PAGE(mem));
5176
5177 vm_page_reset_canonical(mem);
5178 }
5179 }
5180
5181 /*
5182 * vm_page_release:
5183 *
5184 * Return a page to the free list.
5185 *
5186 * Keep in sync with vm_page_free_list().
5187 */
5188
5189 void
vm_page_release(vm_page_t mem,vmp_release_options_t options)5190 vm_page_release(vm_page_t mem, vmp_release_options_t options)
5191 {
5192 if (options & VMP_RELEASE_Q_LOCKED) {
5193 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5194 } else {
5195 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
5196 }
5197
5198 assert(vm_page_is_canonical(mem));
5199 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
5200
5201 if ((options & VMP_RELEASE_SKIP_FREE_CHECK) == 0) {
5202 pmap_recycle_page(VM_PAGE_GET_PHYS_PAGE(mem));
5203 }
5204
5205 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
5206
5207
5208 vm_page_free_queue_enter_list(vm_page_list_for_page(mem), options);
5209 }
5210
5211 /*
5212 * This version of vm_page_release() is used only at startup
5213 * when we are single-threaded and pages are being released
5214 * for the first time. Hence, no locking or unnecessary checks are made.
5215 * Note: VM_CHECK_MEMORYSTATUS invoked by the caller.
5216 */
5217 void
vm_page_release_startup(vm_page_t mem)5218 vm_page_release_startup(vm_page_t mem)
5219 {
5220 vm_page_free_queue_enter_list(vm_page_list_for_page(mem),
5221 VMP_RELEASE_STARTUP);
5222 }
5223
5224 /*
5225 * vm_page_free:
5226 *
5227 * Returns the given page to the free list,
5228 * disassociating it with any VM object.
5229 *
5230 * Object and page queues must be locked prior to entry.
5231 */
5232 void
vm_page_free(vm_page_t mem)5233 vm_page_free(vm_page_t mem)
5234 {
5235 vm_page_free_prepare(mem);
5236
5237 if (vm_page_is_canonical(mem)) {
5238 /* page queues are locked */
5239 vm_page_release(mem, VMP_RELEASE_Q_LOCKED |
5240 VMP_RELEASE_SKIP_FREE_CHECK);
5241 } else {
5242 vm_page_release_fictitious(mem);
5243 }
5244 }
5245
5246
5247 void
vm_page_free_unlocked(vm_page_t mem,boolean_t remove_from_hash)5248 vm_page_free_unlocked(vm_page_t mem, boolean_t remove_from_hash)
5249 {
5250 vm_page_lockspin_queues();
5251 vm_page_free_prepare_queues(mem);
5252 vm_page_unlock_queues();
5253
5254 vm_page_free_prepare_object(mem, remove_from_hash);
5255
5256 if (vm_page_is_canonical(mem)) {
5257 /* page queues are not locked */
5258 vm_page_release(mem, VMP_RELEASE_SKIP_FREE_CHECK);
5259 } else {
5260 vm_page_release_fictitious(mem);
5261 }
5262 }
5263
5264
5265 /*
5266 * Free a list of pages. The list can be up to several hundred pages,
5267 * as blocked up by vm_pageout_scan().
5268 * The big win is not having to take the free list lock once
5269 * per page.
5270 *
5271 * The VM page queues lock (vm_page_queue_lock) should NOT be held.
5272 * The VM page free queues lock (vm_page_queue_free_lock) should NOT be held.
5273 *
5274 * Keep in sync with vm_page_release().
5275 */
5276 void
vm_page_free_list(vm_page_t freeq,bool prepare_object)5277 vm_page_free_list(vm_page_t freeq, bool prepare_object)
5278 {
5279 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
5280 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_NOTOWNED);
5281
5282 while (freeq) {
5283 vm_page_list_t list = { };
5284
5285 while (list.vmpl_count < VMP_FREE_BATCH_SIZE && freeq) {
5286 vm_page_t mem = _vm_page_list_pop(&freeq);
5287
5288 assert((mem->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
5289 (mem->vmp_q_state == VM_PAGE_IS_WIRED));
5290
5291 if (prepare_object) {
5292 vm_page_free_prepare_object(mem, TRUE);
5293 }
5294
5295 if (vm_page_is_fictitious(mem)) {
5296 vm_page_release_fictitious(mem);
5297 continue;
5298 }
5299
5300 if (!prepare_object) {
5301 /* vm_page_free_prepare_object() checked it */
5302 pmap_recycle_page(VM_PAGE_GET_PHYS_PAGE(mem));
5303 }
5304
5305 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
5306
5307
5308 /*
5309 * IMPORTANT: we can't set the page "free" here
5310 * because that would make the page eligible for
5311 * a physically-contiguous allocation (see
5312 * vm_page_find_contiguous()) right away (we don't
5313 * hold the vm_page_queue_free lock). That would
5314 * cause trouble because the page is not actually
5315 * in the free queue yet...
5316 */
5317
5318 vm_page_list_push(&list, mem);
5319 }
5320
5321 if (list.vmpl_count) {
5322 vm_page_free_queue_enter_list(list, VMP_RELEASE_NONE);
5323 }
5324 }
5325 }
5326
5327
5328 /*
5329 * vm_page_wire:
5330 *
5331 * Mark this page as wired down by yet
5332 * another map, removing it from paging queues
5333 * as necessary.
5334 *
5335 * The page's object and the page queues must be locked.
5336 */
5337
5338
5339 void
vm_page_wire(vm_page_t mem,vm_tag_t tag,boolean_t check_memorystatus)5340 vm_page_wire(
5341 vm_page_t mem,
5342 vm_tag_t tag,
5343 boolean_t check_memorystatus)
5344 {
5345 vm_object_t m_object;
5346
5347 m_object = VM_PAGE_OBJECT(mem);
5348
5349 // dbgLog(current_thread(), mem->vmp_offset, m_object, 1); /* (TEST/DEBUG) */
5350
5351 VM_PAGE_CHECK(mem);
5352 if (m_object) {
5353 vm_object_lock_assert_exclusive(m_object);
5354 } else {
5355 /*
5356 * In theory, the page should be in an object before it
5357 * gets wired, since we need to hold the object lock
5358 * to update some fields in the page structure.
5359 * However, some code (i386 pmap, for example) might want
5360 * to wire a page before it gets inserted into an object.
5361 * That's somewhat OK, as long as nobody else can get to
5362 * that page and update it at the same time.
5363 */
5364 }
5365 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5366 if (!VM_PAGE_WIRED(mem)) {
5367 if (mem->vmp_laundry) {
5368 vm_pageout_steal_laundry(mem, TRUE);
5369 }
5370
5371 vm_page_queues_remove(mem, TRUE);
5372
5373 assert(mem->vmp_wire_count == 0);
5374 mem->vmp_q_state = VM_PAGE_IS_WIRED;
5375
5376 #if CONFIG_TRACK_UNMODIFIED_ANON_PAGES
5377 if (mem->vmp_unmodified_ro == true) {
5378 /* Object and PageQ locks are held*/
5379 mem->vmp_unmodified_ro = false;
5380 os_atomic_dec(&compressor_ro_uncompressed, relaxed);
5381 vm_object_compressor_pager_state_clr(VM_PAGE_OBJECT(mem), mem->vmp_offset);
5382 }
5383 #endif /* CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
5384
5385 if (m_object) {
5386 task_t owner;
5387 int ledger_idx_volatile;
5388 int ledger_idx_nonvolatile;
5389 int ledger_idx_volatile_compressed;
5390 int ledger_idx_nonvolatile_compressed;
5391 int ledger_idx_composite;
5392 int ledger_idx_external_wired;
5393 boolean_t do_footprint;
5394
5395 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
5396 VM_OBJECT_WIRED_PAGE_ADD(m_object, mem);
5397 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, tag);
5398
5399 assert(m_object->resident_page_count >=
5400 m_object->wired_page_count);
5401 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
5402 assert(vm_page_purgeable_count > 0);
5403 OSAddAtomic(-1, &vm_page_purgeable_count);
5404 OSAddAtomic(1, &vm_page_purgeable_wired_count);
5405 }
5406 if (m_object->internal &&
5407 m_object->vo_owner != TASK_NULL &&
5408 (m_object->purgable == VM_PURGABLE_VOLATILE ||
5409 m_object->purgable == VM_PURGABLE_EMPTY)) {
5410 owner = VM_OBJECT_OWNER(m_object);
5411 vm_object_ledger_tag_ledgers(
5412 m_object,
5413 &ledger_idx_volatile,
5414 &ledger_idx_nonvolatile,
5415 &ledger_idx_volatile_compressed,
5416 &ledger_idx_nonvolatile_compressed,
5417 &ledger_idx_composite,
5418 &ledger_idx_external_wired,
5419 &do_footprint);
5420 /* less volatile bytes */
5421 ledger_debit(owner->ledger,
5422 ledger_idx_volatile,
5423 PAGE_SIZE);
5424 /* more not-quite-volatile bytes */
5425 ledger_credit(owner->ledger,
5426 ledger_idx_nonvolatile,
5427 PAGE_SIZE);
5428 if (do_footprint) {
5429 /* more footprint */
5430 ledger_credit(owner->ledger,
5431 task_ledgers.phys_footprint,
5432 PAGE_SIZE);
5433 } else if (ledger_idx_composite != -1) {
5434 ledger_credit(owner->ledger,
5435 ledger_idx_composite,
5436 PAGE_SIZE);
5437 }
5438 }
5439
5440 if (m_object->all_reusable) {
5441 /*
5442 * Wired pages are not counted as "re-usable"
5443 * in "all_reusable" VM objects, so nothing
5444 * to do here.
5445 */
5446 } else if (mem->vmp_reusable) {
5447 /*
5448 * This page is not "re-usable" when it's
5449 * wired, so adjust its state and the
5450 * accounting.
5451 */
5452 vm_page_lockconvert_queues();
5453 vm_object_reuse_pages(m_object,
5454 mem->vmp_offset,
5455 mem->vmp_offset + PAGE_SIZE_64,
5456 FALSE);
5457 }
5458 }
5459 assert(!mem->vmp_reusable);
5460
5461 if (vm_page_is_canonical(mem) && !mem->vmp_gobbled) {
5462 vm_page_wire_count++;
5463 }
5464 if (mem->vmp_gobbled) {
5465 vm_page_gobble_count--;
5466 }
5467 mem->vmp_gobbled = FALSE;
5468
5469 if (check_memorystatus == TRUE) {
5470 VM_CHECK_MEMORYSTATUS;
5471 }
5472 }
5473 assert(!mem->vmp_gobbled);
5474 assert(mem->vmp_q_state == VM_PAGE_IS_WIRED);
5475 mem->vmp_wire_count++;
5476
5477
5478 if (__improbable(mem->vmp_wire_count == 0)) {
5479 panic("vm_page_wire(%p): wire_count overflow", mem);
5480 }
5481 VM_PAGE_CHECK(mem);
5482 }
5483
5484 /*
5485 * vm_page_unwire:
5486 *
5487 * Release one wiring of this page, potentially
5488 * enabling it to be paged again.
5489 *
5490 * The page's object and the page queues must be locked.
5491 */
5492 void
vm_page_unwire(vm_page_t mem,boolean_t queueit)5493 vm_page_unwire(
5494 vm_page_t mem,
5495 boolean_t queueit)
5496 {
5497 vm_object_t m_object;
5498
5499 m_object = VM_PAGE_OBJECT(mem);
5500
5501 // dbgLog(current_thread(), mem->vmp_offset, m_object, 0); /* (TEST/DEBUG) */
5502
5503 VM_PAGE_CHECK(mem);
5504 assert(VM_PAGE_WIRED(mem));
5505 assert(mem->vmp_wire_count > 0);
5506 assert(!mem->vmp_gobbled);
5507 assert(m_object != VM_OBJECT_NULL);
5508 vm_object_lock_assert_exclusive(m_object);
5509 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5510 if (--mem->vmp_wire_count == 0) {
5511 task_t owner;
5512 int ledger_idx_volatile;
5513 int ledger_idx_nonvolatile;
5514 int ledger_idx_volatile_compressed;
5515 int ledger_idx_nonvolatile_compressed;
5516 int ledger_idx_composite;
5517 int ledger_idx_external_wired;
5518 boolean_t do_footprint;
5519
5520 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
5521 mem->vmp_iopl_wired = false;
5522
5523 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
5524 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
5525 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
5526 if (vm_page_is_canonical(mem)) {
5527 vm_page_wire_count--;
5528 }
5529
5530
5531 assert(m_object->resident_page_count >=
5532 m_object->wired_page_count);
5533 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
5534 OSAddAtomic(+1, &vm_page_purgeable_count);
5535 assert(vm_page_purgeable_wired_count > 0);
5536 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
5537 }
5538 if (m_object->internal &&
5539 m_object->vo_owner != TASK_NULL &&
5540 (m_object->purgable == VM_PURGABLE_VOLATILE ||
5541 m_object->purgable == VM_PURGABLE_EMPTY)) {
5542 owner = VM_OBJECT_OWNER(m_object);
5543 vm_object_ledger_tag_ledgers(
5544 m_object,
5545 &ledger_idx_volatile,
5546 &ledger_idx_nonvolatile,
5547 &ledger_idx_volatile_compressed,
5548 &ledger_idx_nonvolatile_compressed,
5549 &ledger_idx_composite,
5550 &ledger_idx_external_wired,
5551 &do_footprint);
5552 /* more volatile bytes */
5553 ledger_credit(owner->ledger,
5554 ledger_idx_volatile,
5555 PAGE_SIZE);
5556 /* less not-quite-volatile bytes */
5557 ledger_debit(owner->ledger,
5558 ledger_idx_nonvolatile,
5559 PAGE_SIZE);
5560 if (do_footprint) {
5561 /* less footprint */
5562 ledger_debit(owner->ledger,
5563 task_ledgers.phys_footprint,
5564 PAGE_SIZE);
5565 } else if (ledger_idx_composite != -1) {
5566 ledger_debit(owner->ledger,
5567 ledger_idx_composite,
5568 PAGE_SIZE);
5569 }
5570 }
5571 assert(!is_kernel_object(m_object));
5572 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
5573
5574 if (queueit == TRUE) {
5575 if (m_object->purgable == VM_PURGABLE_EMPTY) {
5576 vm_page_deactivate(mem);
5577 } else {
5578 vm_page_activate(mem);
5579 }
5580 }
5581
5582 VM_CHECK_MEMORYSTATUS;
5583 }
5584 VM_PAGE_CHECK(mem);
5585 }
5586
5587 /*
5588 * vm_page_deactivate:
5589 *
5590 * Returns the given page to the inactive list,
5591 * indicating that no physical maps have access
5592 * to this page. [Used by the physical mapping system.]
5593 *
5594 * The page queues must be locked.
5595 */
5596 void
vm_page_deactivate(vm_page_t m)5597 vm_page_deactivate(
5598 vm_page_t m)
5599 {
5600 vm_page_deactivate_internal(m, TRUE);
5601 }
5602
5603
5604 void
vm_page_deactivate_internal(vm_page_t m,boolean_t clear_hw_reference)5605 vm_page_deactivate_internal(
5606 vm_page_t m,
5607 boolean_t clear_hw_reference)
5608 {
5609 vm_object_t m_object;
5610
5611 m_object = VM_PAGE_OBJECT(m);
5612
5613 VM_PAGE_CHECK(m);
5614 assert(!is_kernel_object(m_object));
5615 assert(!vm_page_is_guard(m));
5616
5617 // dbgLog(VM_PAGE_GET_PHYS_PAGE(m), vm_page_free_count, vm_page_wire_count, 6); /* (TEST/DEBUG) */
5618 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5619 /*
5620 * This page is no longer very interesting. If it was
5621 * interesting (active or inactive/referenced), then we
5622 * clear the reference bit and (re)enter it in the
5623 * inactive queue. Note wired pages should not have
5624 * their reference bit cleared.
5625 */
5626 assert( !(m->vmp_absent && !m->vmp_unusual));
5627
5628 if (m->vmp_gobbled) { /* can this happen? */
5629 assert( !VM_PAGE_WIRED(m));
5630
5631 if (vm_page_is_canonical(m)) {
5632 vm_page_wire_count--;
5633 }
5634 vm_page_gobble_count--;
5635 m->vmp_gobbled = FALSE;
5636 }
5637 /*
5638 * if this page is currently on the pageout queue, we can't do the
5639 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5640 * and we can't remove it manually since we would need the object lock
5641 * (which is not required here) to decrement the activity_in_progress
5642 * reference which is held on the object while the page is in the pageout queue...
5643 * just let the normal laundry processing proceed
5644 */
5645 if (m->vmp_laundry || !vm_page_is_canonical(m) ||
5646 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
5647 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
5648 VM_PAGE_WIRED(m)) {
5649 return;
5650 }
5651 if (!m->vmp_absent && clear_hw_reference == TRUE) {
5652 vm_page_lockconvert_queues();
5653 pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
5654 }
5655
5656 m->vmp_reference = FALSE;
5657 m->vmp_no_cache = FALSE;
5658
5659 if (!VM_PAGE_INACTIVE(m)) {
5660 vm_page_queues_remove(m, FALSE);
5661
5662 if (!VM_DYNAMIC_PAGING_ENABLED() &&
5663 m->vmp_dirty && m_object->internal &&
5664 (m_object->purgable == VM_PURGABLE_DENY ||
5665 m_object->purgable == VM_PURGABLE_NONVOLATILE ||
5666 m_object->purgable == VM_PURGABLE_VOLATILE)) {
5667 vm_page_check_pageable_safe(m);
5668 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
5669 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
5670 vm_page_throttled_count++;
5671 } else {
5672 if (m_object->named &&
5673 os_ref_get_count_raw(&m_object->ref_count) == 1) {
5674 vm_page_speculate(m, FALSE);
5675 #if DEVELOPMENT || DEBUG
5676 vm_page_speculative_recreated++;
5677 #endif
5678 } else {
5679 vm_page_enqueue_inactive(m, FALSE);
5680 }
5681 }
5682 }
5683 }
5684
5685 /*
5686 * vm_page_enqueue_cleaned
5687 *
5688 * Put the page on the cleaned queue, mark it cleaned, etc.
5689 * Being on the cleaned queue (and having m->clean_queue set)
5690 * does ** NOT ** guarantee that the page is clean!
5691 *
5692 * Call with the queues lock held.
5693 */
5694
5695 void
vm_page_enqueue_cleaned(vm_page_t m)5696 vm_page_enqueue_cleaned(vm_page_t m)
5697 {
5698 vm_object_t m_object;
5699
5700 m_object = VM_PAGE_OBJECT(m);
5701
5702 assert(!vm_page_is_guard(m));
5703 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5704 assert(!(m->vmp_absent && !m->vmp_unusual));
5705
5706 if (VM_PAGE_WIRED(m)) {
5707 return;
5708 }
5709
5710 if (m->vmp_gobbled) {
5711 if (vm_page_is_canonical(m)) {
5712 vm_page_wire_count--;
5713 }
5714 vm_page_gobble_count--;
5715 m->vmp_gobbled = FALSE;
5716 }
5717 /*
5718 * if this page is currently on the pageout queue, we can't do the
5719 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5720 * and we can't remove it manually since we would need the object lock
5721 * (which is not required here) to decrement the activity_in_progress
5722 * reference which is held on the object while the page is in the pageout queue...
5723 * just let the normal laundry processing proceed
5724 */
5725 if (m->vmp_laundry || !vm_page_is_canonical(m) ||
5726 (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
5727 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
5728 return;
5729 }
5730 vm_page_queues_remove(m, FALSE);
5731
5732 vm_page_check_pageable_safe(m);
5733 vm_page_queue_enter(&vm_page_queue_cleaned, m, vmp_pageq);
5734 m->vmp_q_state = VM_PAGE_ON_INACTIVE_CLEANED_Q;
5735 vm_page_cleaned_count++;
5736
5737 vm_page_inactive_count++;
5738 if (m_object->internal) {
5739 vm_page_pageable_internal_count++;
5740 } else {
5741 vm_page_pageable_external_count++;
5742 }
5743 vm_page_add_to_specialq(m, TRUE);
5744 VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
5745 }
5746
5747 /*
5748 * vm_page_activate:
5749 *
5750 * Put the specified page on the active list (if appropriate).
5751 *
5752 * The page queues must be locked.
5753 */
5754
5755 void
vm_page_activate(vm_page_t m)5756 vm_page_activate(
5757 vm_page_t m)
5758 {
5759 vm_object_t m_object;
5760
5761 m_object = VM_PAGE_OBJECT(m);
5762
5763 VM_PAGE_CHECK(m);
5764 #ifdef FIXME_4778297
5765 assert(!is_kernel_object(m_object));
5766 #endif
5767 assert(!vm_page_is_guard(m));
5768 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5769 assert( !(m->vmp_absent && !m->vmp_unusual));
5770
5771 if (m->vmp_gobbled) {
5772 assert( !VM_PAGE_WIRED(m));
5773 if (vm_page_is_canonical(m)) {
5774 vm_page_wire_count--;
5775 }
5776 vm_page_gobble_count--;
5777 m->vmp_gobbled = FALSE;
5778 }
5779 /*
5780 * if this page is currently on the pageout queue, we can't do the
5781 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5782 * and we can't remove it manually since we would need the object lock
5783 * (which is not required here) to decrement the activity_in_progress
5784 * reference which is held on the object while the page is in the pageout queue...
5785 * just let the normal laundry processing proceed
5786 */
5787 if (m->vmp_laundry || !vm_page_is_canonical(m) ||
5788 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
5789 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
5790 return;
5791 }
5792
5793 #if DEBUG
5794 if (m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q) {
5795 panic("vm_page_activate: already active");
5796 }
5797 #endif
5798
5799 if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
5800 DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
5801 DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL);
5802 }
5803
5804 /*
5805 * A freshly activated page should be promoted in the donation queue.
5806 * So we remove it here while preserving its hint and we will enqueue
5807 * it again in vm_page_enqueue_active.
5808 */
5809 vm_page_queues_remove(m, ((m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE) ? TRUE : FALSE));
5810
5811 if (!VM_PAGE_WIRED(m)) {
5812 vm_page_check_pageable_safe(m);
5813 if (!VM_DYNAMIC_PAGING_ENABLED() &&
5814 m->vmp_dirty && m_object->internal &&
5815 (m_object->purgable == VM_PURGABLE_DENY ||
5816 m_object->purgable == VM_PURGABLE_NONVOLATILE ||
5817 m_object->purgable == VM_PURGABLE_VOLATILE)) {
5818 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
5819 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
5820 vm_page_throttled_count++;
5821 } else {
5822 #if CONFIG_SECLUDED_MEMORY
5823 if (secluded_for_filecache &&
5824 vm_page_secluded_target != 0 &&
5825 num_tasks_can_use_secluded_mem == 0 &&
5826 m_object->eligible_for_secluded &&
5827 !m->vmp_realtime) {
5828 vm_page_queue_enter(&vm_page_queue_secluded, m, vmp_pageq);
5829 m->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
5830 vm_page_secluded_count++;
5831 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
5832 vm_page_secluded_count_inuse++;
5833 assert(!m_object->internal);
5834 // vm_page_pageable_external_count++;
5835 } else
5836 #endif /* CONFIG_SECLUDED_MEMORY */
5837 vm_page_enqueue_active(m, FALSE);
5838 }
5839 m->vmp_reference = TRUE;
5840 m->vmp_no_cache = FALSE;
5841 }
5842 VM_PAGE_CHECK(m);
5843 }
5844
5845
5846 /*
5847 * vm_page_speculate:
5848 *
5849 * Put the specified page on the speculative list (if appropriate).
5850 *
5851 * The page queues must be locked.
5852 */
5853 void
vm_page_speculate(vm_page_t m,boolean_t new)5854 vm_page_speculate(
5855 vm_page_t m,
5856 boolean_t new)
5857 {
5858 struct vm_speculative_age_q *aq;
5859 vm_object_t m_object;
5860
5861 m_object = VM_PAGE_OBJECT(m);
5862
5863 VM_PAGE_CHECK(m);
5864 vm_page_check_pageable_safe(m);
5865
5866 assert(!vm_page_is_guard(m));
5867 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5868 assert(!(m->vmp_absent && !m->vmp_unusual));
5869 assert(m_object->internal == FALSE);
5870
5871 /*
5872 * if this page is currently on the pageout queue, we can't do the
5873 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5874 * and we can't remove it manually since we would need the object lock
5875 * (which is not required here) to decrement the activity_in_progress
5876 * reference which is held on the object while the page is in the pageout queue...
5877 * just let the normal laundry processing proceed
5878 */
5879 if (m->vmp_laundry || !vm_page_is_canonical(m) ||
5880 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
5881 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
5882 return;
5883 }
5884
5885 vm_page_queues_remove(m, FALSE);
5886
5887 if (!VM_PAGE_WIRED(m)) {
5888 mach_timespec_t ts;
5889 clock_sec_t sec;
5890 clock_nsec_t nsec;
5891
5892 clock_get_system_nanotime(&sec, &nsec);
5893 ts.tv_sec = (unsigned int) sec;
5894 ts.tv_nsec = nsec;
5895
5896 if (vm_page_speculative_count == 0) {
5897 speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5898 speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5899
5900 aq = &vm_page_queue_speculative[speculative_age_index];
5901
5902 /*
5903 * set the timer to begin a new group
5904 */
5905 aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
5906 aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
5907
5908 ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
5909 } else {
5910 aq = &vm_page_queue_speculative[speculative_age_index];
5911
5912 if (CMP_MACH_TIMESPEC(&ts, &aq->age_ts) >= 0) {
5913 speculative_age_index++;
5914
5915 if (speculative_age_index > vm_page_max_speculative_age_q) {
5916 speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5917 }
5918 if (speculative_age_index == speculative_steal_index) {
5919 speculative_steal_index = speculative_age_index + 1;
5920
5921 if (speculative_steal_index > vm_page_max_speculative_age_q) {
5922 speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5923 }
5924 }
5925 aq = &vm_page_queue_speculative[speculative_age_index];
5926
5927 if (!vm_page_queue_empty(&aq->age_q)) {
5928 vm_page_speculate_ageit(aq);
5929 }
5930
5931 aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
5932 aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
5933
5934 ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
5935 }
5936 }
5937 vm_page_enqueue_tail(&aq->age_q, &m->vmp_pageq);
5938 m->vmp_q_state = VM_PAGE_ON_SPECULATIVE_Q;
5939 vm_page_speculative_count++;
5940 vm_page_pageable_external_count++;
5941
5942 if (new == TRUE) {
5943 vm_object_lock_assert_exclusive(m_object);
5944
5945 m_object->pages_created++;
5946 #if DEVELOPMENT || DEBUG
5947 vm_page_speculative_created++;
5948 #endif
5949 }
5950 }
5951 VM_PAGE_CHECK(m);
5952 }
5953
5954
5955 /*
5956 * move pages from the specified aging bin to
5957 * the speculative bin that pageout_scan claims from
5958 *
5959 * The page queues must be locked.
5960 */
5961 void
vm_page_speculate_ageit(struct vm_speculative_age_q * aq)5962 vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
5963 {
5964 struct vm_speculative_age_q *sq;
5965 vm_page_t t;
5966
5967 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
5968
5969 if (vm_page_queue_empty(&sq->age_q)) {
5970 sq->age_q.next = aq->age_q.next;
5971 sq->age_q.prev = aq->age_q.prev;
5972
5973 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.next);
5974 t->vmp_pageq.prev = VM_PAGE_PACK_PTR(&sq->age_q);
5975
5976 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
5977 t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
5978 } else {
5979 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
5980 t->vmp_pageq.next = aq->age_q.next;
5981
5982 t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.next);
5983 t->vmp_pageq.prev = sq->age_q.prev;
5984
5985 t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.prev);
5986 t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
5987
5988 sq->age_q.prev = aq->age_q.prev;
5989 }
5990 vm_page_queue_init(&aq->age_q);
5991 }
5992
5993
5994 void
vm_page_lru(vm_page_t m)5995 vm_page_lru(
5996 vm_page_t m)
5997 {
5998 VM_PAGE_CHECK(m);
5999 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
6000 assert(!vm_page_is_guard(m));
6001
6002 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
6003
6004 if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q) {
6005 /*
6006 * we don't need to do all the other work that
6007 * vm_page_queues_remove and vm_page_enqueue_inactive
6008 * bring along for the ride
6009 */
6010 assert(!m->vmp_laundry);
6011 assert(!vm_page_is_private(m));
6012
6013 m->vmp_no_cache = FALSE;
6014
6015 vm_page_queue_remove(&vm_page_queue_inactive, m, vmp_pageq);
6016 vm_page_queue_enter(&vm_page_queue_inactive, m, vmp_pageq);
6017
6018 return;
6019 }
6020 /*
6021 * if this page is currently on the pageout queue, we can't do the
6022 * vm_page_queues_remove (which doesn't handle the pageout queue case)
6023 * and we can't remove it manually since we would need the object lock
6024 * (which is not required here) to decrement the activity_in_progress
6025 * reference which is held on the object while the page is in the pageout queue...
6026 * just let the normal laundry processing proceed
6027 */
6028 if (m->vmp_laundry || vm_page_is_private(m) ||
6029 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
6030 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
6031 VM_PAGE_WIRED(m)) {
6032 return;
6033 }
6034
6035 m->vmp_no_cache = FALSE;
6036
6037 vm_page_queues_remove(m, FALSE);
6038
6039 vm_page_enqueue_inactive(m, FALSE);
6040 }
6041
6042
6043 void
vm_page_reactivate_all_throttled(void)6044 vm_page_reactivate_all_throttled(void)
6045 {
6046 vm_page_t first_throttled, last_throttled;
6047 vm_page_t first_active;
6048 vm_page_t m;
6049 int extra_active_count;
6050 int extra_internal_count, extra_external_count;
6051 vm_object_t m_object;
6052
6053 if (!VM_DYNAMIC_PAGING_ENABLED()) {
6054 return;
6055 }
6056
6057 extra_active_count = 0;
6058 extra_internal_count = 0;
6059 extra_external_count = 0;
6060 vm_page_lock_queues();
6061 if (!vm_page_queue_empty(&vm_page_queue_throttled)) {
6062 /*
6063 * Switch "throttled" pages to "active".
6064 */
6065 vm_page_queue_iterate(&vm_page_queue_throttled, m, vmp_pageq) {
6066 VM_PAGE_CHECK(m);
6067 assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
6068
6069 m_object = VM_PAGE_OBJECT(m);
6070
6071 extra_active_count++;
6072 if (m_object->internal) {
6073 extra_internal_count++;
6074 } else {
6075 extra_external_count++;
6076 }
6077
6078 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
6079 VM_PAGE_CHECK(m);
6080 vm_page_add_to_specialq(m, FALSE);
6081 }
6082
6083 /*
6084 * Transfer the entire throttled queue to a regular LRU page queues.
6085 * We insert it at the head of the active queue, so that these pages
6086 * get re-evaluated by the LRU algorithm first, since they've been
6087 * completely out of it until now.
6088 */
6089 first_throttled = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
6090 last_throttled = (vm_page_t) vm_page_queue_last(&vm_page_queue_throttled);
6091 first_active = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
6092 if (vm_page_queue_empty(&vm_page_queue_active)) {
6093 vm_page_queue_active.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_throttled);
6094 } else {
6095 first_active->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_throttled);
6096 }
6097 vm_page_queue_active.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_throttled);
6098 first_throttled->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(&vm_page_queue_active);
6099 last_throttled->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_active);
6100
6101 #if DEBUG
6102 printf("reactivated %d throttled pages\n", vm_page_throttled_count);
6103 #endif
6104 vm_page_queue_init(&vm_page_queue_throttled);
6105 /*
6106 * Adjust the global page counts.
6107 */
6108 vm_page_active_count += extra_active_count;
6109 vm_page_pageable_internal_count += extra_internal_count;
6110 vm_page_pageable_external_count += extra_external_count;
6111 vm_page_throttled_count = 0;
6112 }
6113 assert(vm_page_throttled_count == 0);
6114 assert(vm_page_queue_empty(&vm_page_queue_throttled));
6115 vm_page_unlock_queues();
6116 }
6117
6118
6119 /*
6120 * move pages from the indicated local queue to the global active queue
6121 * its ok to fail if we're below the hard limit and force == FALSE
6122 * the nolocks == TRUE case is to allow this function to be run on
6123 * the hibernate path
6124 */
6125
6126 void
vm_page_reactivate_local(uint32_t lid,boolean_t force,boolean_t nolocks)6127 vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
6128 {
6129 struct vpl *lq;
6130 vm_page_t first_local, last_local;
6131 vm_page_t first_active;
6132 vm_page_t m;
6133 uint32_t count = 0;
6134
6135 if (vm_page_local_q == NULL) {
6136 return;
6137 }
6138
6139 lq = zpercpu_get_cpu(vm_page_local_q, lid);
6140
6141 if (nolocks == FALSE) {
6142 if (lq->vpl_count < vm_page_local_q_hard_limit && force == FALSE) {
6143 if (!vm_page_trylockspin_queues()) {
6144 return;
6145 }
6146 } else {
6147 vm_page_lockspin_queues();
6148 }
6149
6150 VPL_LOCK(&lq->vpl_lock);
6151 }
6152 if (lq->vpl_count) {
6153 /*
6154 * Switch "local" pages to "active".
6155 */
6156 assert(!vm_page_queue_empty(&lq->vpl_queue));
6157
6158 vm_page_queue_iterate(&lq->vpl_queue, m, vmp_pageq) {
6159 VM_PAGE_CHECK(m);
6160 vm_page_check_pageable_safe(m);
6161 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q);
6162 assert(!vm_page_is_fictitious(m));
6163
6164 if (m->vmp_local_id != lid) {
6165 panic("vm_page_reactivate_local: found vm_page_t(%p) with wrong cpuid", m);
6166 }
6167
6168 m->vmp_local_id = 0;
6169 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
6170 VM_PAGE_CHECK(m);
6171 vm_page_add_to_specialq(m, FALSE);
6172 count++;
6173 }
6174 if (count != lq->vpl_count) {
6175 panic("vm_page_reactivate_local: count = %d, vm_page_local_count = %d", count, lq->vpl_count);
6176 }
6177
6178 /*
6179 * Transfer the entire local queue to a regular LRU page queues.
6180 */
6181 first_local = (vm_page_t) vm_page_queue_first(&lq->vpl_queue);
6182 last_local = (vm_page_t) vm_page_queue_last(&lq->vpl_queue);
6183 first_active = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
6184
6185 if (vm_page_queue_empty(&vm_page_queue_active)) {
6186 vm_page_queue_active.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
6187 } else {
6188 first_active->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
6189 }
6190 vm_page_queue_active.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
6191 first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(&vm_page_queue_active);
6192 last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_active);
6193
6194 vm_page_queue_init(&lq->vpl_queue);
6195 /*
6196 * Adjust the global page counts.
6197 */
6198 vm_page_active_count += lq->vpl_count;
6199 vm_page_pageable_internal_count += lq->vpl_internal_count;
6200 vm_page_pageable_external_count += lq->vpl_external_count;
6201 lq->vpl_count = 0;
6202 lq->vpl_internal_count = 0;
6203 lq->vpl_external_count = 0;
6204 }
6205 assert(vm_page_queue_empty(&lq->vpl_queue));
6206
6207 if (nolocks == FALSE) {
6208 VPL_UNLOCK(&lq->vpl_lock);
6209
6210 vm_page_balance_inactive(count / 4);
6211 vm_page_unlock_queues();
6212 }
6213 }
6214
6215 /*
6216 * vm_page_part_zero_fill:
6217 *
6218 * Zero-fill a part of the page.
6219 */
6220 #define PMAP_ZERO_PART_PAGE_IMPLEMENTED
6221 void
vm_page_part_zero_fill(vm_page_t m,vm_offset_t m_pa,vm_size_t len)6222 vm_page_part_zero_fill(
6223 vm_page_t m,
6224 vm_offset_t m_pa,
6225 vm_size_t len)
6226 {
6227 #if 0
6228 /*
6229 * we don't hold the page queue lock
6230 * so this check isn't safe to make
6231 */
6232 VM_PAGE_CHECK(m);
6233 #endif
6234
6235 #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED
6236 pmap_zero_part_page(VM_PAGE_GET_PHYS_PAGE(m), m_pa, len);
6237 #else
6238 vm_page_t tmp;
6239 while (1) {
6240 tmp = vm_page_grab();
6241 if (tmp == VM_PAGE_NULL) {
6242 vm_page_wait(THREAD_UNINT);
6243 continue;
6244 }
6245 break;
6246 }
6247 vm_page_zero_fill(
6248 tmp
6249 );
6250 if (m_pa != 0) {
6251 vm_page_part_copy(m, 0, tmp, 0, m_pa);
6252 }
6253 if ((m_pa + len) < PAGE_SIZE) {
6254 vm_page_part_copy(m, m_pa + len, tmp,
6255 m_pa + len, PAGE_SIZE - (m_pa + len));
6256 }
6257 vm_page_copy(tmp, m);
6258 VM_PAGE_FREE(tmp);
6259 #endif
6260 }
6261
6262 /*!
6263 * @function vm_page_zero_fill
6264 *
6265 * @abstract
6266 * Zero-fill the specified page.
6267 *
6268 * @param m the page to be zero-filled.
6269 */
6270 void
vm_page_zero_fill(vm_page_t m)6271 vm_page_zero_fill(
6272 vm_page_t m
6273 )
6274 {
6275 int options = 0;
6276 #if 0
6277 /*
6278 * we don't hold the page queue lock
6279 * so this check isn't safe to make
6280 */
6281 VM_PAGE_CHECK(m);
6282 #endif
6283
6284 // dbgTrace(0xAEAEAEAE, VM_PAGE_GET_PHYS_PAGE(m), 0); /* (BRINGUP) */
6285 pmap_zero_page_with_options(VM_PAGE_GET_PHYS_PAGE(m), options);
6286 }
6287
6288 /*
6289 * vm_page_part_copy:
6290 *
6291 * copy part of one page to another
6292 *
6293 * This function is currently only consumed downstream of a
6294 * vm_map_copy_overwrite(). The implementation has a simpler contract
6295 * than vm_page_copy() as there's a restricted set of cases that
6296 * are allowed to be overwriteable. If vm_map_entry_is_overwriteable()
6297 * is expanded, this function may have to be adjusted.
6298 */
6299 void
vm_page_part_copy(vm_page_t src_m,vm_offset_t src_pa,vm_page_t dst_m,vm_offset_t dst_pa,vm_size_t len)6300 vm_page_part_copy(
6301 vm_page_t src_m,
6302 vm_offset_t src_pa,
6303 vm_page_t dst_m,
6304 vm_offset_t dst_pa,
6305 vm_size_t len)
6306 {
6307 #if 0
6308 /*
6309 * we don't hold the page queue lock
6310 * so this check isn't safe to make
6311 */
6312 VM_PAGE_CHECK(src_m);
6313 VM_PAGE_CHECK(dst_m);
6314 #endif
6315
6316 /*
6317 * Copying from/into restricted pages is a security issue,
6318 * as it allows for restricted pages' policies bypass.
6319 */
6320 if (vm_page_is_restricted(src_m)) {
6321 panic("%s: cannot copy from a restricted page", __func__);
6322 }
6323
6324 if (vm_page_is_restricted(dst_m)) {
6325 panic("%s: cannot copy into a restricted page", __func__);
6326 }
6327
6328
6329 pmap_copy_part_page(VM_PAGE_GET_PHYS_PAGE(src_m), src_pa,
6330 VM_PAGE_GET_PHYS_PAGE(dst_m), dst_pa, len);
6331 }
6332
6333 /*
6334 * vm_page_copy:
6335 *
6336 * Copy one page to another
6337 */
6338
6339 int vm_page_copy_cs_validations = 0;
6340 int vm_page_copy_cs_tainted = 0;
6341
6342 void
vm_page_copy(vm_page_t src_m,vm_page_t dest_m)6343 vm_page_copy(
6344 vm_page_t src_m,
6345 vm_page_t dest_m)
6346 {
6347 vm_object_t src_m_object;
6348 int options = 0;
6349
6350 src_m_object = VM_PAGE_OBJECT(src_m);
6351
6352 #if 0
6353 /*
6354 * we don't hold the page queue lock
6355 * so this check isn't safe to make
6356 */
6357 VM_PAGE_CHECK(src_m);
6358 VM_PAGE_CHECK(dest_m);
6359 #endif
6360 vm_object_lock_assert_held(src_m_object);
6361
6362 /*
6363 * Copying from/into restricted pages is a security issue,
6364 * as it allows for restricted pages' policies bypass.
6365 */
6366 if (vm_page_is_restricted(src_m)) {
6367 panic("%s: cannot copy from a restricted page", __func__);
6368 }
6369
6370 if (vm_page_is_restricted(dest_m)) {
6371 panic("%s: cannot copy into a restricted page", __func__);
6372 }
6373
6374 if (src_m_object != VM_OBJECT_NULL &&
6375 src_m_object->code_signed) {
6376 /*
6377 * We're copying a page from a code-signed object.
6378 * Whoever ends up mapping the copy page might care about
6379 * the original page's integrity, so let's validate the
6380 * source page now.
6381 */
6382 vm_page_copy_cs_validations++;
6383 vm_page_validate_cs(src_m, PAGE_SIZE, 0);
6384 #if DEVELOPMENT || DEBUG
6385 DTRACE_VM4(codesigned_copy,
6386 vm_object_t, src_m_object,
6387 vm_object_offset_t, src_m->vmp_offset,
6388 int, src_m->vmp_cs_validated,
6389 int, src_m->vmp_cs_tainted);
6390 #endif /* DEVELOPMENT || DEBUG */
6391 }
6392
6393 /*
6394 * Propagate the cs_tainted bit to the copy page. Do not propagate
6395 * the cs_validated bit.
6396 */
6397 dest_m->vmp_cs_tainted = src_m->vmp_cs_tainted;
6398 dest_m->vmp_cs_nx = src_m->vmp_cs_nx;
6399 if (dest_m->vmp_cs_tainted) {
6400 vm_page_copy_cs_tainted++;
6401 }
6402
6403
6404 dest_m->vmp_error = VMP_ERROR_GET(src_m); /* sliding src_m might have failed... */
6405 pmap_copy_page(VM_PAGE_GET_PHYS_PAGE(src_m), VM_PAGE_GET_PHYS_PAGE(dest_m), options);
6406 }
6407
6408 #if MACH_ASSERT
6409 static void
_vm_page_print(vm_page_t p)6410 _vm_page_print(
6411 vm_page_t p)
6412 {
6413 printf("vm_page %p: \n", p);
6414 printf(" pageq: next=%p prev=%p\n",
6415 (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.next),
6416 (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev));
6417 printf(" listq: next=%p prev=%p\n",
6418 (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.next)),
6419 (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.prev)));
6420 printf(" next=%p\n", (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m)));
6421 printf(" object=%p offset=0x%llx\n", VM_PAGE_OBJECT(p), p->vmp_offset);
6422 printf(" wire_count=%u\n", p->vmp_wire_count);
6423 printf(" q_state=%u\n", p->vmp_q_state);
6424
6425 printf(" %slaundry, %sref, %sgobbled, %sprivate\n",
6426 (p->vmp_laundry ? "" : "!"),
6427 (p->vmp_reference ? "" : "!"),
6428 (p->vmp_gobbled ? "" : "!"),
6429 (vm_page_is_private(p) ? "" : "!"));
6430 printf(" %sbusy, %swanted, %stabled, %sfictitious, %spmapped, %swpmapped\n",
6431 (p->vmp_busy ? "" : "!"),
6432 (p->vmp_wanted ? "" : "!"),
6433 (p->vmp_tabled ? "" : "!"),
6434 (vm_page_is_fictitious(p) ? "" : "!"),
6435 (p->vmp_pmapped ? "" : "!"),
6436 (p->vmp_wpmapped ? "" : "!"));
6437 printf(" %sfree_when_done, %sabsent, %serror, %sdirty, %scleaning, %sprecious, %sclustered\n",
6438 (p->vmp_free_when_done ? "" : "!"),
6439 (p->vmp_absent ? "" : "!"),
6440 (VMP_ERROR_GET(p) ? "" : "!"),
6441 (p->vmp_dirty ? "" : "!"),
6442 (p->vmp_cleaning ? "" : "!"),
6443 (p->vmp_precious ? "" : "!"),
6444 (p->vmp_clustered ? "" : "!"));
6445 printf(" %soverwriting, %srestart, %sunusual\n",
6446 (p->vmp_overwriting ? "" : "!"),
6447 (p->vmp_restart ? "" : "!"),
6448 (p->vmp_unusual ? "" : "!"));
6449 printf(" cs_validated=%d, cs_tainted=%d, cs_nx=%d, %sno_cache\n",
6450 p->vmp_cs_validated,
6451 p->vmp_cs_tainted,
6452 p->vmp_cs_nx,
6453 (p->vmp_no_cache ? "" : "!"));
6454
6455 printf("phys_page=0x%x\n", VM_PAGE_GET_PHYS_PAGE(p));
6456 }
6457
6458 /*
6459 * Check that the list of pages is ordered by
6460 * ascending physical address and has no holes.
6461 */
6462 static int
vm_page_verify_contiguous(vm_page_t pages,unsigned int npages)6463 vm_page_verify_contiguous(
6464 vm_page_t pages,
6465 unsigned int npages)
6466 {
6467 vm_page_t m;
6468 unsigned int page_count;
6469 vm_offset_t prev_addr;
6470
6471 prev_addr = VM_PAGE_GET_PHYS_PAGE(pages);
6472 page_count = 1;
6473 for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
6474 if (VM_PAGE_GET_PHYS_PAGE(m) != prev_addr + 1) {
6475 printf("m %p prev_addr 0x%lx, current addr 0x%x\n",
6476 m, (long)prev_addr, VM_PAGE_GET_PHYS_PAGE(m));
6477 printf("pages %p page_count %d npages %d\n", pages, page_count, npages);
6478 panic("vm_page_verify_contiguous: not contiguous!");
6479 }
6480 prev_addr = VM_PAGE_GET_PHYS_PAGE(m);
6481 ++page_count;
6482 }
6483 if (page_count != npages) {
6484 printf("pages %p actual count 0x%x but requested 0x%x\n",
6485 pages, page_count, npages);
6486 panic("vm_page_verify_contiguous: count error");
6487 }
6488 return 1;
6489 }
6490
6491
6492 /*
6493 * Check the free lists for proper length etc.
6494 */
6495 static boolean_t vm_page_verify_this_free_list_enabled = FALSE;
6496 static unsigned int
vm_page_verify_free_list(vm_page_queue_head_t * vm_page_queue,unsigned int color,vm_page_t look_for_page,boolean_t expect_page)6497 vm_page_verify_free_list(
6498 vm_page_queue_head_t *vm_page_queue,
6499 unsigned int color,
6500 vm_page_t look_for_page,
6501 boolean_t expect_page)
6502 {
6503 unsigned int npages;
6504 vm_page_t m;
6505 vm_page_t prev_m;
6506 boolean_t found_page;
6507
6508 if (!vm_page_verify_this_free_list_enabled) {
6509 return 0;
6510 }
6511
6512 found_page = FALSE;
6513 npages = 0;
6514 prev_m = (vm_page_t)((uintptr_t)vm_page_queue);
6515
6516 vm_page_queue_iterate(vm_page_queue, m, vmp_pageq) {
6517 if (m == look_for_page) {
6518 found_page = TRUE;
6519 }
6520 if ((vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev) != prev_m) {
6521 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p corrupted prev ptr %p instead of %p",
6522 color, npages, m, (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev), prev_m);
6523 }
6524 if (!m->vmp_busy) {
6525 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p not busy",
6526 color, npages, m);
6527 }
6528 if (color != (unsigned int) -1) {
6529 if (VM_PAGE_GET_COLOR(m) != color) {
6530 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p wrong color %u instead of %u",
6531 color, npages, m, VM_PAGE_GET_COLOR(m), color);
6532 }
6533 if (m->vmp_q_state != VM_PAGE_ON_FREE_Q) {
6534 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p - expecting q_state == VM_PAGE_ON_FREE_Q, found %d",
6535 color, npages, m, m->vmp_q_state);
6536 }
6537 } else {
6538 if (m->vmp_q_state != VM_PAGE_ON_FREE_LOCAL_Q) {
6539 panic("vm_page_verify_free_list(npages=%u): local page %p - expecting q_state == VM_PAGE_ON_FREE_LOCAL_Q, found %d",
6540 npages, m, m->vmp_q_state);
6541 }
6542 }
6543 ++npages;
6544 prev_m = m;
6545 }
6546 if (look_for_page != VM_PAGE_NULL) {
6547 unsigned int other_color;
6548
6549 if (expect_page && !found_page) {
6550 printf("vm_page_verify_free_list(color=%u, npages=%u): page %p not found phys=%u\n",
6551 color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
6552 _vm_page_print(look_for_page);
6553 for (other_color = 0;
6554 other_color < vm_colors;
6555 other_color++) {
6556 if (other_color == color) {
6557 continue;
6558 }
6559 vm_page_verify_free_list(&vm_page_queue_free.vmpfq_queues[other_color].qhead,
6560 other_color, look_for_page, FALSE);
6561 }
6562 #if XNU_VM_HAS_LOPAGE
6563 if (color == (unsigned int) -1) {
6564 vm_page_verify_free_list(&vm_lopage_queue_free,
6565 (unsigned int) -1, look_for_page, FALSE);
6566 }
6567 #endif /* XNU_VM_HAS_LOPAGE */
6568 panic("vm_page_verify_free_list(color=%u)", color);
6569 }
6570 if (!expect_page && found_page) {
6571 printf("vm_page_verify_free_list(color=%u, npages=%u): page %p found phys=%u\n",
6572 color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
6573 }
6574 }
6575 return npages;
6576 }
6577
6578 static boolean_t vm_page_verify_all_free_lists_enabled = FALSE;
6579 static void
vm_page_verify_free_lists(void)6580 vm_page_verify_free_lists( void )
6581 {
6582 unsigned int color, npages, nlopages;
6583 boolean_t toggle = TRUE;
6584
6585 if (!vm_page_verify_all_free_lists_enabled) {
6586 return;
6587 }
6588
6589 npages = 0;
6590 nlopages = 0;
6591
6592 vm_free_page_lock();
6593
6594 if (vm_page_verify_this_free_list_enabled == TRUE) {
6595 /*
6596 * This variable has been set globally for extra checking of
6597 * each free list Q. Since we didn't set it, we don't own it
6598 * and we shouldn't toggle it.
6599 */
6600 toggle = FALSE;
6601 }
6602
6603 if (toggle == TRUE) {
6604 vm_page_verify_this_free_list_enabled = TRUE;
6605 }
6606
6607 for (color = 0; color < vm_colors; color++) {
6608 npages += vm_page_verify_free_list(&vm_page_queue_free.vmpfq_queues[color].qhead,
6609 color, VM_PAGE_NULL, FALSE);
6610 }
6611 #if XNU_VM_HAS_LOPAGE
6612 nlopages = vm_page_verify_free_list(&vm_lopage_queue_free,
6613 (unsigned int) -1,
6614 VM_PAGE_NULL, FALSE);
6615 #endif /* XNU_VM_HAS_LOPAGE */
6616 if (npages != vm_page_free_count || nlopages != vm_lopage_free_count) {
6617 panic("vm_page_verify_free_lists: "
6618 "npages %u free_count %d nlopages %u lo_free_count %u",
6619 npages, vm_page_free_count, nlopages, vm_lopage_free_count);
6620 }
6621
6622 if (toggle == TRUE) {
6623 vm_page_verify_this_free_list_enabled = FALSE;
6624 }
6625
6626 vm_free_page_unlock();
6627 }
6628
6629 #endif /* MACH_ASSERT */
6630
6631 extern boolean_t(*volatile consider_buffer_cache_collect)(int);
6632
6633 /*
6634 * CONTIGUOUS PAGE ALLOCATION AND HELPER FUNCTIONS
6635 */
6636
6637 /*
6638 * Helper function used to determine if a page can be relocated
6639 * A page is relocatable if it is in a stable non-transient state
6640 * and if the page being relocated is compatible with the reason for reloc
6641 * The page queue lock must be held, and the object lock too, if the page
6642 * is in an object.
6643 */
6644 boolean_t
vm_page_is_relocatable(vm_page_t m,vm_relocate_reason_t reloc_reason)6645 vm_page_is_relocatable(vm_page_t m, vm_relocate_reason_t reloc_reason)
6646 {
6647
6648 if (VM_PAGE_WIRED(m) || m->vmp_gobbled || m->vmp_laundry || m->vmp_wanted ||
6649 m->vmp_cleaning || m->vmp_overwriting || m->vmp_free_when_done) {
6650 /*
6651 * Page is in a transient state
6652 * or a state we don't want to deal with.
6653 */
6654 return FALSE;
6655 } else if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
6656 (m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q) ||
6657 #if XNU_VM_HAS_LOPAGE
6658 (m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q) ||
6659 #endif /* XNU_VM_HAS_LOPAGE */
6660 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
6661 /*
6662 * Page needs to be on one of our queues (other then the pageout or special
6663 * free queues) or it needs to belong to the compressor pool (which is now
6664 * indicated by vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR and falls out from
6665 * the check for VM_PAGE_NOT_ON_Q) in order for it to be stable behind the
6666 * locks we hold at this point...
6667 */
6668 return FALSE;
6669 } else if ((m->vmp_q_state != VM_PAGE_ON_FREE_Q) &&
6670 (!m->vmp_tabled || m->vmp_busy)) {
6671 /*
6672 * pages on the free list are always 'busy'
6673 * so we couldn't test for 'busy' in the check
6674 * for the transient states... pages that are
6675 * 'free' are never 'tabled', so we also couldn't
6676 * test for 'tabled'. So we check here to make
6677 * sure that a non-free page is not busy and is
6678 * tabled on an object...
6679 */
6680 return FALSE;
6681 }
6682
6683 /*
6684 * Lastly, check the page against the relocation reason; the page may
6685 * be in a relocatable state, but not be a page we WANT to relocate for
6686 * the caller's use case.
6687 */
6688 switch (reloc_reason) {
6689 case VM_RELOCATE_REASON_CONTIGUOUS:
6690 {
6691 break;
6692 }
6693
6694 default:
6695 {
6696 panic("Invalid relocation reason %u", reloc_reason);
6697 __builtin_unreachable();
6698 }
6699 }
6700
6701 return TRUE;
6702 }
6703
6704 /*
6705 * Free up the given page by possibily relocating its contents to a new page
6706 * If the page is on an object the object lock must be held.
6707 *
6708 * Whether or not the page is considered relocatable is contingent on the
6709 * reason it is being relocated.
6710 *
6711 * Return the new page back to the caller if requested, as done in
6712 * vm_object_iopl_wire_full().
6713 *
6714 * The VM page queues lock must also be held.
6715 *
6716 * @returns
6717 * - KERN_SUCCESS if the relocation was successful.
6718 * - KERN_INVALID_OBJECT if @c m1's object is VM_OBJECT_NULL.
6719 * - KERN_FAILURE if the reolcation failed due to @c m1's state.
6720 * - KERN_RESOURCE_SHORTAGE if no page could be allocated to relocate @c m1.
6721 */
6722 kern_return_t
vm_page_relocate(vm_page_t m1,int * compressed_pages,vm_relocate_reason_t reloc_reason,vm_page_t * new_page)6723 vm_page_relocate(
6724 vm_page_t m1,
6725 int *compressed_pages,
6726 vm_relocate_reason_t reloc_reason,
6727 vm_page_t* new_page)
6728 {
6729 int refmod = 0;
6730 vm_object_t object = VM_PAGE_OBJECT(m1);
6731 kern_return_t kr;
6732
6733 switch (reloc_reason) {
6734 case VM_RELOCATE_REASON_CONTIGUOUS:
6735 {
6736 break;
6737 }
6738 default:
6739 {
6740 panic("Unrecognized relocation reason %u\n", reloc_reason);
6741 break;
6742 }
6743 }
6744
6745 if (object == VM_OBJECT_NULL) {
6746 return KERN_INVALID_OBJECT;
6747 }
6748
6749 vm_object_lock_assert_held(object);
6750
6751 if (VM_PAGE_WIRED(m1) ||
6752 m1->vmp_gobbled ||
6753 m1->vmp_laundry ||
6754 m1->vmp_wanted ||
6755 m1->vmp_cleaning ||
6756 m1->vmp_overwriting ||
6757 m1->vmp_free_when_done ||
6758 m1->vmp_busy ||
6759 m1->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
6760 return KERN_FAILURE;
6761 }
6762
6763 boolean_t disconnected = FALSE;
6764 boolean_t reusable = FALSE;
6765
6766 /*
6767 * Pages from reusable objects can be reclaimed directly.
6768 */
6769 if ((m1->vmp_reusable || object->all_reusable) &&
6770 m1->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q && !m1->vmp_dirty &&
6771 !m1->vmp_reference) {
6772 /*
6773 * reusable page...
6774 */
6775
6776 refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));
6777 disconnected = TRUE;
6778 if (refmod == 0) {
6779 /*
6780 * ... not reused: can steal without relocating contents.
6781 */
6782 reusable = TRUE;
6783 }
6784 }
6785
6786 if ((m1->vmp_pmapped && !reusable) || m1->vmp_dirty || m1->vmp_precious) {
6787 vm_grab_options_t grab_options = VM_PAGE_GRAB_Q_LOCK_HELD;
6788 vm_object_offset_t offset;
6789 int copy_page_options = 0;
6790
6791 /* page is not reusable, we need to allocate a new page
6792 * and move its contents there.
6793 */
6794 vm_page_t m2 = vm_page_grab_options(grab_options);
6795
6796 if (m2 == VM_PAGE_NULL) {
6797 return KERN_RESOURCE_SHORTAGE;
6798 }
6799
6800 if (!disconnected) {
6801 if (m1->vmp_pmapped) {
6802 refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));
6803 } else {
6804 refmod = 0;
6805 }
6806 }
6807
6808 /* copy the page's contents */
6809 pmap_copy_page(VM_PAGE_GET_PHYS_PAGE(m1), VM_PAGE_GET_PHYS_PAGE(m2), copy_page_options);
6810
6811 /* copy the page's state */
6812 assert(!VM_PAGE_WIRED(m1));
6813 assert(m1->vmp_q_state != VM_PAGE_ON_FREE_Q);
6814 assert(m1->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q);
6815 assert(!m1->vmp_laundry);
6816 m2->vmp_reference = m1->vmp_reference;
6817 assert(!m1->vmp_gobbled);
6818 m2->vmp_no_cache = m1->vmp_no_cache;
6819 m2->vmp_xpmapped = 0;
6820 assert(!m1->vmp_busy);
6821 assert(!m1->vmp_wanted);
6822 assert(vm_page_is_canonical(m1));
6823 m2->vmp_pmapped = m1->vmp_pmapped; /* should flush cache ? */
6824 m2->vmp_wpmapped = m1->vmp_wpmapped;
6825 assert(!m1->vmp_free_when_done);
6826 m2->vmp_absent = m1->vmp_absent;
6827 m2->vmp_error = VMP_ERROR_GET(m1);
6828 m2->vmp_dirty = m1->vmp_dirty;
6829 assert(!m1->vmp_cleaning);
6830 m2->vmp_precious = m1->vmp_precious;
6831 m2->vmp_clustered = m1->vmp_clustered;
6832 assert(!m1->vmp_overwriting);
6833 m2->vmp_restart = m1->vmp_restart;
6834 m2->vmp_unusual = m1->vmp_unusual;
6835 m2->vmp_cs_validated = m1->vmp_cs_validated;
6836 m2->vmp_cs_tainted = m1->vmp_cs_tainted;
6837 m2->vmp_cs_nx = m1->vmp_cs_nx;
6838
6839 m2->vmp_realtime = m1->vmp_realtime;
6840 m1->vmp_realtime = false;
6841
6842 /*
6843 * If m1 had really been reusable,
6844 * we would have just stolen it, so
6845 * let's not propagate its "reusable"
6846 * bit and assert that m2 is not
6847 * marked as "reusable".
6848 */
6849 // m2->vmp_reusable = m1->vmp_reusable;
6850 assert(!m2->vmp_reusable);
6851
6852 // assert(!m1->vmp_lopage);
6853
6854 if (m1->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6855 m2->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR;
6856 /*
6857 * We just grabbed m2 up above and so it isn't
6858 * going to be on any special Q as yet and so
6859 * we don't need to 'remove' it from the special
6860 * queues. Just resetting the state should be enough.
6861 */
6862 m2->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
6863 }
6864
6865 /*
6866 * page may need to be flushed if
6867 * it is marshalled into a UPL
6868 * that is going to be used by a device
6869 * that doesn't support coherency
6870 */
6871 m2->vmp_written_by_kernel = TRUE;
6872
6873 /*
6874 * make sure we clear the ref/mod state
6875 * from the pmap layer... else we risk
6876 * inheriting state from the last time
6877 * this page was used...
6878 */
6879 pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m2),
6880 VM_MEM_MODIFIED | VM_MEM_REFERENCED);
6881
6882 if (refmod & VM_MEM_REFERENCED) {
6883 m2->vmp_reference = TRUE;
6884 }
6885 if (refmod & VM_MEM_MODIFIED) {
6886 SET_PAGE_DIRTY(m2, TRUE);
6887 }
6888 offset = m1->vmp_offset;
6889
6890 /*
6891 * completely cleans up the state
6892 * of the page so that it is ready
6893 * to be put onto the free list, or
6894 * for this purpose it looks like it
6895 * just came off of the free list
6896 */
6897 vm_page_free_prepare(m1);
6898
6899 /*
6900 * now put the substitute page on the object
6901 */
6902 vm_page_insert_internal(m2, object, offset, VM_KERN_MEMORY_NONE, TRUE,
6903 TRUE, FALSE, FALSE, NULL);
6904
6905 /*
6906 * Return the relocated vm_page_t if the caller wants to know.
6907 */
6908 if (new_page) {
6909 *new_page = m2;
6910 }
6911
6912 if (m2->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6913 m2->vmp_pmapped = TRUE;
6914 m2->vmp_wpmapped = TRUE;
6915
6916 kr = pmap_enter_check(kernel_pmap, (vm_map_offset_t)m2->vmp_offset, m2,
6917 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, TRUE);
6918
6919 assert(kr == KERN_SUCCESS);
6920
6921 if (compressed_pages) {
6922 ++*compressed_pages;
6923 }
6924 } else {
6925 /* relocated page was not used by the compressor
6926 * put it on either the active or inactive lists */
6927 if (m2->vmp_reference) {
6928 vm_page_activate(m2);
6929 } else {
6930 vm_page_deactivate(m2);
6931 }
6932 }
6933
6934 /* unset the busy flag (pages on the free queue are busy) and notify if wanted */
6935 vm_page_wakeup_done(object, m2);
6936 } else {
6937 assert(m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
6938
6939 /*
6940 * completely cleans up the state
6941 * of the page so that it is ready
6942 * to be put onto the free list, or
6943 * for this purpose it looks like it
6944 * just came off of the free list
6945 */
6946 vm_page_free_prepare(m1);
6947
6948 if (new_page) {
6949 vm_page_t m2;
6950 vm_object_offset_t offset;
6951 vm_grab_options_t grab_options = VM_PAGE_GRAB_Q_LOCK_HELD;
6952
6953 /* The caller still wanted a page, so let's give them a new one. */
6954 offset = m1->vmp_offset;
6955 m2 = vm_page_grab_options(grab_options);
6956
6957 if (m2 == VM_PAGE_NULL) {
6958 return KERN_RESOURCE_SHORTAGE;
6959 }
6960
6961 /*
6962 * make sure we clear the ref/mod state
6963 * from the pmap layer... else we risk
6964 * inheriting state from the last time
6965 * this page was used...
6966 */
6967 pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m2),
6968 VM_MEM_MODIFIED | VM_MEM_REFERENCED);
6969
6970 offset = m1->vmp_offset;
6971
6972 /*
6973 * now put the substitute page on the object
6974 */
6975 vm_page_insert_internal(m2, object, offset, VM_KERN_MEMORY_NONE, TRUE,
6976 TRUE, FALSE, FALSE, NULL);
6977
6978 *new_page = m2;
6979 }
6980 }
6981
6982 /* we're done here */
6983 return KERN_SUCCESS;
6984 }
6985
6986 /*
6987 * CONTIGUOUS PAGE ALLOCATION
6988 *
6989 * Find a region large enough to contain at least n pages
6990 * of contiguous physical memory.
6991 *
6992 * This is done by traversing the vm_page_t array in a linear fashion
6993 * we assume that the vm_page_t array has the avaiable physical pages in an
6994 * ordered, ascending list... this is currently true of all our implementations
6995 * and must remain so... there can be 'holes' in the array... we also can
6996 * no longer tolerate the vm_page_t's in the list being 'freed' and reclaimed
6997 * which use to happen via 'vm_page_convert'... that function was no longer
6998 * being called and was removed...
6999 *
7000 * The basic flow consists of stabilizing some of the interesting state of
7001 * a vm_page_t behind the vm_page_queue and vm_page_free locks... we start our
7002 * sweep at the beginning of the array looking for pages that meet our criterea
7003 * for a 'stealable' page... currently we are pretty conservative... if the page
7004 * meets this criterea and is physically contiguous to the previous page in the 'run'
7005 * we keep developing it. If we hit a page that doesn't fit, we reset our state
7006 * and start to develop a new run... if at this point we've already considered
7007 * at least MAX_CONSIDERED_BEFORE_YIELD pages, we'll drop the 2 locks we hold,
7008 * and mutex_pause (which will yield the processor), to keep the latency low w/r
7009 * to other threads trying to acquire free pages (or move pages from q to q),
7010 * and then continue from the spot we left off... we only make 1 pass through the
7011 * array. Once we have a 'run' that is long enough, we'll go into the loop which
7012 * which steals the pages from the queues they're currently on... pages on the free
7013 * queue can be stolen directly... pages that are on any of the other queues
7014 * must be removed from the object they are tabled on... this requires taking the
7015 * object lock... we do this as a 'try' to prevent deadlocks... if the 'try' fails
7016 * or if the state of the page behind the vm_object lock is no longer viable, we'll
7017 * dump the pages we've currently stolen back to the free list, and pick up our
7018 * scan from the point where we aborted the 'current' run.
7019 *
7020 *
7021 * Requirements:
7022 * - neither vm_page_queue nor vm_free_list lock can be held on entry
7023 *
7024 * Returns a pointer to a list of gobbled/wired pages or VM_PAGE_NULL.
7025 *
7026 * Algorithm:
7027 */
7028
7029 #define MAX_CONSIDERED_BEFORE_YIELD 1000
7030
7031
7032 #define RESET_STATE_OF_RUN() \
7033 MACRO_BEGIN \
7034 prevcontaddr = -2; \
7035 start_pnum = -1; \
7036 free_considered = 0; \
7037 substitute_needed = 0; \
7038 npages = 0; \
7039 MACRO_END
7040
7041 /*
7042 * Can we steal in-use (i.e. not free) pages when searching for
7043 * physically-contiguous pages ?
7044 */
7045 #define VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL 1
7046
7047 static unsigned int vm_page_find_contiguous_last_idx = 0, vm_page_lomem_find_contiguous_last_idx = 0;
7048 #if DEBUG
7049 int vm_page_find_contig_debug = 0;
7050 #endif
7051
7052 static vm_page_t
vm_page_find_contiguous(unsigned int contig_pages,ppnum_t max_pnum,ppnum_t pnum_mask,boolean_t wire,int flags)7053 vm_page_find_contiguous(
7054 unsigned int contig_pages,
7055 ppnum_t max_pnum,
7056 ppnum_t pnum_mask,
7057 boolean_t wire,
7058 int flags)
7059 {
7060 vm_page_list_t list = { };
7061 ppnum_t prevcontaddr = 0;
7062 ppnum_t start_pnum = 0;
7063 unsigned int npages = 0, considered = 0, scanned = 0;
7064 unsigned int page_idx = 0, start_idx = 0, last_idx = 0, orig_last_idx = 0;
7065 unsigned int idx_last_contig_page_found = 0;
7066 int free_considered = 0, free_available = 0;
7067 int substitute_needed = 0;
7068 int zone_gc_called = 0;
7069 boolean_t wrapped;
7070 kern_return_t kr;
7071 #if DEBUG
7072 clock_sec_t tv_start_sec = 0, tv_end_sec = 0;
7073 clock_usec_t tv_start_usec = 0, tv_end_usec = 0;
7074 #endif
7075
7076 int yielded = 0;
7077 int dumped_run = 0;
7078 int stolen_pages = 0;
7079 int compressed_pages = 0;
7080
7081
7082 if (contig_pages == 0) {
7083 return VM_PAGE_NULL;
7084 }
7085
7086 full_scan_again:
7087
7088 #if MACH_ASSERT
7089 vm_page_verify_free_lists();
7090 #endif
7091 #if DEBUG
7092 clock_get_system_microtime(&tv_start_sec, &tv_start_usec);
7093 #endif
7094 PAGE_REPLACEMENT_ALLOWED(TRUE);
7095
7096 #if XNU_VM_HAS_DELAYED_PAGES
7097 /*
7098 * If there are still delayed pages, try to free up some that match.
7099 */
7100 if (__improbable(vm_delayed_count != 0 && contig_pages != 0)) {
7101 vm_free_delayed_pages_contig(contig_pages, max_pnum, pnum_mask);
7102 }
7103 #endif /* XNU_VM_HAS_DELAYED_PAGES */
7104
7105 vm_page_lock_queues();
7106 vm_free_page_lock();
7107
7108 RESET_STATE_OF_RUN();
7109
7110 scanned = 0;
7111 considered = 0;
7112 free_available = vm_page_free_count - vm_page_free_reserved;
7113
7114 wrapped = FALSE;
7115
7116 if (flags & KMA_LOMEM) {
7117 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx;
7118 } else {
7119 idx_last_contig_page_found = vm_page_find_contiguous_last_idx;
7120 }
7121
7122 orig_last_idx = idx_last_contig_page_found;
7123 last_idx = orig_last_idx;
7124
7125 for (page_idx = last_idx, start_idx = last_idx;
7126 npages < contig_pages && page_idx < vm_pages_count;
7127 page_idx++) {
7128 vm_page_t m = NULL;
7129
7130 retry:
7131 if (wrapped &&
7132 npages == 0 &&
7133 page_idx >= orig_last_idx) {
7134 /*
7135 * We're back where we started and we haven't
7136 * found any suitable contiguous range. Let's
7137 * give up.
7138 */
7139 break;
7140 }
7141 scanned++;
7142 m = vm_page_get(page_idx);
7143
7144 assert(vm_page_is_canonical(m));
7145
7146 if (max_pnum && VM_PAGE_GET_PHYS_PAGE(m) > max_pnum) {
7147 /* no more low pages... */
7148 break;
7149 }
7150 if (!npages & ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0)) {
7151 /*
7152 * not aligned
7153 */
7154 RESET_STATE_OF_RUN();
7155 } else if (!vm_page_is_relocatable(m,
7156 VM_RELOCATE_REASON_CONTIGUOUS)) {
7157 /*
7158 * page is not relocatable */
7159 RESET_STATE_OF_RUN();
7160 } else {
7161 if (VM_PAGE_GET_PHYS_PAGE(m) != prevcontaddr + 1) {
7162 if ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0) {
7163 RESET_STATE_OF_RUN();
7164 goto did_consider;
7165 } else {
7166 npages = 1;
7167 start_idx = page_idx;
7168 start_pnum = VM_PAGE_GET_PHYS_PAGE(m);
7169 }
7170 } else {
7171 npages++;
7172 }
7173 prevcontaddr = VM_PAGE_GET_PHYS_PAGE(m);
7174
7175 VM_PAGE_CHECK(m);
7176 if (m->vmp_q_state == VM_PAGE_ON_FREE_Q) {
7177 free_considered++;
7178 } else {
7179 /*
7180 * This page is not free.
7181 * If we can't steal used pages,
7182 * we have to give up this run
7183 * and keep looking.
7184 * Otherwise, we might need to
7185 * move the contents of this page
7186 * into a substitute page.
7187 */
7188 #if VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL
7189 if (m->vmp_pmapped || m->vmp_dirty || m->vmp_precious) {
7190 substitute_needed++;
7191 }
7192 #else
7193 RESET_STATE_OF_RUN();
7194 #endif
7195 }
7196
7197 if ((free_considered + substitute_needed) > free_available) {
7198 /*
7199 * if we let this run continue
7200 * we will end up dropping the vm_page_free_count
7201 * below the reserve limit... we need to abort
7202 * this run, but we can at least re-consider this
7203 * page... thus the jump back to 'retry'
7204 */
7205 RESET_STATE_OF_RUN();
7206
7207 if (free_available && considered <= MAX_CONSIDERED_BEFORE_YIELD) {
7208 considered++;
7209 goto retry;
7210 }
7211 /*
7212 * free_available == 0
7213 * so can't consider any free pages... if
7214 * we went to retry in this case, we'd
7215 * get stuck looking at the same page
7216 * w/o making any forward progress
7217 * we also want to take this path if we've already
7218 * reached our limit that controls the lock latency
7219 */
7220 }
7221 }
7222 did_consider:
7223 if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) {
7224 PAGE_REPLACEMENT_ALLOWED(FALSE);
7225
7226 vm_free_page_unlock();
7227 vm_page_unlock_queues();
7228
7229 mutex_pause(0);
7230
7231 PAGE_REPLACEMENT_ALLOWED(TRUE);
7232
7233 vm_page_lock_queues();
7234 vm_free_page_lock();
7235
7236 RESET_STATE_OF_RUN();
7237 /*
7238 * reset our free page limit since we
7239 * dropped the lock protecting the vm_page_free_queue
7240 */
7241 free_available = vm_page_free_count - vm_page_free_reserved;
7242 considered = 0;
7243
7244 yielded++;
7245
7246 goto retry;
7247 }
7248 considered++;
7249 } /* main for-loop end */
7250
7251 if (npages != contig_pages) {
7252 if (!wrapped) {
7253 /*
7254 * We didn't find a contiguous range but we didn't
7255 * start from the very first page.
7256 * Start again from the very first page.
7257 */
7258 RESET_STATE_OF_RUN();
7259 if (flags & KMA_LOMEM) {
7260 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx = 0;
7261 } else {
7262 idx_last_contig_page_found = vm_page_find_contiguous_last_idx = 0;
7263 }
7264 last_idx = 0;
7265 page_idx = last_idx;
7266 wrapped = TRUE;
7267 goto retry;
7268 }
7269 vm_free_page_unlock();
7270 } else {
7271 vm_page_t m1;
7272 unsigned int cur_idx;
7273 unsigned int tmp_start_idx;
7274 vm_object_t locked_object = VM_OBJECT_NULL;
7275 bool abort_run = false;
7276
7277 assert(page_idx - start_idx == contig_pages);
7278
7279 tmp_start_idx = start_idx;
7280
7281 /*
7282 * first pass through to pull the free pages
7283 * off of the free queue so that in case we
7284 * need substitute pages, we won't grab any
7285 * of the free pages in the run... we'll clear
7286 * the 'free' bit in the 2nd pass, and even in
7287 * an abort_run case, we'll collect all of the
7288 * free pages in this run and return them to the free list
7289 */
7290 while (start_idx < page_idx) {
7291 vm_grab_options_t options = VM_PAGE_GRAB_OPTIONS_NONE;
7292
7293 m1 = vm_page_get(start_idx++);
7294
7295 #if !VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL
7296 assert(m1->vmp_q_state == VM_PAGE_ON_FREE_Q);
7297 #endif
7298 if (m1->vmp_q_state == VM_PAGE_ON_FREE_Q) {
7299 vm_page_free_queue_steal(options, m1);
7300 }
7301 }
7302 if (flags & KMA_LOMEM) {
7303 vm_page_lomem_find_contiguous_last_idx = page_idx;
7304 } else {
7305 vm_page_find_contiguous_last_idx = page_idx;
7306 }
7307
7308 /*
7309 * we can drop the free queue lock at this point since
7310 * we've pulled any 'free' candidates off of the list
7311 * we need it dropped so that we can do a vm_page_grab
7312 * when substituing for pmapped/dirty pages
7313 */
7314 vm_free_page_unlock();
7315
7316 start_idx = tmp_start_idx;
7317 cur_idx = page_idx - 1;
7318
7319 while (start_idx++ < page_idx) {
7320 /*
7321 * must go through the list from back to front
7322 * so that the page list is created in the
7323 * correct order - low -> high phys addresses
7324 */
7325 m1 = vm_page_get(cur_idx--);
7326
7327 if (m1->vmp_object == 0) {
7328 /*
7329 * page has already been removed from
7330 * the free list in the 1st pass
7331 */
7332 assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
7333 assert(m1->vmp_offset == (vm_object_offset_t) -1);
7334 assert(m1->vmp_busy);
7335 assert(!m1->vmp_wanted);
7336 assert(!m1->vmp_laundry);
7337 } else {
7338 /*
7339 * try to relocate/steal the page
7340 */
7341 if (abort_run) {
7342 continue;
7343 }
7344
7345 assert(m1->vmp_q_state != VM_PAGE_NOT_ON_Q);
7346
7347 vm_object_t object = VM_PAGE_OBJECT(m1);
7348
7349 if (object != locked_object) {
7350 if (locked_object) {
7351 vm_object_unlock(locked_object);
7352 locked_object = VM_OBJECT_NULL;
7353 }
7354 if (vm_object_lock_try(object)) {
7355 locked_object = object;
7356 } else {
7357 /* object must be locked to relocate its pages */
7358 tmp_start_idx = cur_idx;
7359 abort_run = true;
7360 continue;
7361 }
7362 }
7363
7364 kr = vm_page_relocate(m1, &compressed_pages, VM_RELOCATE_REASON_CONTIGUOUS, NULL);
7365 if (kr != KERN_SUCCESS) {
7366 if (locked_object) {
7367 vm_object_unlock(locked_object);
7368 locked_object = VM_OBJECT_NULL;
7369 }
7370 tmp_start_idx = cur_idx;
7371 abort_run = true;
7372 continue;
7373 }
7374
7375 stolen_pages++;
7376 }
7377
7378 /* m1 is ours at this point ... */
7379
7380 if (m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR) {
7381 /*
7382 * The Q state is preserved on m1 because vm_page_queues_remove doesn't
7383 * change it for pages marked as used-by-compressor.
7384 */
7385 vm_page_assign_special_state(m1, VM_PAGE_SPECIAL_Q_BG);
7386 }
7387 VM_PAGE_ZERO_PAGEQ_ENTRY(m1);
7388 vm_page_list_push(&list, m1);
7389 }
7390
7391 if (locked_object) {
7392 vm_object_unlock(locked_object);
7393 locked_object = VM_OBJECT_NULL;
7394 }
7395
7396 if (abort_run) {
7397 /*
7398 * want the index of the last
7399 * page in this run that was
7400 * successfully 'stolen', so back
7401 * it up 1 for the auto-decrement on use
7402 * and 1 more to bump back over this page
7403 */
7404 page_idx = tmp_start_idx + 2;
7405 if (page_idx >= vm_pages_count) {
7406 if (wrapped) {
7407 if (list.vmpl_count) {
7408 vm_page_unlock_queues();
7409 vm_page_free_list(list.vmpl_head, FALSE);
7410 vm_page_lock_queues();
7411 list = (vm_page_list_t){ };
7412 }
7413 dumped_run++;
7414 goto done_scanning;
7415 }
7416 page_idx = last_idx = 0;
7417 wrapped = TRUE;
7418 }
7419 abort_run = false;
7420
7421 /*
7422 * We didn't find a contiguous range but we didn't
7423 * start from the very first page.
7424 * Start again from the very first page.
7425 */
7426 RESET_STATE_OF_RUN();
7427
7428 if (flags & KMA_LOMEM) {
7429 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx = page_idx;
7430 } else {
7431 idx_last_contig_page_found = vm_page_find_contiguous_last_idx = page_idx;
7432 }
7433
7434 last_idx = page_idx;
7435
7436 if (list.vmpl_count) {
7437 vm_page_unlock_queues();
7438 vm_page_free_list(list.vmpl_head, FALSE);
7439 vm_page_lock_queues();
7440 list = (vm_page_list_t){ };
7441 }
7442 dumped_run++;
7443
7444 vm_free_page_lock();
7445 /*
7446 * reset our free page limit since we
7447 * dropped the lock protecting the vm_page_free_queue
7448 */
7449 free_available = vm_page_free_count - vm_page_free_reserved;
7450 goto retry;
7451 }
7452
7453 vm_page_list_foreach(m1, list) {
7454 assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
7455 assert(m1->vmp_wire_count == 0);
7456
7457 if (wire == TRUE) {
7458 m1->vmp_wire_count++;
7459 m1->vmp_q_state = VM_PAGE_IS_WIRED;
7460
7461 } else {
7462 m1->vmp_gobbled = TRUE;
7463 }
7464 }
7465 if (wire == FALSE) {
7466 vm_page_gobble_count += npages;
7467 }
7468
7469 /*
7470 * gobbled pages are also counted as wired pages
7471 */
7472 vm_page_wire_count += npages;
7473
7474 assert(vm_page_verify_contiguous(list.vmpl_head, npages));
7475 }
7476 done_scanning:
7477 PAGE_REPLACEMENT_ALLOWED(FALSE);
7478
7479 vm_page_unlock_queues();
7480
7481 #if DEBUG
7482 clock_get_system_microtime(&tv_end_sec, &tv_end_usec);
7483
7484 tv_end_sec -= tv_start_sec;
7485 if (tv_end_usec < tv_start_usec) {
7486 tv_end_sec--;
7487 tv_end_usec += 1000000;
7488 }
7489 tv_end_usec -= tv_start_usec;
7490 if (tv_end_usec >= 1000000) {
7491 tv_end_sec++;
7492 tv_end_sec -= 1000000;
7493 }
7494 if (vm_page_find_contig_debug) {
7495 printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds... started at %d... scanned %d pages... yielded %d times... dumped run %d times... stole %d pages... stole %d compressed pages\n",
7496 __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
7497 (long)tv_end_sec, tv_end_usec, orig_last_idx,
7498 scanned, yielded, dumped_run, stolen_pages, compressed_pages);
7499 }
7500
7501 #endif
7502 #if MACH_ASSERT
7503 vm_page_verify_free_lists();
7504 #endif
7505 if (list.vmpl_count == 0 && zone_gc_called < 2) {
7506 printf("%s(num=%d,low=%d): found %d pages at 0x%llx...scanned %d pages... yielded %d times... dumped run %d times... stole %d pages... stole %d compressed pages... wired count is %d\n",
7507 __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
7508 scanned, yielded, dumped_run, stolen_pages, compressed_pages, vm_page_wire_count);
7509
7510 if (consider_buffer_cache_collect != NULL) {
7511 (void)(*consider_buffer_cache_collect)(1);
7512 }
7513
7514 zone_gc(zone_gc_called ? ZONE_GC_DRAIN : ZONE_GC_TRIM);
7515
7516 zone_gc_called++;
7517
7518 printf("vm_page_find_contiguous: zone_gc called... wired count is %d\n", vm_page_wire_count);
7519 goto full_scan_again;
7520 }
7521
7522 return list.vmpl_head;
7523 }
7524
7525 /*
7526 * Allocate a list of contiguous, wired pages.
7527 */
7528 kern_return_t
cpm_allocate(vm_size_t size,vm_page_t * list,ppnum_t max_pnum,ppnum_t pnum_mask,boolean_t wire,int flags)7529 cpm_allocate(
7530 vm_size_t size,
7531 vm_page_t *list,
7532 ppnum_t max_pnum,
7533 ppnum_t pnum_mask,
7534 boolean_t wire,
7535 int flags)
7536 {
7537 vm_page_t pages;
7538 unsigned int npages;
7539
7540 if (size % PAGE_SIZE != 0) {
7541 return KERN_INVALID_ARGUMENT;
7542 }
7543
7544 npages = (unsigned int) (size / PAGE_SIZE);
7545 if (npages != size / PAGE_SIZE) {
7546 /* 32-bit overflow */
7547 return KERN_INVALID_ARGUMENT;
7548 }
7549
7550 /*
7551 * Obtain a pointer to a subset of the free
7552 * list large enough to satisfy the request;
7553 * the region will be physically contiguous.
7554 */
7555 pages = vm_page_find_contiguous(npages, max_pnum, pnum_mask, wire, flags);
7556
7557 if (pages == VM_PAGE_NULL) {
7558 return KERN_NO_SPACE;
7559 }
7560 /*
7561 * determine need for wakeups
7562 */
7563 if (vm_page_free_count < vm_page_free_min) {
7564 vm_free_page_lock();
7565 if (vm_pageout_running == FALSE) {
7566 vm_free_page_unlock();
7567 thread_wakeup((event_t) &vm_page_free_wanted);
7568 } else {
7569 vm_free_page_unlock();
7570 }
7571 }
7572
7573 VM_CHECK_MEMORYSTATUS;
7574
7575 /*
7576 * The CPM pages should now be available and
7577 * ordered by ascending physical address.
7578 */
7579 assert(vm_page_verify_contiguous(pages, npages));
7580
7581 if (flags & KMA_ZERO) {
7582 for (vm_page_t m = pages; m; m = NEXT_PAGE(m)) {
7583 vm_page_zero_fill(
7584 m
7585 );
7586 }
7587 }
7588
7589 *list = pages;
7590 return KERN_SUCCESS;
7591 }
7592
7593
7594 unsigned int vm_max_delayed_work_limit = DEFAULT_DELAYED_WORK_LIMIT;
7595
7596 /*
7597 * when working on a 'run' of pages, it is necessary to hold
7598 * the vm_page_queue_lock (a hot global lock) for certain operations
7599 * on the page... however, the majority of the work can be done
7600 * while merely holding the object lock... in fact there are certain
7601 * collections of pages that don't require any work brokered by the
7602 * vm_page_queue_lock... to mitigate the time spent behind the global
7603 * lock, go to a 2 pass algorithm... collect pages up to DELAYED_WORK_LIMIT
7604 * while doing all of the work that doesn't require the vm_page_queue_lock...
7605 * then call vm_page_do_delayed_work to acquire the vm_page_queue_lock and do the
7606 * necessary work for each page... we will grab the busy bit on the page
7607 * if it's not already held so that vm_page_do_delayed_work can drop the object lock
7608 * if it can't immediately take the vm_page_queue_lock in order to compete
7609 * for the locks in the same order that vm_pageout_scan takes them.
7610 * the operation names are modeled after the names of the routines that
7611 * need to be called in order to make the changes very obvious in the
7612 * original loop
7613 *
7614 * On certain configurations, this function may return failure if any of
7615 * the pages in the run has a mapping state that doesn't allow the specified
7616 * operation. In that case, it will still fully process the run of pages
7617 * in order to avoid requiring the caller to partially undo the work done
7618 * here.
7619 */
7620
7621 kern_return_t
vm_page_do_delayed_work(vm_object_t object,vm_tag_t tag,struct vm_page_delayed_work * dwp,int dw_count)7622 vm_page_do_delayed_work(
7623 vm_object_t object,
7624 vm_tag_t tag,
7625 struct vm_page_delayed_work *dwp,
7626 int dw_count)
7627 {
7628 kern_return_t kr = KERN_SUCCESS;
7629 int j;
7630 vm_page_t m;
7631 vm_page_t local_free_q = VM_PAGE_NULL;
7632
7633 /*
7634 * pageout_scan takes the vm_page_lock_queues first
7635 * then tries for the object lock... to avoid what
7636 * is effectively a lock inversion, we'll go to the
7637 * trouble of taking them in that same order... otherwise
7638 * if this object contains the majority of the pages resident
7639 * in the UBC (or a small set of large objects actively being
7640 * worked on contain the majority of the pages), we could
7641 * cause the pageout_scan thread to 'starve' in its attempt
7642 * to find pages to move to the free queue, since it has to
7643 * successfully acquire the object lock of any candidate page
7644 * before it can steal/clean it.
7645 */
7646 if (!vm_page_trylock_queues()) {
7647 vm_object_unlock(object);
7648
7649 /*
7650 * "Turnstile enabled vm_pageout_scan" can be runnable
7651 * for a very long time without getting on a core.
7652 * If this is a higher priority thread it could be
7653 * waiting here for a very long time respecting the fact
7654 * that pageout_scan would like its object after VPS does
7655 * a mutex_pause(0).
7656 * So we cap the number of yields in the vm_object_lock_avoid()
7657 * case to a single mutex_pause(0) which will give vm_pageout_scan
7658 * 10us to run and grab the object if needed.
7659 */
7660 vm_page_lock_queues();
7661
7662 for (j = 0;; j++) {
7663 if ((!vm_object_lock_avoid(object) ||
7664 (vps_dynamic_priority_enabled && (j > 0))) &&
7665 _vm_object_lock_try(object)) {
7666 break;
7667 }
7668 vm_page_unlock_queues();
7669 mutex_pause(j);
7670 vm_page_lock_queues();
7671 }
7672 }
7673 for (j = 0; j < dw_count; j++, dwp++) {
7674 m = dwp->dw_m;
7675
7676 if (dwp->dw_mask & DW_vm_pageout_throttle_up) {
7677 vm_pageout_throttle_up(m);
7678 }
7679 #if CONFIG_PHANTOM_CACHE
7680 if (dwp->dw_mask & DW_vm_phantom_cache_update) {
7681 vm_phantom_cache_update(m);
7682 }
7683 #endif
7684 if (dwp->dw_mask & DW_vm_page_wire) {
7685 vm_page_wire(m, tag, FALSE);
7686 if (dwp->dw_mask & DW_vm_page_iopl_wire) {
7687 #if CONFIG_SPTM
7688 /*
7689 * The SPTM's security model prevents us from allowing writable I/O
7690 * mappings of executable pages. We need to check that here,
7691 * in the same place that we set vmp_iopl_wired, because this
7692 * function may have transiently dropped the VM object lock
7693 * before reaching this point, which means that frontloading
7694 * this check in the caller may not work in all cases.
7695 */
7696 if ((dwp->dw_mask & DW_vm_page_iopl_wire_write) && PMAP_PAGE_IS_USER_EXECUTABLE(m)) {
7697 if (kr == KERN_SUCCESS) {
7698 kr = KERN_PROTECTION_FAILURE;
7699 vm_map_guard_exception(VM_PAGE_GET_PHYS_PAGE(m), kGUARD_EXC_SEC_IOPL_ON_EXEC_PAGE);
7700 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM,
7701 KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_IOPL_ON_EXEC_PAGE),
7702 (uintptr_t)(VM_PAGE_GET_PHYS_PAGE(m)));
7703 }
7704 } else {
7705 m->vmp_iopl_wired = true;
7706 }
7707 #else
7708 m->vmp_iopl_wired = true;
7709 #endif /* CONFIG_SPTM */
7710 }
7711 } else if (dwp->dw_mask & DW_vm_page_unwire) {
7712 boolean_t queueit;
7713
7714 queueit = (dwp->dw_mask & (DW_vm_page_free | DW_vm_page_deactivate_internal)) ? FALSE : TRUE;
7715
7716 vm_page_unwire(m, queueit);
7717 }
7718 if (dwp->dw_mask & DW_vm_page_free) {
7719 vm_page_free_prepare_queues(m);
7720
7721 assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
7722 /*
7723 * Add this page to our list of reclaimed pages,
7724 * to be freed later.
7725 */
7726 m->vmp_snext = local_free_q;
7727 local_free_q = m;
7728 } else {
7729 if (dwp->dw_mask & DW_vm_page_deactivate_internal) {
7730 vm_page_deactivate_internal(m, FALSE);
7731 } else if (dwp->dw_mask & DW_vm_page_activate) {
7732 if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
7733 vm_page_activate(m);
7734 }
7735 } else if (dwp->dw_mask & DW_vm_page_speculate) {
7736 vm_page_speculate(m, TRUE);
7737 } else if (dwp->dw_mask & DW_enqueue_cleaned) {
7738 /*
7739 * if we didn't hold the object lock and did this,
7740 * we might disconnect the page, then someone might
7741 * soft fault it back in, then we would put it on the
7742 * cleaned queue, and so we would have a referenced (maybe even dirty)
7743 * page on that queue, which we don't want
7744 */
7745 int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
7746
7747 if ((refmod_state & VM_MEM_REFERENCED)) {
7748 /*
7749 * this page has been touched since it got cleaned; let's activate it
7750 * if it hasn't already been
7751 */
7752 VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
7753 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
7754
7755 if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
7756 vm_page_activate(m);
7757 }
7758 } else {
7759 m->vmp_reference = FALSE;
7760 vm_page_enqueue_cleaned(m);
7761 }
7762 } else if (dwp->dw_mask & DW_vm_page_lru) {
7763 vm_page_lru(m);
7764 } else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) {
7765 if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {
7766 vm_page_queues_remove(m, TRUE);
7767 }
7768 }
7769 if (dwp->dw_mask & DW_set_reference) {
7770 m->vmp_reference = TRUE;
7771 } else if (dwp->dw_mask & DW_clear_reference) {
7772 m->vmp_reference = FALSE;
7773 }
7774
7775 if (dwp->dw_mask & DW_move_page) {
7776 if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {
7777 vm_page_queues_remove(m, FALSE);
7778
7779 assert(!is_kernel_object(VM_PAGE_OBJECT(m)));
7780
7781 vm_page_enqueue_inactive(m, FALSE);
7782 }
7783 }
7784 if (dwp->dw_mask & DW_clear_busy) {
7785 m->vmp_busy = FALSE;
7786 }
7787
7788 if (dwp->dw_mask & DW_PAGE_WAKEUP) {
7789 vm_page_wakeup(object, m);
7790 }
7791 }
7792 }
7793 vm_page_unlock_queues();
7794
7795 if (local_free_q) {
7796 vm_page_free_list(local_free_q, TRUE);
7797 }
7798
7799 VM_CHECK_MEMORYSTATUS;
7800
7801 return kr;
7802 }
7803
7804 __abortlike
7805 static void
__vm_page_alloc_list_failed_panic(vm_size_t page_count,kma_flags_t flags,kern_return_t kr)7806 __vm_page_alloc_list_failed_panic(
7807 vm_size_t page_count,
7808 kma_flags_t flags,
7809 kern_return_t kr)
7810 {
7811 panic("vm_page_alloc_list(%zd, 0x%x) failed unexpectedly with %d",
7812 (size_t)page_count, flags, kr);
7813 }
7814
7815 kern_return_t
vm_page_alloc_list(vm_size_t page_count,kma_flags_t flags,vm_page_t * list)7816 vm_page_alloc_list(vm_size_t page_count, kma_flags_t flags, vm_page_t *list)
7817 {
7818 vm_page_t page_list = VM_PAGE_NULL;
7819 vm_page_t mem;
7820 kern_return_t kr = KERN_SUCCESS;
7821 int page_grab_count = 0;
7822 task_t task;
7823
7824 for (vm_size_t i = 0; i < page_count; i++) {
7825 for (;;) {
7826 vm_grab_options_t options = VM_PAGE_GRAB_OPTIONS_NONE;
7827
7828 if (flags & KMA_NOPAGEWAIT) {
7829 options |= VM_PAGE_GRAB_NOPAGEWAIT;
7830 }
7831 if (flags & KMA_LOMEM) {
7832 mem = vm_page_grablo(options);
7833 } else {
7834 mem = vm_page_grab_options(options);
7835 }
7836
7837 if (mem != VM_PAGE_NULL) {
7838 break;
7839 }
7840
7841 if (flags & KMA_NOPAGEWAIT) {
7842 kr = KERN_RESOURCE_SHORTAGE;
7843 goto out;
7844 }
7845 if ((flags & KMA_LOMEM) && vm_lopage_needed) {
7846 kr = KERN_RESOURCE_SHORTAGE;
7847 goto out;
7848 }
7849
7850 /* VM privileged threads should have waited in vm_page_grab() and not get here. */
7851 assert(!(current_thread()->options & TH_OPT_VMPRIV));
7852
7853 if ((flags & KMA_NOFAIL) == 0 && ptoa_64(page_count) > max_mem / 4) {
7854 uint64_t unavailable = ptoa_64(vm_page_wire_count + vm_page_free_target);
7855 if (unavailable > max_mem || ptoa_64(page_count) > (max_mem - unavailable)) {
7856 kr = KERN_RESOURCE_SHORTAGE;
7857 goto out;
7858 }
7859 }
7860 VM_PAGE_WAIT();
7861 }
7862
7863 page_grab_count++;
7864 mem->vmp_snext = page_list;
7865 page_list = mem;
7866 }
7867
7868 if ((KMA_ZERO | KMA_NOENCRYPT) & flags) {
7869 for (mem = page_list; mem; mem = mem->vmp_snext) {
7870 vm_page_zero_fill(
7871 mem
7872 );
7873 }
7874 }
7875
7876 out:
7877 task = current_task_early();
7878 if (task != NULL) {
7879 ledger_credit(task->ledger, task_ledgers.pages_grabbed_kern, page_grab_count);
7880 }
7881 counter_add(&vm_page_grab_count_kern, page_grab_count);
7882
7883 if (kr == KERN_SUCCESS) {
7884 *list = page_list;
7885 } else if (flags & KMA_NOFAIL) {
7886 __vm_page_alloc_list_failed_panic(page_count, flags, kr);
7887 } else {
7888 vm_page_free_list(page_list, FALSE);
7889 }
7890
7891 return kr;
7892 }
7893
7894 void
vm_page_set_offset(vm_page_t page,vm_object_offset_t offset)7895 vm_page_set_offset(vm_page_t page, vm_object_offset_t offset)
7896 {
7897 page->vmp_offset = offset;
7898 }
7899
7900 vm_page_t
vm_page_get_next(vm_page_t page)7901 vm_page_get_next(vm_page_t page)
7902 {
7903 return page->vmp_snext;
7904 }
7905
7906 vm_object_offset_t
vm_page_get_offset(vm_page_t page)7907 vm_page_get_offset(vm_page_t page)
7908 {
7909 return page->vmp_offset;
7910 }
7911
7912 ppnum_t
vm_page_get_phys_page(vm_page_t page)7913 vm_page_get_phys_page(vm_page_t page)
7914 {
7915 return VM_PAGE_GET_PHYS_PAGE(page);
7916 }
7917
7918
7919 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
7920
7921 #if HIBERNATION
7922
7923 static uint32_t hibernate_teardown_vm_structs(hibernate_page_list_t *, hibernate_page_list_t *);
7924
7925 struct hibernate_statistics {
7926 int hibernate_considered;
7927 int hibernate_reentered_on_q;
7928 int hibernate_found_dirty;
7929 int hibernate_skipped_cleaning;
7930 int hibernate_skipped_transient;
7931 int hibernate_skipped_precious;
7932 int hibernate_skipped_external;
7933 int hibernate_queue_nolock;
7934 int hibernate_queue_paused;
7935 int hibernate_throttled;
7936 int hibernate_throttle_timeout;
7937 int hibernate_drained;
7938 int hibernate_drain_timeout;
7939 int cd_lock_failed;
7940 int cd_found_precious;
7941 int cd_found_wired;
7942 int cd_found_busy;
7943 int cd_found_unusual;
7944 int cd_found_cleaning;
7945 int cd_found_laundry;
7946 int cd_found_dirty;
7947 int cd_found_xpmapped;
7948 int cd_skipped_xpmapped;
7949 int cd_local_free;
7950 int cd_total_free;
7951 int cd_vm_page_wire_count;
7952 int cd_vm_struct_pages_unneeded;
7953 int cd_pages;
7954 int cd_discarded;
7955 int cd_count_wire;
7956 } hibernate_stats;
7957
7958 #if CONFIG_SPTM
7959 /**
7960 * On SPTM-based systems don't save any executable pages into the hibernation
7961 * image. The SPTM has stronger guarantees around not allowing write access to
7962 * the executable pages than on older systems, which prevents XNU from being
7963 * able to restore any pages mapped as executable.
7964 */
7965 #define HIBERNATE_XPMAPPED_LIMIT 0ULL
7966 #else /* CONFIG_SPTM */
7967 /*
7968 * clamp the number of 'xpmapped' pages we'll sweep into the hibernation image
7969 * so that we don't overrun the estimated image size, which would
7970 * result in a hibernation failure.
7971 *
7972 * We use a size value instead of pages because we don't want to take up more space
7973 * on disk if the system has a 16K page size vs 4K. Also, we are not guaranteed
7974 * to have that additional space available.
7975 *
7976 * Since this was set at 40000 pages on X86 we are going to use 160MB as our
7977 * xpmapped size.
7978 */
7979 #define HIBERNATE_XPMAPPED_LIMIT ((160 * 1024 * 1024ULL) / PAGE_SIZE)
7980 #endif /* CONFIG_SPTM */
7981
7982 static int
hibernate_drain_pageout_queue(struct vm_pageout_queue * q)7983 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
7984 {
7985 wait_result_t wait_result;
7986
7987 vm_page_lock_queues();
7988
7989 while (!vm_page_queue_empty(&q->pgo_pending)) {
7990 q->pgo_draining = TRUE;
7991
7992 assert_wait_timeout((event_t) (&q->pgo_laundry + 1), THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
7993
7994 vm_page_unlock_queues();
7995
7996 wait_result = thread_block(THREAD_CONTINUE_NULL);
7997
7998 if (wait_result == THREAD_TIMED_OUT && !vm_page_queue_empty(&q->pgo_pending)) {
7999 hibernate_stats.hibernate_drain_timeout++;
8000
8001 if (q == &vm_pageout_queue_external) {
8002 return 0;
8003 }
8004
8005 return 1;
8006 }
8007 vm_page_lock_queues();
8008
8009 hibernate_stats.hibernate_drained++;
8010 }
8011 vm_page_unlock_queues();
8012
8013 return 0;
8014 }
8015
8016
8017 boolean_t hibernate_skip_external = FALSE;
8018
8019 static int
hibernate_flush_queue(vm_page_queue_head_t * q,int qcount)8020 hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
8021 {
8022 vm_page_t m;
8023 vm_object_t l_object = NULL;
8024 vm_object_t m_object = NULL;
8025 int refmod_state = 0;
8026 int try_failed_count = 0;
8027 int retval = 0;
8028 int current_run = 0;
8029 struct vm_pageout_queue *iq;
8030 struct vm_pageout_queue *eq;
8031 struct vm_pageout_queue *tq;
8032
8033 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START,
8034 VM_KERNEL_UNSLIDE_OR_PERM(q), qcount);
8035
8036 iq = &vm_pageout_queue_internal;
8037 eq = &vm_pageout_queue_external;
8038
8039 vm_page_lock_queues();
8040
8041 while (qcount && !vm_page_queue_empty(q)) {
8042 if (current_run++ == 1000) {
8043 if (hibernate_should_abort()) {
8044 retval = 1;
8045 break;
8046 }
8047 current_run = 0;
8048 }
8049
8050 m = (vm_page_t) vm_page_queue_first(q);
8051 m_object = VM_PAGE_OBJECT(m);
8052
8053 /*
8054 * check to see if we currently are working
8055 * with the same object... if so, we've
8056 * already got the lock
8057 */
8058 if (m_object != l_object) {
8059 /*
8060 * the object associated with candidate page is
8061 * different from the one we were just working
8062 * with... dump the lock if we still own it
8063 */
8064 if (l_object != NULL) {
8065 vm_object_unlock(l_object);
8066 l_object = NULL;
8067 }
8068 /*
8069 * Try to lock object; since we've alread got the
8070 * page queues lock, we can only 'try' for this one.
8071 * if the 'try' fails, we need to do a mutex_pause
8072 * to allow the owner of the object lock a chance to
8073 * run...
8074 */
8075 if (!vm_object_lock_try_scan(m_object)) {
8076 if (try_failed_count > 20) {
8077 hibernate_stats.hibernate_queue_nolock++;
8078
8079 goto reenter_pg_on_q;
8080 }
8081
8082 vm_page_unlock_queues();
8083 mutex_pause(try_failed_count++);
8084 vm_page_lock_queues();
8085
8086 hibernate_stats.hibernate_queue_paused++;
8087 continue;
8088 } else {
8089 l_object = m_object;
8090 }
8091 }
8092 if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || VMP_ERROR_GET(m)) {
8093 /*
8094 * page is not to be cleaned
8095 * put it back on the head of its queue
8096 */
8097 if (m->vmp_cleaning) {
8098 hibernate_stats.hibernate_skipped_cleaning++;
8099 } else {
8100 hibernate_stats.hibernate_skipped_transient++;
8101 }
8102
8103 goto reenter_pg_on_q;
8104 }
8105 if (m_object->vo_copy == VM_OBJECT_NULL) {
8106 if (m_object->purgable == VM_PURGABLE_VOLATILE || m_object->purgable == VM_PURGABLE_EMPTY) {
8107 /*
8108 * let the normal hibernate image path
8109 * deal with these
8110 */
8111 goto reenter_pg_on_q;
8112 }
8113 }
8114 if (!m->vmp_dirty && m->vmp_pmapped) {
8115 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
8116
8117 if ((refmod_state & VM_MEM_MODIFIED)) {
8118 SET_PAGE_DIRTY(m, FALSE);
8119 }
8120 } else {
8121 refmod_state = 0;
8122 }
8123
8124 if (!m->vmp_dirty) {
8125 /*
8126 * page is not to be cleaned
8127 * put it back on the head of its queue
8128 */
8129 if (m->vmp_precious) {
8130 hibernate_stats.hibernate_skipped_precious++;
8131 }
8132
8133 goto reenter_pg_on_q;
8134 }
8135
8136 if (hibernate_skip_external == TRUE && !m_object->internal) {
8137 hibernate_stats.hibernate_skipped_external++;
8138
8139 goto reenter_pg_on_q;
8140 }
8141 tq = NULL;
8142
8143 if (m_object->internal) {
8144 if (VM_PAGE_Q_THROTTLED(iq)) {
8145 tq = iq;
8146 }
8147 } else if (VM_PAGE_Q_THROTTLED(eq)) {
8148 tq = eq;
8149 }
8150
8151 if (tq != NULL) {
8152 wait_result_t wait_result;
8153 int wait_count = 5;
8154
8155 if (l_object != NULL) {
8156 vm_object_unlock(l_object);
8157 l_object = NULL;
8158 }
8159
8160 while (retval == 0) {
8161 tq->pgo_throttled = TRUE;
8162
8163 assert_wait_timeout((event_t) &tq->pgo_laundry, THREAD_INTERRUPTIBLE, 1000, 1000 * NSEC_PER_USEC);
8164
8165 vm_page_unlock_queues();
8166
8167 wait_result = thread_block(THREAD_CONTINUE_NULL);
8168
8169 vm_page_lock_queues();
8170
8171 if (wait_result != THREAD_TIMED_OUT) {
8172 break;
8173 }
8174 if (!VM_PAGE_Q_THROTTLED(tq)) {
8175 break;
8176 }
8177
8178 if (hibernate_should_abort()) {
8179 retval = 1;
8180 }
8181
8182 if (--wait_count == 0) {
8183 hibernate_stats.hibernate_throttle_timeout++;
8184
8185 if (tq == eq) {
8186 hibernate_skip_external = TRUE;
8187 break;
8188 }
8189 retval = 1;
8190 }
8191 }
8192 if (retval) {
8193 break;
8194 }
8195
8196 hibernate_stats.hibernate_throttled++;
8197
8198 continue;
8199 }
8200 /*
8201 * we've already factored out pages in the laundry which
8202 * means this page can't be on the pageout queue so it's
8203 * safe to do the vm_page_queues_remove
8204 */
8205 vm_page_queues_remove(m, TRUE);
8206
8207 if (m_object->internal == TRUE) {
8208 pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), PMAP_OPTIONS_COMPRESSOR, NULL);
8209 }
8210
8211 vm_pageout_cluster(m);
8212
8213 hibernate_stats.hibernate_found_dirty++;
8214
8215 goto next_pg;
8216
8217 reenter_pg_on_q:
8218 vm_page_queue_remove(q, m, vmp_pageq);
8219 vm_page_queue_enter(q, m, vmp_pageq);
8220
8221 hibernate_stats.hibernate_reentered_on_q++;
8222 next_pg:
8223 hibernate_stats.hibernate_considered++;
8224
8225 qcount--;
8226 try_failed_count = 0;
8227 }
8228 if (l_object != NULL) {
8229 vm_object_unlock(l_object);
8230 l_object = NULL;
8231 }
8232
8233 vm_page_unlock_queues();
8234
8235 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_END, hibernate_stats.hibernate_found_dirty, retval, 0, 0, 0);
8236
8237 return retval;
8238 }
8239
8240
8241 static int
hibernate_flush_dirty_pages(int pass)8242 hibernate_flush_dirty_pages(int pass)
8243 {
8244 struct vm_speculative_age_q *aq;
8245 uint32_t i;
8246
8247 if (vm_page_local_q) {
8248 zpercpu_foreach_cpu(lid) {
8249 vm_page_reactivate_local(lid, TRUE, FALSE);
8250 }
8251 }
8252
8253 for (i = 0; i <= vm_page_max_speculative_age_q; i++) {
8254 int qcount;
8255 vm_page_t m;
8256
8257 aq = &vm_page_queue_speculative[i];
8258
8259 if (vm_page_queue_empty(&aq->age_q)) {
8260 continue;
8261 }
8262 qcount = 0;
8263
8264 vm_page_lockspin_queues();
8265
8266 vm_page_queue_iterate(&aq->age_q, m, vmp_pageq) {
8267 qcount++;
8268 }
8269 vm_page_unlock_queues();
8270
8271 if (qcount) {
8272 if (hibernate_flush_queue(&aq->age_q, qcount)) {
8273 return 1;
8274 }
8275 }
8276 }
8277 if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_page_anonymous_count - vm_page_cleaned_count)) {
8278 return 1;
8279 }
8280 /* XXX FBDP TODO: flush secluded queue */
8281 if (hibernate_flush_queue(&vm_page_queue_anonymous, vm_page_anonymous_count)) {
8282 return 1;
8283 }
8284 if (hibernate_flush_queue(&vm_page_queue_cleaned, vm_page_cleaned_count)) {
8285 return 1;
8286 }
8287 if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
8288 return 1;
8289 }
8290
8291 if (pass == 1) {
8292 vm_compressor_record_warmup_start();
8293 }
8294
8295 if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) {
8296 if (pass == 1) {
8297 vm_compressor_record_warmup_end();
8298 }
8299 return 1;
8300 }
8301 if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
8302 if (pass == 1) {
8303 vm_compressor_record_warmup_end();
8304 }
8305 return 1;
8306 }
8307 if (pass == 1) {
8308 vm_compressor_record_warmup_end();
8309 }
8310
8311 if (hibernate_skip_external == FALSE && hibernate_drain_pageout_queue(&vm_pageout_queue_external)) {
8312 return 1;
8313 }
8314
8315 return 0;
8316 }
8317
8318
8319 void
hibernate_reset_stats(void)8320 hibernate_reset_stats(void)
8321 {
8322 bzero(&hibernate_stats, sizeof(struct hibernate_statistics));
8323 }
8324
8325
8326 int
hibernate_flush_memory(void)8327 hibernate_flush_memory(void)
8328 {
8329 int retval;
8330
8331 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
8332
8333 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_START, vm_page_free_count, 0, 0, 0, 0);
8334
8335 hibernate_cleaning_in_progress = TRUE;
8336 hibernate_skip_external = FALSE;
8337
8338 if ((retval = hibernate_flush_dirty_pages(1)) == 0) {
8339 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_START, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
8340
8341 vm_compressor_flush();
8342
8343 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_END, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
8344
8345 if (consider_buffer_cache_collect != NULL) {
8346 unsigned int orig_wire_count;
8347
8348 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, 0, 0, 0, 0, 0);
8349 orig_wire_count = vm_page_wire_count;
8350
8351 (void)(*consider_buffer_cache_collect)(1);
8352 zone_gc(ZONE_GC_DRAIN);
8353
8354 HIBLOG("hibernate_flush_memory: buffer_cache_gc freed up %d wired pages\n", orig_wire_count - vm_page_wire_count);
8355
8356 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_END, orig_wire_count - vm_page_wire_count, 0, 0, 0, 0);
8357 }
8358 }
8359 hibernate_cleaning_in_progress = FALSE;
8360
8361 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_END, vm_page_free_count, hibernate_stats.hibernate_found_dirty, retval, 0, 0);
8362
8363 if (retval) {
8364 HIBLOG("hibernate_flush_memory() failed to finish - vm_page_compressor_count(%d)\n", VM_PAGE_COMPRESSOR_COUNT);
8365 }
8366
8367
8368 HIBPRINT("hibernate_flush_memory() considered(%d) reentered_on_q(%d) found_dirty(%d)\n",
8369 hibernate_stats.hibernate_considered,
8370 hibernate_stats.hibernate_reentered_on_q,
8371 hibernate_stats.hibernate_found_dirty);
8372 HIBPRINT(" skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) skipped_external(%d) queue_nolock(%d)\n",
8373 hibernate_stats.hibernate_skipped_cleaning,
8374 hibernate_stats.hibernate_skipped_transient,
8375 hibernate_stats.hibernate_skipped_precious,
8376 hibernate_stats.hibernate_skipped_external,
8377 hibernate_stats.hibernate_queue_nolock);
8378 HIBPRINT(" queue_paused(%d) throttled(%d) throttle_timeout(%d) drained(%d) drain_timeout(%d)\n",
8379 hibernate_stats.hibernate_queue_paused,
8380 hibernate_stats.hibernate_throttled,
8381 hibernate_stats.hibernate_throttle_timeout,
8382 hibernate_stats.hibernate_drained,
8383 hibernate_stats.hibernate_drain_timeout);
8384
8385 return retval;
8386 }
8387
8388
8389 static void
hibernate_page_list_zero(hibernate_page_list_t * list)8390 hibernate_page_list_zero(hibernate_page_list_t *list)
8391 {
8392 uint32_t bank;
8393 hibernate_bitmap_t * bitmap;
8394
8395 bitmap = &list->bank_bitmap[0];
8396 for (bank = 0; bank < list->bank_count; bank++) {
8397 uint32_t last_bit;
8398
8399 bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2);
8400 // set out-of-bound bits at end of bitmap.
8401 last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
8402 if (last_bit) {
8403 bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
8404 }
8405
8406 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
8407 }
8408 }
8409
8410 static boolean_t
hibernate_consider_discard(vm_page_t m,boolean_t preflight)8411 hibernate_consider_discard(vm_page_t m, boolean_t preflight)
8412 {
8413 vm_object_t object = NULL;
8414 int refmod_state;
8415 boolean_t discard = FALSE;
8416
8417 do{
8418 if (vm_page_is_private(m)) {
8419 panic("hibernate_consider_discard: private");
8420 }
8421
8422 object = VM_PAGE_OBJECT(m);
8423
8424 if (!vm_object_lock_try(object)) {
8425 object = NULL;
8426 if (!preflight) {
8427 hibernate_stats.cd_lock_failed++;
8428 }
8429 break;
8430 }
8431 if (VM_PAGE_WIRED(m)) {
8432 if (!preflight) {
8433 hibernate_stats.cd_found_wired++;
8434 }
8435 break;
8436 }
8437 if (m->vmp_precious) {
8438 if (!preflight) {
8439 hibernate_stats.cd_found_precious++;
8440 }
8441 break;
8442 }
8443 if (m->vmp_busy || !object->alive) {
8444 /*
8445 * Somebody is playing with this page.
8446 */
8447 if (!preflight) {
8448 hibernate_stats.cd_found_busy++;
8449 }
8450 break;
8451 }
8452 if (m->vmp_absent || m->vmp_unusual || VMP_ERROR_GET(m)) {
8453 /*
8454 * If it's unusual in anyway, ignore it
8455 */
8456 if (!preflight) {
8457 hibernate_stats.cd_found_unusual++;
8458 }
8459 break;
8460 }
8461 if (m->vmp_cleaning) {
8462 if (!preflight) {
8463 hibernate_stats.cd_found_cleaning++;
8464 }
8465 break;
8466 }
8467 if (m->vmp_laundry) {
8468 if (!preflight) {
8469 hibernate_stats.cd_found_laundry++;
8470 }
8471 break;
8472 }
8473 if (!m->vmp_dirty) {
8474 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
8475
8476 if (refmod_state & VM_MEM_REFERENCED) {
8477 m->vmp_reference = TRUE;
8478 }
8479 if (refmod_state & VM_MEM_MODIFIED) {
8480 SET_PAGE_DIRTY(m, FALSE);
8481 }
8482 }
8483
8484 /*
8485 * If it's clean or purgeable we can discard the page on wakeup.
8486 */
8487 discard = (!m->vmp_dirty)
8488 || (VM_PURGABLE_VOLATILE == object->purgable)
8489 || (VM_PURGABLE_EMPTY == object->purgable);
8490
8491
8492 if (discard == FALSE) {
8493 if (!preflight) {
8494 hibernate_stats.cd_found_dirty++;
8495 }
8496 } else if (m->vmp_xpmapped && m->vmp_reference && !object->internal) {
8497 if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) {
8498 if (!preflight) {
8499 hibernate_stats.cd_found_xpmapped++;
8500 }
8501 discard = FALSE;
8502 } else {
8503 if (!preflight) {
8504 hibernate_stats.cd_skipped_xpmapped++;
8505 }
8506 }
8507 }
8508 }while (FALSE);
8509
8510 if (object) {
8511 vm_object_unlock(object);
8512 }
8513
8514 return discard;
8515 }
8516
8517
8518 static void
hibernate_discard_page(vm_page_t m)8519 hibernate_discard_page(vm_page_t m)
8520 {
8521 vm_object_t m_object;
8522
8523 if (m->vmp_absent || m->vmp_unusual || VMP_ERROR_GET(m)) {
8524 /*
8525 * If it's unusual in anyway, ignore
8526 */
8527 return;
8528 }
8529
8530 m_object = VM_PAGE_OBJECT(m);
8531
8532 #if MACH_ASSERT || DEBUG
8533 if (!vm_object_lock_try(m_object)) {
8534 panic("hibernate_discard_page(%p) !vm_object_lock_try", m);
8535 }
8536 #else
8537 /* No need to lock page queue for token delete, hibernate_vm_unlock()
8538 * makes sure these locks are uncontended before sleep */
8539 #endif /* MACH_ASSERT || DEBUG */
8540
8541 if (m->vmp_pmapped == TRUE) {
8542 __unused int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
8543 }
8544
8545 if (m->vmp_laundry) {
8546 panic("hibernate_discard_page(%p) laundry", m);
8547 }
8548 if (vm_page_is_private(m)) {
8549 panic("hibernate_discard_page(%p) private", m);
8550 }
8551 if (vm_page_is_fictitious(m)) {
8552 panic("hibernate_discard_page(%p) fictitious", m);
8553 }
8554
8555 if (VM_PURGABLE_VOLATILE == m_object->purgable) {
8556 /* object should be on a queue */
8557 assert((m_object->objq.next != NULL) && (m_object->objq.prev != NULL));
8558 purgeable_q_t old_queue = vm_purgeable_object_remove(m_object);
8559 assert(old_queue);
8560 if (m_object->purgeable_when_ripe) {
8561 vm_purgeable_token_delete_first(old_queue);
8562 }
8563 vm_object_lock_assert_exclusive(m_object);
8564 VM_OBJECT_SET_PURGABLE(m_object, VM_PURGABLE_EMPTY);
8565
8566 /*
8567 * Purgeable ledgers: pages of VOLATILE and EMPTY objects are
8568 * accounted in the "volatile" ledger, so no change here.
8569 * We have to update vm_page_purgeable_count, though, since we're
8570 * effectively purging this object.
8571 */
8572 unsigned int delta;
8573 assert(m_object->resident_page_count >= m_object->wired_page_count);
8574 delta = (m_object->resident_page_count - m_object->wired_page_count);
8575 assert(vm_page_purgeable_count >= delta);
8576 assert(delta > 0);
8577 OSAddAtomic(-delta, (SInt32 *)&vm_page_purgeable_count);
8578 }
8579
8580 vm_page_free(m);
8581
8582 #if MACH_ASSERT || DEBUG
8583 vm_object_unlock(m_object);
8584 #endif /* MACH_ASSERT || DEBUG */
8585 }
8586
8587 /*
8588 * Grab locks for hibernate_page_list_setall()
8589 */
8590 void
hibernate_vm_lock_queues(void)8591 hibernate_vm_lock_queues(void)
8592 {
8593 vm_object_lock(compressor_object);
8594 vm_page_lock_queues();
8595 vm_free_page_lock();
8596 lck_mtx_lock(&vm_purgeable_queue_lock);
8597
8598 if (vm_page_local_q) {
8599 zpercpu_foreach(lq, vm_page_local_q) {
8600 VPL_LOCK(&lq->vpl_lock);
8601 }
8602 }
8603 }
8604
8605 void
hibernate_vm_unlock_queues(void)8606 hibernate_vm_unlock_queues(void)
8607 {
8608 if (vm_page_local_q) {
8609 zpercpu_foreach(lq, vm_page_local_q) {
8610 VPL_UNLOCK(&lq->vpl_lock);
8611 }
8612 }
8613 lck_mtx_unlock(&vm_purgeable_queue_lock);
8614 vm_free_page_unlock();
8615 vm_page_unlock_queues();
8616 vm_object_unlock(compressor_object);
8617 }
8618
8619 #if CONFIG_SPTM
8620 static bool
hibernate_sptm_should_force_page_to_wired_pagelist(vm_page_t vmp)8621 hibernate_sptm_should_force_page_to_wired_pagelist(vm_page_t vmp)
8622 {
8623 const sptm_paddr_t paddr = ptoa_64(VM_PAGE_GET_PHYS_PAGE(vmp));
8624 const sptm_frame_type_t frame_type = sptm_get_frame_type(paddr);
8625 const vm_object_t vmp_objp = VM_PAGE_OBJECT(vmp);
8626
8627 return frame_type == XNU_USER_JIT || frame_type == XNU_USER_DEBUG ||
8628 (frame_type == XNU_USER_EXEC && vmp_objp->internal == TRUE);
8629 }
8630 #endif
8631
8632 /*
8633 * Bits zero in the bitmaps => page needs to be saved. All pages default to be saved,
8634 * pages known to VM to not need saving are subtracted.
8635 * Wired pages to be saved are present in page_list_wired, pageable in page_list.
8636 */
8637
8638 void
hibernate_page_list_setall(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired,hibernate_page_list_t * page_list_pal,boolean_t preflight,boolean_t will_discard,uint32_t * pagesOut)8639 hibernate_page_list_setall(hibernate_page_list_t * page_list,
8640 hibernate_page_list_t * page_list_wired,
8641 hibernate_page_list_t * page_list_pal,
8642 boolean_t preflight,
8643 boolean_t will_discard,
8644 uint32_t * pagesOut)
8645 {
8646 uint64_t start, end, nsec;
8647 vm_page_t m;
8648 vm_page_t next;
8649 __block uint32_t pages = page_list->page_count;
8650 __block uint32_t count_wire = pages;
8651 uint32_t count_anonymous = 0, count_throttled = 0, count_compressor = 0;
8652 uint32_t count_inactive = 0, count_active = 0, count_speculative = 0, count_cleaned = 0;
8653 uint32_t count_discard_active = 0;
8654 uint32_t count_discard_inactive = 0;
8655 uint32_t count_retired = 0;
8656 uint32_t count_discard_cleaned = 0;
8657 uint32_t count_discard_purgeable = 0;
8658 uint32_t count_discard_speculative = 0;
8659 uint32_t count_discard_vm_struct_pages = 0;
8660 uint32_t bank;
8661 hibernate_bitmap_t * bitmap;
8662 hibernate_bitmap_t * bitmap_wired;
8663 boolean_t discard_all;
8664 boolean_t discard = FALSE;
8665
8666 HIBLOG("hibernate_page_list_setall(preflight %d) start\n", preflight);
8667
8668 if (preflight) {
8669 page_list = NULL;
8670 page_list_wired = NULL;
8671 page_list_pal = NULL;
8672 discard_all = FALSE;
8673 } else {
8674 discard_all = will_discard;
8675 }
8676
8677 #if MACH_ASSERT || DEBUG
8678 if (!preflight) {
8679 assert(hibernate_vm_locks_are_safe());
8680 vm_page_lock_queues();
8681 if (vm_page_local_q) {
8682 zpercpu_foreach(lq, vm_page_local_q) {
8683 VPL_LOCK(&lq->vpl_lock);
8684 }
8685 }
8686 }
8687 #endif /* MACH_ASSERT || DEBUG */
8688
8689
8690 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_START, count_wire, 0, 0, 0, 0);
8691
8692 clock_get_uptime(&start);
8693
8694 if (!preflight) {
8695 hibernate_page_list_zero(page_list);
8696 hibernate_page_list_zero(page_list_wired);
8697 hibernate_page_list_zero(page_list_pal);
8698
8699 hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count;
8700 hibernate_stats.cd_pages = pages;
8701 }
8702
8703 if (vm_page_local_q) {
8704 zpercpu_foreach_cpu(lid) {
8705 vm_page_reactivate_local(lid, TRUE, !preflight);
8706 }
8707 }
8708
8709 if (preflight) {
8710 vm_object_lock(compressor_object);
8711 vm_page_lock_queues();
8712 vm_free_page_lock();
8713 }
8714
8715 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
8716
8717 hibernation_vmqueues_inspection = TRUE;
8718
8719 __auto_type hib_free_boilerplate = ^(vm_page_t page) {
8720 assert((page->vmp_q_state == VM_PAGE_ON_FREE_Q) ||
8721 #if XNU_VM_HAS_LOPAGE
8722 (page->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q) ||
8723 #endif /* XNU_VM_HAS_LOPAGE */
8724 (page->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q));
8725
8726 pages--;
8727 count_wire--;
8728
8729 if (!preflight) {
8730 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(page));
8731 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(page));
8732
8733 hibernate_stats.cd_total_free++;
8734
8735 if (page->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q) {
8736 hibernate_stats.cd_local_free++;
8737 }
8738 }
8739 };
8740
8741 if (!preflight) {
8742 percpu_foreach(free_pages_head, free_pages) {
8743 _vm_page_list_foreach(m, *free_pages_head) {
8744 assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
8745 hib_free_boilerplate(m);
8746 }
8747 }
8748 }
8749
8750 #if CONFIG_SPTM
8751 if (vm_pages_free_masks()) {
8752 uint32_t bits = vm_pages_free_mask_len() * MAX_COLORS;
8753 bitmap_t *map = vm_pages_free_masks_as_bitmap(0);
8754
8755 for (int bit = bitmap_first(map, bits);
8756 bit >= 0; bit = bitmap_next(map, bit)) {
8757 ppnum_t pnum = pmap_first_pnum + bit;
8758 vm_page_t mem = vm_page_find_canonical(pnum);
8759
8760 hib_free_boilerplate(mem);
8761 }
8762 } else
8763 #endif /* CONFIG_SPTM */
8764 {
8765 vm_page_free_queue_foreach(&vm_page_queue_free, hib_free_boilerplate);
8766 }
8767 #if XNU_VM_HAS_LOPAGE
8768 vm_page_free_queue_foreach(&vm_lopage_queue_free, hib_free_boilerplate);
8769 #endif /* XNU_VM_HAS_LOPAGE */
8770
8771 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
8772 while (m && !vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t)m)) {
8773 assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
8774
8775 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8776 discard = FALSE;
8777 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode)
8778 && hibernate_consider_discard(m, preflight)) {
8779 if (!preflight) {
8780 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8781 }
8782 count_discard_inactive++;
8783 discard = discard_all;
8784 } else {
8785 count_throttled++;
8786 }
8787 count_wire--;
8788 if (!preflight) {
8789 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8790 }
8791
8792 if (discard) {
8793 hibernate_discard_page(m);
8794 }
8795 m = next;
8796 }
8797
8798 m = (vm_page_t)vm_page_queue_first(&vm_page_queue_anonymous);
8799 while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
8800 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
8801 bool force_to_wired_list = false; /* Default to NOT forcing page into the wired page list */
8802 #if CONFIG_SPTM
8803 force_to_wired_list = hibernate_sptm_should_force_page_to_wired_pagelist(m);
8804 #endif
8805 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8806 discard = FALSE;
8807 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
8808 hibernate_consider_discard(m, preflight)) {
8809 if (!preflight) {
8810 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8811 }
8812 if (m->vmp_dirty) {
8813 count_discard_purgeable++;
8814 } else {
8815 count_discard_inactive++;
8816 }
8817 discard = discard_all;
8818 } else {
8819 /*
8820 * If the page must be force-added to the wired page list, prevent it from appearing
8821 * in the unwired page list.
8822 */
8823 if (force_to_wired_list) {
8824 if (!preflight) {
8825 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8826 }
8827 } else {
8828 count_anonymous++;
8829 }
8830 }
8831 /*
8832 * If the page is NOT being forced into the wired page list, remove it from the
8833 * wired page list here.
8834 */
8835 if (!force_to_wired_list) {
8836 count_wire--;
8837 if (!preflight) {
8838 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8839 }
8840 }
8841 if (discard) {
8842 hibernate_discard_page(m);
8843 }
8844 m = next;
8845 }
8846
8847 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
8848 while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
8849 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
8850
8851 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8852 discard = FALSE;
8853 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
8854 hibernate_consider_discard(m, preflight)) {
8855 if (!preflight) {
8856 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8857 }
8858 if (m->vmp_dirty) {
8859 count_discard_purgeable++;
8860 } else {
8861 count_discard_cleaned++;
8862 }
8863 discard = discard_all;
8864 } else {
8865 count_cleaned++;
8866 }
8867 count_wire--;
8868 if (!preflight) {
8869 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8870 }
8871 if (discard) {
8872 hibernate_discard_page(m);
8873 }
8874 m = next;
8875 }
8876
8877 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
8878 while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
8879 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
8880 bool force_to_wired_list = false; /* Default to NOT forcing page into the wired page list */
8881 #if CONFIG_SPTM
8882 force_to_wired_list = hibernate_sptm_should_force_page_to_wired_pagelist(m);
8883 #endif
8884 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8885 discard = FALSE;
8886 if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) &&
8887 hibernate_consider_discard(m, preflight)) {
8888 if (!preflight) {
8889 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8890 }
8891 if (m->vmp_dirty) {
8892 count_discard_purgeable++;
8893 } else {
8894 count_discard_active++;
8895 }
8896 discard = discard_all;
8897 } else {
8898 /*
8899 * If the page must be force-added to the wired page list, prevent it from appearing
8900 * in the unwired page list.
8901 */
8902 if (force_to_wired_list) {
8903 if (!preflight) {
8904 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8905 }
8906 } else {
8907 count_active++;
8908 }
8909 }
8910 /*
8911 * If the page is NOT being forced into the wired page list, remove it from the
8912 * wired page list here.
8913 */
8914 if (!force_to_wired_list) {
8915 count_wire--;
8916 if (!preflight) {
8917 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8918 }
8919 }
8920 if (discard) {
8921 hibernate_discard_page(m);
8922 }
8923 m = next;
8924 }
8925
8926 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
8927 while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
8928 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
8929 bool force_to_wired_list = false; /* Default to NOT forcing page into the wired page list */
8930 #if CONFIG_SPTM
8931 force_to_wired_list = hibernate_sptm_should_force_page_to_wired_pagelist(m);
8932 #endif
8933 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8934 discard = FALSE;
8935 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
8936 hibernate_consider_discard(m, preflight)) {
8937 if (!preflight) {
8938 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8939 }
8940 if (m->vmp_dirty) {
8941 count_discard_purgeable++;
8942 } else {
8943 count_discard_inactive++;
8944 }
8945 discard = discard_all;
8946 } else {
8947 /*
8948 * If the page must be force-added to the wired page list, prevent it from appearing
8949 * in the unwired page list.
8950 */
8951 if (force_to_wired_list) {
8952 if (!preflight) {
8953 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8954 }
8955 } else {
8956 count_inactive++;
8957 }
8958 }
8959 /*
8960 * If the page is NOT being forced into the wired page list, remove it from the
8961 * wired page list here.
8962 */
8963 if (!force_to_wired_list) {
8964 count_wire--;
8965 if (!preflight) {
8966 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8967 }
8968 }
8969 if (discard) {
8970 hibernate_discard_page(m);
8971 }
8972 m = next;
8973 }
8974 /* XXX FBDP TODO: secluded queue */
8975
8976 for (uint32_t i = 0; i <= vm_page_max_speculative_age_q; i++) {
8977 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
8978 while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
8979 assertf(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q,
8980 "Bad page: %p (0x%x:0x%x) on queue %d has state: %d (Discard: %d, Preflight: %d)",
8981 m, m->vmp_pageq.next, m->vmp_pageq.prev, i, m->vmp_q_state, discard, preflight);
8982
8983 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8984 discard = FALSE;
8985 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
8986 hibernate_consider_discard(m, preflight)) {
8987 if (!preflight) {
8988 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8989 }
8990 count_discard_speculative++;
8991 discard = discard_all;
8992 } else {
8993 count_speculative++;
8994 }
8995 count_wire--;
8996 if (!preflight) {
8997 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
8998 }
8999 if (discard) {
9000 hibernate_discard_page(m);
9001 }
9002 m = next;
9003 }
9004 }
9005
9006 vm_page_queue_iterate(&compressor_object->memq, m, vmp_listq) {
9007 assert(m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR);
9008
9009 count_compressor++;
9010 count_wire--;
9011 if (!preflight) {
9012 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
9013 }
9014 }
9015
9016
9017 if (preflight == FALSE && discard_all == TRUE) {
9018 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START);
9019
9020 HIBLOG("hibernate_teardown started\n");
9021 count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired);
9022 HIBLOG("hibernate_teardown completed - discarded %d\n", count_discard_vm_struct_pages);
9023
9024 pages -= count_discard_vm_struct_pages;
9025 count_wire -= count_discard_vm_struct_pages;
9026
9027 hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages;
9028
9029 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_END);
9030 }
9031
9032 if (!preflight) {
9033 // pull wired from hibernate_bitmap
9034 bitmap = &page_list->bank_bitmap[0];
9035 bitmap_wired = &page_list_wired->bank_bitmap[0];
9036 for (bank = 0; bank < page_list->bank_count; bank++) {
9037 for (uint32_t i = 0; i < bitmap->bitmapwords; i++) {
9038 bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
9039 }
9040 bitmap = (hibernate_bitmap_t *)&bitmap->bitmap[bitmap->bitmapwords];
9041 bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
9042 }
9043 }
9044
9045 // machine dependent adjustments
9046 hibernate_page_list_setall_machine(page_list, page_list_wired, preflight, &pages);
9047
9048 if (!preflight) {
9049 hibernate_stats.cd_count_wire = count_wire;
9050 hibernate_stats.cd_discarded = count_discard_active +
9051 count_discard_inactive + count_discard_purgeable +
9052 count_discard_speculative + count_discard_cleaned +
9053 count_discard_vm_struct_pages;
9054 }
9055
9056 clock_get_uptime(&end);
9057 absolutetime_to_nanoseconds(end - start, &nsec);
9058 HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
9059
9060 HIBLOG("pages %d, wire %d, act %d, inact %d, cleaned %d spec %d, "
9061 "zf %d, throt %d, compr %d, xpmapped %d\n"
9062 " %s discard act %d inact %d purgeable %d "
9063 "spec %d cleaned %d retired %d\n",
9064 pages, count_wire, count_active, count_inactive, count_cleaned, count_speculative,
9065 count_anonymous, count_throttled, count_compressor, hibernate_stats.cd_found_xpmapped,
9066 discard_all ? "did" : "could",
9067 count_discard_active, count_discard_inactive, count_discard_purgeable,
9068 count_discard_speculative, count_discard_cleaned, count_retired);
9069
9070 if (hibernate_stats.cd_skipped_xpmapped) {
9071 HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n",
9072 hibernate_stats.cd_skipped_xpmapped);
9073 }
9074
9075 *pagesOut = pages - count_discard_active - count_discard_inactive -
9076 count_discard_purgeable - count_discard_speculative -
9077 count_discard_cleaned - count_retired;
9078
9079 if (preflight && will_discard) {
9080 *pagesOut -= count_compressor + count_throttled +
9081 count_anonymous + count_inactive + count_cleaned +
9082 count_speculative + count_active;
9083
9084 /*
9085 * We try to keep max HIBERNATE_XPMAPPED_LIMIT pages around in the hibernation image
9086 * even if these are clean and so we need to size the hibernation image accordingly.
9087 *
9088 * NB: We have to assume all HIBERNATE_XPMAPPED_LIMIT pages might show up because 'dirty'
9089 * xpmapped pages aren't distinguishable from other 'dirty' pages in preflight. So we might
9090 * only see part of the xpmapped pages if we look at 'cd_found_xpmapped' which solely tracks
9091 * clean xpmapped pages.
9092 *
9093 * Since these pages are all cleaned by the time we are in the post-preflight phase, we might
9094 * see a much larger number in 'cd_found_xpmapped' now than we did in the preflight phase
9095 */
9096 *pagesOut += HIBERNATE_XPMAPPED_LIMIT;
9097 }
9098
9099 hibernation_vmqueues_inspection = FALSE;
9100
9101 #if MACH_ASSERT || DEBUG
9102 if (!preflight) {
9103 if (vm_page_local_q) {
9104 zpercpu_foreach(lq, vm_page_local_q) {
9105 VPL_UNLOCK(&lq->vpl_lock);
9106 }
9107 }
9108 vm_page_unlock_queues();
9109 }
9110 #endif /* MACH_ASSERT || DEBUG */
9111
9112 if (preflight) {
9113 vm_free_page_unlock();
9114 vm_page_unlock_queues();
9115 vm_object_unlock(compressor_object);
9116 }
9117
9118 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_END, count_wire, *pagesOut, 0, 0, 0);
9119 }
9120
9121 void
hibernate_page_list_discard(hibernate_page_list_t * page_list)9122 hibernate_page_list_discard(hibernate_page_list_t * page_list)
9123 {
9124 uint64_t start, end, nsec;
9125 vm_page_t m;
9126 vm_page_t next;
9127 uint32_t i;
9128 uint32_t count_discard_active = 0;
9129 uint32_t count_discard_inactive = 0;
9130 uint32_t count_discard_purgeable = 0;
9131 uint32_t count_discard_cleaned = 0;
9132 uint32_t count_discard_speculative = 0;
9133
9134
9135 #if MACH_ASSERT || DEBUG
9136 vm_page_lock_queues();
9137 if (vm_page_local_q) {
9138 zpercpu_foreach(lq, vm_page_local_q) {
9139 VPL_LOCK(&lq->vpl_lock);
9140 }
9141 }
9142 #endif /* MACH_ASSERT || DEBUG */
9143
9144 clock_get_uptime(&start);
9145
9146 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
9147 while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
9148 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
9149
9150 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
9151 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
9152 if (m->vmp_dirty) {
9153 count_discard_purgeable++;
9154 } else {
9155 count_discard_inactive++;
9156 }
9157 hibernate_discard_page(m);
9158 }
9159 m = next;
9160 }
9161
9162 for (i = 0; i <= vm_page_max_speculative_age_q; i++) {
9163 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
9164 while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
9165 assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
9166
9167 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
9168 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
9169 count_discard_speculative++;
9170 hibernate_discard_page(m);
9171 }
9172 m = next;
9173 }
9174 }
9175
9176 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
9177 while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
9178 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
9179
9180 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
9181 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
9182 if (m->vmp_dirty) {
9183 count_discard_purgeable++;
9184 } else {
9185 count_discard_inactive++;
9186 }
9187 hibernate_discard_page(m);
9188 }
9189 m = next;
9190 }
9191 /* XXX FBDP TODO: secluded queue */
9192
9193 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
9194 while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
9195 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
9196
9197 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
9198 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
9199 if (m->vmp_dirty) {
9200 count_discard_purgeable++;
9201 } else {
9202 count_discard_active++;
9203 }
9204 hibernate_discard_page(m);
9205 }
9206 m = next;
9207 }
9208
9209 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
9210 while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
9211 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
9212
9213 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
9214 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
9215 if (m->vmp_dirty) {
9216 count_discard_purgeable++;
9217 } else {
9218 count_discard_cleaned++;
9219 }
9220 hibernate_discard_page(m);
9221 }
9222 m = next;
9223 }
9224
9225 #if MACH_ASSERT || DEBUG
9226 if (vm_page_local_q) {
9227 zpercpu_foreach(lq, vm_page_local_q) {
9228 VPL_UNLOCK(&lq->vpl_lock);
9229 }
9230 }
9231 vm_page_unlock_queues();
9232 #endif /* MACH_ASSERT || DEBUG */
9233
9234 clock_get_uptime(&end);
9235 absolutetime_to_nanoseconds(end - start, &nsec);
9236 HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d cleaned %d\n",
9237 nsec / 1000000ULL,
9238 count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
9239 }
9240
9241 boolean_t hibernate_paddr_map_inited = FALSE;
9242 unsigned int hibernate_teardown_last_valid_compact_indx = -1;
9243 vm_page_t hibernate_rebuild_hash_list = NULL;
9244
9245 unsigned int hibernate_teardown_found_tabled_pages = 0;
9246 unsigned int hibernate_teardown_found_created_pages = 0;
9247 unsigned int hibernate_teardown_found_free_pages = 0;
9248 unsigned int hibernate_teardown_vm_page_free_count;
9249
9250
9251 struct ppnum_mapping {
9252 struct ppnum_mapping *ppnm_next;
9253 ppnum_t ppnm_base_paddr;
9254 unsigned int ppnm_sindx;
9255 unsigned int ppnm_eindx;
9256 };
9257
9258 struct ppnum_mapping *ppnm_head;
9259 struct ppnum_mapping *ppnm_last_found = NULL;
9260
9261
9262 void
hibernate_create_paddr_map(void)9263 hibernate_create_paddr_map(void)
9264 {
9265 unsigned int i;
9266 ppnum_t next_ppnum_in_run = 0;
9267 struct ppnum_mapping *ppnm = NULL;
9268
9269 if (hibernate_paddr_map_inited == FALSE) {
9270 for (i = 0; i < vm_pages_count; i++) {
9271 if (ppnm) {
9272 ppnm->ppnm_eindx = i;
9273 }
9274
9275 if (ppnm == NULL || VM_PAGE_GET_PHYS_PAGE(vm_page_get(i)) != next_ppnum_in_run) {
9276 ppnm = zalloc_permanent_type(struct ppnum_mapping);
9277
9278 ppnm->ppnm_next = ppnm_head;
9279 ppnm_head = ppnm;
9280
9281 ppnm->ppnm_sindx = i;
9282 ppnm->ppnm_base_paddr = VM_PAGE_GET_PHYS_PAGE(vm_page_get(i));
9283 }
9284 next_ppnum_in_run = VM_PAGE_GET_PHYS_PAGE(vm_page_get(i)) + 1;
9285 }
9286 ppnm->ppnm_eindx = vm_pages_count;
9287
9288 hibernate_paddr_map_inited = TRUE;
9289 }
9290 }
9291
9292 static ppnum_t
hibernate_lookup_paddr(unsigned int indx)9293 hibernate_lookup_paddr(unsigned int indx)
9294 {
9295 struct ppnum_mapping *ppnm = NULL;
9296
9297 ppnm = ppnm_last_found;
9298
9299 if (ppnm) {
9300 if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
9301 goto done;
9302 }
9303 }
9304 for (ppnm = ppnm_head; ppnm; ppnm = ppnm->ppnm_next) {
9305 if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
9306 ppnm_last_found = ppnm;
9307 break;
9308 }
9309 }
9310 if (ppnm == NULL) {
9311 panic("hibernate_lookup_paddr of %d failed", indx);
9312 }
9313 done:
9314 return ppnm->ppnm_base_paddr + (indx - ppnm->ppnm_sindx);
9315 }
9316
9317
9318 static uint32_t
hibernate_mark_as_unneeded(addr64_t saddr,addr64_t eaddr,hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired)9319 hibernate_mark_as_unneeded(addr64_t saddr, addr64_t eaddr, hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
9320 {
9321 addr64_t saddr_aligned;
9322 addr64_t eaddr_aligned;
9323 addr64_t addr;
9324 ppnum_t paddr;
9325 unsigned int mark_as_unneeded_pages = 0;
9326
9327 saddr_aligned = (saddr + PAGE_MASK_64) & ~PAGE_MASK_64;
9328 eaddr_aligned = eaddr & ~PAGE_MASK_64;
9329
9330 for (addr = saddr_aligned; addr < eaddr_aligned; addr += PAGE_SIZE_64) {
9331 paddr = pmap_find_phys(kernel_pmap, addr);
9332
9333 assert(paddr);
9334
9335 hibernate_page_bitset(page_list, TRUE, paddr);
9336 hibernate_page_bitset(page_list_wired, TRUE, paddr);
9337
9338 mark_as_unneeded_pages++;
9339 }
9340 return mark_as_unneeded_pages;
9341 }
9342
9343
9344 static void
hibernate_hash_insert_page(vm_page_t mem)9345 hibernate_hash_insert_page(vm_page_t mem)
9346 {
9347 vm_page_bucket_t *bucket;
9348 int hash_id;
9349 vm_object_t m_object;
9350
9351 m_object = VM_PAGE_OBJECT(mem);
9352
9353 assert(mem->vmp_hashed);
9354 assert(m_object);
9355 assert(mem->vmp_offset != (vm_object_offset_t) -1);
9356
9357 /*
9358 * Insert it into the object_object/offset hash table
9359 */
9360 hash_id = vm_page_hash(m_object, mem->vmp_offset);
9361 bucket = &vm_page_buckets[hash_id];
9362
9363 mem->vmp_next_m = bucket->page_list;
9364 bucket->page_list = VM_PAGE_PACK_PTR(mem);
9365 }
9366
9367
9368 static void
hibernate_free_range_flush(vm_page_list_t * list)9369 hibernate_free_range_flush(vm_page_list_t *list)
9370 {
9371 vm_page_free_queue_enter_list(*list, VMP_RELEASE_HIBERNATE);
9372 *list = (vm_page_list_t){ };
9373 }
9374
9375 static void
hibernate_free_range(vm_page_list_t * list,int sindx,int eindx)9376 hibernate_free_range(vm_page_list_t *list, int sindx, int eindx)
9377 {
9378 for (; sindx < eindx; sindx++) {
9379 vm_page_t mem = vm_page_get(sindx);
9380 ppnum_t pnum = hibernate_lookup_paddr(sindx);
9381
9382 vm_page_init(mem, pnum);
9383 vm_page_list_push(list, mem);
9384
9385 /* Max batch size of these lists is 255 due to vmp_free_list_result_t */
9386 if (list->vmpl_count >= UINT8_MAX) {
9387 hibernate_free_range_flush(list);
9388 }
9389 }
9390 }
9391
9392 void
hibernate_rebuild_vm_structs(void)9393 hibernate_rebuild_vm_structs(void)
9394 {
9395 int cindx, sindx, eindx;
9396 vm_page_list_t list = { };
9397 vm_page_t mem, tmem, mem_next;
9398 AbsoluteTime startTime, endTime;
9399 uint64_t nsec;
9400
9401 if (!hibernate_rebuild_needed) {
9402 return;
9403 }
9404
9405 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START);
9406 HIBLOG("hibernate_rebuild started\n");
9407
9408 clock_get_uptime(&startTime);
9409
9410 pal_hib_rebuild_pmap_structs();
9411
9412 bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t));
9413 eindx = vm_pages_count;
9414
9415 /*
9416 * Mark all the vm_pages[] that have not been initialized yet as being
9417 * transient. This is needed to ensure that buddy page search is corrrect.
9418 * Without this random data in these vm_pages[] can trip the buddy search
9419 */
9420 for (int i = hibernate_teardown_last_valid_compact_indx + 1; i < eindx; ++i) {
9421 vm_page_get(i)->vmp_q_state = VM_PAGE_NOT_ON_Q;
9422 }
9423
9424 for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) {
9425 mem = vm_page_get(cindx);
9426 assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
9427 /*
9428 * hibernate_teardown_vm_structs leaves the location where
9429 * this vm_page_t must be located in "next".
9430 */
9431 tmem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
9432 mem->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
9433 assert(tmem >= mem);
9434
9435 sindx = (int)(tmem - vm_page_get(0));
9436
9437 if (mem != tmem) {
9438 /*
9439 * this vm_page_t was moved by hibernate_teardown_vm_structs,
9440 * so move it back to its real location
9441 */
9442 *tmem = *mem;
9443 mem = tmem;
9444 }
9445 if (mem->vmp_hashed) {
9446 hibernate_hash_insert_page(mem);
9447 }
9448 /*
9449 * the 'hole' between this vm_page_t and the previous
9450 * vm_page_t we moved needs to be initialized as
9451 * a range of free vm_page_t's
9452 */
9453 hibernate_free_range(&list, sindx + 1, eindx);
9454
9455 eindx = sindx;
9456 }
9457 hibernate_free_range(&list, 0, sindx);
9458 hibernate_free_range_flush(&list);
9459
9460 VM_CHECK_MEMORYSTATUS;
9461
9462 assert(vm_page_free_count == hibernate_teardown_vm_page_free_count);
9463
9464 /*
9465 * process the list of vm_page_t's that were entered in the hash,
9466 * but were not located in the vm_pages arrary... these are
9467 * vm_page_t's that were created on the fly (i.e. fictitious)
9468 */
9469 for (mem = hibernate_rebuild_hash_list; mem; mem = mem_next) {
9470 mem_next = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
9471
9472 mem->vmp_next_m = 0;
9473 hibernate_hash_insert_page(mem);
9474 }
9475 hibernate_rebuild_hash_list = NULL;
9476
9477 clock_get_uptime(&endTime);
9478 SUB_ABSOLUTETIME(&endTime, &startTime);
9479 absolutetime_to_nanoseconds(endTime, &nsec);
9480
9481 HIBLOG("hibernate_rebuild completed - took %qd msecs\n", nsec / 1000000ULL);
9482
9483 hibernate_rebuild_needed = false;
9484
9485 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END);
9486 }
9487
9488 static uint32_t
hibernate_teardown_vm_structs(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired)9489 hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
9490 {
9491 unsigned int compact_target_indx;
9492 unsigned int mark_as_unneeded_pages = 0;
9493 unsigned int unneeded_vm_page_bucket_pages = 0;
9494 unsigned int unneeded_vm_pages_pages = 0;
9495 unsigned int unneeded_pmap_pages = 0;
9496 addr64_t start_of_unneeded = 0;
9497 addr64_t end_of_unneeded = 0;
9498
9499
9500 if (hibernate_should_abort()) {
9501 return 0;
9502 }
9503
9504 hibernate_rebuild_needed = true;
9505
9506 HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, "
9507 "active_pages %d, inactive_pages %d, speculative_pages %d, "
9508 "cleaned_pages %d, compressor_pages %d\n",
9509 vm_page_wire_count, vm_page_free_count,
9510 vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count,
9511 vm_page_cleaned_count, compressor_object->resident_page_count);
9512
9513 for (uint32_t i = 0; i < vm_page_bucket_count; i++) {
9514 vm_page_bucket_t *bucket = &vm_page_buckets[i];
9515 vm_page_t mem, mem_next;
9516
9517 for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)); mem != VM_PAGE_NULL; mem = mem_next) {
9518 assert(mem->vmp_hashed);
9519
9520 mem_next = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
9521
9522 if (!vm_page_in_array(mem)) {
9523 mem->vmp_next_m = VM_PAGE_PACK_PTR(hibernate_rebuild_hash_list);
9524 hibernate_rebuild_hash_list = mem;
9525 }
9526 }
9527 }
9528 unneeded_vm_page_bucket_pages = hibernate_mark_as_unneeded((addr64_t)&vm_page_buckets[0],
9529 (addr64_t)&vm_page_buckets[vm_page_bucket_count], page_list, page_list_wired);
9530 mark_as_unneeded_pages += unneeded_vm_page_bucket_pages;
9531
9532 hibernate_teardown_vm_page_free_count = vm_page_free_count;
9533
9534 compact_target_indx = 0;
9535
9536 vm_free_page_lock();
9537
9538 for (uint32_t i = 0; i < vm_pages_count; i++) {
9539 vm_page_t mem = vm_page_get(i);
9540 ppnum_t pnum = VM_PAGE_GET_PHYS_PAGE(mem);
9541 vm_memory_class_t class = vm_page_get_memory_class(mem, pnum);
9542
9543 if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
9544 vm_page_free_queue_remove(class, mem, pnum,
9545 VM_PAGE_ON_FREE_Q);
9546 hibernate_teardown_found_free_pages++;
9547
9548 if (vm_page_get(compact_target_indx)->vmp_q_state != VM_PAGE_ON_FREE_Q) {
9549 compact_target_indx = i;
9550 }
9551 } else {
9552 /*
9553 * record this vm_page_t's original location
9554 * we need this even if it doesn't get moved
9555 * as an indicator to the rebuild function that
9556 * we don't have to move it
9557 */
9558 mem->vmp_next_m = VM_PAGE_PACK_PTR(mem);
9559
9560 if (vm_page_get(compact_target_indx)->vmp_q_state == VM_PAGE_ON_FREE_Q) {
9561 /*
9562 * we've got a hole to fill, so
9563 * move this vm_page_t to it's new home
9564 */
9565 *vm_page_get(compact_target_indx) = *mem;
9566 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
9567
9568 hibernate_teardown_last_valid_compact_indx = compact_target_indx;
9569 compact_target_indx++;
9570 } else {
9571 hibernate_teardown_last_valid_compact_indx = i;
9572 }
9573 }
9574 }
9575
9576 vm_free_page_unlock();
9577
9578 unneeded_vm_pages_pages = hibernate_mark_as_unneeded(
9579 (addr64_t)vm_page_get(hibernate_teardown_last_valid_compact_indx + 1),
9580 (addr64_t)vm_page_get(vm_pages_count - 1),
9581 page_list, page_list_wired);
9582 mark_as_unneeded_pages += unneeded_vm_pages_pages;
9583
9584 pal_hib_teardown_pmap_structs(&start_of_unneeded, &end_of_unneeded);
9585
9586 if (start_of_unneeded) {
9587 unneeded_pmap_pages = hibernate_mark_as_unneeded(start_of_unneeded,
9588 end_of_unneeded, page_list, page_list_wired);
9589 mark_as_unneeded_pages += unneeded_pmap_pages;
9590 }
9591 HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n",
9592 unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages);
9593
9594 return mark_as_unneeded_pages;
9595 }
9596
9597 #endif /* HIBERNATION */
9598
9599 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9600
9601 #include <mach_vm_debug.h>
9602 #if MACH_VM_DEBUG
9603
9604 #include <mach_debug/hash_info.h>
9605 #include <vm/vm_debug_internal.h>
9606
9607 /*
9608 * Routine: vm_page_info
9609 * Purpose:
9610 * Return information about the global VP table.
9611 * Fills the buffer with as much information as possible
9612 * and returns the desired size of the buffer.
9613 * Conditions:
9614 * Nothing locked. The caller should provide
9615 * possibly-pageable memory.
9616 */
9617
9618 unsigned int
vm_page_info(hash_info_bucket_t * info,unsigned int count)9619 vm_page_info(
9620 hash_info_bucket_t *info,
9621 unsigned int count)
9622 {
9623 unsigned int i;
9624 lck_ticket_t *bucket_lock;
9625
9626 if (vm_page_bucket_count < count) {
9627 count = vm_page_bucket_count;
9628 }
9629
9630 for (i = 0; i < count; i++) {
9631 vm_page_bucket_t *bucket = &vm_page_buckets[i];
9632 unsigned int bucket_count = 0;
9633 vm_page_t m;
9634
9635 bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
9636 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
9637
9638 for (m = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
9639 m != VM_PAGE_NULL;
9640 m = (vm_page_t)(VM_PAGE_UNPACK_PTR(m->vmp_next_m))) {
9641 bucket_count++;
9642 }
9643
9644 lck_ticket_unlock(bucket_lock);
9645
9646 /* don't touch pageable memory while holding locks */
9647 info[i].hib_count = bucket_count;
9648 }
9649
9650 return vm_page_bucket_count;
9651 }
9652 #endif /* MACH_VM_DEBUG */
9653
9654 #if VM_PAGE_BUCKETS_CHECK
9655 void
vm_page_buckets_check(void)9656 vm_page_buckets_check(void)
9657 {
9658 unsigned int i;
9659 vm_page_t p;
9660 unsigned int p_hash;
9661 vm_page_bucket_t *bucket;
9662 lck_ticket_t *bucket_lock;
9663
9664 if (!vm_page_buckets_check_ready) {
9665 return;
9666 }
9667
9668 #if HIBERNATION
9669 if (hibernate_rebuild_needed ||
9670 hibernate_rebuild_hash_list) {
9671 panic("BUCKET_CHECK: hibernation in progress: "
9672 "rebuild_needed=%d rebuild_hash_list=%p\n",
9673 hibernate_rebuild_needed,
9674 hibernate_rebuild_hash_list);
9675 }
9676 #endif /* HIBERNATION */
9677
9678 #if VM_PAGE_FAKE_BUCKETS
9679 char *cp;
9680 for (cp = (char *) vm_page_fake_buckets_start;
9681 cp < (char *) vm_page_fake_buckets_end;
9682 cp++) {
9683 if (*cp != 0x5a) {
9684 panic("BUCKET_CHECK: corruption at %p in fake buckets "
9685 "[0x%llx:0x%llx]\n",
9686 cp,
9687 (uint64_t) vm_page_fake_buckets_start,
9688 (uint64_t) vm_page_fake_buckets_end);
9689 }
9690 }
9691 #endif /* VM_PAGE_FAKE_BUCKETS */
9692
9693 for (i = 0; i < vm_page_bucket_count; i++) {
9694 vm_object_t p_object;
9695
9696 bucket = &vm_page_buckets[i];
9697 if (!bucket->page_list) {
9698 continue;
9699 }
9700
9701 bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
9702 lck_ticket_lock(bucket_lock, &vm_page_lck_grp_bucket);
9703 p = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
9704
9705 while (p != VM_PAGE_NULL) {
9706 p_object = VM_PAGE_OBJECT(p);
9707
9708 if (!p->vmp_hashed) {
9709 panic("BUCKET_CHECK: page %p (%p,0x%llx) "
9710 "hash %d in bucket %d at %p "
9711 "is not hashed\n",
9712 p, p_object, p->vmp_offset,
9713 p_hash, i, bucket);
9714 }
9715 p_hash = vm_page_hash(p_object, p->vmp_offset);
9716 if (p_hash != i) {
9717 panic("BUCKET_CHECK: corruption in bucket %d "
9718 "at %p: page %p object %p offset 0x%llx "
9719 "hash %d\n",
9720 i, bucket, p, p_object, p->vmp_offset,
9721 p_hash);
9722 }
9723 p = (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m));
9724 }
9725 lck_ticket_unlock(bucket_lock);
9726 }
9727
9728 // printf("BUCKET_CHECK: checked buckets\n");
9729 }
9730 #endif /* VM_PAGE_BUCKETS_CHECK */
9731
9732 /*
9733 * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
9734 * local queues if they exist... its the only spot in the system where we add pages
9735 * to those queues... once on those queues, those pages can only move to one of the
9736 * global page queues or the free queues... they NEVER move from local q to local q.
9737 * the 'local' state is stable when vm_page_queues_remove is called since we're behind
9738 * the global vm_page_queue_lock at this point... we still need to take the local lock
9739 * in case this operation is being run on a different CPU then the local queue's identity,
9740 * but we don't have to worry about the page moving to a global queue or becoming wired
9741 * while we're grabbing the local lock since those operations would require the global
9742 * vm_page_queue_lock to be held, and we already own it.
9743 *
9744 * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
9745 * 'wired' and local are ALWAYS mutually exclusive conditions.
9746 */
9747
9748 void
vm_page_queues_remove(vm_page_t mem,boolean_t remove_from_specialq)9749 vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_specialq)
9750 {
9751 boolean_t was_pageable = TRUE;
9752 vm_object_t m_object;
9753
9754 m_object = VM_PAGE_OBJECT(mem);
9755
9756 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
9757
9758 if (mem->vmp_q_state == VM_PAGE_NOT_ON_Q) {
9759 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
9760 if (remove_from_specialq == TRUE) {
9761 vm_page_remove_from_specialq(mem);
9762 }
9763 /*if (mem->vmp_on_specialq != VM_PAGE_SPECIAL_Q_EMPTY) {
9764 * assert(mem->vmp_specialq.next != 0);
9765 * assert(mem->vmp_specialq.prev != 0);
9766 * } else {*/
9767 if (mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY) {
9768 assert(mem->vmp_specialq.next == 0);
9769 assert(mem->vmp_specialq.prev == 0);
9770 }
9771 return;
9772 }
9773
9774 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
9775 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
9776 assert(mem->vmp_specialq.next == 0 &&
9777 mem->vmp_specialq.prev == 0 &&
9778 mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
9779 return;
9780 }
9781 if (mem->vmp_q_state == VM_PAGE_IS_WIRED) {
9782 /*
9783 * might put these guys on a list for debugging purposes
9784 * if we do, we'll need to remove this assert
9785 */
9786 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
9787 assert(mem->vmp_specialq.next == 0 &&
9788 mem->vmp_specialq.prev == 0);
9789 /*
9790 * Recall that vmp_on_specialq also means a request to put
9791 * it on the special Q. So we don't want to reset that bit
9792 * just because a wiring request came in. We might want to
9793 * put it on the special queue post-unwiring.
9794 *
9795 * &&
9796 * mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
9797 */
9798 return;
9799 }
9800
9801 assert(m_object != compressor_object);
9802 assert(!is_kernel_object(m_object));
9803 assert(!vm_page_is_fictitious(mem));
9804
9805 switch (mem->vmp_q_state) {
9806 case VM_PAGE_ON_ACTIVE_LOCAL_Q:
9807 {
9808 struct vpl *lq;
9809
9810 lq = zpercpu_get_cpu(vm_page_local_q, mem->vmp_local_id);
9811 VPL_LOCK(&lq->vpl_lock);
9812 vm_page_queue_remove(&lq->vpl_queue, mem, vmp_pageq);
9813 mem->vmp_local_id = 0;
9814 lq->vpl_count--;
9815 if (m_object->internal) {
9816 lq->vpl_internal_count--;
9817 } else {
9818 lq->vpl_external_count--;
9819 }
9820 VPL_UNLOCK(&lq->vpl_lock);
9821 was_pageable = FALSE;
9822 break;
9823 }
9824 case VM_PAGE_ON_ACTIVE_Q:
9825 {
9826 vm_page_queue_remove(&vm_page_queue_active, mem, vmp_pageq);
9827 vm_page_active_count--;
9828 break;
9829 }
9830
9831 case VM_PAGE_ON_INACTIVE_INTERNAL_Q:
9832 {
9833 assert(m_object->internal == TRUE);
9834
9835 vm_page_inactive_count--;
9836 vm_page_queue_remove(&vm_page_queue_anonymous, mem, vmp_pageq);
9837 vm_page_anonymous_count--;
9838
9839 vm_purgeable_q_advance_all();
9840 vm_page_balance_inactive(3);
9841 break;
9842 }
9843
9844 case VM_PAGE_ON_INACTIVE_EXTERNAL_Q:
9845 {
9846 assert(m_object->internal == FALSE);
9847
9848 vm_page_inactive_count--;
9849 vm_page_queue_remove(&vm_page_queue_inactive, mem, vmp_pageq);
9850 vm_purgeable_q_advance_all();
9851 vm_page_balance_inactive(3);
9852 break;
9853 }
9854
9855 case VM_PAGE_ON_INACTIVE_CLEANED_Q:
9856 {
9857 assert(m_object->internal == FALSE);
9858
9859 vm_page_inactive_count--;
9860 vm_page_queue_remove(&vm_page_queue_cleaned, mem, vmp_pageq);
9861 vm_page_cleaned_count--;
9862 vm_page_balance_inactive(3);
9863 break;
9864 }
9865
9866 case VM_PAGE_ON_THROTTLED_Q:
9867 {
9868 assert(m_object->internal == TRUE);
9869
9870 vm_page_queue_remove(&vm_page_queue_throttled, mem, vmp_pageq);
9871 vm_page_throttled_count--;
9872 was_pageable = FALSE;
9873 break;
9874 }
9875
9876 case VM_PAGE_ON_SPECULATIVE_Q:
9877 {
9878 assert(m_object->internal == FALSE);
9879
9880 vm_page_remque(&mem->vmp_pageq);
9881 vm_page_speculative_count--;
9882 vm_page_balance_inactive(3);
9883 break;
9884 }
9885
9886 #if CONFIG_SECLUDED_MEMORY
9887 case VM_PAGE_ON_SECLUDED_Q:
9888 {
9889 vm_page_queue_remove(&vm_page_queue_secluded, mem, vmp_pageq);
9890 vm_page_secluded_count--;
9891 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
9892 if (m_object == VM_OBJECT_NULL) {
9893 vm_page_secluded_count_free--;
9894 was_pageable = FALSE;
9895 } else {
9896 assert(!m_object->internal);
9897 vm_page_secluded_count_inuse--;
9898 was_pageable = FALSE;
9899 // was_pageable = TRUE;
9900 }
9901 break;
9902 }
9903 #endif /* CONFIG_SECLUDED_MEMORY */
9904
9905 default:
9906 {
9907 /*
9908 * if (mem->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)
9909 * NOTE: vm_page_queues_remove does not deal with removing pages from the pageout queue...
9910 * the caller is responsible for determing if the page is on that queue, and if so, must
9911 * either first remove it (it needs both the page queues lock and the object lock to do
9912 * this via vm_pageout_steal_laundry), or avoid the call to vm_page_queues_remove
9913 *
9914 * we also don't expect to encounter VM_PAGE_ON_FREE_Q, VM_PAGE_ON_FREE_LOCAL_Q, VM_PAGE_ON_FREE_LOPAGE_Q
9915 * or any of the undefined states
9916 */
9917 panic("vm_page_queues_remove - bad page q_state (%p, %d)", mem, mem->vmp_q_state);
9918 break;
9919 }
9920 }
9921 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
9922 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
9923
9924 if (remove_from_specialq == TRUE) {
9925 vm_page_remove_from_specialq(mem);
9926 }
9927 if (was_pageable) {
9928 if (m_object->internal) {
9929 vm_page_pageable_internal_count--;
9930 } else {
9931 vm_page_pageable_external_count--;
9932 }
9933 }
9934 }
9935
9936 void
vm_page_remove_internal(vm_page_t page)9937 vm_page_remove_internal(vm_page_t page)
9938 {
9939 vm_object_t __object = VM_PAGE_OBJECT(page);
9940 if (page == __object->memq_hint) {
9941 vm_page_t __new_hint;
9942 vm_page_queue_entry_t __qe;
9943 __qe = (vm_page_queue_entry_t)vm_page_queue_next(&page->vmp_listq);
9944 if (vm_page_queue_end(&__object->memq, __qe)) {
9945 __qe = (vm_page_queue_entry_t)vm_page_queue_prev(&page->vmp_listq);
9946 if (vm_page_queue_end(&__object->memq, __qe)) {
9947 __qe = NULL;
9948 }
9949 }
9950 __new_hint = (vm_page_t)((uintptr_t) __qe);
9951 __object->memq_hint = __new_hint;
9952 }
9953 vm_page_queue_remove(&__object->memq, page, vmp_listq);
9954 #if CONFIG_SECLUDED_MEMORY
9955 if (__object->eligible_for_secluded) {
9956 vm_page_secluded.eligible_for_secluded--;
9957 }
9958 #endif /* CONFIG_SECLUDED_MEMORY */
9959 }
9960
9961 void
vm_page_enqueue_inactive(vm_page_t mem,boolean_t first)9962 vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
9963 {
9964 vm_object_t m_object;
9965
9966 m_object = VM_PAGE_OBJECT(mem);
9967
9968 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
9969 assert(!vm_page_is_fictitious(mem));
9970 assert(!mem->vmp_laundry);
9971 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
9972 vm_page_check_pageable_safe(mem);
9973
9974 if (m_object->internal) {
9975 mem->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
9976
9977 if (first == TRUE) {
9978 vm_page_queue_enter_first(&vm_page_queue_anonymous, mem, vmp_pageq);
9979 } else {
9980 vm_page_queue_enter(&vm_page_queue_anonymous, mem, vmp_pageq);
9981 }
9982
9983 vm_page_anonymous_count++;
9984 vm_page_pageable_internal_count++;
9985 } else {
9986 mem->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
9987
9988 if (first == TRUE) {
9989 vm_page_queue_enter_first(&vm_page_queue_inactive, mem, vmp_pageq);
9990 } else {
9991 vm_page_queue_enter(&vm_page_queue_inactive, mem, vmp_pageq);
9992 }
9993
9994 vm_page_pageable_external_count++;
9995 }
9996 vm_page_inactive_count++;
9997 token_new_pagecount++;
9998
9999 vm_page_add_to_specialq(mem, FALSE);
10000 }
10001
10002 void
vm_page_enqueue_active(vm_page_t mem,boolean_t first)10003 vm_page_enqueue_active(vm_page_t mem, boolean_t first)
10004 {
10005 vm_object_t m_object;
10006
10007 m_object = VM_PAGE_OBJECT(mem);
10008
10009 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
10010 assert(!vm_page_is_fictitious(mem));
10011 assert(!mem->vmp_laundry);
10012 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
10013 vm_page_check_pageable_safe(mem);
10014
10015 mem->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
10016 if (first == TRUE) {
10017 vm_page_queue_enter_first(&vm_page_queue_active, mem, vmp_pageq);
10018 } else {
10019 vm_page_queue_enter(&vm_page_queue_active, mem, vmp_pageq);
10020 }
10021 vm_page_active_count++;
10022
10023 if (m_object->internal) {
10024 vm_page_pageable_internal_count++;
10025 } else {
10026 vm_page_pageable_external_count++;
10027 }
10028
10029 vm_page_add_to_specialq(mem, FALSE);
10030 vm_page_balance_inactive(3);
10031 }
10032
10033 /*
10034 * Pages from special kernel objects shouldn't
10035 * be placed on pageable queues.
10036 */
10037 void
vm_page_check_pageable_safe(vm_page_t page)10038 vm_page_check_pageable_safe(vm_page_t page)
10039 {
10040 vm_object_t page_object;
10041
10042 page_object = VM_PAGE_OBJECT(page);
10043
10044 if (is_kernel_object(page_object)) {
10045 panic("vm_page_check_pageable_safe: trying to add page"
10046 "from a kernel object to pageable queue");
10047 }
10048
10049 if (page_object == compressor_object) {
10050 panic("vm_page_check_pageable_safe: trying to add page"
10051 "from compressor object (%p) to pageable queue", compressor_object);
10052 }
10053 }
10054
10055 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
10056 * wired page diagnose
10057 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
10058
10059 #include <libkern/OSKextLibPrivate.h>
10060
10061 #define KA_SIZE(namelen, subtotalscount) \
10062 (sizeof(struct vm_allocation_site) + (namelen) + 1 + ((subtotalscount) * sizeof(struct vm_allocation_total)))
10063
10064 #define KA_NAME(alloc) \
10065 ((char *)(&(alloc)->subtotals[(alloc->subtotalscount)]))
10066
10067 #define KA_NAME_LEN(alloc) \
10068 (VM_TAG_NAME_LEN_MAX & (alloc->flags >> VM_TAG_NAME_LEN_SHIFT))
10069
10070 vm_tag_t
vm_tag_bt(void)10071 vm_tag_bt(void)
10072 {
10073 uintptr_t* frameptr;
10074 uintptr_t* frameptr_next;
10075 uintptr_t retaddr;
10076 uintptr_t kstackb, kstackt;
10077 const vm_allocation_site_t * site;
10078 thread_t cthread;
10079 kern_allocation_name_t name;
10080
10081 cthread = current_thread();
10082 if (__improbable(cthread == NULL)) {
10083 return VM_KERN_MEMORY_OSFMK;
10084 }
10085
10086 if ((name = thread_get_kernel_state(cthread)->allocation_name)) {
10087 if (!name->tag) {
10088 vm_tag_alloc(name);
10089 }
10090 return name->tag;
10091 }
10092
10093 kstackb = cthread->kernel_stack;
10094 kstackt = kstackb + kernel_stack_size;
10095
10096 /* Load stack frame pointer (EBP on x86) into frameptr */
10097 frameptr = __builtin_frame_address(0);
10098 site = NULL;
10099 while (frameptr != NULL) {
10100 /* Verify thread stack bounds */
10101 if (((uintptr_t)(frameptr + 2) > kstackt) || ((uintptr_t)frameptr < kstackb)) {
10102 break;
10103 }
10104
10105 /* Next frame pointer is pointed to by the previous one */
10106 frameptr_next = (uintptr_t*) *frameptr;
10107 #if defined(HAS_APPLE_PAC)
10108 frameptr_next = ptrauth_strip(frameptr_next, ptrauth_key_frame_pointer);
10109 #endif
10110
10111 /* Pull return address from one spot above the frame pointer */
10112 retaddr = *(frameptr + 1);
10113
10114 #if defined(HAS_APPLE_PAC)
10115 retaddr = (uintptr_t) ptrauth_strip((void *)retaddr, ptrauth_key_return_address);
10116 #endif
10117
10118 if (((retaddr < vm_kernel_builtinkmod_text_end) && (retaddr >= vm_kernel_builtinkmod_text))
10119 || (retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top)) {
10120 site = OSKextGetAllocationSiteForCaller(retaddr);
10121 break;
10122 }
10123 frameptr = frameptr_next;
10124 }
10125
10126 if (site) {
10127 return site->tag;
10128 }
10129
10130 #if MACH_ASSERT
10131 /*
10132 * Kernel tests appear here as unrecognized call sites and would get
10133 * no memory tag. Give them a default tag to prevent panics later.
10134 */
10135 if (thread_get_test_option(test_option_vm_prevent_wire_tag_panic)) {
10136 return VM_KERN_MEMORY_OSFMK;
10137 }
10138 #endif
10139
10140 return VM_KERN_MEMORY_NONE;
10141 }
10142
10143 static uint64_t free_tag_bits[VM_MAX_TAG_VALUE / 64];
10144
10145 void
vm_tag_alloc_locked(vm_allocation_site_t * site,vm_allocation_site_t ** releasesiteP)10146 vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP)
10147 {
10148 vm_tag_t tag;
10149 uint64_t avail;
10150 uint32_t idx;
10151 vm_allocation_site_t * prev;
10152
10153 if (site->tag) {
10154 return;
10155 }
10156
10157 idx = 0;
10158 while (TRUE) {
10159 avail = free_tag_bits[idx];
10160 if (avail) {
10161 tag = (vm_tag_t)__builtin_clzll(avail);
10162 avail &= ~(1ULL << (63 - tag));
10163 free_tag_bits[idx] = avail;
10164 tag += (idx << 6);
10165 break;
10166 }
10167 idx++;
10168 if (idx >= ARRAY_COUNT(free_tag_bits)) {
10169 for (idx = 0; idx < ARRAY_COUNT(vm_allocation_sites); idx++) {
10170 prev = vm_allocation_sites[idx];
10171 if (!prev) {
10172 continue;
10173 }
10174 if (!KA_NAME_LEN(prev)) {
10175 continue;
10176 }
10177 if (!prev->tag) {
10178 continue;
10179 }
10180 if (prev->total) {
10181 continue;
10182 }
10183 if (1 != prev->refcount) {
10184 continue;
10185 }
10186
10187 assert(idx == prev->tag);
10188 tag = (vm_tag_t)idx;
10189 prev->tag = VM_KERN_MEMORY_NONE;
10190 *releasesiteP = prev;
10191 break;
10192 }
10193 if (idx >= ARRAY_COUNT(vm_allocation_sites)) {
10194 tag = VM_KERN_MEMORY_ANY;
10195 }
10196 break;
10197 }
10198 }
10199 site->tag = tag;
10200
10201 OSAddAtomic16(1, &site->refcount);
10202
10203 if (VM_KERN_MEMORY_ANY != tag) {
10204 vm_allocation_sites[tag] = site;
10205 }
10206
10207 if (tag > vm_allocation_tag_highest) {
10208 vm_allocation_tag_highest = tag;
10209 }
10210 }
10211
10212 static void
vm_tag_free_locked(vm_tag_t tag)10213 vm_tag_free_locked(vm_tag_t tag)
10214 {
10215 uint64_t avail;
10216 uint32_t idx;
10217 uint64_t bit;
10218
10219 if (VM_KERN_MEMORY_ANY == tag) {
10220 return;
10221 }
10222
10223 idx = (tag >> 6);
10224 avail = free_tag_bits[idx];
10225 tag &= 63;
10226 bit = (1ULL << (63 - tag));
10227 assert(!(avail & bit));
10228 free_tag_bits[idx] = (avail | bit);
10229 }
10230
10231 static void
vm_tag_init(void)10232 vm_tag_init(void)
10233 {
10234 vm_tag_t tag;
10235 for (tag = VM_KERN_MEMORY_FIRST_DYNAMIC; tag < VM_KERN_MEMORY_ANY; tag++) {
10236 vm_tag_free_locked(tag);
10237 }
10238
10239 for (tag = VM_KERN_MEMORY_ANY + 1; tag < VM_MAX_TAG_VALUE; tag++) {
10240 vm_tag_free_locked(tag);
10241 }
10242 }
10243
10244 vm_tag_t
vm_tag_alloc(vm_allocation_site_t * site)10245 vm_tag_alloc(vm_allocation_site_t * site)
10246 {
10247 vm_allocation_site_t * releasesite;
10248
10249 if (!site->tag) {
10250 releasesite = NULL;
10251 lck_ticket_lock(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
10252 vm_tag_alloc_locked(site, &releasesite);
10253 lck_ticket_unlock(&vm_allocation_sites_lock);
10254 if (releasesite) {
10255 kern_allocation_name_release(releasesite);
10256 }
10257 }
10258
10259 return site->tag;
10260 }
10261
10262 #ifndef ARRAY_SIZE
10263 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
10264 #endif /* ARRAY_SIZE */
10265 #define VM_KERN_MEMORY_ELEM(name) [VM_KERN_MEMORY_##name] = "VM_KERN_MEMORY_" #name
10266 const char *vm_kern_memory_names[] = {
10267 VM_KERN_MEMORY_ELEM(NONE),
10268 VM_KERN_MEMORY_ELEM(OSFMK),
10269 VM_KERN_MEMORY_ELEM(BSD),
10270 VM_KERN_MEMORY_ELEM(IOKIT),
10271 VM_KERN_MEMORY_ELEM(LIBKERN),
10272 VM_KERN_MEMORY_ELEM(OSKEXT),
10273 VM_KERN_MEMORY_ELEM(KEXT),
10274 VM_KERN_MEMORY_ELEM(IPC),
10275 VM_KERN_MEMORY_ELEM(STACK),
10276 VM_KERN_MEMORY_ELEM(CPU),
10277 VM_KERN_MEMORY_ELEM(PMAP),
10278 VM_KERN_MEMORY_ELEM(PTE),
10279 VM_KERN_MEMORY_ELEM(ZONE),
10280 VM_KERN_MEMORY_ELEM(KALLOC),
10281 VM_KERN_MEMORY_ELEM(COMPRESSOR),
10282 VM_KERN_MEMORY_ELEM(COMPRESSED_DATA),
10283 VM_KERN_MEMORY_ELEM(PHANTOM_CACHE),
10284 VM_KERN_MEMORY_ELEM(WAITQ),
10285 VM_KERN_MEMORY_ELEM(DIAG),
10286 VM_KERN_MEMORY_ELEM(LOG),
10287 VM_KERN_MEMORY_ELEM(FILE),
10288 VM_KERN_MEMORY_ELEM(MBUF),
10289 VM_KERN_MEMORY_ELEM(UBC),
10290 VM_KERN_MEMORY_ELEM(SECURITY),
10291 VM_KERN_MEMORY_ELEM(MLOCK),
10292 VM_KERN_MEMORY_ELEM(REASON),
10293 VM_KERN_MEMORY_ELEM(SKYWALK),
10294 VM_KERN_MEMORY_ELEM(LTABLE),
10295 VM_KERN_MEMORY_ELEM(HV),
10296 VM_KERN_MEMORY_ELEM(KALLOC_DATA),
10297 VM_KERN_MEMORY_ELEM(RETIRED),
10298 VM_KERN_MEMORY_ELEM(KALLOC_TYPE),
10299 VM_KERN_MEMORY_ELEM(TRIAGE),
10300 VM_KERN_MEMORY_ELEM(RECOUNT),
10301 "VM_KERN_MEMORY_34",
10302 VM_KERN_MEMORY_ELEM(EXCLAVES),
10303 VM_KERN_MEMORY_ELEM(EXCLAVES_SHARED),
10304 VM_KERN_MEMORY_ELEM(KALLOC_SHARED),
10305 VM_KERN_MEMORY_ELEM(CPUTRACE),
10306 };
10307
10308 _Static_assert(ARRAY_SIZE(vm_kern_memory_names) == VM_KERN_MEMORY_FIRST_DYNAMIC,
10309 "vm_kern_memory_names must map all counter tags");
10310
10311 #define VM_KERN_COUNT_ELEM(name) [VM_KERN_COUNT_##name] = "VM_KERN_COUNT_" #name
10312 const char *vm_kern_count_names[] = {
10313 VM_KERN_COUNT_ELEM(MANAGED),
10314 VM_KERN_COUNT_ELEM(RESERVED),
10315 VM_KERN_COUNT_ELEM(WIRED),
10316 VM_KERN_COUNT_ELEM(WIRED_MANAGED),
10317 VM_KERN_COUNT_ELEM(STOLEN),
10318 VM_KERN_COUNT_ELEM(LOPAGE),
10319 VM_KERN_COUNT_ELEM(MAP_KERNEL),
10320 VM_KERN_COUNT_ELEM(MAP_ZONE),
10321 VM_KERN_COUNT_ELEM(MAP_KALLOC_LARGE),
10322 VM_KERN_COUNT_ELEM(WIRED_BOOT),
10323 VM_KERN_COUNT_ELEM(BOOT_STOLEN),
10324 VM_KERN_COUNT_ELEM(WIRED_STATIC_KERNELCACHE),
10325 VM_KERN_COUNT_ELEM(MAP_KALLOC_LARGE_DATA),
10326 VM_KERN_COUNT_ELEM(MAP_KERNEL_DATA),
10327 VM_KERN_COUNT_ELEM(EXCLAVES_CARVEOUT),
10328 };
10329
10330 #if VM_BTLOG_TAGS
10331 #define VM_KERN_MEMORY_STR_MAX_LEN (32)
10332 TUNABLE_STR(vmtaglog, VM_KERN_MEMORY_STR_MAX_LEN, "vmtaglog", "");
10333 #define VM_TAG_BTLOG_SIZE (16u << 10)
10334
10335 btlog_t vmtaglog_btlog;
10336 vm_tag_t vmtaglog_tag;
10337
10338 static void
vm_tag_log(vm_object_t object,int64_t delta,void * fp)10339 vm_tag_log(vm_object_t object, int64_t delta, void *fp)
10340 {
10341 if (is_kernel_object(object)) {
10342 /* kernel object backtraces are tracked in vm entries */
10343 return;
10344 }
10345 if (delta > 0) {
10346 btref_t ref = btref_get(fp, BTREF_GET_NOWAIT);
10347 btlog_record(vmtaglog_btlog, object, 0, ref);
10348 } else if (object->wired_page_count == 0) {
10349 btlog_erase(vmtaglog_btlog, object);
10350 }
10351 }
10352
10353 _Static_assert(ARRAY_SIZE(vm_kern_count_names) == VM_KERN_COUNTER_COUNT,
10354 "vm_kern_count_names must map all counter tags");
10355
10356 static vm_tag_t
vm_tag_str_to_idx(char tagstr[VM_KERN_MEMORY_STR_MAX_LEN])10357 vm_tag_str_to_idx(char tagstr[VM_KERN_MEMORY_STR_MAX_LEN])
10358 {
10359 for (vm_tag_t i = VM_KERN_MEMORY_OSFMK; i < ARRAY_SIZE(vm_kern_memory_names); i++) {
10360 if (!strncmp(vm_kern_memory_names[i], tagstr, VM_KERN_MEMORY_STR_MAX_LEN)) {
10361 return i;
10362 }
10363 }
10364
10365 if (!strncmp("dynamic", tagstr, VM_KERN_MEMORY_STR_MAX_LEN)) {
10366 return VM_KERN_MEMORY_FIRST_DYNAMIC;
10367 }
10368
10369 if (!strncmp("any", tagstr, VM_KERN_MEMORY_STR_MAX_LEN)) {
10370 return VM_KERN_MEMORY_ANY;
10371 }
10372
10373 printf("Unable to find vm tag %s for btlog\n", tagstr);
10374 return VM_KERN_MEMORY_NONE;
10375 }
10376
10377 __startup_func
10378 static void
vm_btlog_init(void)10379 vm_btlog_init(void)
10380 {
10381 vmtaglog_tag = vm_tag_str_to_idx(vmtaglog);
10382
10383 if (vmtaglog_tag != VM_KERN_MEMORY_NONE) {
10384 vmtaglog_btlog = btlog_create(BTLOG_HASH, VM_TAG_BTLOG_SIZE, 0);
10385 }
10386 }
10387 STARTUP(ZALLOC, STARTUP_RANK_FIRST, vm_btlog_init);
10388 #endif /* VM_BTLOG_TAGS */
10389
10390 void
vm_tag_update_size(vm_tag_t tag,int64_t delta,vm_object_t object)10391 vm_tag_update_size(vm_tag_t tag, int64_t delta, vm_object_t object)
10392 {
10393 assert(VM_KERN_MEMORY_NONE != tag && tag < VM_MAX_TAG_VALUE);
10394
10395 kern_allocation_update_size(vm_allocation_sites[tag], delta, object);
10396 }
10397
10398 uint64_t
vm_tag_get_size(vm_tag_t tag)10399 vm_tag_get_size(vm_tag_t tag)
10400 {
10401 vm_allocation_site_t *allocation;
10402
10403 assert(VM_KERN_MEMORY_NONE != tag && tag < VM_MAX_TAG_VALUE);
10404
10405 allocation = vm_allocation_sites[tag];
10406 return allocation ? os_atomic_load(&allocation->total, relaxed) : 0;
10407 }
10408
10409 void
kern_allocation_update_size(kern_allocation_name_t allocation,int64_t delta,__unused vm_object_t object)10410 kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta, __unused vm_object_t object)
10411 {
10412 uint64_t value;
10413
10414 value = os_atomic_add(&allocation->total, delta, relaxed);
10415 if (delta < 0) {
10416 assertf(value + (uint64_t)-delta > value,
10417 "tag %d, site %p", allocation->tag, allocation);
10418 }
10419
10420 #if DEBUG || DEVELOPMENT
10421 /* release to publish the new total */
10422 os_atomic_max(&allocation->peak, value, release);
10423 #endif /* DEBUG || DEVELOPMENT */
10424
10425 if (value == (uint64_t)delta && !allocation->tag) {
10426 vm_tag_alloc(allocation);
10427 }
10428
10429 #if VM_BTLOG_TAGS
10430 if (vmtaglog_matches(allocation->tag) && object) {
10431 vm_tag_log(object, delta, __builtin_frame_address(0));
10432 }
10433 #endif /* VM_BTLOG_TAGS */
10434 }
10435
10436 #if DEBUG || DEVELOPMENT
10437
10438 void
vm_tag_reset_all_peaks(void)10439 vm_tag_reset_all_peaks(void)
10440 {
10441 vm_log("resetting peak size for all kernel tags\n");
10442 for (vm_tag_t tag = 0; tag <= vm_allocation_tag_highest; tag++) {
10443 vm_tag_reset_peak(tag);
10444 }
10445 }
10446
10447 kern_return_t
vm_tag_reset_peak(vm_tag_t tag)10448 vm_tag_reset_peak(vm_tag_t tag)
10449 {
10450 if (tag > vm_allocation_tag_highest) {
10451 return KERN_INVALID_ARGUMENT;
10452 }
10453
10454 vm_allocation_site_t *site = vm_allocation_sites[tag];
10455 vm_log_info("resetting peak size for kernel tag %s\n",
10456 KA_NAME(site));
10457
10458 uint64_t new_peak = os_atomic_load(&site->total, relaxed);
10459 /* acquire updates to the total */
10460 os_atomic_min(&site->peak, new_peak, acquire);
10461
10462 return KERN_SUCCESS;
10463 }
10464
10465 #endif /* DEBUG || DEVELOPMENT */
10466
10467 #if VM_TAG_SIZECLASSES
10468
10469 void
vm_allocation_zones_init(void)10470 vm_allocation_zones_init(void)
10471 {
10472 vm_offset_t addr;
10473 vm_size_t size;
10474
10475 const vm_tag_t early_tags[] = {
10476 VM_KERN_MEMORY_DIAG,
10477 VM_KERN_MEMORY_KALLOC,
10478 VM_KERN_MEMORY_KALLOC_DATA,
10479 VM_KERN_MEMORY_KALLOC_TYPE,
10480 VM_KERN_MEMORY_LIBKERN,
10481 VM_KERN_MEMORY_OSFMK,
10482 VM_KERN_MEMORY_RECOUNT,
10483 };
10484
10485 size = VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *)
10486 + ARRAY_COUNT(early_tags) * VM_TAG_SIZECLASSES * sizeof(vm_allocation_zone_total_t);
10487
10488 kmem_alloc(kernel_map, &addr, round_page(size),
10489 KMA_NOFAIL | KMA_KOBJECT | KMA_ZERO | KMA_PERMANENT,
10490 VM_KERN_MEMORY_DIAG);
10491
10492 vm_allocation_zone_totals = (vm_allocation_zone_total_t **) addr;
10493 addr += VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *);
10494
10495 // prepopulate early tag ranges so allocations
10496 // in vm_tag_update_zone_size() and early boot won't recurse
10497 for (size_t i = 0; i < ARRAY_COUNT(early_tags); i++) {
10498 vm_allocation_zone_totals[early_tags[i]] = (vm_allocation_zone_total_t *)addr;
10499 addr += VM_TAG_SIZECLASSES * sizeof(vm_allocation_zone_total_t);
10500 }
10501 }
10502
10503 __attribute__((noinline))
10504 static vm_tag_t
vm_tag_zone_stats_alloc(vm_tag_t tag,zalloc_flags_t flags)10505 vm_tag_zone_stats_alloc(vm_tag_t tag, zalloc_flags_t flags)
10506 {
10507 vm_allocation_zone_total_t *stats;
10508 vm_size_t size = sizeof(*stats) * VM_TAG_SIZECLASSES;
10509
10510 flags = Z_VM_TAG(Z_ZERO | flags, VM_KERN_MEMORY_DIAG);
10511 stats = kalloc_data(size, flags);
10512 if (!stats) {
10513 return VM_KERN_MEMORY_NONE;
10514 }
10515 if (!os_atomic_cmpxchg(&vm_allocation_zone_totals[tag], NULL, stats, release)) {
10516 kfree_data(stats, size);
10517 }
10518 return tag;
10519 }
10520
10521 vm_tag_t
vm_tag_will_update_zone(vm_tag_t tag,uint32_t zflags)10522 vm_tag_will_update_zone(vm_tag_t tag, uint32_t zflags)
10523 {
10524 assert(VM_KERN_MEMORY_NONE != tag);
10525 assert(tag < VM_MAX_TAG_VALUE);
10526
10527 if (__probable(vm_allocation_zone_totals[tag])) {
10528 return tag;
10529 }
10530 return vm_tag_zone_stats_alloc(tag, zflags);
10531 }
10532
10533 void
vm_tag_update_zone_size(vm_tag_t tag,uint32_t zidx,long delta)10534 vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, long delta)
10535 {
10536 vm_allocation_zone_total_t *stats;
10537 vm_size_t value;
10538
10539 assert(VM_KERN_MEMORY_NONE != tag);
10540 assert(tag < VM_MAX_TAG_VALUE);
10541
10542 if (zidx >= VM_TAG_SIZECLASSES) {
10543 return;
10544 }
10545
10546 stats = vm_allocation_zone_totals[tag];
10547 assert(stats);
10548 stats += zidx;
10549
10550 value = os_atomic_add(&stats->vazt_total, delta, relaxed);
10551 if (delta < 0) {
10552 assertf((long)value >= 0, "zidx %d, tag %d, %p", zidx, tag, stats);
10553 return;
10554 } else if (os_atomic_load(&stats->vazt_peak, relaxed) < value) {
10555 os_atomic_max(&stats->vazt_peak, value, relaxed);
10556 }
10557 }
10558
10559 #endif /* VM_TAG_SIZECLASSES */
10560
10561 void
kern_allocation_update_subtotal(kern_allocation_name_t allocation,vm_tag_t subtag,int64_t delta)10562 kern_allocation_update_subtotal(kern_allocation_name_t allocation, vm_tag_t subtag, int64_t delta)
10563 {
10564 kern_allocation_name_t other;
10565 struct vm_allocation_total * total;
10566 uint32_t subidx;
10567
10568 assert(VM_KERN_MEMORY_NONE != subtag);
10569 lck_ticket_lock(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
10570 for (subidx = 0; subidx < allocation->subtotalscount; subidx++) {
10571 total = &allocation->subtotals[subidx];
10572 if (subtag == total->tag) {
10573 break;
10574 }
10575 }
10576 if (subidx >= allocation->subtotalscount) {
10577 for (subidx = 0; subidx < allocation->subtotalscount; subidx++) {
10578 total = &allocation->subtotals[subidx];
10579 if ((VM_KERN_MEMORY_NONE == total->tag)
10580 || !total->total) {
10581 total->tag = (vm_tag_t)subtag;
10582 break;
10583 }
10584 }
10585 }
10586 assert(subidx < allocation->subtotalscount);
10587 if (subidx >= allocation->subtotalscount) {
10588 lck_ticket_unlock(&vm_allocation_sites_lock);
10589 return;
10590 }
10591 if (delta < 0) {
10592 assertf(total->total >= ((uint64_t)-delta), "name %p", allocation);
10593 }
10594 OSAddAtomic64(delta, &total->total);
10595 lck_ticket_unlock(&vm_allocation_sites_lock);
10596
10597 other = vm_allocation_sites[subtag];
10598 assert(other);
10599 if (delta < 0) {
10600 assertf(other->mapped >= ((uint64_t)-delta), "other %p", other);
10601 }
10602 OSAddAtomic64(delta, &other->mapped);
10603 }
10604
10605 const char *
kern_allocation_get_name(kern_allocation_name_t allocation)10606 kern_allocation_get_name(kern_allocation_name_t allocation)
10607 {
10608 return KA_NAME(allocation);
10609 }
10610
10611 kern_allocation_name_t
kern_allocation_name_allocate(const char * name,uint16_t subtotalscount)10612 kern_allocation_name_allocate(const char * name, uint16_t subtotalscount)
10613 {
10614 kern_allocation_name_t allocation;
10615 uint16_t namelen;
10616
10617 namelen = (uint16_t)strnlen(name, MACH_MEMORY_INFO_NAME_MAX_LEN - 1);
10618
10619 allocation = kalloc_data(KA_SIZE(namelen, subtotalscount), Z_WAITOK | Z_ZERO);
10620 allocation->refcount = 1;
10621 allocation->subtotalscount = subtotalscount;
10622 allocation->flags = (uint16_t)(namelen << VM_TAG_NAME_LEN_SHIFT);
10623 strlcpy(KA_NAME(allocation), name, namelen + 1);
10624
10625 vm_tag_alloc(allocation);
10626 return allocation;
10627 }
10628
10629 void
kern_allocation_name_release(kern_allocation_name_t allocation)10630 kern_allocation_name_release(kern_allocation_name_t allocation)
10631 {
10632 assert(allocation->refcount > 0);
10633 if (1 == OSAddAtomic16(-1, &allocation->refcount)) {
10634 kfree_data(allocation,
10635 KA_SIZE(KA_NAME_LEN(allocation), allocation->subtotalscount));
10636 }
10637 }
10638
10639 #if !VM_TAG_ACTIVE_UPDATE
10640 static void
vm_page_count_object(mach_memory_info_t * info,unsigned int __unused num_info,vm_object_t object)10641 vm_page_count_object(mach_memory_info_t * info, unsigned int __unused num_info, vm_object_t object)
10642 {
10643 if (!object->wired_page_count) {
10644 return;
10645 }
10646 if (!is_kernel_object(object)) {
10647 assert(object->wire_tag < num_info);
10648 info[object->wire_tag].size += ptoa_64(object->wired_page_count);
10649 }
10650 }
10651
10652 typedef void (*vm_page_iterate_proc)(mach_memory_info_t * info,
10653 unsigned int num_info, vm_object_t object);
10654
10655 static void
vm_page_iterate_purgeable_objects(mach_memory_info_t * info,unsigned int num_info,vm_page_iterate_proc proc,purgeable_q_t queue,int group)10656 vm_page_iterate_purgeable_objects(mach_memory_info_t * info, unsigned int num_info,
10657 vm_page_iterate_proc proc, purgeable_q_t queue,
10658 int group)
10659 {
10660 vm_object_t object;
10661
10662 for (object = (vm_object_t) queue_first(&queue->objq[group]);
10663 !queue_end(&queue->objq[group], (queue_entry_t) object);
10664 object = (vm_object_t) queue_next(&object->objq)) {
10665 proc(info, num_info, object);
10666 }
10667 }
10668
10669 static void
vm_page_iterate_objects(mach_memory_info_t * info,unsigned int num_info,vm_page_iterate_proc proc)10670 vm_page_iterate_objects(mach_memory_info_t * info, unsigned int num_info,
10671 vm_page_iterate_proc proc)
10672 {
10673 vm_object_t object;
10674
10675 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket);
10676 queue_iterate(&vm_objects_wired,
10677 object,
10678 vm_object_t,
10679 wired_objq)
10680 {
10681 proc(info, num_info, object);
10682 }
10683 lck_spin_unlock(&vm_objects_wired_lock);
10684 }
10685 #endif /* ! VM_TAG_ACTIVE_UPDATE */
10686
10687 static uint64_t
process_account(mach_memory_info_t * info,unsigned int num_info,uint64_t zones_collectable_bytes,boolean_t iterated,bool redact_info __unused)10688 process_account(mach_memory_info_t * info, unsigned int num_info,
10689 uint64_t zones_collectable_bytes, boolean_t iterated, bool redact_info __unused)
10690 {
10691 size_t namelen;
10692 unsigned int idx, count, nextinfo;
10693 vm_allocation_site_t * site;
10694 lck_ticket_lock(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
10695
10696 for (idx = 0; idx <= vm_allocation_tag_highest; idx++) {
10697 site = vm_allocation_sites[idx];
10698 if (!site) {
10699 continue;
10700 }
10701 info[idx].mapped = site->mapped;
10702 info[idx].tag = site->tag;
10703 if (!iterated) {
10704 info[idx].size = site->total;
10705 #if DEBUG || DEVELOPMENT
10706 info[idx].peak = site->peak;
10707 #endif /* DEBUG || DEVELOPMENT */
10708 } else {
10709 if (!site->subtotalscount && (site->total != info[idx].size)) {
10710 printf("tag mismatch[%d] 0x%qx, iter 0x%qx\n", idx, site->total, info[idx].size);
10711 info[idx].size = site->total;
10712 }
10713 }
10714 info[idx].flags |= VM_KERN_SITE_WIRED;
10715 if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC) {
10716 info[idx].site = idx;
10717 info[idx].flags |= VM_KERN_SITE_TAG;
10718 if (VM_KERN_MEMORY_ZONE == idx) {
10719 info[idx].flags |= VM_KERN_SITE_HIDE;
10720 info[idx].flags &= ~VM_KERN_SITE_WIRED;
10721 info[idx].collectable_bytes = zones_collectable_bytes;
10722 }
10723 info[idx].flags |= VM_KERN_SITE_NAMED;
10724 strlcpy(info[idx].name, vm_kern_memory_names[idx], MACH_MEMORY_INFO_NAME_MAX_LEN);
10725 } else if ((namelen = (VM_TAG_NAME_LEN_MAX & (site->flags >> VM_TAG_NAME_LEN_SHIFT)))) {
10726 info[idx].site = 0;
10727 info[idx].flags |= VM_KERN_SITE_NAMED;
10728 if (namelen > sizeof(info[idx].name)) {
10729 namelen = sizeof(info[idx].name);
10730 }
10731 strncpy(&info[idx].name[0], KA_NAME(site), namelen);
10732 } else if (VM_TAG_KMOD & site->flags) {
10733 info[idx].site = OSKextGetKmodIDForSite(site, NULL, 0);
10734 info[idx].flags |= VM_KERN_SITE_KMOD;
10735 } else {
10736 info[idx].site = VM_KERNEL_UNSLIDE(site);
10737 info[idx].flags |= VM_KERN_SITE_KERNEL;
10738 }
10739 }
10740
10741 nextinfo = (vm_allocation_tag_highest + 1);
10742 count = nextinfo;
10743 if (count >= num_info) {
10744 count = num_info;
10745 }
10746
10747 for (idx = 0; idx < count; idx++) {
10748 site = vm_allocation_sites[idx];
10749 if (!site) {
10750 continue;
10751 }
10752 #if VM_TAG_SIZECLASSES
10753 vm_allocation_zone_total_t * zone;
10754 unsigned int zidx;
10755
10756 if (!redact_info
10757 && vm_allocation_zone_totals
10758 && (zone = vm_allocation_zone_totals[idx])
10759 && (nextinfo < num_info)) {
10760 for (zidx = 0; zidx < VM_TAG_SIZECLASSES; zidx++) {
10761 if (!zone[zidx].vazt_peak) {
10762 continue;
10763 }
10764 info[nextinfo] = info[idx];
10765 info[nextinfo].zone = zone_index_from_tag_index(zidx);
10766 info[nextinfo].flags &= ~VM_KERN_SITE_WIRED;
10767 info[nextinfo].flags |= VM_KERN_SITE_ZONE;
10768 info[nextinfo].flags |= VM_KERN_SITE_KALLOC;
10769 info[nextinfo].size = zone[zidx].vazt_total;
10770 info[nextinfo].peak = zone[zidx].vazt_peak;
10771 info[nextinfo].mapped = 0;
10772 nextinfo++;
10773 }
10774 }
10775 #endif /* VM_TAG_SIZECLASSES */
10776 if (site->subtotalscount) {
10777 uint64_t mapped, mapcost, take;
10778 uint32_t sub;
10779 vm_tag_t alloctag;
10780
10781 info[idx].size = site->total;
10782 mapped = info[idx].size;
10783 info[idx].mapped = mapped;
10784 mapcost = 0;
10785 for (sub = 0; sub < site->subtotalscount; sub++) {
10786 alloctag = site->subtotals[sub].tag;
10787 assert(alloctag < num_info);
10788 if (info[alloctag].name[0] && alloctag >= VM_KERN_MEMORY_FIRST_DYNAMIC) {
10789 continue;
10790 }
10791 take = site->subtotals[sub].total;
10792 if (take > info[alloctag].size) {
10793 take = info[alloctag].size;
10794 }
10795 if (take > mapped) {
10796 take = mapped;
10797 }
10798 info[alloctag].mapped -= take;
10799 info[alloctag].size -= take;
10800 mapped -= take;
10801 mapcost += take;
10802 }
10803 info[idx].size = mapcost;
10804 }
10805 }
10806 lck_ticket_unlock(&vm_allocation_sites_lock);
10807
10808 return 0;
10809 }
10810
10811 uint32_t
vm_page_diagnose_estimate(void)10812 vm_page_diagnose_estimate(void)
10813 {
10814 vm_allocation_site_t * site;
10815 uint32_t count = zone_view_count;
10816 uint32_t idx;
10817
10818 lck_ticket_lock(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
10819 for (idx = 0; idx < VM_MAX_TAG_VALUE; idx++) {
10820 site = vm_allocation_sites[idx];
10821 if (!site) {
10822 continue;
10823 }
10824 count++;
10825 #if VM_TAG_SIZECLASSES
10826 if (vm_allocation_zone_totals) {
10827 vm_allocation_zone_total_t * zone;
10828 zone = vm_allocation_zone_totals[idx];
10829 if (!zone) {
10830 continue;
10831 }
10832 for (uint32_t zidx = 0; zidx < VM_TAG_SIZECLASSES; zidx++) {
10833 count += (zone[zidx].vazt_peak != 0);
10834 }
10835 }
10836 #endif
10837 }
10838 lck_ticket_unlock(&vm_allocation_sites_lock);
10839
10840 /* some slop for new tags created */
10841 count += 8;
10842 count += VM_KERN_COUNTER_COUNT;
10843
10844 return count;
10845 }
10846
10847 static void
vm_page_diagnose_zone_stats(mach_memory_info_t * info,zone_stats_t zstats,bool percpu)10848 vm_page_diagnose_zone_stats(mach_memory_info_t *info, zone_stats_t zstats,
10849 bool percpu)
10850 {
10851 zpercpu_foreach(zs, zstats) {
10852 info->size += zs->zs_mem_allocated - zs->zs_mem_freed;
10853 }
10854 if (percpu) {
10855 info->size *= zpercpu_count();
10856 }
10857 info->flags |= VM_KERN_SITE_NAMED | VM_KERN_SITE_ZONE_VIEW;
10858 }
10859
10860 static void
vm_page_add_info(mach_memory_info_t * info,zone_stats_t stats,bool per_cpu,const char * parent_heap_name,const char * parent_zone_name,const char * view_name)10861 vm_page_add_info(
10862 mach_memory_info_t *info,
10863 zone_stats_t stats,
10864 bool per_cpu,
10865 const char *parent_heap_name,
10866 const char *parent_zone_name,
10867 const char *view_name)
10868 {
10869 vm_page_diagnose_zone_stats(info, stats, per_cpu);
10870 snprintf(info->name, sizeof(info->name),
10871 "%s%s[%s]", parent_heap_name, parent_zone_name, view_name);
10872 }
10873
10874 static void
vm_page_diagnose_zone(mach_memory_info_t * info,zone_t z)10875 vm_page_diagnose_zone(mach_memory_info_t *info, zone_t z)
10876 {
10877 vm_page_add_info(info, z->z_stats, z->z_percpu, zone_heap_name(z),
10878 z->z_name, "raw");
10879 }
10880
10881 static void
vm_page_add_view(mach_memory_info_t * info,zone_stats_t stats,const char * parent_heap_name,const char * parent_zone_name,const char * view_name)10882 vm_page_add_view(
10883 mach_memory_info_t *info,
10884 zone_stats_t stats,
10885 const char *parent_heap_name,
10886 const char *parent_zone_name,
10887 const char *view_name)
10888 {
10889 vm_page_add_info(info, stats, false, parent_heap_name, parent_zone_name,
10890 view_name);
10891 }
10892
10893 static uint32_t
vm_page_diagnose_heap_views(mach_memory_info_t * info,kalloc_heap_t kh,const char * parent_heap_name,const char * parent_zone_name)10894 vm_page_diagnose_heap_views(
10895 mach_memory_info_t *info,
10896 kalloc_heap_t kh,
10897 const char *parent_heap_name,
10898 const char *parent_zone_name)
10899 {
10900 uint32_t i = 0;
10901
10902 while (kh) {
10903 vm_page_add_view(info + i, kh->kh_stats, parent_heap_name,
10904 parent_zone_name, kh->kh_name);
10905 kh = kh->kh_views;
10906 i++;
10907 }
10908 return i;
10909 }
10910
10911 static uint32_t
vm_page_diagnose_heap(mach_memory_info_t * info,kalloc_heap_t kheap)10912 vm_page_diagnose_heap(mach_memory_info_t *info, kalloc_heap_t kheap)
10913 {
10914 uint32_t i = 0;
10915
10916 for (; i < KHEAP_NUM_ZONES; i++) {
10917 vm_page_diagnose_zone(info + i, zone_by_id(kheap->kh_zstart + i));
10918 }
10919
10920 i += vm_page_diagnose_heap_views(info + i, kheap->kh_views, kheap->kh_name,
10921 NULL);
10922 return i;
10923 }
10924
10925 static int
vm_page_diagnose_kt_heaps(mach_memory_info_t * info)10926 vm_page_diagnose_kt_heaps(mach_memory_info_t *info)
10927 {
10928 uint32_t idx = 0;
10929 vm_page_add_view(info + idx, KHEAP_KT_VAR->kh_stats, KHEAP_KT_VAR->kh_name,
10930 "", "raw");
10931 idx++;
10932
10933 for (uint32_t i = 0; i < KT_VAR_MAX_HEAPS; i++) {
10934 struct kheap_info heap = kalloc_type_heap_array[i];
10935 char heap_num_tmp[MAX_ZONE_NAME] = "";
10936 const char *heap_num;
10937
10938 snprintf(&heap_num_tmp[0], MAX_ZONE_NAME, "%u", i);
10939 heap_num = &heap_num_tmp[0];
10940
10941 for (kalloc_type_var_view_t ktv = heap.kt_views; ktv;
10942 ktv = (kalloc_type_var_view_t) ktv->kt_next) {
10943 if (ktv->kt_stats && ktv->kt_stats != KHEAP_KT_VAR->kh_stats) {
10944 vm_page_add_view(info + idx, ktv->kt_stats, KHEAP_KT_VAR->kh_name,
10945 heap_num, ktv->kt_name);
10946 idx++;
10947 }
10948 }
10949
10950 idx += vm_page_diagnose_heap_views(info + idx, heap.kh_views,
10951 KHEAP_KT_VAR->kh_name, heap_num);
10952 }
10953
10954 return idx;
10955 }
10956
10957 kern_return_t
vm_page_diagnose(mach_memory_info_t * info,unsigned int num_info,uint64_t zones_collectable_bytes,bool redact_info)10958 vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes, bool redact_info)
10959 {
10960 uint64_t wired_size;
10961 uint64_t wired_managed_size;
10962 uint64_t wired_reserved_size;
10963 boolean_t iterate;
10964 mach_memory_info_t * counts;
10965 uint32_t i;
10966
10967 vmlp_api_start(VM_PAGE_DIAGNOSE);
10968
10969 bzero(info, num_info * sizeof(mach_memory_info_t));
10970
10971 if (!vm_page_wire_count_initial) {
10972 vmlp_api_end(VM_PAGE_DIAGNOSE, KERN_ABORTED);
10973 return KERN_ABORTED;
10974 }
10975
10976 wired_size = ptoa_64(vm_page_wire_count);
10977 wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count);
10978 #if XNU_TARGET_OS_OSX
10979 wired_size += ptoa_64(vm_lopage_free_count + vm_page_throttled_count);
10980 wired_reserved_size += ptoa_64(vm_page_throttled_count);
10981 #endif /* XNU_TARGET_OS_OSX */
10982 wired_managed_size = ptoa_64(vm_page_wire_count - vm_page_wire_count_initial);
10983
10984 wired_size += booter_size;
10985
10986 assert(num_info >= VM_KERN_COUNTER_COUNT);
10987 num_info -= VM_KERN_COUNTER_COUNT;
10988 counts = &info[num_info];
10989
10990 #define SET_COUNT(xcount, xsize, xflags) MACRO_BEGIN \
10991 counts[xcount].tag = VM_MAX_TAG_VALUE + xcount; \
10992 counts[xcount].site = (xcount); \
10993 counts[xcount].size = (xsize); \
10994 counts[xcount].mapped = (xsize); \
10995 counts[xcount].flags = VM_KERN_SITE_COUNTER | VM_KERN_SITE_NAMED | xflags; \
10996 strlcpy(counts[xcount].name, vm_kern_count_names[xcount], MACH_MEMORY_INFO_NAME_MAX_LEN); \
10997 MACRO_END;
10998
10999 SET_COUNT(VM_KERN_COUNT_MANAGED, ptoa_64(vm_page_pages), 0);
11000 SET_COUNT(VM_KERN_COUNT_WIRED, wired_size, 0);
11001 SET_COUNT(VM_KERN_COUNT_WIRED_MANAGED, wired_managed_size, 0);
11002 SET_COUNT(VM_KERN_COUNT_RESERVED, wired_reserved_size, VM_KERN_SITE_WIRED);
11003 SET_COUNT(VM_KERN_COUNT_STOLEN, ptoa_64(vm_page_stolen_count), VM_KERN_SITE_WIRED);
11004 SET_COUNT(VM_KERN_COUNT_LOPAGE, ptoa_64(vm_lopage_free_count), VM_KERN_SITE_WIRED);
11005 SET_COUNT(VM_KERN_COUNT_WIRED_BOOT, ptoa_64(vm_page_wire_count_on_boot), 0);
11006 SET_COUNT(VM_KERN_COUNT_BOOT_STOLEN, booter_size, VM_KERN_SITE_WIRED);
11007 SET_COUNT(VM_KERN_COUNT_WIRED_STATIC_KERNELCACHE, ptoa_64(vm_page_kernelcache_count), 0);
11008 #if CONFIG_SPTM
11009 SET_COUNT(VM_KERN_COUNT_EXCLAVES_CARVEOUT, SPTMArgs->sk_carveout_size, 0);
11010 #endif
11011
11012 #define SET_MAP(xcount, xsize, xfree, xlargest) MACRO_BEGIN \
11013 counts[xcount].site = (xcount); \
11014 counts[xcount].size = (xsize); \
11015 counts[xcount].mapped = (xsize); \
11016 counts[xcount].free = (xfree); \
11017 counts[xcount].largest = (xlargest); \
11018 counts[xcount].flags = VM_KERN_SITE_COUNTER | VM_KERN_SITE_NAMED; \
11019 strlcpy(counts[xcount].name, vm_kern_count_names[xcount], MACH_MEMORY_INFO_NAME_MAX_LEN); \
11020 MACRO_END;
11021
11022 vm_map_size_t map_size, map_free, map_largest;
11023
11024 vm_map_sizes(kernel_map, &map_size, &map_free, &map_largest);
11025 SET_MAP(VM_KERN_COUNT_MAP_KERNEL, map_size, map_free, map_largest);
11026
11027 zone_map_sizes(&map_size, &map_free, &map_largest);
11028 SET_MAP(VM_KERN_COUNT_MAP_ZONE, map_size, map_free, map_largest);
11029
11030 assert(num_info >= zone_view_count);
11031 num_info -= zone_view_count;
11032 counts = &info[num_info];
11033 i = 0;
11034
11035 if (!redact_info) {
11036 if (zone_is_data_buffers_kheap(KHEAP_DATA_BUFFERS->kh_heap_id)) {
11037 i += vm_page_diagnose_heap(counts + i, KHEAP_DATA_BUFFERS);
11038 }
11039 if (zone_is_data_shared_kheap(KHEAP_DATA_SHARED->kh_heap_id)) {
11040 i += vm_page_diagnose_heap(counts + i, KHEAP_DATA_SHARED);
11041 }
11042
11043 if (KHEAP_KT_VAR->kh_heap_id == KHEAP_ID_KT_VAR) {
11044 i += vm_page_diagnose_kt_heaps(counts + i);
11045 }
11046 assert(i <= zone_view_count);
11047
11048 zone_index_foreach(zidx) {
11049 zone_t z = &zone_array[zidx];
11050 zone_security_flags_t zsflags = zone_security_array[zidx];
11051 zone_view_t zv = z->z_views;
11052
11053 if (zv == NULL) {
11054 continue;
11055 }
11056
11057 zone_stats_t zv_stats_head = z->z_stats;
11058 bool has_raw_view = false;
11059
11060 for (; zv; zv = zv->zv_next) {
11061 /*
11062 * kalloc_types that allocate from the same zone are linked
11063 * as views. Only print the ones that have their own stats.
11064 */
11065 if (zv->zv_stats == zv_stats_head) {
11066 continue;
11067 }
11068 has_raw_view = true;
11069 vm_page_diagnose_zone_stats(counts + i, zv->zv_stats,
11070 z->z_percpu);
11071 snprintf(counts[i].name, sizeof(counts[i].name), "%s%s[%s]",
11072 zone_heap_name(z), z->z_name, zv->zv_name);
11073 i++;
11074 assert(i <= zone_view_count);
11075 }
11076
11077 /*
11078 * Print raw views for non kalloc or kalloc_type zones
11079 */
11080 bool kalloc_type = zsflags.z_kalloc_type;
11081 if ((zsflags.z_kheap_id == KHEAP_ID_NONE && !kalloc_type) ||
11082 (kalloc_type && has_raw_view)) {
11083 vm_page_diagnose_zone(counts + i, z);
11084 i++;
11085 assert(i <= zone_view_count);
11086 }
11087 }
11088 }
11089
11090 iterate = !VM_TAG_ACTIVE_UPDATE;
11091 if (iterate) {
11092 enum { kMaxKernelDepth = 1 };
11093 vm_map_t maps[kMaxKernelDepth];
11094 vm_map_entry_t entries[kMaxKernelDepth];
11095 vm_map_t map;
11096 vm_map_entry_t entry;
11097 vm_object_offset_t offset;
11098 vm_page_t page;
11099 int stackIdx, count;
11100
11101 #if !VM_TAG_ACTIVE_UPDATE
11102 vm_page_iterate_objects(info, num_info, &vm_page_count_object);
11103 #endif /* ! VM_TAG_ACTIVE_UPDATE */
11104
11105 map = kernel_map;
11106 stackIdx = 0;
11107 while (map) {
11108 vm_map_lock(map);
11109 for (entry = map->hdr.links.next; map; entry = entry->vme_next) {
11110 if (entry->is_sub_map) {
11111 assert(stackIdx < kMaxKernelDepth);
11112 maps[stackIdx] = map;
11113 entries[stackIdx] = entry;
11114 stackIdx++;
11115 map = VME_SUBMAP(entry);
11116 entry = NULL;
11117 break;
11118 }
11119
11120 vmlp_range_event_entry(map, entry);
11121
11122 if (is_kernel_object(VME_OBJECT(entry))) {
11123 count = 0;
11124 vm_object_lock(VME_OBJECT(entry));
11125 for (offset = entry->vme_start; offset < entry->vme_end; offset += page_size) {
11126 page = vm_page_lookup(VME_OBJECT(entry), offset);
11127 if (page && VM_PAGE_WIRED(page)) {
11128 count++;
11129 }
11130 }
11131 vm_object_unlock(VME_OBJECT(entry));
11132
11133 if (count) {
11134 assert(VME_ALIAS(entry) != VM_KERN_MEMORY_NONE);
11135 assert(VME_ALIAS(entry) < num_info);
11136 info[VME_ALIAS(entry)].size += ptoa_64(count);
11137 }
11138 }
11139 while (map && (entry == vm_map_last_entry(map))) {
11140 vm_map_unlock(map);
11141 if (!stackIdx) {
11142 map = NULL;
11143 } else {
11144 --stackIdx;
11145 map = maps[stackIdx];
11146 entry = entries[stackIdx];
11147 }
11148 }
11149 }
11150 }
11151 }
11152
11153 process_account(info, num_info, zones_collectable_bytes, iterate, redact_info);
11154
11155 vmlp_api_end(VM_PAGE_DIAGNOSE, KERN_SUCCESS);
11156 return KERN_SUCCESS;
11157 }
11158
11159 #if DEBUG || DEVELOPMENT
11160
11161 kern_return_t
vm_kern_allocation_info(uintptr_t addr,vm_size_t * size,vm_tag_t * tag,vm_size_t * zone_size)11162 vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size)
11163 {
11164 kern_return_t ret;
11165 vm_size_t zsize;
11166 vm_map_t map;
11167 vm_map_entry_t entry;
11168
11169 vmlp_api_start(VM_KERN_ALLOCATION_INFO);
11170
11171 zsize = zone_element_info((void *) addr, tag);
11172 if (zsize) {
11173 *zone_size = *size = zsize;
11174 vmlp_api_end(VM_KERN_ALLOCATION_INFO, KERN_SUCCESS);
11175 return KERN_SUCCESS;
11176 }
11177
11178 *zone_size = 0;
11179 ret = KERN_INVALID_ADDRESS;
11180 for (map = kernel_map; map;) {
11181 vm_map_lock(map);
11182 if (!vm_map_lookup_entry(map, addr, &entry)) {
11183 break;
11184 }
11185 if (entry->is_sub_map) {
11186 if (map != kernel_map) {
11187 break;
11188 }
11189 map = VME_SUBMAP(entry);
11190 continue;
11191 }
11192 if (entry->vme_start != addr) {
11193 break;
11194 }
11195
11196 vmlp_range_event_entry(map, entry);
11197
11198 *tag = (vm_tag_t)VME_ALIAS(entry);
11199 *size = (entry->vme_end - addr);
11200 ret = KERN_SUCCESS;
11201 break;
11202 }
11203 if (map != kernel_map) {
11204 vm_map_unlock(map);
11205 }
11206 vm_map_unlock(kernel_map);
11207
11208 vmlp_api_end(VM_KERN_ALLOCATION_INFO, ret);
11209 return ret;
11210 }
11211
11212 // some DEBUG/DEVELOPMENT code to get a process to page out its shared cache TEXT pages,
11213 // only used for DK driver in LPW testing
11214 uint64_t
vm_task_evict_shared_cache(task_t task)11215 vm_task_evict_shared_cache(task_t task)
11216 {
11217 enum { kMaxKernelDepth = 3 };
11218 vm_map_t maps[kMaxKernelDepth];
11219 vm_map_entry_t entries[kMaxKernelDepth];
11220 vm_map_t map;
11221 vm_object_t textObject, shadow;
11222 vm_map_entry_t entry;
11223 vm_object_offset_t textOffset, textSize;
11224 int stackIdx;
11225 uint64_t count;
11226
11227 count = counter_load(&task->pageins);
11228 map = get_task_map(task);
11229 textObject = NULL;
11230 stackIdx = 0;
11231 while (map) {
11232 vm_map_lock_read(map);
11233 for (entry = map->hdr.links.next; map; entry = entry->vme_next) {
11234 if (entry->is_sub_map) {
11235 assert(stackIdx < kMaxKernelDepth);
11236 maps[stackIdx] = map;
11237 entries[stackIdx] = entry;
11238 stackIdx++;
11239 map = VME_SUBMAP(entry);
11240 entry = NULL;
11241 break;
11242 }
11243 if (stackIdx && (VM_PROT_EXECUTE | VM_PROT_READ) == entry->protection) {
11244 textObject = VME_OBJECT(entry);
11245 vm_object_lock(textObject);
11246 while ((shadow = textObject->shadow)) {
11247 vm_object_lock(shadow);
11248 vm_object_unlock(textObject);
11249 textObject = shadow;
11250 }
11251 vm_object_reference_locked(textObject);
11252 vm_object_unlock(textObject);
11253 textOffset = VME_OFFSET(entry);
11254 textSize = entry->vme_end - entry->vme_start;
11255 entry = vm_map_last_entry(map);
11256 }
11257 while (map && (entry == vm_map_last_entry(map))) {
11258 vm_map_unlock_read(map);
11259 if (!stackIdx) {
11260 map = NULL;
11261 } else {
11262 --stackIdx;
11263 map = maps[stackIdx];
11264 entry = entries[stackIdx];
11265 if (textObject) {
11266 entry = vm_map_last_entry(map);
11267 }
11268 }
11269 }
11270 }
11271 }
11272
11273 if (textObject) {
11274 vm_object_sync(textObject, textOffset, textSize, true, false, false);
11275 vm_object_deallocate(textObject);
11276 }
11277 return count;
11278 }
11279
11280 uint64_t
vm_task_pageins(task_t task)11281 vm_task_pageins(task_t task)
11282 {
11283 return counter_load(&task->pageins);
11284 }
11285
11286 #endif /* DEBUG || DEVELOPMENT */
11287
11288 uint32_t
vm_tag_get_kext(vm_tag_t tag,char * name,vm_size_t namelen)11289 vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen)
11290 {
11291 vm_allocation_site_t * site;
11292 uint32_t kmodId;
11293
11294 kmodId = 0;
11295 lck_ticket_lock(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
11296 if ((site = vm_allocation_sites[tag])) {
11297 if (VM_TAG_KMOD & site->flags) {
11298 kmodId = OSKextGetKmodIDForSite(site, name, namelen);
11299 }
11300 }
11301 lck_ticket_unlock(&vm_allocation_sites_lock);
11302
11303 return kmodId;
11304 }
11305
11306
11307 #if CONFIG_SECLUDED_MEMORY
11308 /*
11309 * Note that there's no locking around other accesses to vm_page_secluded_target.
11310 * That should be OK, since these are the only place where it can be changed after
11311 * initialization. Other users (like vm_pageout) may see the wrong value briefly,
11312 * but will eventually get the correct value. This brief mismatch is OK as pageout
11313 * and page freeing will auto-adjust the vm_page_secluded_count to match the target
11314 * over time.
11315 */
11316 unsigned int vm_page_secluded_suppress_cnt = 0;
11317 unsigned int vm_page_secluded_save_target;
11318
11319 LCK_GRP_DECLARE(secluded_suppress_slock_grp, "secluded_suppress_slock");
11320 LCK_SPIN_DECLARE(secluded_suppress_slock, &secluded_suppress_slock_grp);
11321
11322 void
start_secluded_suppression(task_t task)11323 start_secluded_suppression(task_t task)
11324 {
11325 if (task->task_suppressed_secluded) {
11326 return;
11327 }
11328 lck_spin_lock(&secluded_suppress_slock);
11329 if (!task->task_suppressed_secluded && vm_page_secluded_suppress_cnt++ == 0) {
11330 task->task_suppressed_secluded = TRUE;
11331 vm_page_secluded_save_target = vm_page_secluded_target;
11332 vm_page_secluded_target = 0;
11333 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
11334 }
11335 lck_spin_unlock(&secluded_suppress_slock);
11336 }
11337
11338 void
stop_secluded_suppression(task_t task)11339 stop_secluded_suppression(task_t task)
11340 {
11341 lck_spin_lock(&secluded_suppress_slock);
11342 if (task->task_suppressed_secluded && --vm_page_secluded_suppress_cnt == 0) {
11343 task->task_suppressed_secluded = FALSE;
11344 vm_page_secluded_target = vm_page_secluded_save_target;
11345 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
11346 }
11347 lck_spin_unlock(&secluded_suppress_slock);
11348 }
11349
11350 #endif /* CONFIG_SECLUDED_MEMORY */
11351
11352 /*
11353 * Move the list of retired pages on the vm_page_queue_retired to
11354 * their final resting place on retired_pages_object.
11355 */
11356 void
vm_retire_boot_pages(void)11357 vm_retire_boot_pages(void)
11358 {
11359 }
11360
11361 /*
11362 * This holds the reported physical address if an ECC error leads to a panic.
11363 * SMC will store it in PMU SRAM under the 'sECC' key.
11364 */
11365 uint64_t ecc_panic_physical_address = 0;
11366
11367