1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * Resident memory management module.
63 */
64
65 #include <debug.h>
66 #include <libkern/OSAtomic.h>
67 #include <libkern/OSDebug.h>
68
69 #include <mach/clock_types.h>
70 #include <mach/vm_prot.h>
71 #include <mach/vm_statistics.h>
72 #include <mach/sdt.h>
73 #include <kern/counter.h>
74 #include <kern/host_statistics.h>
75 #include <kern/sched_prim.h>
76 #include <kern/policy_internal.h>
77 #include <kern/task.h>
78 #include <kern/thread.h>
79 #include <kern/kalloc.h>
80 #include <kern/zalloc_internal.h>
81 #include <kern/ledger.h>
82 #include <kern/ecc.h>
83 #include <vm/pmap.h>
84 #include <vm/vm_init.h>
85 #include <vm/vm_map.h>
86 #include <vm/vm_page.h>
87 #include <vm/vm_pageout.h>
88 #include <vm/vm_kern.h> /* kmem_alloc() */
89 #include <kern/misc_protos.h>
90 #include <mach_debug/zone_info.h>
91 #include <vm/cpm.h>
92 #include <pexpert/pexpert.h>
93 #include <pexpert/device_tree.h>
94 #include <san/kasan.h>
95
96 #include <vm/vm_protos.h>
97 #include <vm/memory_object.h>
98 #include <vm/vm_purgeable_internal.h>
99 #include <vm/vm_compressor.h>
100 #if defined (__x86_64__)
101 #include <i386/misc_protos.h>
102 #endif
103
104 #if CONFIG_PHANTOM_CACHE
105 #include <vm/vm_phantom_cache.h>
106 #endif
107
108 #if HIBERNATION
109 #include <IOKit/IOHibernatePrivate.h>
110 #include <machine/pal_hibernate.h>
111 #endif /* HIBERNATION */
112
113 #include <sys/kdebug.h>
114
115 #if defined(HAS_APPLE_PAC)
116 #include <ptrauth.h>
117 #endif
118 #if defined(__arm64__)
119 #include <arm/cpu_internal.h>
120 #endif /* defined(__arm64__) */
121
122 #if MACH_ASSERT
123
124 #define ASSERT_PMAP_FREE(mem) pmap_assert_free(VM_PAGE_GET_PHYS_PAGE(mem))
125
126 #else /* MACH_ASSERT */
127
128 #define ASSERT_PMAP_FREE(mem) /* nothing */
129
130 #endif /* MACH_ASSERT */
131
132 extern boolean_t vm_pageout_running;
133 extern thread_t vm_pageout_scan_thread;
134 extern boolean_t vps_dynamic_priority_enabled;
135
136 char vm_page_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
137 char vm_page_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
138 char vm_page_non_speculative_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
139 char vm_page_active_or_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
140
141 #if CONFIG_SECLUDED_MEMORY
142 struct vm_page_secluded_data vm_page_secluded;
143 #endif /* CONFIG_SECLUDED_MEMORY */
144
145 #if DEVELOPMENT || DEBUG
146 extern struct memory_object_pager_ops shared_region_pager_ops;
147 unsigned int shared_region_pagers_resident_count = 0;
148 unsigned int shared_region_pagers_resident_peak = 0;
149 #endif /* DEVELOPMENT || DEBUG */
150
151
152
153 int PERCPU_DATA(start_color);
154 vm_page_t PERCPU_DATA(free_pages);
155 boolean_t hibernate_cleaning_in_progress = FALSE;
156 boolean_t vm_page_free_verify = TRUE;
157
158 uint32_t vm_lopage_free_count = 0;
159 uint32_t vm_lopage_free_limit = 0;
160 uint32_t vm_lopage_lowater = 0;
161 boolean_t vm_lopage_refill = FALSE;
162 boolean_t vm_lopage_needed = FALSE;
163
164 int speculative_age_index = 0;
165 int speculative_steal_index = 0;
166 struct vm_speculative_age_q vm_page_queue_speculative[VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1];
167
168 boolean_t hibernation_vmqueues_inspection = FALSE; /* Tracks if the hibernation code is looking at the VM queues.
169 * Updated and checked behind the vm_page_queues_lock. */
170
171 static void vm_page_free_prepare(vm_page_t page);
172 static vm_page_t vm_page_grab_fictitious_common(ppnum_t, boolean_t);
173
174 static void vm_tag_init(void);
175
176 /* for debugging purposes */
177 SECURITY_READ_ONLY_EARLY(uint32_t) vm_packed_from_vm_pages_array_mask =
178 VM_PAGE_PACKED_FROM_ARRAY;
179 SECURITY_READ_ONLY_EARLY(vm_packing_params_t) vm_page_packing_params =
180 VM_PACKING_PARAMS(VM_PAGE_PACKED_PTR);
181
182 /*
183 * Associated with page of user-allocatable memory is a
184 * page structure.
185 */
186
187 /*
188 * These variables record the values returned by vm_page_bootstrap,
189 * for debugging purposes. The implementation of pmap_steal_memory
190 * and pmap_startup here also uses them internally.
191 */
192
193 vm_offset_t virtual_space_start;
194 vm_offset_t virtual_space_end;
195 uint32_t vm_page_pages;
196
197 /*
198 * The vm_page_lookup() routine, which provides for fast
199 * (virtual memory object, offset) to page lookup, employs
200 * the following hash table. The vm_page_{insert,remove}
201 * routines install and remove associations in the table.
202 * [This table is often called the virtual-to-physical,
203 * or VP, table.]
204 */
205 typedef struct {
206 vm_page_packed_t page_list;
207 #if MACH_PAGE_HASH_STATS
208 int cur_count; /* current count */
209 int hi_count; /* high water mark */
210 #endif /* MACH_PAGE_HASH_STATS */
211 } vm_page_bucket_t;
212
213
214 #define BUCKETS_PER_LOCK 16
215
216 SECURITY_READ_ONLY_LATE(vm_page_bucket_t *) vm_page_buckets; /* Array of buckets */
217 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_bucket_count = 0; /* How big is array? */
218 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_hash_mask; /* Mask for hash function */
219 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_hash_shift; /* Shift for hash function */
220 SECURITY_READ_ONLY_LATE(uint32_t) vm_page_bucket_hash; /* Basic bucket hash */
221 SECURITY_READ_ONLY_LATE(unsigned int) vm_page_bucket_lock_count = 0; /* How big is array of locks? */
222
223 #ifndef VM_TAG_ACTIVE_UPDATE
224 #error VM_TAG_ACTIVE_UPDATE
225 #endif
226 #ifndef VM_TAG_SIZECLASSES
227 #error VM_TAG_SIZECLASSES
228 #endif
229
230 /* for debugging */
231 SECURITY_READ_ONLY_LATE(bool) vm_tag_active_update = VM_TAG_ACTIVE_UPDATE;
232 SECURITY_READ_ONLY_LATE(lck_spin_t *) vm_page_bucket_locks;
233
234 vm_allocation_site_t vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC + 1];
235 vm_allocation_site_t * vm_allocation_sites[VM_MAX_TAG_VALUE];
236 #if VM_TAG_SIZECLASSES
237 static vm_allocation_zone_total_t **vm_allocation_zone_totals;
238 #endif /* VM_TAG_SIZECLASSES */
239
240 vm_tag_t vm_allocation_tag_highest;
241
242 #if VM_PAGE_BUCKETS_CHECK
243 boolean_t vm_page_buckets_check_ready = FALSE;
244 #if VM_PAGE_FAKE_BUCKETS
245 vm_page_bucket_t *vm_page_fake_buckets; /* decoy buckets */
246 vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
247 #endif /* VM_PAGE_FAKE_BUCKETS */
248 #endif /* VM_PAGE_BUCKETS_CHECK */
249
250 #if MACH_PAGE_HASH_STATS
251 /* This routine is only for debug. It is intended to be called by
252 * hand by a developer using a kernel debugger. This routine prints
253 * out vm_page_hash table statistics to the kernel debug console.
254 */
255 void
hash_debug(void)256 hash_debug(void)
257 {
258 int i;
259 int numbuckets = 0;
260 int highsum = 0;
261 int maxdepth = 0;
262
263 for (i = 0; i < vm_page_bucket_count; i++) {
264 if (vm_page_buckets[i].hi_count) {
265 numbuckets++;
266 highsum += vm_page_buckets[i].hi_count;
267 if (vm_page_buckets[i].hi_count > maxdepth) {
268 maxdepth = vm_page_buckets[i].hi_count;
269 }
270 }
271 }
272 printf("Total number of buckets: %d\n", vm_page_bucket_count);
273 printf("Number used buckets: %d = %d%%\n",
274 numbuckets, 100 * numbuckets / vm_page_bucket_count);
275 printf("Number unused buckets: %d = %d%%\n",
276 vm_page_bucket_count - numbuckets,
277 100 * (vm_page_bucket_count - numbuckets) / vm_page_bucket_count);
278 printf("Sum of bucket max depth: %d\n", highsum);
279 printf("Average bucket depth: %d.%2d\n",
280 highsum / vm_page_bucket_count,
281 highsum % vm_page_bucket_count);
282 printf("Maximum bucket depth: %d\n", maxdepth);
283 }
284 #endif /* MACH_PAGE_HASH_STATS */
285
286 /*
287 * The virtual page size is currently implemented as a runtime
288 * variable, but is constant once initialized using vm_set_page_size.
289 * This initialization must be done in the machine-dependent
290 * bootstrap sequence, before calling other machine-independent
291 * initializations.
292 *
293 * All references to the virtual page size outside this
294 * module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT
295 * constants.
296 */
297 #if defined(__arm64__)
298 vm_size_t page_size;
299 vm_size_t page_mask;
300 int page_shift;
301 #else
302 vm_size_t page_size = PAGE_SIZE;
303 vm_size_t page_mask = PAGE_MASK;
304 int page_shift = PAGE_SHIFT;
305 #endif
306
307 SECURITY_READ_ONLY_LATE(vm_page_t) vm_pages = VM_PAGE_NULL;
308 SECURITY_READ_ONLY_LATE(vm_page_t) vm_page_array_beginning_addr;
309 vm_page_t vm_page_array_ending_addr;
310
311 unsigned int vm_pages_count = 0;
312
313 /*
314 * Resident pages that represent real memory
315 * are allocated from a set of free lists,
316 * one per color.
317 */
318 unsigned int vm_colors;
319 unsigned int vm_color_mask; /* mask is == (vm_colors-1) */
320 unsigned int vm_cache_geometry_colors = 0; /* set by hw dependent code during startup */
321 unsigned int vm_free_magazine_refill_limit = 0;
322
323
324 struct vm_page_queue_free_head {
325 vm_page_queue_head_t qhead;
326 } VM_PAGE_PACKED_ALIGNED;
327
328 struct vm_page_queue_free_head vm_page_queue_free[MAX_COLORS];
329
330
331 unsigned int vm_page_free_wanted;
332 unsigned int vm_page_free_wanted_privileged;
333 #if CONFIG_SECLUDED_MEMORY
334 unsigned int vm_page_free_wanted_secluded;
335 #endif /* CONFIG_SECLUDED_MEMORY */
336 unsigned int vm_page_free_count;
337
338 /*
339 * Occasionally, the virtual memory system uses
340 * resident page structures that do not refer to
341 * real pages, for example to leave a page with
342 * important state information in the VP table.
343 *
344 * These page structures are allocated the way
345 * most other kernel structures are.
346 */
347 SECURITY_READ_ONLY_LATE(zone_t) vm_page_zone;
348 vm_locks_array_t vm_page_locks;
349
350 LCK_ATTR_DECLARE(vm_page_lck_attr, 0, 0);
351 LCK_GRP_DECLARE(vm_page_lck_grp_free, "vm_page_free");
352 LCK_GRP_DECLARE(vm_page_lck_grp_queue, "vm_page_queue");
353 LCK_GRP_DECLARE(vm_page_lck_grp_local, "vm_page_queue_local");
354 LCK_GRP_DECLARE(vm_page_lck_grp_purge, "vm_page_purge");
355 LCK_GRP_DECLARE(vm_page_lck_grp_alloc, "vm_page_alloc");
356 LCK_GRP_DECLARE(vm_page_lck_grp_bucket, "vm_page_bucket");
357 LCK_SPIN_DECLARE_ATTR(vm_objects_wired_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
358 LCK_TICKET_DECLARE(vm_allocation_sites_lock, &vm_page_lck_grp_bucket);
359
360 unsigned int vm_page_local_q_soft_limit = 250;
361 unsigned int vm_page_local_q_hard_limit = 500;
362 struct vpl *__zpercpu vm_page_local_q;
363
364 /* N.B. Guard and fictitious pages must not
365 * be assigned a zero phys_page value.
366 */
367 /*
368 * Fictitious pages don't have a physical address,
369 * but we must initialize phys_page to something.
370 * For debugging, this should be a strange value
371 * that the pmap module can recognize in assertions.
372 */
373 const ppnum_t vm_page_fictitious_addr = (ppnum_t) -1;
374
375 /*
376 * Guard pages are not accessible so they don't
377 * need a physical address, but we need to enter
378 * one in the pmap.
379 * Let's make it recognizable and make sure that
380 * we don't use a real physical page with that
381 * physical address.
382 */
383 const ppnum_t vm_page_guard_addr = (ppnum_t) -2;
384
385 /*
386 * Resident page structures are also chained on
387 * queues that are used by the page replacement
388 * system (pageout daemon). These queues are
389 * defined here, but are shared by the pageout
390 * module. The inactive queue is broken into
391 * file backed and anonymous for convenience as the
392 * pageout daemon often assignes a higher
393 * importance to anonymous pages (less likely to pick)
394 */
395 vm_page_queue_head_t vm_page_queue_active VM_PAGE_PACKED_ALIGNED;
396 vm_page_queue_head_t vm_page_queue_inactive VM_PAGE_PACKED_ALIGNED;
397 #if CONFIG_SECLUDED_MEMORY
398 vm_page_queue_head_t vm_page_queue_secluded VM_PAGE_PACKED_ALIGNED;
399 #endif /* CONFIG_SECLUDED_MEMORY */
400 vm_page_queue_head_t vm_page_queue_anonymous VM_PAGE_PACKED_ALIGNED; /* inactive memory queue for anonymous pages */
401 vm_page_queue_head_t vm_page_queue_throttled VM_PAGE_PACKED_ALIGNED;
402
403 queue_head_t vm_objects_wired;
404
405 void vm_update_darkwake_mode(boolean_t);
406
407 vm_page_queue_head_t vm_page_queue_donate VM_PAGE_PACKED_ALIGNED;
408 uint32_t vm_page_donate_mode;
409 uint32_t vm_page_donate_target, vm_page_donate_target_high, vm_page_donate_target_low;
410 uint32_t vm_page_donate_count;
411 bool vm_page_donate_queue_ripe;
412
413
414 vm_page_queue_head_t vm_page_queue_background VM_PAGE_PACKED_ALIGNED;
415 uint32_t vm_page_background_target;
416 uint32_t vm_page_background_target_snapshot;
417 uint32_t vm_page_background_count;
418 uint64_t vm_page_background_promoted_count;
419
420 uint32_t vm_page_background_internal_count;
421 uint32_t vm_page_background_external_count;
422
423 uint32_t vm_page_background_mode;
424 uint32_t vm_page_background_exclude_external;
425
426 unsigned int vm_page_active_count;
427 unsigned int vm_page_inactive_count;
428 unsigned int vm_page_kernelcache_count;
429 #if CONFIG_SECLUDED_MEMORY
430 unsigned int vm_page_secluded_count;
431 unsigned int vm_page_secluded_count_free;
432 unsigned int vm_page_secluded_count_inuse;
433 unsigned int vm_page_secluded_count_over_target;
434 #endif /* CONFIG_SECLUDED_MEMORY */
435 unsigned int vm_page_anonymous_count;
436 unsigned int vm_page_throttled_count;
437 unsigned int vm_page_speculative_count;
438
439 unsigned int vm_page_wire_count;
440 unsigned int vm_page_wire_count_on_boot = 0;
441 unsigned int vm_page_stolen_count = 0;
442 unsigned int vm_page_wire_count_initial;
443 unsigned int vm_page_gobble_count = 0;
444 unsigned int vm_page_kern_lpage_count = 0;
445
446 uint64_t booter_size; /* external so it can be found in core dumps */
447
448 #define VM_PAGE_WIRE_COUNT_WARNING 0
449 #define VM_PAGE_GOBBLE_COUNT_WARNING 0
450
451 unsigned int vm_page_purgeable_count = 0; /* # of pages purgeable now */
452 unsigned int vm_page_purgeable_wired_count = 0; /* # of purgeable pages that are wired now */
453 uint64_t vm_page_purged_count = 0; /* total count of purged pages */
454
455 unsigned int vm_page_xpmapped_external_count = 0;
456 unsigned int vm_page_external_count = 0;
457 unsigned int vm_page_internal_count = 0;
458 unsigned int vm_page_pageable_external_count = 0;
459 unsigned int vm_page_pageable_internal_count = 0;
460
461 #if DEVELOPMENT || DEBUG
462 unsigned int vm_page_speculative_recreated = 0;
463 unsigned int vm_page_speculative_created = 0;
464 unsigned int vm_page_speculative_used = 0;
465 #endif
466
467 vm_page_queue_head_t vm_page_queue_cleaned VM_PAGE_PACKED_ALIGNED;
468
469 unsigned int vm_page_cleaned_count = 0;
470
471 uint64_t max_valid_dma_address = 0xffffffffffffffffULL;
472 ppnum_t max_valid_low_ppnum = PPNUM_MAX;
473
474
475 /*
476 * Several page replacement parameters are also
477 * shared with this module, so that page allocation
478 * (done here in vm_page_alloc) can trigger the
479 * pageout daemon.
480 */
481 unsigned int vm_page_free_target = 0;
482 unsigned int vm_page_free_min = 0;
483 unsigned int vm_page_throttle_limit = 0;
484 unsigned int vm_page_inactive_target = 0;
485 #if CONFIG_SECLUDED_MEMORY
486 unsigned int vm_page_secluded_target = 0;
487 #endif /* CONFIG_SECLUDED_MEMORY */
488 unsigned int vm_page_anonymous_min = 0;
489 unsigned int vm_page_free_reserved = 0;
490
491
492 /*
493 * The VM system has a couple of heuristics for deciding
494 * that pages are "uninteresting" and should be placed
495 * on the inactive queue as likely candidates for replacement.
496 * These variables let the heuristics be controlled at run-time
497 * to make experimentation easier.
498 */
499
500 boolean_t vm_page_deactivate_hint = TRUE;
501
502 struct vm_page_stats_reusable vm_page_stats_reusable;
503
504 /*
505 * vm_set_page_size:
506 *
507 * Sets the page size, perhaps based upon the memory
508 * size. Must be called before any use of page-size
509 * dependent functions.
510 *
511 * Sets page_shift and page_mask from page_size.
512 */
513 void
vm_set_page_size(void)514 vm_set_page_size(void)
515 {
516 page_size = PAGE_SIZE;
517 page_mask = PAGE_MASK;
518 page_shift = PAGE_SHIFT;
519
520 if ((page_mask & page_size) != 0) {
521 panic("vm_set_page_size: page size not a power of two");
522 }
523
524 for (page_shift = 0;; page_shift++) {
525 if ((1U << page_shift) == page_size) {
526 break;
527 }
528 }
529 }
530
531 #if defined (__x86_64__)
532
533 #define MAX_CLUMP_SIZE 16
534 #define DEFAULT_CLUMP_SIZE 4
535
536 unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
537
538 #if DEVELOPMENT || DEBUG
539 unsigned long vm_clump_stats[MAX_CLUMP_SIZE + 1];
540 unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
541
542 static inline void
vm_clump_update_stats(unsigned int c)543 vm_clump_update_stats(unsigned int c)
544 {
545 assert(c <= vm_clump_size);
546 if (c > 0 && c <= vm_clump_size) {
547 vm_clump_stats[c] += c;
548 }
549 vm_clump_allocs += c;
550 }
551 #endif /* if DEVELOPMENT || DEBUG */
552
553 /* Called once to setup the VM clump knobs */
554 static void
vm_page_setup_clump(void)555 vm_page_setup_clump( void )
556 {
557 unsigned int override, n;
558
559 vm_clump_size = DEFAULT_CLUMP_SIZE;
560 if (PE_parse_boot_argn("clump_size", &override, sizeof(override))) {
561 vm_clump_size = override;
562 }
563
564 if (vm_clump_size > MAX_CLUMP_SIZE) {
565 panic("vm_page_setup_clump:: clump_size is too large!");
566 }
567 if (vm_clump_size < 1) {
568 panic("vm_page_setup_clump:: clump_size must be >= 1");
569 }
570 if ((vm_clump_size & (vm_clump_size - 1)) != 0) {
571 panic("vm_page_setup_clump:: clump_size must be a power of 2");
572 }
573
574 vm_clump_promote_threshold = vm_clump_size;
575 vm_clump_mask = vm_clump_size - 1;
576 for (vm_clump_shift = 0, n = vm_clump_size; n > 1; n >>= 1, vm_clump_shift++) {
577 ;
578 }
579
580 #if DEVELOPMENT || DEBUG
581 bzero(vm_clump_stats, sizeof(vm_clump_stats));
582 vm_clump_allocs = vm_clump_inserts = vm_clump_inrange = vm_clump_promotes = 0;
583 #endif /* if DEVELOPMENT || DEBUG */
584 }
585
586 #endif /* #if defined (__x86_64__) */
587
588 #define COLOR_GROUPS_TO_STEAL 4
589
590 /* Called once during statup, once the cache geometry is known.
591 */
592 static void
vm_page_set_colors(void)593 vm_page_set_colors( void )
594 {
595 unsigned int n, override;
596
597 #if defined (__x86_64__)
598 /* adjust #colors because we need to color outside the clump boundary */
599 vm_cache_geometry_colors >>= vm_clump_shift;
600 #endif
601 if (PE_parse_boot_argn("colors", &override, sizeof(override))) { /* colors specified as a boot-arg? */
602 n = override;
603 } else if (vm_cache_geometry_colors) { /* do we know what the cache geometry is? */
604 n = vm_cache_geometry_colors;
605 } else {
606 n = DEFAULT_COLORS; /* use default if all else fails */
607 }
608 if (n == 0) {
609 n = 1;
610 }
611 if (n > MAX_COLORS) {
612 n = MAX_COLORS;
613 }
614
615 /* the count must be a power of 2 */
616 if ((n & (n - 1)) != 0) {
617 n = DEFAULT_COLORS; /* use default if all else fails */
618 }
619 vm_colors = n;
620 vm_color_mask = n - 1;
621
622 vm_free_magazine_refill_limit = vm_colors * COLOR_GROUPS_TO_STEAL;
623
624 #if defined (__x86_64__)
625 /* adjust for reduction in colors due to clumping and multiple cores */
626 if (real_ncpus) {
627 vm_free_magazine_refill_limit *= (vm_clump_size * real_ncpus);
628 }
629 #endif
630 }
631
632 /*
633 * During single threaded early boot we don't initialize all pages.
634 * This avoids some delay during boot. They'll be initialized and
635 * added to the free list as needed or after we are multithreaded by
636 * what becomes the pageout thread.
637 */
638 static boolean_t fill = FALSE;
639 static unsigned int fillval;
640 uint_t vm_delayed_count = 0; /* when non-zero, indicates we may have more pages to init */
641 ppnum_t delay_above_pnum = PPNUM_MAX;
642
643 /*
644 * For x86 first 8 Gig initializes quickly and gives us lots of lowmem + mem above to start off with.
645 * If ARM ever uses delayed page initialization, this value may need to be quite different.
646 */
647 #define DEFAULT_DELAY_ABOVE_PHYS_GB (8)
648
649 /*
650 * When we have to dip into more delayed pages due to low memory, free up
651 * a large chunk to get things back to normal. This avoids contention on the
652 * delayed code allocating page by page.
653 */
654 #define VM_DELAY_PAGE_CHUNK ((1024 * 1024 * 1024) / PAGE_SIZE)
655
656 /*
657 * Get and initialize the next delayed page.
658 */
659 static vm_page_t
vm_get_delayed_page(int grab_options)660 vm_get_delayed_page(int grab_options)
661 {
662 vm_page_t p;
663 ppnum_t pnum;
664
665 /*
666 * Get a new page if we have one.
667 */
668 vm_free_page_lock();
669 if (vm_delayed_count == 0) {
670 vm_free_page_unlock();
671 return NULL;
672 }
673
674 if (!pmap_next_page(&pnum)) {
675 vm_delayed_count = 0;
676 vm_free_page_unlock();
677 return NULL;
678 }
679
680
681 assert(vm_delayed_count > 0);
682 --vm_delayed_count;
683
684 #if defined(__x86_64__)
685 /* x86 cluster code requires increasing phys_page in vm_pages[] */
686 if (vm_pages_count > 0) {
687 assert(pnum > vm_pages[vm_pages_count - 1].vmp_phys_page);
688 }
689 #endif
690 p = &vm_pages[vm_pages_count];
691 assert(p < vm_page_array_ending_addr);
692 vm_page_init(p, pnum, FALSE);
693 ++vm_pages_count;
694 ++vm_page_pages;
695 vm_free_page_unlock();
696
697 /*
698 * These pages were initially counted as wired, undo that now.
699 */
700 if (grab_options & VM_PAGE_GRAB_Q_LOCK_HELD) {
701 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
702 } else {
703 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
704 vm_page_lockspin_queues();
705 }
706 --vm_page_wire_count;
707 --vm_page_wire_count_initial;
708 if (vm_page_wire_count_on_boot != 0) {
709 --vm_page_wire_count_on_boot;
710 }
711 if (!(grab_options & VM_PAGE_GRAB_Q_LOCK_HELD)) {
712 vm_page_unlock_queues();
713 }
714
715
716 if (fill) {
717 fillPage(pnum, fillval);
718 }
719 return p;
720 }
721
722 static void vm_page_module_init_delayed(void);
723
724 /*
725 * Free all remaining delayed pages to the free lists.
726 */
727 void
vm_free_delayed_pages(void)728 vm_free_delayed_pages(void)
729 {
730 vm_page_t p;
731 vm_page_t list = NULL;
732 uint_t cnt = 0;
733 vm_offset_t start_free_va;
734 int64_t free_size;
735
736 while ((p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE)) != NULL) {
737 if (vm_himemory_mode) {
738 vm_page_release(p, FALSE);
739 } else {
740 p->vmp_snext = list;
741 list = p;
742 }
743 ++cnt;
744 }
745
746 /*
747 * Free the pages in reverse order if not himemory mode.
748 * Hence the low memory pages will be first on free lists. (LIFO)
749 */
750 while (list != NULL) {
751 p = list;
752 list = p->vmp_snext;
753 p->vmp_snext = NULL;
754 vm_page_release(p, FALSE);
755 }
756 #if DEVELOPMENT || DEBUG
757 kprintf("vm_free_delayed_pages: initialized %d free pages\n", cnt);
758 #endif
759
760 /*
761 * Free up any unused full pages at the end of the vm_pages[] array
762 */
763 start_free_va = round_page((vm_offset_t)&vm_pages[vm_pages_count]);
764
765 #if defined(__x86_64__)
766 /*
767 * Since x86 might have used large pages for vm_pages[], we can't
768 * free starting in the middle of a partially used large page.
769 */
770 if (pmap_query_pagesize(kernel_pmap, start_free_va) == I386_LPGBYTES) {
771 start_free_va = ((start_free_va + I386_LPGMASK) & ~I386_LPGMASK);
772 }
773 #endif
774 if (start_free_va < (vm_offset_t)vm_page_array_ending_addr) {
775 free_size = trunc_page((vm_offset_t)vm_page_array_ending_addr - start_free_va);
776 if (free_size > 0) {
777 ml_static_mfree(start_free_va, (vm_offset_t)free_size);
778 vm_page_array_ending_addr = (void *)start_free_va;
779
780 /*
781 * Note there's no locking here, as only this thread will ever change this value.
782 * The reader, vm_page_diagnose, doesn't grab any locks for the counts it looks at.
783 */
784 vm_page_stolen_count -= (free_size >> PAGE_SHIFT);
785
786 #if DEVELOPMENT || DEBUG
787 kprintf("Freeing final unused %ld bytes from vm_pages[] at 0x%lx\n",
788 (long)free_size, (long)start_free_va);
789 #endif
790 }
791 }
792
793
794 /*
795 * now we can create the VM page array zone
796 */
797 vm_page_module_init_delayed();
798 }
799
800 /*
801 * Try and free up enough delayed pages to match a contig memory allocation.
802 */
803 static void
vm_free_delayed_pages_contig(uint_t npages,ppnum_t max_pnum,ppnum_t pnum_mask)804 vm_free_delayed_pages_contig(
805 uint_t npages,
806 ppnum_t max_pnum,
807 ppnum_t pnum_mask)
808 {
809 vm_page_t p;
810 ppnum_t pnum;
811 uint_t cnt = 0;
812
813 /*
814 * Treat 0 as the absolute max page number.
815 */
816 if (max_pnum == 0) {
817 max_pnum = PPNUM_MAX;
818 }
819
820 /*
821 * Free till we get a properly aligned start page
822 */
823 for (;;) {
824 p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
825 if (p == NULL) {
826 return;
827 }
828 pnum = VM_PAGE_GET_PHYS_PAGE(p);
829 vm_page_release(p, FALSE);
830 if (pnum >= max_pnum) {
831 return;
832 }
833 if ((pnum & pnum_mask) == 0) {
834 break;
835 }
836 }
837
838 /*
839 * Having a healthy pool of free pages will help performance. We don't
840 * want to fall back to the delayed code for every page allocation.
841 */
842 if (vm_page_free_count < VM_DELAY_PAGE_CHUNK) {
843 npages += VM_DELAY_PAGE_CHUNK;
844 }
845
846 /*
847 * Now free up the pages
848 */
849 for (cnt = 1; cnt < npages; ++cnt) {
850 p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
851 if (p == NULL) {
852 return;
853 }
854 vm_page_release(p, FALSE);
855 }
856 }
857
858 #define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1)))
859
860 void
vm_page_init_local_q(unsigned int num_cpus)861 vm_page_init_local_q(unsigned int num_cpus)
862 {
863 struct vpl *t_local_q;
864
865 /*
866 * no point in this for a uni-processor system
867 */
868 if (num_cpus >= 2) {
869 ml_cpu_info_t cpu_info;
870
871 /*
872 * Force the allocation alignment to a cacheline,
873 * because the `vpl` struct has a lock and will be taken
874 * cross CPU so we want to isolate the rest of the per-CPU
875 * data to avoid false sharing due to this lock being taken.
876 */
877
878 ml_cpu_get_info(&cpu_info);
879
880 t_local_q = zalloc_percpu_permanent(sizeof(struct vpl),
881 cpu_info.cache_line_size - 1);
882
883 zpercpu_foreach(lq, t_local_q) {
884 VPL_LOCK_INIT(lq, &vm_page_lck_grp_local, &vm_page_lck_attr);
885 vm_page_queue_init(&lq->vpl_queue);
886 }
887
888 /* make the initialization visible to all cores */
889 os_atomic_store(&vm_page_local_q, t_local_q, release);
890 }
891 }
892
893 /*
894 * vm_init_before_launchd
895 *
896 * This should be called right before launchd is loaded.
897 */
898 void
vm_init_before_launchd()899 vm_init_before_launchd()
900 {
901 vm_page_lockspin_queues();
902 vm_page_wire_count_on_boot = vm_page_wire_count;
903 vm_page_unlock_queues();
904 }
905
906
907 /*
908 * vm_page_bootstrap:
909 *
910 * Initializes the resident memory module.
911 *
912 * Allocates memory for the page cells, and
913 * for the object/offset-to-page hash table headers.
914 * Each page cell is initialized and placed on the free list.
915 * Returns the range of available kernel virtual memory.
916 */
917 __startup_func
918 void
vm_page_bootstrap(vm_offset_t * startp,vm_offset_t * endp)919 vm_page_bootstrap(
920 vm_offset_t *startp,
921 vm_offset_t *endp)
922 {
923 unsigned int i;
924 unsigned int log1;
925 unsigned int log2;
926 unsigned int size;
927
928 /*
929 * Initialize the page queues.
930 */
931
932 lck_mtx_init(&vm_page_queue_free_lock, &vm_page_lck_grp_free, &vm_page_lck_attr);
933 lck_mtx_init(&vm_page_queue_lock, &vm_page_lck_grp_queue, &vm_page_lck_attr);
934 lck_mtx_init(&vm_purgeable_queue_lock, &vm_page_lck_grp_purge, &vm_page_lck_attr);
935
936 for (i = 0; i < PURGEABLE_Q_TYPE_MAX; i++) {
937 int group;
938
939 purgeable_queues[i].token_q_head = 0;
940 purgeable_queues[i].token_q_tail = 0;
941 for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
942 queue_init(&purgeable_queues[i].objq[group]);
943 }
944
945 purgeable_queues[i].type = i;
946 purgeable_queues[i].new_pages = 0;
947 #if MACH_ASSERT
948 purgeable_queues[i].debug_count_tokens = 0;
949 purgeable_queues[i].debug_count_objects = 0;
950 #endif
951 }
952 ;
953 purgeable_nonvolatile_count = 0;
954 queue_init(&purgeable_nonvolatile_queue);
955
956 for (i = 0; i < MAX_COLORS; i++) {
957 vm_page_queue_init(&vm_page_queue_free[i].qhead);
958 }
959
960 vm_page_queue_init(&vm_lopage_queue_free);
961 vm_page_queue_init(&vm_page_queue_active);
962 vm_page_queue_init(&vm_page_queue_inactive);
963 #if CONFIG_SECLUDED_MEMORY
964 vm_page_queue_init(&vm_page_queue_secluded);
965 #endif /* CONFIG_SECLUDED_MEMORY */
966 vm_page_queue_init(&vm_page_queue_cleaned);
967 vm_page_queue_init(&vm_page_queue_throttled);
968 vm_page_queue_init(&vm_page_queue_anonymous);
969 queue_init(&vm_objects_wired);
970
971 for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
972 vm_page_queue_init(&vm_page_queue_speculative[i].age_q);
973
974 vm_page_queue_speculative[i].age_ts.tv_sec = 0;
975 vm_page_queue_speculative[i].age_ts.tv_nsec = 0;
976 }
977
978 vm_page_queue_init(&vm_page_queue_donate);
979 vm_page_queue_init(&vm_page_queue_background);
980
981 vm_page_background_count = 0;
982 vm_page_background_internal_count = 0;
983 vm_page_background_external_count = 0;
984 vm_page_background_promoted_count = 0;
985
986 vm_page_background_target = (unsigned int)(atop_64(max_mem) / 25);
987
988 if (vm_page_background_target > VM_PAGE_BACKGROUND_TARGET_MAX) {
989 vm_page_background_target = VM_PAGE_BACKGROUND_TARGET_MAX;
990 }
991
992 #if defined(__LP64__)
993 vm_page_background_mode = VM_PAGE_BG_ENABLED;
994 vm_page_donate_mode = VM_PAGE_DONATE_ENABLED;
995 #else
996 vm_page_background_mode = VM_PAGE_BG_DISABLED;
997 vm_page_donate_mode = VM_PAGE_DONATE_DISABLED;
998 #endif
999 vm_page_background_exclude_external = 0;
1000
1001 PE_parse_boot_argn("vm_page_bg_mode", &vm_page_background_mode, sizeof(vm_page_background_mode));
1002 PE_parse_boot_argn("vm_page_bg_exclude_external", &vm_page_background_exclude_external, sizeof(vm_page_background_exclude_external));
1003 PE_parse_boot_argn("vm_page_bg_target", &vm_page_background_target, sizeof(vm_page_background_target));
1004
1005 if (vm_page_background_mode != VM_PAGE_BG_DISABLED && vm_page_background_mode != VM_PAGE_BG_ENABLED) {
1006 vm_page_background_mode = VM_PAGE_BG_DISABLED;
1007 }
1008
1009 PE_parse_boot_argn("vm_page_donate_mode", &vm_page_donate_mode, sizeof(vm_page_donate_mode));
1010 if (vm_page_donate_mode != VM_PAGE_DONATE_DISABLED && vm_page_donate_mode != VM_PAGE_DONATE_ENABLED) {
1011 vm_page_donate_mode = VM_PAGE_DONATE_DISABLED;
1012 }
1013
1014 vm_page_donate_target_high = VM_PAGE_DONATE_TARGET_HIGHWATER;
1015 vm_page_donate_target_low = VM_PAGE_DONATE_TARGET_LOWWATER;
1016 vm_page_donate_target = vm_page_donate_target_high;
1017 vm_page_donate_count = 0;
1018
1019 vm_page_free_wanted = 0;
1020 vm_page_free_wanted_privileged = 0;
1021 #if CONFIG_SECLUDED_MEMORY
1022 vm_page_free_wanted_secluded = 0;
1023 #endif /* CONFIG_SECLUDED_MEMORY */
1024
1025 #if defined (__x86_64__)
1026 /* this must be called before vm_page_set_colors() */
1027 vm_page_setup_clump();
1028 #endif
1029
1030 vm_page_set_colors();
1031
1032 bzero(vm_page_inactive_states, sizeof(vm_page_inactive_states));
1033 vm_page_inactive_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
1034 vm_page_inactive_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;
1035 vm_page_inactive_states[VM_PAGE_ON_INACTIVE_CLEANED_Q] = 1;
1036
1037 bzero(vm_page_pageable_states, sizeof(vm_page_pageable_states));
1038 vm_page_pageable_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
1039 vm_page_pageable_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;
1040 vm_page_pageable_states[VM_PAGE_ON_INACTIVE_CLEANED_Q] = 1;
1041 vm_page_pageable_states[VM_PAGE_ON_ACTIVE_Q] = 1;
1042 vm_page_pageable_states[VM_PAGE_ON_SPECULATIVE_Q] = 1;
1043 vm_page_pageable_states[VM_PAGE_ON_THROTTLED_Q] = 1;
1044 #if CONFIG_SECLUDED_MEMORY
1045 vm_page_pageable_states[VM_PAGE_ON_SECLUDED_Q] = 1;
1046 #endif /* CONFIG_SECLUDED_MEMORY */
1047
1048 bzero(vm_page_non_speculative_pageable_states, sizeof(vm_page_non_speculative_pageable_states));
1049 vm_page_non_speculative_pageable_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
1050 vm_page_non_speculative_pageable_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;
1051 vm_page_non_speculative_pageable_states[VM_PAGE_ON_INACTIVE_CLEANED_Q] = 1;
1052 vm_page_non_speculative_pageable_states[VM_PAGE_ON_ACTIVE_Q] = 1;
1053 vm_page_non_speculative_pageable_states[VM_PAGE_ON_THROTTLED_Q] = 1;
1054 #if CONFIG_SECLUDED_MEMORY
1055 vm_page_non_speculative_pageable_states[VM_PAGE_ON_SECLUDED_Q] = 1;
1056 #endif /* CONFIG_SECLUDED_MEMORY */
1057
1058 bzero(vm_page_active_or_inactive_states, sizeof(vm_page_active_or_inactive_states));
1059 vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
1060 vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;
1061 vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_CLEANED_Q] = 1;
1062 vm_page_active_or_inactive_states[VM_PAGE_ON_ACTIVE_Q] = 1;
1063 #if CONFIG_SECLUDED_MEMORY
1064 vm_page_active_or_inactive_states[VM_PAGE_ON_SECLUDED_Q] = 1;
1065 #endif /* CONFIG_SECLUDED_MEMORY */
1066
1067 for (vm_tag_t t = 0; t < VM_KERN_MEMORY_FIRST_DYNAMIC; t++) {
1068 vm_allocation_sites_static[t].refcount = 2;
1069 vm_allocation_sites_static[t].tag = t;
1070 vm_allocation_sites[t] = &vm_allocation_sites_static[t];
1071 }
1072 vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].refcount = 2;
1073 vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].tag = VM_KERN_MEMORY_ANY;
1074 vm_allocation_sites[VM_KERN_MEMORY_ANY] = &vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC];
1075
1076 /*
1077 * Steal memory for the map and zone subsystems.
1078 */
1079 kernel_startup_initialize_upto(STARTUP_SUB_PMAP_STEAL);
1080
1081 /*
1082 * Allocate (and initialize) the virtual-to-physical
1083 * table hash buckets.
1084 *
1085 * The number of buckets should be a power of two to
1086 * get a good hash function. The following computation
1087 * chooses the first power of two that is greater
1088 * than the number of physical pages in the system.
1089 */
1090
1091 if (vm_page_bucket_count == 0) {
1092 unsigned int npages = pmap_free_pages();
1093
1094 vm_page_bucket_count = 1;
1095 while (vm_page_bucket_count < npages) {
1096 vm_page_bucket_count <<= 1;
1097 }
1098 }
1099 vm_page_bucket_lock_count = (vm_page_bucket_count + BUCKETS_PER_LOCK - 1) / BUCKETS_PER_LOCK;
1100
1101 vm_page_hash_mask = vm_page_bucket_count - 1;
1102
1103 /*
1104 * Calculate object shift value for hashing algorithm:
1105 * O = log2(sizeof(struct vm_object))
1106 * B = log2(vm_page_bucket_count)
1107 * hash shifts the object left by
1108 * B/2 - O
1109 */
1110 size = vm_page_bucket_count;
1111 for (log1 = 0; size > 1; log1++) {
1112 size /= 2;
1113 }
1114 size = sizeof(struct vm_object);
1115 for (log2 = 0; size > 1; log2++) {
1116 size /= 2;
1117 }
1118 vm_page_hash_shift = log1 / 2 - log2 + 1;
1119
1120 vm_page_bucket_hash = 1 << ((log1 + 1) >> 1); /* Get (ceiling of sqrt of table size) */
1121 vm_page_bucket_hash |= 1 << ((log1 + 1) >> 2); /* Get (ceiling of quadroot of table size) */
1122 vm_page_bucket_hash |= 1; /* Set bit and add 1 - always must be 1 to insure unique series */
1123
1124 if (vm_page_hash_mask & vm_page_bucket_count) {
1125 printf("vm_page_bootstrap: WARNING -- strange page hash\n");
1126 }
1127
1128 #if VM_PAGE_BUCKETS_CHECK
1129 #if VM_PAGE_FAKE_BUCKETS
1130 /*
1131 * Allocate a decoy set of page buckets, to detect
1132 * any stomping there.
1133 */
1134 vm_page_fake_buckets = (vm_page_bucket_t *)
1135 pmap_steal_memory(vm_page_bucket_count *
1136 sizeof(vm_page_bucket_t));
1137 vm_page_fake_buckets_start = (vm_map_offset_t) vm_page_fake_buckets;
1138 vm_page_fake_buckets_end =
1139 vm_map_round_page((vm_page_fake_buckets_start +
1140 (vm_page_bucket_count *
1141 sizeof(vm_page_bucket_t))),
1142 PAGE_MASK);
1143 char *cp;
1144 for (cp = (char *)vm_page_fake_buckets_start;
1145 cp < (char *)vm_page_fake_buckets_end;
1146 cp++) {
1147 *cp = 0x5a;
1148 }
1149 #endif /* VM_PAGE_FAKE_BUCKETS */
1150 #endif /* VM_PAGE_BUCKETS_CHECK */
1151
1152 kernel_debug_string_early("vm_page_buckets");
1153 vm_page_buckets = (vm_page_bucket_t *)
1154 pmap_steal_memory(vm_page_bucket_count *
1155 sizeof(vm_page_bucket_t));
1156
1157 kernel_debug_string_early("vm_page_bucket_locks");
1158 vm_page_bucket_locks = (lck_spin_t *)
1159 pmap_steal_memory(vm_page_bucket_lock_count *
1160 sizeof(lck_spin_t));
1161
1162 for (i = 0; i < vm_page_bucket_count; i++) {
1163 vm_page_bucket_t *bucket = &vm_page_buckets[i];
1164
1165 bucket->page_list = VM_PAGE_PACK_PTR(VM_PAGE_NULL);
1166 #if MACH_PAGE_HASH_STATS
1167 bucket->cur_count = 0;
1168 bucket->hi_count = 0;
1169 #endif /* MACH_PAGE_HASH_STATS */
1170 }
1171
1172 for (i = 0; i < vm_page_bucket_lock_count; i++) {
1173 lck_spin_init(&vm_page_bucket_locks[i], &vm_page_lck_grp_bucket, &vm_page_lck_attr);
1174 }
1175
1176 vm_tag_init();
1177
1178 #if VM_PAGE_BUCKETS_CHECK
1179 vm_page_buckets_check_ready = TRUE;
1180 #endif /* VM_PAGE_BUCKETS_CHECK */
1181
1182 /*
1183 * Machine-dependent code allocates the resident page table.
1184 * It uses vm_page_init to initialize the page frames.
1185 * The code also returns to us the virtual space available
1186 * to the kernel. We don't trust the pmap module
1187 * to get the alignment right.
1188 */
1189
1190 kernel_debug_string_early("pmap_startup");
1191 pmap_startup(&virtual_space_start, &virtual_space_end);
1192 virtual_space_start = round_page(virtual_space_start);
1193 virtual_space_end = trunc_page(virtual_space_end);
1194
1195 *startp = virtual_space_start;
1196 *endp = virtual_space_end;
1197
1198 /*
1199 * Compute the initial "wire" count.
1200 * Up until now, the pages which have been set aside are not under
1201 * the VM system's control, so although they aren't explicitly
1202 * wired, they nonetheless can't be moved. At this moment,
1203 * all VM managed pages are "free", courtesy of pmap_startup.
1204 */
1205 assert((unsigned int) atop_64(max_mem) == atop_64(max_mem));
1206 vm_page_wire_count = ((unsigned int) atop_64(max_mem)) -
1207 vm_page_free_count - vm_lopage_free_count;
1208 #if CONFIG_SECLUDED_MEMORY
1209 vm_page_wire_count -= vm_page_secluded_count;
1210 #endif
1211 vm_page_wire_count_initial = vm_page_wire_count;
1212
1213 /* capture this for later use */
1214 booter_size = ml_get_booter_memory_size();
1215
1216 printf("vm_page_bootstrap: %d free pages, %d wired pages, (up to %d of which are delayed free)\n",
1217 vm_page_free_count, vm_page_wire_count, vm_delayed_count);
1218
1219 kernel_debug_string_early("vm_page_bootstrap complete");
1220 }
1221
1222 #ifndef MACHINE_PAGES
1223 /*
1224 * This is the early boot time allocator for data structures needed to bootstrap the VM system.
1225 * On x86 it will allocate large pages if size is sufficiently large. We don't need to do this
1226 * on ARM yet, due to the combination of a large base page size and smaller RAM devices.
1227 */
1228 static void *
pmap_steal_memory_internal(vm_size_t size,boolean_t might_free)1229 pmap_steal_memory_internal(
1230 vm_size_t size,
1231 boolean_t might_free)
1232 {
1233 kern_return_t kr;
1234 vm_offset_t addr;
1235 vm_offset_t map_addr;
1236 ppnum_t phys_page;
1237
1238 /*
1239 * Size needs to be aligned to word size.
1240 */
1241 size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
1242
1243 /*
1244 * On the first call, get the initial values for virtual address space
1245 * and page align them.
1246 */
1247 if (virtual_space_start == virtual_space_end) {
1248 pmap_virtual_space(&virtual_space_start, &virtual_space_end);
1249 virtual_space_start = round_page(virtual_space_start);
1250 virtual_space_end = trunc_page(virtual_space_end);
1251
1252 #if defined(__x86_64__)
1253 /*
1254 * Release remaining unused section of preallocated KVA and the 4K page tables
1255 * that map it. This makes the VA available for large page mappings.
1256 */
1257 Idle_PTs_release(virtual_space_start, virtual_space_end);
1258 #endif
1259 }
1260
1261 /*
1262 * Allocate the virtual space for this request. On x86, we'll align to a large page
1263 * address if the size is big enough to back with at least 1 large page.
1264 */
1265 #if defined(__x86_64__)
1266 if (size >= I386_LPGBYTES) {
1267 virtual_space_start = ((virtual_space_start + I386_LPGMASK) & ~I386_LPGMASK);
1268 }
1269 #endif
1270 addr = virtual_space_start;
1271 virtual_space_start += size;
1272
1273 //kprintf("pmap_steal_memory: %08lX - %08lX; size=%08lX\n", (long)addr, (long)virtual_space_start, (long)size); /* (TEST/DEBUG) */
1274
1275 /*
1276 * Allocate and map physical pages to back the new virtual space.
1277 */
1278 map_addr = round_page(addr);
1279 while (map_addr < addr + size) {
1280 #if defined(__x86_64__)
1281 /*
1282 * Back with a large page if properly aligned on x86
1283 */
1284 if ((map_addr & I386_LPGMASK) == 0 &&
1285 map_addr + I386_LPGBYTES <= addr + size &&
1286 pmap_pre_expand_large(kernel_pmap, map_addr) == KERN_SUCCESS &&
1287 pmap_next_page_large(&phys_page) == KERN_SUCCESS) {
1288 kr = pmap_enter(kernel_pmap, map_addr, phys_page,
1289 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
1290 VM_WIMG_USE_DEFAULT | VM_MEM_SUPERPAGE, FALSE);
1291
1292 if (kr != KERN_SUCCESS) {
1293 panic("pmap_steal_memory: pmap_enter() large failed, new_addr=%#lx, phys_page=%u",
1294 (unsigned long)map_addr, phys_page);
1295 }
1296 map_addr += I386_LPGBYTES;
1297 vm_page_wire_count += I386_LPGBYTES >> PAGE_SHIFT;
1298 vm_page_stolen_count += I386_LPGBYTES >> PAGE_SHIFT;
1299 vm_page_kern_lpage_count++;
1300 continue;
1301 }
1302 #endif
1303
1304 if (!pmap_next_page_hi(&phys_page, might_free)) {
1305 panic("pmap_steal_memory() size: 0x%llx", (uint64_t)size);
1306 }
1307
1308 #if defined(__x86_64__)
1309 pmap_pre_expand(kernel_pmap, map_addr);
1310 #endif
1311
1312 kr = pmap_enter(kernel_pmap, map_addr, phys_page,
1313 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
1314 VM_WIMG_USE_DEFAULT, FALSE);
1315
1316 if (kr != KERN_SUCCESS) {
1317 panic("pmap_steal_memory() pmap_enter failed, map_addr=%#lx, phys_page=%u",
1318 (unsigned long)map_addr, phys_page);
1319 }
1320 map_addr += PAGE_SIZE;
1321
1322 /*
1323 * Account for newly stolen memory
1324 */
1325 vm_page_wire_count++;
1326 vm_page_stolen_count++;
1327 }
1328
1329 #if defined(__x86_64__)
1330 /*
1331 * The call with might_free is currently the last use of pmap_steal_memory*().
1332 * Notify the pmap layer to record which high pages were allocated so far.
1333 */
1334 if (might_free) {
1335 pmap_hi_pages_done();
1336 }
1337 #endif
1338 #if KASAN
1339 kasan_notify_address(round_page(addr), size);
1340 #endif
1341 return (void *) addr;
1342 }
1343
1344 void *
pmap_steal_memory(vm_size_t size)1345 pmap_steal_memory(
1346 vm_size_t size)
1347 {
1348 return pmap_steal_memory_internal(size, FALSE);
1349 }
1350
1351 void *
pmap_steal_freeable_memory(vm_size_t size)1352 pmap_steal_freeable_memory(
1353 vm_size_t size)
1354 {
1355 return pmap_steal_memory_internal(size, TRUE);
1356 }
1357
1358
1359 #if CONFIG_SECLUDED_MEMORY
1360 /* boot-args to control secluded memory */
1361 unsigned int secluded_mem_mb = 0; /* # of MBs of RAM to seclude */
1362 int secluded_for_iokit = 1; /* IOKit can use secluded memory */
1363 int secluded_for_apps = 1; /* apps can use secluded memory */
1364 int secluded_for_filecache = 2; /* filecache can use seclude memory */
1365 #if 11
1366 int secluded_for_fbdp = 0;
1367 #endif
1368 uint64_t secluded_shutoff_trigger = 0;
1369 uint64_t secluded_shutoff_headroom = 150 * 1024 * 1024; /* original value from N56 */
1370 #endif /* CONFIG_SECLUDED_MEMORY */
1371
1372
1373 #if defined(__arm64__)
1374 extern void patch_low_glo_vm_page_info(void *, void *, uint32_t);
1375 unsigned int vm_first_phys_ppnum = 0;
1376 #endif
1377
1378 void vm_page_release_startup(vm_page_t mem);
1379 void
pmap_startup(vm_offset_t * startp,vm_offset_t * endp)1380 pmap_startup(
1381 vm_offset_t *startp,
1382 vm_offset_t *endp)
1383 {
1384 unsigned int i, npages;
1385 ppnum_t phys_page;
1386 uint64_t mem_sz;
1387 uint64_t start_ns;
1388 uint64_t now_ns;
1389 uint_t low_page_count = 0;
1390
1391 #if defined(__LP64__)
1392 /*
1393 * make sure we are aligned on a 64 byte boundary
1394 * for VM_PAGE_PACK_PTR (it clips off the low-order
1395 * 6 bits of the pointer)
1396 */
1397 if (virtual_space_start != virtual_space_end) {
1398 virtual_space_start = round_page(virtual_space_start);
1399 }
1400 #endif
1401
1402 /*
1403 * We calculate how many page frames we will have
1404 * and then allocate the page structures in one chunk.
1405 *
1406 * Note that the calculation here doesn't take into account
1407 * the memory needed to map what's being allocated, i.e. the page
1408 * table entries. So the actual number of pages we get will be
1409 * less than this. To do someday: include that in the computation.
1410 *
1411 * Also for ARM, we don't use the count of free_pages, but rather the
1412 * range from last page to first page (ignore holes due to retired pages).
1413 */
1414 #if defined(__arm64__)
1415 mem_sz = pmap_free_pages_span() * (uint64_t)PAGE_SIZE;
1416 #else /* defined(__arm64__) */
1417 mem_sz = pmap_free_pages() * (uint64_t)PAGE_SIZE;
1418 #endif /* defined(__arm64__) */
1419 mem_sz += round_page(virtual_space_start) - virtual_space_start; /* Account for any slop */
1420 npages = (uint_t)(mem_sz / (PAGE_SIZE + sizeof(*vm_pages))); /* scaled to include the vm_page_ts */
1421
1422
1423 vm_pages = (vm_page_t) pmap_steal_freeable_memory(npages * sizeof *vm_pages);
1424
1425 /*
1426 * Check if we want to initialize pages to a known value
1427 */
1428 if (PE_parse_boot_argn("fill", &fillval, sizeof(fillval))) {
1429 fill = TRUE;
1430 }
1431 #if DEBUG
1432 /* This slows down booting the DEBUG kernel, particularly on
1433 * large memory systems, but is worthwhile in deterministically
1434 * trapping uninitialized memory usage.
1435 */
1436 if (!fill) {
1437 fill = TRUE;
1438 fillval = 0xDEB8F177;
1439 }
1440 #endif
1441 if (fill) {
1442 kprintf("Filling vm_pages with pattern: 0x%x\n", fillval);
1443 }
1444
1445 #if CONFIG_SECLUDED_MEMORY
1446 /*
1447 * Figure out how much secluded memory to have before we start
1448 * release pages to free lists.
1449 * The default, if specified nowhere else, is no secluded mem.
1450 */
1451 secluded_mem_mb = 0;
1452 if (max_mem > 1 * 1024 * 1024 * 1024) {
1453 /* default to 90MB for devices with > 1GB of RAM */
1454 secluded_mem_mb = 90;
1455 }
1456 /* override with value from device tree, if provided */
1457 PE_get_default("kern.secluded_mem_mb",
1458 &secluded_mem_mb, sizeof(secluded_mem_mb));
1459 /* override with value from boot-args, if provided */
1460 PE_parse_boot_argn("secluded_mem_mb",
1461 &secluded_mem_mb,
1462 sizeof(secluded_mem_mb));
1463
1464 vm_page_secluded_target = (unsigned int)
1465 ((secluded_mem_mb * 1024ULL * 1024ULL) / PAGE_SIZE);
1466 PE_parse_boot_argn("secluded_for_iokit",
1467 &secluded_for_iokit,
1468 sizeof(secluded_for_iokit));
1469 PE_parse_boot_argn("secluded_for_apps",
1470 &secluded_for_apps,
1471 sizeof(secluded_for_apps));
1472 PE_parse_boot_argn("secluded_for_filecache",
1473 &secluded_for_filecache,
1474 sizeof(secluded_for_filecache));
1475 #if 11
1476 PE_parse_boot_argn("secluded_for_fbdp",
1477 &secluded_for_fbdp,
1478 sizeof(secluded_for_fbdp));
1479 #endif
1480
1481 /*
1482 * Allow a really large app to effectively use secluded memory until it exits.
1483 */
1484 if (vm_page_secluded_target != 0) {
1485 /*
1486 * Get an amount from boot-args, else use 1/2 of max_mem.
1487 * 1/2 max_mem was chosen from a Peace daemon tentpole test which
1488 * used munch to induce jetsam thrashing of false idle daemons on N56.
1489 */
1490 int secluded_shutoff_mb;
1491 if (PE_parse_boot_argn("secluded_shutoff_mb", &secluded_shutoff_mb,
1492 sizeof(secluded_shutoff_mb))) {
1493 secluded_shutoff_trigger = (uint64_t)secluded_shutoff_mb * 1024 * 1024;
1494 } else {
1495 secluded_shutoff_trigger = max_mem / 2;
1496 }
1497
1498 /* ensure the headroom value is sensible and avoid underflows */
1499 assert(secluded_shutoff_trigger == 0 || secluded_shutoff_trigger > secluded_shutoff_headroom);
1500 }
1501
1502 #endif /* CONFIG_SECLUDED_MEMORY */
1503
1504 #if defined(__x86_64__)
1505
1506 /*
1507 * Decide how much memory we delay freeing at boot time.
1508 */
1509 uint32_t delay_above_gb;
1510 if (!PE_parse_boot_argn("delay_above_gb", &delay_above_gb, sizeof(delay_above_gb))) {
1511 delay_above_gb = DEFAULT_DELAY_ABOVE_PHYS_GB;
1512 }
1513
1514 if (delay_above_gb == 0) {
1515 delay_above_pnum = PPNUM_MAX;
1516 } else {
1517 delay_above_pnum = delay_above_gb * (1024 * 1024 * 1024 / PAGE_SIZE);
1518 }
1519
1520 /* make sure we have sane breathing room: 1G above low memory */
1521 if (delay_above_pnum <= max_valid_low_ppnum) {
1522 delay_above_pnum = max_valid_low_ppnum + ((1024 * 1024 * 1024) >> PAGE_SHIFT);
1523 }
1524
1525 if (delay_above_pnum < PPNUM_MAX) {
1526 printf("pmap_startup() delaying init/free of page nums > 0x%x\n", delay_above_pnum);
1527 }
1528
1529 #endif /* defined(__x86_64__) */
1530
1531 /*
1532 * Initialize and release the page frames.
1533 */
1534 kernel_debug_string_early("page_frame_init");
1535
1536 vm_page_array_beginning_addr = &vm_pages[0];
1537 vm_page_array_ending_addr = &vm_pages[npages]; /* used by ptr packing/unpacking code */
1538 #if VM_PAGE_PACKED_FROM_ARRAY
1539 if (npages >= VM_PAGE_PACKED_FROM_ARRAY) {
1540 panic("pmap_startup(): too many pages to support vm_page packing");
1541 }
1542 #endif
1543
1544 vm_delayed_count = 0;
1545
1546 absolutetime_to_nanoseconds(mach_absolute_time(), &start_ns);
1547 vm_pages_count = 0;
1548 for (i = 0; i < npages; i++) {
1549 /* Did we run out of pages? */
1550 if (!pmap_next_page(&phys_page)) {
1551 break;
1552 }
1553
1554 if (phys_page < max_valid_low_ppnum) {
1555 ++low_page_count;
1556 }
1557
1558 /* Are we at high enough pages to delay the rest? */
1559 if (low_page_count > vm_lopage_free_limit && phys_page > delay_above_pnum) {
1560 vm_delayed_count = pmap_free_pages();
1561 break;
1562 }
1563
1564 #if defined(__arm64__)
1565 if (i == 0) {
1566 vm_first_phys_ppnum = phys_page;
1567 patch_low_glo_vm_page_info((void *)vm_page_array_beginning_addr,
1568 (void *)vm_page_array_ending_addr, vm_first_phys_ppnum);
1569 }
1570 #endif /* defined(__arm64__) */
1571
1572 #if defined(__x86_64__)
1573 /* The x86 clump freeing code requires increasing ppn's to work correctly */
1574 if (i > 0) {
1575 assert(phys_page > vm_pages[i - 1].vmp_phys_page);
1576 }
1577 #endif
1578 ++vm_pages_count;
1579 vm_page_init(&vm_pages[i], phys_page, FALSE);
1580 if (fill) {
1581 fillPage(phys_page, fillval);
1582 }
1583 if (vm_himemory_mode) {
1584 vm_page_release_startup(&vm_pages[i]);
1585 }
1586 }
1587 vm_page_pages = vm_pages_count; /* used to report to user space */
1588
1589 if (!vm_himemory_mode) {
1590 do {
1591 if (!VMP_ERROR_GET(&vm_pages[--i])) { /* skip retired pages */
1592 vm_page_release_startup(&vm_pages[i]);
1593 }
1594 } while (i != 0);
1595 }
1596
1597 absolutetime_to_nanoseconds(mach_absolute_time(), &now_ns);
1598 printf("pmap_startup() init/release time: %lld microsec\n", (now_ns - start_ns) / NSEC_PER_USEC);
1599 printf("pmap_startup() delayed init/release of %d pages\n", vm_delayed_count);
1600
1601 #if defined(__LP64__)
1602 if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[0]))) != &vm_pages[0]) {
1603 panic("VM_PAGE_PACK_PTR failed on &vm_pages[0] - %p", (void *)&vm_pages[0]);
1604 }
1605
1606 if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[vm_pages_count - 1]))) != &vm_pages[vm_pages_count - 1]) {
1607 panic("VM_PAGE_PACK_PTR failed on &vm_pages[vm_pages_count-1] - %p", (void *)&vm_pages[vm_pages_count - 1]);
1608 }
1609 #endif
1610
1611 VM_CHECK_MEMORYSTATUS;
1612
1613 /*
1614 * We have to re-align virtual_space_start,
1615 * because pmap_steal_memory has been using it.
1616 */
1617 virtual_space_start = round_page(virtual_space_start);
1618 *startp = virtual_space_start;
1619 *endp = virtual_space_end;
1620 }
1621 #endif /* MACHINE_PAGES */
1622
1623 /*
1624 * Create the zone that represents the vm_pages[] array. Nothing ever allocates
1625 * or frees to this zone. It's just here for reporting purposes via zprint command.
1626 * This needs to be done after all initially delayed pages are put on the free lists.
1627 */
1628 static void
vm_page_module_init_delayed(void)1629 vm_page_module_init_delayed(void)
1630 {
1631 (void)zone_create_ext("vm pages array", sizeof(struct vm_page),
1632 ZC_NONE, ZONE_ID_VM_PAGES, ^(zone_t z) {
1633 uint64_t vm_page_zone_pages, vm_page_array_zone_data_size;
1634
1635 zone_set_exhaustible(z, 0);
1636 /*
1637 * Reflect size and usage information for vm_pages[].
1638 */
1639
1640 z->z_elems_avail = (uint32_t)(vm_page_array_ending_addr - vm_pages);
1641 z->z_elems_free = z->z_elems_avail - vm_pages_count;
1642 zpercpu_get_cpu(z->z_stats, 0)->zs_mem_allocated =
1643 vm_pages_count * sizeof(struct vm_page);
1644 vm_page_array_zone_data_size = (uint64_t)vm_page_array_ending_addr - (uint64_t)vm_pages;
1645 vm_page_zone_pages = atop(round_page((vm_offset_t)vm_page_array_zone_data_size));
1646 z->z_wired_cur += vm_page_zone_pages;
1647 z->z_wired_hwm = z->z_wired_cur;
1648 z->z_va_cur = z->z_wired_cur;
1649 /* since zone accounts for these, take them out of stolen */
1650 VM_PAGE_MOVE_STOLEN(vm_page_zone_pages);
1651 });
1652 }
1653
1654 /*
1655 * Create the vm_pages zone. This is used for the vm_page structures for the pages
1656 * that are scavanged from other boot time usages by ml_static_mfree(). As such,
1657 * this needs to happen in early VM bootstrap.
1658 */
1659
1660 __startup_func
1661 static void
vm_page_module_init(void)1662 vm_page_module_init(void)
1663 {
1664 vm_size_t vm_page_with_ppnum_size;
1665
1666 /*
1667 * Since the pointers to elements in this zone will be packed, they
1668 * must have appropriate size. Not strictly what sizeof() reports.
1669 */
1670 vm_page_with_ppnum_size =
1671 (sizeof(struct vm_page_with_ppnum) + (VM_PAGE_PACKED_PTR_ALIGNMENT - 1)) &
1672 ~(VM_PAGE_PACKED_PTR_ALIGNMENT - 1);
1673
1674 vm_page_zone = zone_create_ext("vm pages", vm_page_with_ppnum_size,
1675 ZC_ALIGNMENT_REQUIRED | ZC_VM_LP64 | ZC_NOTBITAG,
1676 ZONE_ID_ANY, ^(zone_t z) {
1677 /*
1678 * The number "10" is a small number that is larger than the number
1679 * of fictitious pages that any single caller will attempt to allocate
1680 * without blocking.
1681 *
1682 * The largest such number at the moment is kmem_alloc()
1683 * when 2 guard pages are asked. 10 is simply a somewhat larger number,
1684 * taking into account the 50% hysteresis the zone allocator uses.
1685 *
1686 * Note: this works at all because the zone allocator
1687 * doesn't ever allocate fictitious pages.
1688 */
1689 zone_raise_reserve(z, 10);
1690 });
1691 }
1692 STARTUP(ZALLOC, STARTUP_RANK_SECOND, vm_page_module_init);
1693
1694 /*
1695 * Routine: vm_page_create
1696 * Purpose:
1697 * After the VM system is up, machine-dependent code
1698 * may stumble across more physical memory. For example,
1699 * memory that it was reserving for a frame buffer.
1700 * vm_page_create turns this memory into available pages.
1701 */
1702
1703 void
vm_page_create(ppnum_t start,ppnum_t end)1704 vm_page_create(
1705 ppnum_t start,
1706 ppnum_t end)
1707 {
1708 ppnum_t phys_page;
1709 vm_page_t m;
1710
1711 for (phys_page = start;
1712 phys_page < end;
1713 phys_page++) {
1714 m = vm_page_grab_fictitious_common(phys_page, TRUE);
1715 m->vmp_fictitious = FALSE;
1716 pmap_clear_noencrypt(phys_page);
1717
1718
1719 vm_free_page_lock();
1720 vm_page_pages++;
1721 vm_free_page_unlock();
1722 vm_page_release(m, FALSE);
1723 }
1724 }
1725
1726
1727 /*
1728 * vm_page_hash:
1729 *
1730 * Distributes the object/offset key pair among hash buckets.
1731 *
1732 * NOTE: The bucket count must be a power of 2
1733 */
1734 #define vm_page_hash(object, offset) (\
1735 ( (natural_t)((uintptr_t)object * vm_page_bucket_hash) + ((uint32_t)atop_64(offset) ^ vm_page_bucket_hash))\
1736 & vm_page_hash_mask)
1737
1738
1739 /*
1740 * vm_page_insert: [ internal use only ]
1741 *
1742 * Inserts the given mem entry into the object/object-page
1743 * table and object list.
1744 *
1745 * The object must be locked.
1746 */
1747 void
vm_page_insert(vm_page_t mem,vm_object_t object,vm_object_offset_t offset)1748 vm_page_insert(
1749 vm_page_t mem,
1750 vm_object_t object,
1751 vm_object_offset_t offset)
1752 {
1753 vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, FALSE, FALSE, NULL);
1754 }
1755
1756 void
vm_page_insert_wired(vm_page_t mem,vm_object_t object,vm_object_offset_t offset,vm_tag_t tag)1757 vm_page_insert_wired(
1758 vm_page_t mem,
1759 vm_object_t object,
1760 vm_object_offset_t offset,
1761 vm_tag_t tag)
1762 {
1763 vm_page_insert_internal(mem, object, offset, tag, FALSE, TRUE, FALSE, FALSE, NULL);
1764 }
1765
1766 void
vm_page_insert_internal(vm_page_t mem,vm_object_t object,vm_object_offset_t offset,vm_tag_t tag,boolean_t queues_lock_held,boolean_t insert_in_hash,boolean_t batch_pmap_op,boolean_t batch_accounting,uint64_t * delayed_ledger_update)1767 vm_page_insert_internal(
1768 vm_page_t mem,
1769 vm_object_t object,
1770 vm_object_offset_t offset,
1771 vm_tag_t tag,
1772 boolean_t queues_lock_held,
1773 boolean_t insert_in_hash,
1774 boolean_t batch_pmap_op,
1775 boolean_t batch_accounting,
1776 uint64_t *delayed_ledger_update)
1777 {
1778 vm_page_bucket_t *bucket;
1779 lck_spin_t *bucket_lock;
1780 int hash_id;
1781 task_t owner;
1782 int ledger_idx_volatile;
1783 int ledger_idx_nonvolatile;
1784 int ledger_idx_volatile_compressed;
1785 int ledger_idx_nonvolatile_compressed;
1786 boolean_t do_footprint;
1787
1788 #if 0
1789 /*
1790 * we may not hold the page queue lock
1791 * so this check isn't safe to make
1792 */
1793 VM_PAGE_CHECK(mem);
1794 #endif
1795
1796 assertf(page_aligned(offset), "0x%llx\n", offset);
1797
1798 assert(!VM_PAGE_WIRED(mem) || mem->vmp_private || mem->vmp_fictitious || (tag != VM_KERN_MEMORY_NONE));
1799
1800 vm_object_lock_assert_exclusive(object);
1801 LCK_MTX_ASSERT(&vm_page_queue_lock,
1802 queues_lock_held ? LCK_MTX_ASSERT_OWNED
1803 : LCK_MTX_ASSERT_NOTOWNED);
1804
1805 if (queues_lock_held == FALSE) {
1806 assert(!VM_PAGE_PAGEABLE(mem));
1807 }
1808
1809 if (insert_in_hash == TRUE) {
1810 #if DEBUG || VM_PAGE_BUCKETS_CHECK
1811 if (mem->vmp_tabled || mem->vmp_object) {
1812 panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) "
1813 "already in (obj=%p,off=0x%llx)",
1814 mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
1815 }
1816 #endif
1817 if (object->internal && (offset >= object->vo_size)) {
1818 panic("vm_page_insert_internal: (page=%p,obj=%p,off=0x%llx,size=0x%llx) inserted at offset past object bounds",
1819 mem, object, offset, object->vo_size);
1820 }
1821
1822 assert(vm_page_lookup(object, offset) == VM_PAGE_NULL);
1823
1824 /*
1825 * Record the object/offset pair in this page
1826 */
1827
1828 mem->vmp_object = VM_PAGE_PACK_OBJECT(object);
1829 mem->vmp_offset = offset;
1830
1831 #if CONFIG_SECLUDED_MEMORY
1832 if (object->eligible_for_secluded) {
1833 vm_page_secluded.eligible_for_secluded++;
1834 }
1835 #endif /* CONFIG_SECLUDED_MEMORY */
1836
1837 /*
1838 * Insert it into the object_object/offset hash table
1839 */
1840 hash_id = vm_page_hash(object, offset);
1841 bucket = &vm_page_buckets[hash_id];
1842 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
1843
1844 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
1845
1846 mem->vmp_next_m = bucket->page_list;
1847 bucket->page_list = VM_PAGE_PACK_PTR(mem);
1848 assert(mem == (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)));
1849
1850 #if MACH_PAGE_HASH_STATS
1851 if (++bucket->cur_count > bucket->hi_count) {
1852 bucket->hi_count = bucket->cur_count;
1853 }
1854 #endif /* MACH_PAGE_HASH_STATS */
1855 mem->vmp_hashed = TRUE;
1856 lck_spin_unlock(bucket_lock);
1857 }
1858
1859 {
1860 unsigned int cache_attr;
1861
1862 cache_attr = object->wimg_bits & VM_WIMG_MASK;
1863
1864 if (cache_attr != VM_WIMG_USE_DEFAULT) {
1865 PMAP_SET_CACHE_ATTR(mem, object, cache_attr, batch_pmap_op);
1866 }
1867 }
1868 /*
1869 * Now link into the object's list of backed pages.
1870 */
1871 vm_page_queue_enter(&object->memq, mem, vmp_listq);
1872 object->memq_hint = mem;
1873 mem->vmp_tabled = TRUE;
1874
1875 /*
1876 * Show that the object has one more resident page.
1877 */
1878
1879 object->resident_page_count++;
1880 if (VM_PAGE_WIRED(mem)) {
1881 assert(mem->vmp_wire_count > 0);
1882 VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
1883 VM_OBJECT_WIRED_PAGE_ADD(object, mem);
1884 VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
1885 }
1886 assert(object->resident_page_count >= object->wired_page_count);
1887
1888 #if DEVELOPMENT || DEBUG
1889 if (object->object_is_shared_cache &&
1890 object->pager != NULL &&
1891 object->pager->mo_pager_ops == &shared_region_pager_ops) {
1892 int new, old;
1893 assert(!object->internal);
1894 new = OSAddAtomic(+1, &shared_region_pagers_resident_count);
1895 do {
1896 old = shared_region_pagers_resident_peak;
1897 } while (old < new &&
1898 !OSCompareAndSwap(old, new, &shared_region_pagers_resident_peak));
1899 }
1900 #endif /* DEVELOPMENT || DEBUG */
1901
1902 if (batch_accounting == FALSE) {
1903 if (object->internal) {
1904 OSAddAtomic(1, &vm_page_internal_count);
1905 } else {
1906 OSAddAtomic(1, &vm_page_external_count);
1907 }
1908 }
1909
1910 /*
1911 * It wouldn't make sense to insert a "reusable" page in
1912 * an object (the page would have been marked "reusable" only
1913 * at the time of a madvise(MADV_FREE_REUSABLE) if it was already
1914 * in the object at that time).
1915 * But a page could be inserted in a "all_reusable" object, if
1916 * something faults it in (a vm_read() from another task or a
1917 * "use-after-free" issue in user space, for example). It can
1918 * also happen if we're relocating a page from that object to
1919 * a different physical page during a physically-contiguous
1920 * allocation.
1921 */
1922 assert(!mem->vmp_reusable);
1923 if (object->all_reusable) {
1924 OSAddAtomic(+1, &vm_page_stats_reusable.reusable_count);
1925 }
1926
1927 if (object->purgable == VM_PURGABLE_DENY &&
1928 !object->vo_ledger_tag) {
1929 owner = TASK_NULL;
1930 } else {
1931 owner = VM_OBJECT_OWNER(object);
1932 vm_object_ledger_tag_ledgers(object,
1933 &ledger_idx_volatile,
1934 &ledger_idx_nonvolatile,
1935 &ledger_idx_volatile_compressed,
1936 &ledger_idx_nonvolatile_compressed,
1937 &do_footprint);
1938 }
1939 if (owner &&
1940 (object->purgable == VM_PURGABLE_NONVOLATILE ||
1941 object->purgable == VM_PURGABLE_DENY ||
1942 VM_PAGE_WIRED(mem))) {
1943 if (delayed_ledger_update) {
1944 *delayed_ledger_update += PAGE_SIZE;
1945 } else {
1946 /* more non-volatile bytes */
1947 ledger_credit(owner->ledger,
1948 ledger_idx_nonvolatile,
1949 PAGE_SIZE);
1950 if (do_footprint) {
1951 /* more footprint */
1952 ledger_credit(owner->ledger,
1953 task_ledgers.phys_footprint,
1954 PAGE_SIZE);
1955 }
1956 }
1957 } else if (owner &&
1958 (object->purgable == VM_PURGABLE_VOLATILE ||
1959 object->purgable == VM_PURGABLE_EMPTY)) {
1960 assert(!VM_PAGE_WIRED(mem));
1961 /* more volatile bytes */
1962 ledger_credit(owner->ledger,
1963 ledger_idx_volatile,
1964 PAGE_SIZE);
1965 }
1966
1967 if (object->purgable == VM_PURGABLE_VOLATILE) {
1968 if (VM_PAGE_WIRED(mem)) {
1969 OSAddAtomic(+1, &vm_page_purgeable_wired_count);
1970 } else {
1971 OSAddAtomic(+1, &vm_page_purgeable_count);
1972 }
1973 } else if (object->purgable == VM_PURGABLE_EMPTY &&
1974 mem->vmp_q_state == VM_PAGE_ON_THROTTLED_Q) {
1975 /*
1976 * This page belongs to a purged VM object but hasn't
1977 * been purged (because it was "busy").
1978 * It's in the "throttled" queue and hence not
1979 * visible to vm_pageout_scan(). Move it to a pageable
1980 * queue, so that it can eventually be reclaimed, instead
1981 * of lingering in the "empty" object.
1982 */
1983 if (queues_lock_held == FALSE) {
1984 vm_page_lockspin_queues();
1985 }
1986 vm_page_deactivate(mem);
1987 if (queues_lock_held == FALSE) {
1988 vm_page_unlock_queues();
1989 }
1990 }
1991
1992 #if VM_OBJECT_TRACKING_OP_MODIFIED
1993 if (vm_object_tracking_btlog &&
1994 object->internal &&
1995 object->resident_page_count == 0 &&
1996 object->pager == NULL &&
1997 object->shadow != NULL &&
1998 object->shadow->copy == object) {
1999 btlog_record(vm_object_tracking_btlog, object,
2000 VM_OBJECT_TRACKING_OP_MODIFIED,
2001 btref_get(__builtin_frame_address(0), 0));
2002 }
2003 #endif /* VM_OBJECT_TRACKING_OP_MODIFIED */
2004 }
2005
2006 /*
2007 * vm_page_replace:
2008 *
2009 * Exactly like vm_page_insert, except that we first
2010 * remove any existing page at the given offset in object.
2011 *
2012 * The object must be locked.
2013 */
2014 void
vm_page_replace(vm_page_t mem,vm_object_t object,vm_object_offset_t offset)2015 vm_page_replace(
2016 vm_page_t mem,
2017 vm_object_t object,
2018 vm_object_offset_t offset)
2019 {
2020 vm_page_bucket_t *bucket;
2021 vm_page_t found_m = VM_PAGE_NULL;
2022 lck_spin_t *bucket_lock;
2023 int hash_id;
2024
2025 #if 0
2026 /*
2027 * we don't hold the page queue lock
2028 * so this check isn't safe to make
2029 */
2030 VM_PAGE_CHECK(mem);
2031 #endif
2032 vm_object_lock_assert_exclusive(object);
2033 #if DEBUG || VM_PAGE_BUCKETS_CHECK
2034 if (mem->vmp_tabled || mem->vmp_object) {
2035 panic("vm_page_replace: page %p for (obj=%p,off=0x%llx) "
2036 "already in (obj=%p,off=0x%llx)",
2037 mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
2038 }
2039 #endif
2040 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
2041
2042 assert(!VM_PAGE_PAGEABLE(mem));
2043
2044 /*
2045 * Record the object/offset pair in this page
2046 */
2047 mem->vmp_object = VM_PAGE_PACK_OBJECT(object);
2048 mem->vmp_offset = offset;
2049
2050 /*
2051 * Insert it into the object_object/offset hash table,
2052 * replacing any page that might have been there.
2053 */
2054
2055 hash_id = vm_page_hash(object, offset);
2056 bucket = &vm_page_buckets[hash_id];
2057 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
2058
2059 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
2060
2061 if (bucket->page_list) {
2062 vm_page_packed_t *mp = &bucket->page_list;
2063 vm_page_t m = (vm_page_t)(VM_PAGE_UNPACK_PTR(*mp));
2064
2065 do {
2066 /*
2067 * compare packed object pointers
2068 */
2069 if (m->vmp_object == mem->vmp_object && m->vmp_offset == offset) {
2070 /*
2071 * Remove old page from hash list
2072 */
2073 *mp = m->vmp_next_m;
2074 m->vmp_hashed = FALSE;
2075 m->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
2076
2077 found_m = m;
2078 break;
2079 }
2080 mp = &m->vmp_next_m;
2081 } while ((m = (vm_page_t)(VM_PAGE_UNPACK_PTR(*mp))));
2082
2083 mem->vmp_next_m = bucket->page_list;
2084 } else {
2085 mem->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
2086 }
2087 /*
2088 * insert new page at head of hash list
2089 */
2090 bucket->page_list = VM_PAGE_PACK_PTR(mem);
2091 mem->vmp_hashed = TRUE;
2092
2093 lck_spin_unlock(bucket_lock);
2094
2095 if (found_m) {
2096 /*
2097 * there was already a page at the specified
2098 * offset for this object... remove it from
2099 * the object and free it back to the free list
2100 */
2101 vm_page_free_unlocked(found_m, FALSE);
2102 }
2103 vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, FALSE, FALSE, FALSE, NULL);
2104 }
2105
2106 /*
2107 * vm_page_remove: [ internal use only ]
2108 *
2109 * Removes the given mem entry from the object/offset-page
2110 * table and the object page list.
2111 *
2112 * The object must be locked.
2113 */
2114
2115 void
vm_page_remove(vm_page_t mem,boolean_t remove_from_hash)2116 vm_page_remove(
2117 vm_page_t mem,
2118 boolean_t remove_from_hash)
2119 {
2120 vm_page_bucket_t *bucket;
2121 vm_page_t this;
2122 lck_spin_t *bucket_lock;
2123 int hash_id;
2124 task_t owner;
2125 vm_object_t m_object;
2126 int ledger_idx_volatile;
2127 int ledger_idx_nonvolatile;
2128 int ledger_idx_volatile_compressed;
2129 int ledger_idx_nonvolatile_compressed;
2130 int do_footprint;
2131
2132 m_object = VM_PAGE_OBJECT(mem);
2133
2134 vm_object_lock_assert_exclusive(m_object);
2135 assert(mem->vmp_tabled);
2136 assert(!mem->vmp_cleaning);
2137 assert(!mem->vmp_laundry);
2138
2139 if (VM_PAGE_PAGEABLE(mem)) {
2140 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
2141 }
2142 #if 0
2143 /*
2144 * we don't hold the page queue lock
2145 * so this check isn't safe to make
2146 */
2147 VM_PAGE_CHECK(mem);
2148 #endif
2149 if (remove_from_hash == TRUE) {
2150 /*
2151 * Remove from the object_object/offset hash table
2152 */
2153 hash_id = vm_page_hash(m_object, mem->vmp_offset);
2154 bucket = &vm_page_buckets[hash_id];
2155 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
2156
2157 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
2158
2159 if ((this = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list))) == mem) {
2160 /* optimize for common case */
2161
2162 bucket->page_list = mem->vmp_next_m;
2163 } else {
2164 vm_page_packed_t *prev;
2165
2166 for (prev = &this->vmp_next_m;
2167 (this = (vm_page_t)(VM_PAGE_UNPACK_PTR(*prev))) != mem;
2168 prev = &this->vmp_next_m) {
2169 continue;
2170 }
2171 *prev = this->vmp_next_m;
2172 }
2173 #if MACH_PAGE_HASH_STATS
2174 bucket->cur_count--;
2175 #endif /* MACH_PAGE_HASH_STATS */
2176 mem->vmp_hashed = FALSE;
2177 this->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
2178 lck_spin_unlock(bucket_lock);
2179 }
2180 /*
2181 * Now remove from the object's list of backed pages.
2182 */
2183
2184 vm_page_remove_internal(mem);
2185
2186 /*
2187 * And show that the object has one fewer resident
2188 * page.
2189 */
2190
2191 assert(m_object->resident_page_count > 0);
2192 m_object->resident_page_count--;
2193
2194 #if DEVELOPMENT || DEBUG
2195 if (m_object->object_is_shared_cache &&
2196 m_object->pager != NULL &&
2197 m_object->pager->mo_pager_ops == &shared_region_pager_ops) {
2198 assert(!m_object->internal);
2199 OSAddAtomic(-1, &shared_region_pagers_resident_count);
2200 }
2201 #endif /* DEVELOPMENT || DEBUG */
2202
2203 if (m_object->internal) {
2204 #if DEBUG
2205 assert(vm_page_internal_count);
2206 #endif /* DEBUG */
2207
2208 OSAddAtomic(-1, &vm_page_internal_count);
2209 } else {
2210 assert(vm_page_external_count);
2211 OSAddAtomic(-1, &vm_page_external_count);
2212
2213 if (mem->vmp_xpmapped) {
2214 assert(vm_page_xpmapped_external_count);
2215 OSAddAtomic(-1, &vm_page_xpmapped_external_count);
2216 }
2217 }
2218 if (!m_object->internal &&
2219 m_object->cached_list.next &&
2220 m_object->cached_list.prev) {
2221 if (m_object->resident_page_count == 0) {
2222 vm_object_cache_remove(m_object);
2223 }
2224 }
2225
2226 if (VM_PAGE_WIRED(mem)) {
2227 assert(mem->vmp_wire_count > 0);
2228 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
2229 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
2230 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
2231 }
2232 assert(m_object->resident_page_count >=
2233 m_object->wired_page_count);
2234 if (mem->vmp_reusable) {
2235 assert(m_object->reusable_page_count > 0);
2236 m_object->reusable_page_count--;
2237 assert(m_object->reusable_page_count <=
2238 m_object->resident_page_count);
2239 mem->vmp_reusable = FALSE;
2240 OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
2241 vm_page_stats_reusable.reused_remove++;
2242 } else if (m_object->all_reusable) {
2243 OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
2244 vm_page_stats_reusable.reused_remove++;
2245 }
2246
2247 if (m_object->purgable == VM_PURGABLE_DENY &&
2248 !m_object->vo_ledger_tag) {
2249 owner = TASK_NULL;
2250 } else {
2251 owner = VM_OBJECT_OWNER(m_object);
2252 vm_object_ledger_tag_ledgers(m_object,
2253 &ledger_idx_volatile,
2254 &ledger_idx_nonvolatile,
2255 &ledger_idx_volatile_compressed,
2256 &ledger_idx_nonvolatile_compressed,
2257 &do_footprint);
2258 }
2259 if (owner &&
2260 (m_object->purgable == VM_PURGABLE_NONVOLATILE ||
2261 m_object->purgable == VM_PURGABLE_DENY ||
2262 VM_PAGE_WIRED(mem))) {
2263 /* less non-volatile bytes */
2264 ledger_debit(owner->ledger,
2265 ledger_idx_nonvolatile,
2266 PAGE_SIZE);
2267 if (do_footprint) {
2268 /* less footprint */
2269 ledger_debit(owner->ledger,
2270 task_ledgers.phys_footprint,
2271 PAGE_SIZE);
2272 }
2273 } else if (owner &&
2274 (m_object->purgable == VM_PURGABLE_VOLATILE ||
2275 m_object->purgable == VM_PURGABLE_EMPTY)) {
2276 assert(!VM_PAGE_WIRED(mem));
2277 /* less volatile bytes */
2278 ledger_debit(owner->ledger,
2279 ledger_idx_volatile,
2280 PAGE_SIZE);
2281 }
2282 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
2283 if (VM_PAGE_WIRED(mem)) {
2284 assert(vm_page_purgeable_wired_count > 0);
2285 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
2286 } else {
2287 assert(vm_page_purgeable_count > 0);
2288 OSAddAtomic(-1, &vm_page_purgeable_count);
2289 }
2290 }
2291
2292 if (m_object->set_cache_attr == TRUE) {
2293 pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), 0);
2294 }
2295
2296 mem->vmp_tabled = FALSE;
2297 mem->vmp_object = 0;
2298 mem->vmp_offset = (vm_object_offset_t) -1;
2299 }
2300
2301
2302 /*
2303 * vm_page_lookup:
2304 *
2305 * Returns the page associated with the object/offset
2306 * pair specified; if none is found, VM_PAGE_NULL is returned.
2307 *
2308 * The object must be locked. No side effects.
2309 */
2310
2311 #define VM_PAGE_HASH_LOOKUP_THRESHOLD 10
2312
2313 #if DEBUG_VM_PAGE_LOOKUP
2314
2315 struct {
2316 uint64_t vpl_total;
2317 uint64_t vpl_empty_obj;
2318 uint64_t vpl_bucket_NULL;
2319 uint64_t vpl_hit_hint;
2320 uint64_t vpl_hit_hint_next;
2321 uint64_t vpl_hit_hint_prev;
2322 uint64_t vpl_fast;
2323 uint64_t vpl_slow;
2324 uint64_t vpl_hit;
2325 uint64_t vpl_miss;
2326
2327 uint64_t vpl_fast_elapsed;
2328 uint64_t vpl_slow_elapsed;
2329 } vm_page_lookup_stats __attribute__((aligned(8)));
2330
2331 #endif
2332
2333 #define KDP_VM_PAGE_WALK_MAX 1000
2334
2335 vm_page_t
kdp_vm_page_lookup(vm_object_t object,vm_object_offset_t offset)2336 kdp_vm_page_lookup(
2337 vm_object_t object,
2338 vm_object_offset_t offset)
2339 {
2340 vm_page_t cur_page;
2341 int num_traversed = 0;
2342
2343 if (not_in_kdp) {
2344 panic("panic: kdp_vm_page_lookup done outside of kernel debugger");
2345 }
2346
2347 vm_page_queue_iterate(&object->memq, cur_page, vmp_listq) {
2348 if (cur_page->vmp_offset == offset) {
2349 return cur_page;
2350 }
2351 num_traversed++;
2352
2353 if (num_traversed >= KDP_VM_PAGE_WALK_MAX) {
2354 return VM_PAGE_NULL;
2355 }
2356 }
2357
2358 return VM_PAGE_NULL;
2359 }
2360
2361 vm_page_t
vm_page_lookup(vm_object_t object,vm_object_offset_t offset)2362 vm_page_lookup(
2363 vm_object_t object,
2364 vm_object_offset_t offset)
2365 {
2366 vm_page_t mem;
2367 vm_page_bucket_t *bucket;
2368 vm_page_queue_entry_t qe;
2369 lck_spin_t *bucket_lock = NULL;
2370 int hash_id;
2371 #if DEBUG_VM_PAGE_LOOKUP
2372 uint64_t start, elapsed;
2373
2374 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_total);
2375 #endif
2376
2377 #if CONFIG_KERNEL_TBI
2378 if (VM_KERNEL_ADDRESS(offset)) {
2379 offset = VM_KERNEL_STRIP_UPTR(offset);
2380 }
2381 #endif /* CONFIG_KERNEL_TBI */
2382
2383 vm_object_lock_assert_held(object);
2384 assertf(page_aligned(offset), "offset 0x%llx\n", offset);
2385
2386 if (object->resident_page_count == 0) {
2387 #if DEBUG_VM_PAGE_LOOKUP
2388 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_empty_obj);
2389 #endif
2390 return VM_PAGE_NULL;
2391 }
2392
2393 mem = object->memq_hint;
2394
2395 if (mem != VM_PAGE_NULL) {
2396 assert(VM_PAGE_OBJECT(mem) == object);
2397
2398 if (mem->vmp_offset == offset) {
2399 #if DEBUG_VM_PAGE_LOOKUP
2400 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint);
2401 #endif
2402 return mem;
2403 }
2404 qe = (vm_page_queue_entry_t)vm_page_queue_next(&mem->vmp_listq);
2405
2406 if (!vm_page_queue_end(&object->memq, qe)) {
2407 vm_page_t next_page;
2408
2409 next_page = (vm_page_t)((uintptr_t)qe);
2410 assert(VM_PAGE_OBJECT(next_page) == object);
2411
2412 if (next_page->vmp_offset == offset) {
2413 object->memq_hint = next_page; /* new hint */
2414 #if DEBUG_VM_PAGE_LOOKUP
2415 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_next);
2416 #endif
2417 return next_page;
2418 }
2419 }
2420 qe = (vm_page_queue_entry_t)vm_page_queue_prev(&mem->vmp_listq);
2421
2422 if (!vm_page_queue_end(&object->memq, qe)) {
2423 vm_page_t prev_page;
2424
2425 prev_page = (vm_page_t)((uintptr_t)qe);
2426 assert(VM_PAGE_OBJECT(prev_page) == object);
2427
2428 if (prev_page->vmp_offset == offset) {
2429 object->memq_hint = prev_page; /* new hint */
2430 #if DEBUG_VM_PAGE_LOOKUP
2431 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_prev);
2432 #endif
2433 return prev_page;
2434 }
2435 }
2436 }
2437 /*
2438 * Search the hash table for this object/offset pair
2439 */
2440 hash_id = vm_page_hash(object, offset);
2441 bucket = &vm_page_buckets[hash_id];
2442
2443 /*
2444 * since we hold the object lock, we are guaranteed that no
2445 * new pages can be inserted into this object... this in turn
2446 * guarantess that the page we're looking for can't exist
2447 * if the bucket it hashes to is currently NULL even when looked
2448 * at outside the scope of the hash bucket lock... this is a
2449 * really cheap optimiztion to avoid taking the lock
2450 */
2451 if (!bucket->page_list) {
2452 #if DEBUG_VM_PAGE_LOOKUP
2453 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_bucket_NULL);
2454 #endif
2455 return VM_PAGE_NULL;
2456 }
2457
2458 #if DEBUG_VM_PAGE_LOOKUP
2459 start = mach_absolute_time();
2460 #endif
2461 if (object->resident_page_count <= VM_PAGE_HASH_LOOKUP_THRESHOLD) {
2462 /*
2463 * on average, it's roughly 3 times faster to run a short memq list
2464 * than to take the spin lock and go through the hash list
2465 */
2466 mem = (vm_page_t)vm_page_queue_first(&object->memq);
2467
2468 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {
2469 if (mem->vmp_offset == offset) {
2470 break;
2471 }
2472
2473 mem = (vm_page_t)vm_page_queue_next(&mem->vmp_listq);
2474 }
2475 if (vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {
2476 mem = NULL;
2477 }
2478 } else {
2479 vm_page_object_t packed_object;
2480
2481 packed_object = VM_PAGE_PACK_OBJECT(object);
2482
2483 bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
2484
2485 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
2486
2487 for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
2488 mem != VM_PAGE_NULL;
2489 mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m))) {
2490 #if 0
2491 /*
2492 * we don't hold the page queue lock
2493 * so this check isn't safe to make
2494 */
2495 VM_PAGE_CHECK(mem);
2496 #endif
2497 if ((mem->vmp_object == packed_object) && (mem->vmp_offset == offset)) {
2498 break;
2499 }
2500 }
2501 lck_spin_unlock(bucket_lock);
2502 }
2503
2504 #if DEBUG_VM_PAGE_LOOKUP
2505 elapsed = mach_absolute_time() - start;
2506
2507 if (bucket_lock) {
2508 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_slow);
2509 OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_slow_elapsed);
2510 } else {
2511 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_fast);
2512 OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_fast_elapsed);
2513 }
2514 if (mem != VM_PAGE_NULL) {
2515 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit);
2516 } else {
2517 OSAddAtomic64(1, &vm_page_lookup_stats.vpl_miss);
2518 }
2519 #endif
2520 if (mem != VM_PAGE_NULL) {
2521 assert(VM_PAGE_OBJECT(mem) == object);
2522
2523 object->memq_hint = mem;
2524 }
2525 return mem;
2526 }
2527
2528
2529 /*
2530 * vm_page_rename:
2531 *
2532 * Move the given memory entry from its
2533 * current object to the specified target object/offset.
2534 *
2535 * The object must be locked.
2536 */
2537 void
vm_page_rename(vm_page_t mem,vm_object_t new_object,vm_object_offset_t new_offset)2538 vm_page_rename(
2539 vm_page_t mem,
2540 vm_object_t new_object,
2541 vm_object_offset_t new_offset)
2542 {
2543 boolean_t internal_to_external, external_to_internal;
2544 vm_tag_t tag;
2545 vm_object_t m_object;
2546
2547 m_object = VM_PAGE_OBJECT(mem);
2548
2549 assert(m_object != new_object);
2550 assert(m_object);
2551
2552 /*
2553 * Changes to mem->vmp_object require the page lock because
2554 * the pageout daemon uses that lock to get the object.
2555 */
2556 vm_page_lockspin_queues();
2557
2558 internal_to_external = FALSE;
2559 external_to_internal = FALSE;
2560
2561 if (mem->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q) {
2562 /*
2563 * it's much easier to get the vm_page_pageable_xxx accounting correct
2564 * if we first move the page to the active queue... it's going to end
2565 * up there anyway, and we don't do vm_page_rename's frequently enough
2566 * for this to matter.
2567 */
2568 vm_page_queues_remove(mem, FALSE);
2569 vm_page_activate(mem);
2570 }
2571 if (VM_PAGE_PAGEABLE(mem)) {
2572 if (m_object->internal && !new_object->internal) {
2573 internal_to_external = TRUE;
2574 }
2575 if (!m_object->internal && new_object->internal) {
2576 external_to_internal = TRUE;
2577 }
2578 }
2579
2580 tag = m_object->wire_tag;
2581 vm_page_remove(mem, TRUE);
2582 vm_page_insert_internal(mem, new_object, new_offset, tag, TRUE, TRUE, FALSE, FALSE, NULL);
2583
2584 if (internal_to_external) {
2585 vm_page_pageable_internal_count--;
2586 vm_page_pageable_external_count++;
2587 } else if (external_to_internal) {
2588 vm_page_pageable_external_count--;
2589 vm_page_pageable_internal_count++;
2590 }
2591
2592 vm_page_unlock_queues();
2593 }
2594
2595 /*
2596 * vm_page_init:
2597 *
2598 * Initialize the fields in a new page.
2599 * This takes a structure with random values and initializes it
2600 * so that it can be given to vm_page_release or vm_page_insert.
2601 */
2602 void
vm_page_init(vm_page_t mem,ppnum_t phys_page,boolean_t lopage)2603 vm_page_init(
2604 vm_page_t mem,
2605 ppnum_t phys_page,
2606 boolean_t lopage)
2607 {
2608 uint_t i;
2609 uintptr_t *p;
2610
2611 assert(phys_page);
2612
2613 #if DEBUG
2614 if ((phys_page != vm_page_fictitious_addr) && (phys_page != vm_page_guard_addr)) {
2615 if (!(pmap_valid_page(phys_page))) {
2616 panic("vm_page_init: non-DRAM phys_page 0x%x", phys_page);
2617 }
2618 }
2619 #endif /* DEBUG */
2620
2621 /*
2622 * Initialize the fields of the vm_page. If adding any new fields to vm_page,
2623 * try to use initial values which match 0. This minimizes the number of writes
2624 * needed for boot-time initialization.
2625 *
2626 * Kernel bzero() isn't an inline yet, so do it by hand for performance.
2627 */
2628 assert(VM_PAGE_NOT_ON_Q == 0);
2629 assert(sizeof(*mem) % sizeof(uintptr_t) == 0);
2630 for (p = (uintptr_t *)(void *)mem, i = sizeof(*mem) / sizeof(uintptr_t); i != 0; --i) {
2631 *p++ = 0;
2632 }
2633 mem->vmp_offset = (vm_object_offset_t)-1;
2634 mem->vmp_busy = TRUE;
2635 mem->vmp_lopage = lopage;
2636
2637 VM_PAGE_SET_PHYS_PAGE(mem, phys_page);
2638 #if 0
2639 /*
2640 * we're leaving this turned off for now... currently pages
2641 * come off the free list and are either immediately dirtied/referenced
2642 * due to zero-fill or COW faults, or are used to read or write files...
2643 * in the file I/O case, the UPL mechanism takes care of clearing
2644 * the state of the HW ref/mod bits in a somewhat fragile way.
2645 * Since we may change the way this works in the future (to toughen it up),
2646 * I'm leaving this as a reminder of where these bits could get cleared
2647 */
2648
2649 /*
2650 * make sure both the h/w referenced and modified bits are
2651 * clear at this point... we are especially dependent on
2652 * not finding a 'stale' h/w modified in a number of spots
2653 * once this page goes back into use
2654 */
2655 pmap_clear_refmod(phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
2656 #endif
2657 }
2658
2659 /*
2660 * vm_page_grab_fictitious:
2661 *
2662 * Remove a fictitious page from the free list.
2663 * Returns VM_PAGE_NULL if there are no free pages.
2664 */
2665
2666 static vm_page_t
vm_page_grab_fictitious_common(ppnum_t phys_addr,boolean_t canwait)2667 vm_page_grab_fictitious_common(ppnum_t phys_addr, boolean_t canwait)
2668 {
2669 vm_page_t m;
2670
2671 m = zalloc_flags(vm_page_zone, canwait ? Z_WAITOK : Z_NOWAIT);
2672 if (m) {
2673 vm_page_init(m, phys_addr, FALSE);
2674 m->vmp_fictitious = TRUE;
2675 }
2676 return m;
2677 }
2678
2679 vm_page_t
vm_page_grab_fictitious(boolean_t canwait)2680 vm_page_grab_fictitious(boolean_t canwait)
2681 {
2682 return vm_page_grab_fictitious_common(vm_page_fictitious_addr, canwait);
2683 }
2684
2685 int vm_guard_count;
2686
2687
2688 vm_page_t
vm_page_grab_guard(boolean_t canwait)2689 vm_page_grab_guard(boolean_t canwait)
2690 {
2691 vm_page_t page;
2692 page = vm_page_grab_fictitious_common(vm_page_guard_addr, canwait);
2693 if (page) {
2694 OSAddAtomic(1, &vm_guard_count);
2695 }
2696 return page;
2697 }
2698
2699
2700 /*
2701 * vm_page_release_fictitious:
2702 *
2703 * Release a fictitious page to the zone pool
2704 */
2705 void
vm_page_release_fictitious(vm_page_t m)2706 vm_page_release_fictitious(
2707 vm_page_t m)
2708 {
2709 assert((m->vmp_q_state == VM_PAGE_NOT_ON_Q) || (m->vmp_q_state == VM_PAGE_IS_WIRED));
2710 assert(m->vmp_fictitious);
2711 assert(VM_PAGE_GET_PHYS_PAGE(m) == vm_page_fictitious_addr ||
2712 VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr);
2713
2714
2715 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
2716 OSAddAtomic(-1, &vm_guard_count);
2717 }
2718
2719 zfree(vm_page_zone, m);
2720 }
2721
2722 /*
2723 * vm_pool_low():
2724 *
2725 * Return true if it is not likely that a non-vm_privileged thread
2726 * can get memory without blocking. Advisory only, since the
2727 * situation may change under us.
2728 */
2729 bool
vm_pool_low(void)2730 vm_pool_low(void)
2731 {
2732 /* No locking, at worst we will fib. */
2733 return vm_page_free_count <= vm_page_free_reserved;
2734 }
2735
2736 boolean_t vm_darkwake_mode = FALSE;
2737
2738 /*
2739 * vm_update_darkwake_mode():
2740 *
2741 * Tells the VM that the system is in / out of darkwake.
2742 *
2743 * Today, the VM only lowers/raises the background queue target
2744 * so as to favor consuming more/less background pages when
2745 * darwake is ON/OFF.
2746 *
2747 * We might need to do more things in the future.
2748 */
2749
2750 void
vm_update_darkwake_mode(boolean_t darkwake_mode)2751 vm_update_darkwake_mode(boolean_t darkwake_mode)
2752 {
2753 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2754 #pragma unused(darkwake_mode)
2755 assert(vm_darkwake_mode == FALSE);
2756 /*
2757 * Darkwake mode isn't supported for AS macOS.
2758 */
2759 return;
2760 #else /* XNU_TARGET_OS_OSX && __arm64__ */
2761 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
2762
2763 vm_page_lockspin_queues();
2764
2765 if (vm_darkwake_mode == darkwake_mode) {
2766 /*
2767 * No change.
2768 */
2769 vm_page_unlock_queues();
2770 return;
2771 }
2772
2773 vm_darkwake_mode = darkwake_mode;
2774
2775 if (vm_darkwake_mode == TRUE) {
2776 /* save background target to restore later */
2777 vm_page_background_target_snapshot = vm_page_background_target;
2778
2779 /* target is set to 0...no protection for background pages */
2780 vm_page_background_target = 0;
2781 } else if (vm_darkwake_mode == FALSE) {
2782 if (vm_page_background_target_snapshot) {
2783 vm_page_background_target = vm_page_background_target_snapshot;
2784 }
2785 }
2786 vm_page_unlock_queues();
2787 #endif
2788 }
2789
2790 void
vm_page_update_special_state(vm_page_t mem)2791 vm_page_update_special_state(vm_page_t mem)
2792 {
2793 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR || mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY) {
2794 return;
2795 }
2796
2797 int mode = mem->vmp_on_specialq;
2798
2799 switch (mode) {
2800 case VM_PAGE_SPECIAL_Q_BG:
2801 {
2802 task_t my_task = current_task_early();
2803
2804 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
2805 return;
2806 }
2807
2808 if (my_task) {
2809 if (task_get_darkwake_mode(my_task)) {
2810 return;
2811 }
2812 }
2813
2814 if (my_task) {
2815 if (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG)) {
2816 return;
2817 }
2818 }
2819 vm_page_lockspin_queues();
2820
2821 vm_page_background_promoted_count++;
2822
2823 vm_page_remove_from_specialq(mem);
2824 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
2825
2826 vm_page_unlock_queues();
2827 break;
2828 }
2829
2830 case VM_PAGE_SPECIAL_Q_DONATE:
2831 {
2832 task_t my_task = current_task_early();
2833
2834 if (vm_page_donate_mode == VM_PAGE_DONATE_DISABLED) {
2835 return;
2836 }
2837
2838 if (my_task->donates_own_pages == false) {
2839 vm_page_lockspin_queues();
2840
2841 vm_page_remove_from_specialq(mem);
2842 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
2843
2844 vm_page_unlock_queues();
2845 }
2846 break;
2847 }
2848
2849 default:
2850 {
2851 assert(VM_PAGE_UNPACK_PTR(mem->vmp_specialq.next) == (uintptr_t)NULL &&
2852 VM_PAGE_UNPACK_PTR(mem->vmp_specialq.prev) == (uintptr_t)NULL);
2853 break;
2854 }
2855 }
2856 }
2857
2858
2859 void
vm_page_assign_special_state(vm_page_t mem,int mode)2860 vm_page_assign_special_state(vm_page_t mem, int mode)
2861 {
2862 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
2863 return;
2864 }
2865
2866 switch (mode) {
2867 case VM_PAGE_SPECIAL_Q_BG:
2868 {
2869 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
2870 return;
2871 }
2872
2873 task_t my_task = current_task_early();
2874
2875 if (my_task) {
2876 if (task_get_darkwake_mode(my_task)) {
2877 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_BG;
2878 return;
2879 }
2880 }
2881
2882 if (my_task) {
2883 mem->vmp_on_specialq = (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG) ? VM_PAGE_SPECIAL_Q_BG : VM_PAGE_SPECIAL_Q_EMPTY);
2884 }
2885 break;
2886 }
2887
2888 case VM_PAGE_SPECIAL_Q_DONATE:
2889 {
2890 if (vm_page_donate_mode == VM_PAGE_DONATE_DISABLED) {
2891 return;
2892 }
2893 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
2894 break;
2895 }
2896
2897 default:
2898 break;
2899 }
2900 }
2901
2902
2903 void
vm_page_remove_from_specialq(vm_page_t mem)2904 vm_page_remove_from_specialq(
2905 vm_page_t mem)
2906 {
2907 vm_object_t m_object;
2908 unsigned short mode;
2909
2910 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
2911
2912 mode = mem->vmp_on_specialq;
2913
2914 switch (mode) {
2915 case VM_PAGE_SPECIAL_Q_BG:
2916 {
2917 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
2918 vm_page_queue_remove(&vm_page_queue_background, mem, vmp_specialq);
2919
2920 mem->vmp_specialq.next = 0;
2921 mem->vmp_specialq.prev = 0;
2922
2923 vm_page_background_count--;
2924
2925 m_object = VM_PAGE_OBJECT(mem);
2926
2927 if (m_object->internal) {
2928 vm_page_background_internal_count--;
2929 } else {
2930 vm_page_background_external_count--;
2931 }
2932 }
2933 break;
2934 }
2935
2936 case VM_PAGE_SPECIAL_Q_DONATE:
2937 {
2938 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
2939 vm_page_queue_remove((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
2940 mem->vmp_specialq.next = 0;
2941 mem->vmp_specialq.prev = 0;
2942 vm_page_donate_count--;
2943 if (vm_page_donate_queue_ripe && (vm_page_donate_count < vm_page_donate_target)) {
2944 assert(vm_page_donate_target == vm_page_donate_target_low);
2945 vm_page_donate_target = vm_page_donate_target_high;
2946 vm_page_donate_queue_ripe = false;
2947 }
2948 }
2949
2950 break;
2951 }
2952
2953 default:
2954 {
2955 assert(VM_PAGE_UNPACK_PTR(mem->vmp_specialq.next) == (uintptr_t)NULL &&
2956 VM_PAGE_UNPACK_PTR(mem->vmp_specialq.prev) == (uintptr_t)NULL);
2957 break;
2958 }
2959 }
2960 }
2961
2962
2963 void
vm_page_add_to_specialq(vm_page_t mem,boolean_t first)2964 vm_page_add_to_specialq(
2965 vm_page_t mem,
2966 boolean_t first)
2967 {
2968 vm_object_t m_object;
2969
2970 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
2971
2972 if (mem->vmp_specialq.next && mem->vmp_specialq.prev) {
2973 return;
2974 }
2975
2976 int mode = mem->vmp_on_specialq;
2977
2978 switch (mode) {
2979 case VM_PAGE_SPECIAL_Q_BG:
2980 {
2981 if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {
2982 return;
2983 }
2984
2985 m_object = VM_PAGE_OBJECT(mem);
2986
2987 if (vm_page_background_exclude_external && !m_object->internal) {
2988 return;
2989 }
2990
2991 if (first == TRUE) {
2992 vm_page_queue_enter_first(&vm_page_queue_background, mem, vmp_specialq);
2993 } else {
2994 vm_page_queue_enter(&vm_page_queue_background, mem, vmp_specialq);
2995 }
2996 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_BG;
2997
2998 vm_page_background_count++;
2999
3000 if (m_object->internal) {
3001 vm_page_background_internal_count++;
3002 } else {
3003 vm_page_background_external_count++;
3004 }
3005 break;
3006 }
3007
3008 case VM_PAGE_SPECIAL_Q_DONATE:
3009 {
3010 if (first == TRUE) {
3011 vm_page_queue_enter_first((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
3012 } else {
3013 vm_page_queue_enter((vm_page_queue_head_t*)&vm_page_queue_donate, mem, vmp_specialq);
3014 }
3015 vm_page_donate_count++;
3016 if (!vm_page_donate_queue_ripe && (vm_page_donate_count > vm_page_donate_target)) {
3017 assert(vm_page_donate_target == vm_page_donate_target_high);
3018 vm_page_donate_target = vm_page_donate_target_low;
3019 vm_page_donate_queue_ripe = true;
3020 }
3021 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_DONATE;
3022 break;
3023 }
3024
3025 default:
3026 break;
3027 }
3028 }
3029
3030 /*
3031 * This can be switched to FALSE to help debug drivers
3032 * that are having problems with memory > 4G.
3033 */
3034 boolean_t vm_himemory_mode = TRUE;
3035
3036 /*
3037 * this interface exists to support hardware controllers
3038 * incapable of generating DMAs with more than 32 bits
3039 * of address on platforms with physical memory > 4G...
3040 */
3041 unsigned int vm_lopages_allocated_q = 0;
3042 unsigned int vm_lopages_allocated_cpm_success = 0;
3043 unsigned int vm_lopages_allocated_cpm_failed = 0;
3044 vm_page_queue_head_t vm_lopage_queue_free VM_PAGE_PACKED_ALIGNED;
3045
3046 vm_page_t
vm_page_grablo(void)3047 vm_page_grablo(void)
3048 {
3049 vm_page_t mem;
3050
3051 if (vm_lopage_needed == FALSE) {
3052 return vm_page_grab();
3053 }
3054
3055 vm_free_page_lock_spin();
3056
3057 if (!vm_page_queue_empty(&vm_lopage_queue_free)) {
3058 vm_page_queue_remove_first(&vm_lopage_queue_free, mem, vmp_pageq);
3059 assert(vm_lopage_free_count);
3060 assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);
3061 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
3062
3063 vm_lopage_free_count--;
3064 vm_lopages_allocated_q++;
3065
3066 if (vm_lopage_free_count < vm_lopage_lowater) {
3067 vm_lopage_refill = TRUE;
3068 }
3069
3070 vm_free_page_unlock();
3071
3072 if (current_task()->donates_own_pages) {
3073 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_DONATE);
3074 } else {
3075 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_BG);
3076 }
3077 } else {
3078 vm_free_page_unlock();
3079
3080 if (cpm_allocate(PAGE_SIZE, &mem, atop(PPNUM_MAX), 0, FALSE, KMA_LOMEM) != KERN_SUCCESS) {
3081 vm_free_page_lock_spin();
3082 vm_lopages_allocated_cpm_failed++;
3083 vm_free_page_unlock();
3084
3085 return VM_PAGE_NULL;
3086 }
3087 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
3088
3089 mem->vmp_busy = TRUE;
3090
3091 vm_page_lockspin_queues();
3092
3093 mem->vmp_gobbled = FALSE;
3094 vm_page_gobble_count--;
3095 vm_page_wire_count--;
3096
3097 vm_lopages_allocated_cpm_success++;
3098 vm_page_unlock_queues();
3099 }
3100 assert(mem->vmp_busy);
3101 assert(!mem->vmp_pmapped);
3102 assert(!mem->vmp_wpmapped);
3103 assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)));
3104
3105 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
3106
3107 counter_inc(&vm_page_grab_count);
3108 VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, 0, 1, 0, 0);
3109
3110 return mem;
3111 }
3112
3113 /*
3114 * vm_page_grab:
3115 *
3116 * first try to grab a page from the per-cpu free list...
3117 * this must be done while pre-emption is disabled... if
3118 * a page is available, we're done...
3119 * if no page is available, grab the vm_page_queue_free_lock
3120 * and see if current number of free pages would allow us
3121 * to grab at least 1... if not, return VM_PAGE_NULL as before...
3122 * if there are pages available, disable preemption and
3123 * recheck the state of the per-cpu free list... we could
3124 * have been preempted and moved to a different cpu, or
3125 * some other thread could have re-filled it... if still
3126 * empty, figure out how many pages we can steal from the
3127 * global free queue and move to the per-cpu queue...
3128 * return 1 of these pages when done... only wakeup the
3129 * pageout_scan thread if we moved pages from the global
3130 * list... no need for the wakeup if we've satisfied the
3131 * request from the per-cpu queue.
3132 */
3133
3134 #if CONFIG_SECLUDED_MEMORY
3135 vm_page_t vm_page_grab_secluded(void);
3136 #endif /* CONFIG_SECLUDED_MEMORY */
3137
3138 static inline void
3139 vm_page_grab_diags(void);
3140
3141 vm_page_t
vm_page_grab(void)3142 vm_page_grab(void)
3143 {
3144 return vm_page_grab_options(VM_PAGE_GRAB_OPTIONS_NONE);
3145 }
3146
3147 #if HIBERNATION
3148 boolean_t hibernate_rebuild_needed = FALSE;
3149 #endif /* HIBERNATION */
3150
3151 vm_page_t
vm_page_grab_options(int grab_options)3152 vm_page_grab_options(
3153 int grab_options)
3154 {
3155 vm_page_t mem;
3156
3157 restart:
3158 disable_preemption();
3159
3160 if ((mem = *PERCPU_GET(free_pages))) {
3161 assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
3162
3163 #if HIBERNATION
3164 if (hibernate_rebuild_needed) {
3165 panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
3166 }
3167 #endif /* HIBERNATION */
3168
3169 vm_page_grab_diags();
3170
3171 vm_offset_t pcpu_base = current_percpu_base();
3172 counter_inc_preemption_disabled(&vm_page_grab_count);
3173 *PERCPU_GET_WITH_BASE(pcpu_base, free_pages) = mem->vmp_snext;
3174 VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
3175
3176 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
3177 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
3178 enable_preemption();
3179
3180 assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
3181 assert(mem->vmp_tabled == FALSE);
3182 assert(mem->vmp_object == 0);
3183 assert(!mem->vmp_laundry);
3184 ASSERT_PMAP_FREE(mem);
3185 assert(mem->vmp_busy);
3186 assert(!mem->vmp_pmapped);
3187 assert(!mem->vmp_wpmapped);
3188 assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)));
3189
3190 task_t cur_task = current_task_early();
3191 if (cur_task && cur_task != kernel_task) {
3192 if (cur_task->donates_own_pages) {
3193 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_DONATE);
3194 } else {
3195 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_BG);
3196 }
3197 }
3198 return mem;
3199 }
3200 enable_preemption();
3201
3202
3203 /*
3204 * Optionally produce warnings if the wire or gobble
3205 * counts exceed some threshold.
3206 */
3207 #if VM_PAGE_WIRE_COUNT_WARNING
3208 if (vm_page_wire_count >= VM_PAGE_WIRE_COUNT_WARNING) {
3209 printf("mk: vm_page_grab(): high wired page count of %d\n",
3210 vm_page_wire_count);
3211 }
3212 #endif
3213 #if VM_PAGE_GOBBLE_COUNT_WARNING
3214 if (vm_page_gobble_count >= VM_PAGE_GOBBLE_COUNT_WARNING) {
3215 printf("mk: vm_page_grab(): high gobbled page count of %d\n",
3216 vm_page_gobble_count);
3217 }
3218 #endif
3219
3220 /*
3221 * If free count is low and we have delayed pages from early boot,
3222 * get one of those instead.
3223 */
3224 if (__improbable(vm_delayed_count > 0 &&
3225 vm_page_free_count <= vm_page_free_target &&
3226 (mem = vm_get_delayed_page(grab_options)) != NULL)) {
3227 return mem;
3228 }
3229
3230 vm_free_page_lock_spin();
3231
3232 /*
3233 * Only let privileged threads (involved in pageout)
3234 * dip into the reserved pool.
3235 */
3236 if ((vm_page_free_count < vm_page_free_reserved) &&
3237 !(current_thread()->options & TH_OPT_VMPRIV)) {
3238 /* no page for us in the free queue... */
3239 vm_free_page_unlock();
3240 mem = VM_PAGE_NULL;
3241
3242 #if CONFIG_SECLUDED_MEMORY
3243 /* ... but can we try and grab from the secluded queue? */
3244 if (vm_page_secluded_count > 0 &&
3245 ((grab_options & VM_PAGE_GRAB_SECLUDED) ||
3246 task_can_use_secluded_mem(current_task(), TRUE))) {
3247 mem = vm_page_grab_secluded();
3248 if (grab_options & VM_PAGE_GRAB_SECLUDED) {
3249 vm_page_secluded.grab_for_iokit++;
3250 if (mem) {
3251 vm_page_secluded.grab_for_iokit_success++;
3252 }
3253 }
3254 if (mem) {
3255 VM_CHECK_MEMORYSTATUS;
3256
3257 vm_page_grab_diags();
3258 counter_inc(&vm_page_grab_count);
3259 VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
3260
3261
3262 return mem;
3263 }
3264 }
3265 #else /* CONFIG_SECLUDED_MEMORY */
3266 (void) grab_options;
3267 #endif /* CONFIG_SECLUDED_MEMORY */
3268 } else {
3269 vm_page_t head;
3270 vm_page_t tail;
3271 unsigned int pages_to_steal;
3272 unsigned int color;
3273 unsigned int clump_end, sub_count;
3274
3275 while (vm_page_free_count == 0) {
3276 vm_free_page_unlock();
3277 /*
3278 * must be a privileged thread to be
3279 * in this state since a non-privileged
3280 * thread would have bailed if we were
3281 * under the vm_page_free_reserved mark
3282 */
3283 VM_PAGE_WAIT();
3284 vm_free_page_lock_spin();
3285 }
3286
3287 /*
3288 * Need to repopulate the per-CPU free list from the global free list.
3289 * Note we don't do any processing of pending retirement pages here.
3290 * That'll happen in the code above when the page comes off the per-CPU list.
3291 */
3292 disable_preemption();
3293
3294 /*
3295 * If we got preempted the cache might now have pages.
3296 */
3297 if ((mem = *PERCPU_GET(free_pages))) {
3298 vm_free_page_unlock();
3299 enable_preemption();
3300 goto restart;
3301 }
3302
3303 if (vm_page_free_count <= vm_page_free_reserved) {
3304 pages_to_steal = 1;
3305 } else {
3306 if (vm_free_magazine_refill_limit <= (vm_page_free_count - vm_page_free_reserved)) {
3307 pages_to_steal = vm_free_magazine_refill_limit;
3308 } else {
3309 pages_to_steal = (vm_page_free_count - vm_page_free_reserved);
3310 }
3311 }
3312 color = *PERCPU_GET(start_color);
3313 head = tail = NULL;
3314
3315 vm_page_free_count -= pages_to_steal;
3316 clump_end = sub_count = 0;
3317
3318 while (pages_to_steal--) {
3319 while (vm_page_queue_empty(&vm_page_queue_free[color].qhead)) {
3320 color = (color + 1) & vm_color_mask;
3321 }
3322 #if defined(__x86_64__)
3323 vm_page_queue_remove_first_with_clump(&vm_page_queue_free[color].qhead,
3324 mem, clump_end);
3325 #else
3326 vm_page_queue_remove_first(&vm_page_queue_free[color].qhead,
3327 mem, vmp_pageq);
3328 #endif
3329
3330 assert(mem->vmp_q_state == VM_PAGE_ON_FREE_Q);
3331
3332 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
3333
3334 #if defined(__arm64__)
3335 color = (color + 1) & vm_color_mask;
3336 #else
3337
3338 #if DEVELOPMENT || DEBUG
3339
3340 sub_count++;
3341 if (clump_end) {
3342 vm_clump_update_stats(sub_count);
3343 sub_count = 0;
3344 color = (color + 1) & vm_color_mask;
3345 }
3346 #else
3347 if (clump_end) {
3348 color = (color + 1) & vm_color_mask;
3349 }
3350
3351 #endif /* if DEVELOPMENT || DEBUG */
3352
3353 #endif /* if defined(__arm64__) */
3354
3355 if (head == NULL) {
3356 head = mem;
3357 } else {
3358 tail->vmp_snext = mem;
3359 }
3360 tail = mem;
3361
3362 assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
3363 assert(mem->vmp_tabled == FALSE);
3364 assert(mem->vmp_object == 0);
3365 assert(!mem->vmp_laundry);
3366
3367 mem->vmp_q_state = VM_PAGE_ON_FREE_LOCAL_Q;
3368
3369 ASSERT_PMAP_FREE(mem);
3370 assert(mem->vmp_busy);
3371 assert(!mem->vmp_pmapped);
3372 assert(!mem->vmp_wpmapped);
3373 assert(!pmap_is_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)));
3374 }
3375 #if defined (__x86_64__) && (DEVELOPMENT || DEBUG)
3376 vm_clump_update_stats(sub_count);
3377 #endif
3378
3379 #if HIBERNATION
3380 if (hibernate_rebuild_needed) {
3381 panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
3382 }
3383 #endif /* HIBERNATION */
3384 vm_offset_t pcpu_base = current_percpu_base();
3385 *PERCPU_GET_WITH_BASE(pcpu_base, free_pages) = head;
3386 *PERCPU_GET_WITH_BASE(pcpu_base, start_color) = color;
3387
3388 vm_free_page_unlock();
3389 enable_preemption();
3390 goto restart;
3391 }
3392
3393 /*
3394 * Decide if we should poke the pageout daemon.
3395 * We do this if the free count is less than the low
3396 * water mark. VM Pageout Scan will keep running till
3397 * the free_count > free_target (& hence above free_min).
3398 * This wakeup is to catch the possibility of the counts
3399 * dropping between VM Pageout Scan parking and this check.
3400 *
3401 * We don't have the counts locked ... if they change a little,
3402 * it doesn't really matter.
3403 */
3404 if (vm_page_free_count < vm_page_free_min) {
3405 vm_free_page_lock();
3406 if (vm_pageout_running == FALSE) {
3407 vm_free_page_unlock();
3408 thread_wakeup((event_t) &vm_page_free_wanted);
3409 } else {
3410 vm_free_page_unlock();
3411 }
3412 }
3413
3414 VM_CHECK_MEMORYSTATUS;
3415
3416 if (mem) {
3417 // dbgLog(VM_PAGE_GET_PHYS_PAGE(mem), vm_page_free_count, vm_page_wire_count, 4); /* (TEST/DEBUG) */
3418
3419 task_t cur_task = current_task_early();
3420 if (cur_task && cur_task != kernel_task) {
3421 if (cur_task->donates_own_pages) {
3422 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_DONATE);
3423 } else {
3424 vm_page_assign_special_state(mem, VM_PAGE_SPECIAL_Q_BG);
3425 }
3426 }
3427 }
3428 return mem;
3429 }
3430
3431 #if CONFIG_SECLUDED_MEMORY
3432 vm_page_t
vm_page_grab_secluded(void)3433 vm_page_grab_secluded(void)
3434 {
3435 vm_page_t mem;
3436 vm_object_t object;
3437 int refmod_state;
3438
3439 if (vm_page_secluded_count == 0) {
3440 /* no secluded pages to grab... */
3441 return VM_PAGE_NULL;
3442 }
3443
3444 /* secluded queue is protected by the VM page queue lock */
3445 vm_page_lock_queues();
3446
3447 if (vm_page_secluded_count == 0) {
3448 /* no secluded pages to grab... */
3449 vm_page_unlock_queues();
3450 return VM_PAGE_NULL;
3451 }
3452
3453 #if 00
3454 /* can we grab from the secluded queue? */
3455 if (vm_page_secluded_count > vm_page_secluded_target ||
3456 (vm_page_secluded_count > 0 &&
3457 task_can_use_secluded_mem(current_task(), TRUE))) {
3458 /* OK */
3459 } else {
3460 /* can't grab from secluded queue... */
3461 vm_page_unlock_queues();
3462 return VM_PAGE_NULL;
3463 }
3464 #endif
3465
3466 /* we can grab a page from secluded queue! */
3467 assert((vm_page_secluded_count_free +
3468 vm_page_secluded_count_inuse) ==
3469 vm_page_secluded_count);
3470 if (current_task()->task_can_use_secluded_mem) {
3471 assert(num_tasks_can_use_secluded_mem > 0);
3472 }
3473 assert(!vm_page_queue_empty(&vm_page_queue_secluded));
3474 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3475 mem = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
3476 assert(mem->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
3477 vm_page_queues_remove(mem, TRUE);
3478
3479 object = VM_PAGE_OBJECT(mem);
3480
3481 assert(!mem->vmp_fictitious);
3482 assert(!VM_PAGE_WIRED(mem));
3483 if (object == VM_OBJECT_NULL) {
3484 /* free for grab! */
3485 vm_page_unlock_queues();
3486 vm_page_secluded.grab_success_free++;
3487
3488 assert(mem->vmp_busy);
3489 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
3490 assert(VM_PAGE_OBJECT(mem) == VM_OBJECT_NULL);
3491 assert(mem->vmp_pageq.next == 0);
3492 assert(mem->vmp_pageq.prev == 0);
3493 assert(mem->vmp_listq.next == 0);
3494 assert(mem->vmp_listq.prev == 0);
3495 assert(mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
3496 assert(mem->vmp_specialq.next == 0);
3497 assert(mem->vmp_specialq.prev == 0);
3498 return mem;
3499 }
3500
3501 assert(!object->internal);
3502 // vm_page_pageable_external_count--;
3503
3504 if (!vm_object_lock_try(object)) {
3505 // printf("SECLUDED: page %p: object %p locked\n", mem, object);
3506 vm_page_secluded.grab_failure_locked++;
3507 reactivate_secluded_page:
3508 vm_page_activate(mem);
3509 vm_page_unlock_queues();
3510 return VM_PAGE_NULL;
3511 }
3512 if (mem->vmp_busy ||
3513 mem->vmp_cleaning ||
3514 mem->vmp_laundry) {
3515 /* can't steal page in this state... */
3516 vm_object_unlock(object);
3517 vm_page_secluded.grab_failure_state++;
3518 goto reactivate_secluded_page;
3519 }
3520
3521 mem->vmp_busy = TRUE;
3522 refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(mem));
3523 if (refmod_state & VM_MEM_REFERENCED) {
3524 mem->vmp_reference = TRUE;
3525 }
3526 if (refmod_state & VM_MEM_MODIFIED) {
3527 SET_PAGE_DIRTY(mem, FALSE);
3528 }
3529 if (mem->vmp_dirty || mem->vmp_precious) {
3530 /* can't grab a dirty page; re-activate */
3531 // printf("SECLUDED: dirty page %p\n", mem);
3532 PAGE_WAKEUP_DONE(mem);
3533 vm_page_secluded.grab_failure_dirty++;
3534 vm_object_unlock(object);
3535 goto reactivate_secluded_page;
3536 }
3537 if (mem->vmp_reference) {
3538 /* it's been used but we do need to grab a page... */
3539 }
3540
3541 vm_page_unlock_queues();
3542
3543
3544 /* finish what vm_page_free() would have done... */
3545 vm_page_free_prepare_object(mem, TRUE);
3546 vm_object_unlock(object);
3547 object = VM_OBJECT_NULL;
3548 if (vm_page_free_verify) {
3549 ASSERT_PMAP_FREE(mem);
3550 }
3551 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
3552 vm_page_secluded.grab_success_other++;
3553
3554 assert(mem->vmp_busy);
3555 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
3556 assert(VM_PAGE_OBJECT(mem) == VM_OBJECT_NULL);
3557 assert(mem->vmp_pageq.next == 0);
3558 assert(mem->vmp_pageq.prev == 0);
3559 assert(mem->vmp_listq.next == 0);
3560 assert(mem->vmp_listq.prev == 0);
3561 assert(mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
3562 assert(mem->vmp_specialq.next == 0);
3563 assert(mem->vmp_specialq.prev == 0);
3564
3565 return mem;
3566 }
3567
3568 uint64_t
vm_page_secluded_drain(void)3569 vm_page_secluded_drain(void)
3570 {
3571 vm_page_t local_freeq;
3572 int local_freed;
3573 uint64_t num_reclaimed;
3574 unsigned int saved_secluded_count, saved_secluded_target;
3575
3576 num_reclaimed = 0;
3577 local_freeq = NULL;
3578 local_freed = 0;
3579
3580 vm_page_lock_queues();
3581
3582 saved_secluded_count = vm_page_secluded_count;
3583 saved_secluded_target = vm_page_secluded_target;
3584 vm_page_secluded_target = 0;
3585 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
3586 while (vm_page_secluded_count) {
3587 vm_page_t secluded_page;
3588
3589 assert((vm_page_secluded_count_free +
3590 vm_page_secluded_count_inuse) ==
3591 vm_page_secluded_count);
3592 secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
3593 assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
3594
3595 vm_page_queues_remove(secluded_page, FALSE);
3596 assert(!secluded_page->vmp_fictitious);
3597 assert(!VM_PAGE_WIRED(secluded_page));
3598
3599 if (secluded_page->vmp_object == 0) {
3600 /* transfer to free queue */
3601 assert(secluded_page->vmp_busy);
3602 secluded_page->vmp_snext = local_freeq;
3603 local_freeq = secluded_page;
3604 local_freed += 1;
3605 } else {
3606 /* transfer to head of active queue */
3607 vm_page_enqueue_active(secluded_page, FALSE);
3608 secluded_page = VM_PAGE_NULL;
3609 }
3610 num_reclaimed++;
3611 }
3612 vm_page_secluded_target = saved_secluded_target;
3613 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
3614
3615 // printf("FBDP %s:%d secluded_count %d->%d, target %d, reclaimed %lld\n", __FUNCTION__, __LINE__, saved_secluded_count, vm_page_secluded_count, vm_page_secluded_target, num_reclaimed);
3616
3617 vm_page_unlock_queues();
3618
3619 if (local_freed) {
3620 vm_page_free_list(local_freeq, TRUE);
3621 local_freeq = NULL;
3622 local_freed = 0;
3623 }
3624
3625 return num_reclaimed;
3626 }
3627 #endif /* CONFIG_SECLUDED_MEMORY */
3628
3629
3630 static inline void
vm_page_grab_diags()3631 vm_page_grab_diags()
3632 {
3633 #if DEVELOPMENT || DEBUG
3634 task_t task = current_task_early();
3635 if (task == NULL) {
3636 return;
3637 }
3638
3639 ledger_credit(task->ledger, task_ledgers.pages_grabbed, 1);
3640 #endif /* DEVELOPMENT || DEBUG */
3641 }
3642
3643 /*
3644 * vm_page_release:
3645 *
3646 * Return a page to the free list.
3647 */
3648
3649 void
vm_page_release(vm_page_t mem,boolean_t page_queues_locked)3650 vm_page_release(
3651 vm_page_t mem,
3652 boolean_t page_queues_locked)
3653 {
3654 unsigned int color;
3655 int need_wakeup = 0;
3656 int need_priv_wakeup = 0;
3657 #if CONFIG_SECLUDED_MEMORY
3658 int need_secluded_wakeup = 0;
3659 #endif /* CONFIG_SECLUDED_MEMORY */
3660 event_t wakeup_event = NULL;
3661
3662 if (page_queues_locked) {
3663 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3664 } else {
3665 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
3666 }
3667
3668 assert(!mem->vmp_private && !mem->vmp_fictitious);
3669 if (vm_page_free_verify) {
3670 ASSERT_PMAP_FREE(mem);
3671 }
3672 // dbgLog(VM_PAGE_GET_PHYS_PAGE(mem), vm_page_free_count, vm_page_wire_count, 5); /* (TEST/DEBUG) */
3673
3674 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
3675
3676 vm_free_page_lock_spin();
3677
3678 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
3679 assert(mem->vmp_busy);
3680 assert(!mem->vmp_laundry);
3681 assert(mem->vmp_object == 0);
3682 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
3683 assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
3684 assert(mem->vmp_specialq.next == 0 && mem->vmp_specialq.prev == 0);
3685
3686 /* Clear any specialQ hints before releasing page to the free pool*/
3687 mem->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
3688
3689 if ((mem->vmp_lopage == TRUE || vm_lopage_refill == TRUE) &&
3690 vm_lopage_free_count < vm_lopage_free_limit &&
3691 VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {
3692 /*
3693 * this exists to support hardware controllers
3694 * incapable of generating DMAs with more than 32 bits
3695 * of address on platforms with physical memory > 4G...
3696 */
3697 vm_page_queue_enter_first(&vm_lopage_queue_free, mem, vmp_pageq);
3698 vm_lopage_free_count++;
3699
3700 if (vm_lopage_free_count >= vm_lopage_free_limit) {
3701 vm_lopage_refill = FALSE;
3702 }
3703
3704 mem->vmp_q_state = VM_PAGE_ON_FREE_LOPAGE_Q;
3705 mem->vmp_lopage = TRUE;
3706 #if CONFIG_SECLUDED_MEMORY
3707 } else if (vm_page_free_count > vm_page_free_reserved &&
3708 vm_page_secluded_count < vm_page_secluded_target &&
3709 num_tasks_can_use_secluded_mem == 0) {
3710 /*
3711 * XXX FBDP TODO: also avoid refilling secluded queue
3712 * when some IOKit objects are already grabbing from it...
3713 */
3714 if (!page_queues_locked) {
3715 if (!vm_page_trylock_queues()) {
3716 /* take locks in right order */
3717 vm_free_page_unlock();
3718 vm_page_lock_queues();
3719 vm_free_page_lock_spin();
3720 }
3721 }
3722 mem->vmp_lopage = FALSE;
3723 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
3724 vm_page_queue_enter_first(&vm_page_queue_secluded, mem, vmp_pageq);
3725 mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
3726 vm_page_secluded_count++;
3727 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
3728 vm_page_secluded_count_free++;
3729 if (!page_queues_locked) {
3730 vm_page_unlock_queues();
3731 }
3732 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
3733 if (vm_page_free_wanted_secluded > 0) {
3734 vm_page_free_wanted_secluded--;
3735 need_secluded_wakeup = 1;
3736 }
3737 #endif /* CONFIG_SECLUDED_MEMORY */
3738 } else {
3739 mem->vmp_lopage = FALSE;
3740 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
3741
3742 color = VM_PAGE_GET_COLOR(mem);
3743 #if defined(__x86_64__)
3744 vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);
3745 #else
3746 vm_page_queue_enter(&vm_page_queue_free[color].qhead, mem, vmp_pageq);
3747 #endif
3748 vm_page_free_count++;
3749 /*
3750 * Check if we should wake up someone waiting for page.
3751 * But don't bother waking them unless they can allocate.
3752 *
3753 * We wakeup only one thread, to prevent starvation.
3754 * Because the scheduling system handles wait queues FIFO,
3755 * if we wakeup all waiting threads, one greedy thread
3756 * can starve multiple niceguy threads. When the threads
3757 * all wakeup, the greedy threads runs first, grabs the page,
3758 * and waits for another page. It will be the first to run
3759 * when the next page is freed.
3760 *
3761 * However, there is a slight danger here.
3762 * The thread we wake might not use the free page.
3763 * Then the other threads could wait indefinitely
3764 * while the page goes unused. To forestall this,
3765 * the pageout daemon will keep making free pages
3766 * as long as vm_page_free_wanted is non-zero.
3767 */
3768
3769 assert(vm_page_free_count > 0);
3770 if (vm_page_free_wanted_privileged > 0) {
3771 vm_page_free_wanted_privileged--;
3772 need_priv_wakeup = 1;
3773 #if CONFIG_SECLUDED_MEMORY
3774 } else if (vm_page_free_wanted_secluded > 0 &&
3775 vm_page_free_count > vm_page_free_reserved) {
3776 vm_page_free_wanted_secluded--;
3777 need_secluded_wakeup = 1;
3778 #endif /* CONFIG_SECLUDED_MEMORY */
3779 } else if (vm_page_free_wanted > 0 &&
3780 vm_page_free_count > vm_page_free_reserved) {
3781 vm_page_free_wanted--;
3782 need_wakeup = 1;
3783 }
3784 }
3785 vm_pageout_vminfo.vm_page_pages_freed++;
3786
3787 vm_free_page_unlock();
3788
3789 VM_DEBUG_CONSTANT_EVENT(vm_page_release, VM_PAGE_RELEASE, DBG_FUNC_NONE, 1, 0, 0, 0);
3790
3791 if (need_priv_wakeup) {
3792 wakeup_event = &vm_page_free_wanted_privileged;
3793 }
3794 #if CONFIG_SECLUDED_MEMORY
3795 else if (need_secluded_wakeup) {
3796 wakeup_event = &vm_page_free_wanted_secluded;
3797 }
3798 #endif /* CONFIG_SECLUDED_MEMORY */
3799 else if (need_wakeup) {
3800 wakeup_event = &vm_page_free_count;
3801 }
3802
3803 if (wakeup_event) {
3804 if (vps_dynamic_priority_enabled == TRUE) {
3805 thread_t thread_woken = NULL;
3806 wakeup_one_with_inheritor((event_t) wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
3807 /*
3808 * (80947592) if this is the last reference on this
3809 * thread, calling thread_deallocate() here
3810 * might take the tasks_threads_lock,
3811 * sadly thread_create_internal is doing several
3812 * allocations under this lock, which can result in
3813 * deadlocks with the pageout scan daemon.
3814 *
3815 * FIXME: we should disallow allocations under the
3816 * task_thread_locks, but that is a larger fix to make.
3817 */
3818 thread_deallocate_safe(thread_woken);
3819 } else {
3820 thread_wakeup_one((event_t) wakeup_event);
3821 }
3822 }
3823
3824 VM_CHECK_MEMORYSTATUS;
3825 }
3826
3827 /*
3828 * This version of vm_page_release() is used only at startup
3829 * when we are single-threaded and pages are being released
3830 * for the first time. Hence, no locking or unnecessary checks are made.
3831 * Note: VM_CHECK_MEMORYSTATUS invoked by the caller.
3832 */
3833 void
vm_page_release_startup(vm_page_t mem)3834 vm_page_release_startup(
3835 vm_page_t mem)
3836 {
3837 vm_page_queue_t queue_free;
3838
3839 if (vm_lopage_free_count < vm_lopage_free_limit &&
3840 VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {
3841 mem->vmp_lopage = TRUE;
3842 mem->vmp_q_state = VM_PAGE_ON_FREE_LOPAGE_Q;
3843 vm_lopage_free_count++;
3844 queue_free = &vm_lopage_queue_free;
3845 #if CONFIG_SECLUDED_MEMORY
3846 } else if (vm_page_secluded_count < vm_page_secluded_target) {
3847 mem->vmp_lopage = FALSE;
3848 mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
3849 vm_page_secluded_count++;
3850 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
3851 vm_page_secluded_count_free++;
3852 queue_free = &vm_page_queue_secluded;
3853 #endif /* CONFIG_SECLUDED_MEMORY */
3854 } else {
3855 mem->vmp_lopage = FALSE;
3856 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
3857 vm_page_free_count++;
3858 queue_free = &vm_page_queue_free[VM_PAGE_GET_COLOR(mem)].qhead;
3859 }
3860 if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
3861 #if defined(__x86_64__)
3862 vm_page_queue_enter_clump(queue_free, mem);
3863 #else
3864 vm_page_queue_enter(queue_free, mem, vmp_pageq);
3865 #endif
3866 } else {
3867 vm_page_queue_enter_first(queue_free, mem, vmp_pageq);
3868 }
3869 }
3870
3871 /*
3872 * vm_page_wait:
3873 *
3874 * Wait for a page to become available.
3875 * If there are plenty of free pages, then we don't sleep.
3876 *
3877 * Returns:
3878 * TRUE: There may be another page, try again
3879 * FALSE: We were interrupted out of our wait, don't try again
3880 */
3881
3882 boolean_t
vm_page_wait(int interruptible)3883 vm_page_wait(
3884 int interruptible )
3885 {
3886 /*
3887 * We can't use vm_page_free_reserved to make this
3888 * determination. Consider: some thread might
3889 * need to allocate two pages. The first allocation
3890 * succeeds, the second fails. After the first page is freed,
3891 * a call to vm_page_wait must really block.
3892 */
3893 kern_return_t wait_result;
3894 int need_wakeup = 0;
3895 int is_privileged = current_thread()->options & TH_OPT_VMPRIV;
3896 event_t wait_event = NULL;
3897
3898 vm_free_page_lock_spin();
3899
3900 if (is_privileged && vm_page_free_count) {
3901 vm_free_page_unlock();
3902 return TRUE;
3903 }
3904
3905 if (vm_page_free_count >= vm_page_free_target) {
3906 vm_free_page_unlock();
3907 return TRUE;
3908 }
3909
3910 if (is_privileged) {
3911 if (vm_page_free_wanted_privileged++ == 0) {
3912 need_wakeup = 1;
3913 }
3914 wait_event = (event_t)&vm_page_free_wanted_privileged;
3915 #if CONFIG_SECLUDED_MEMORY
3916 } else if (secluded_for_apps &&
3917 task_can_use_secluded_mem(current_task(), FALSE)) {
3918 #if 00
3919 /* XXX FBDP: need pageq lock for this... */
3920 /* XXX FBDP: might wait even if pages available, */
3921 /* XXX FBDP: hopefully not for too long... */
3922 if (vm_page_secluded_count > 0) {
3923 vm_free_page_unlock();
3924 return TRUE;
3925 }
3926 #endif
3927 if (vm_page_free_wanted_secluded++ == 0) {
3928 need_wakeup = 1;
3929 }
3930 wait_event = (event_t)&vm_page_free_wanted_secluded;
3931 #endif /* CONFIG_SECLUDED_MEMORY */
3932 } else {
3933 if (vm_page_free_wanted++ == 0) {
3934 need_wakeup = 1;
3935 }
3936 wait_event = (event_t)&vm_page_free_count;
3937 }
3938
3939 /*
3940 * We don't do a vm_pageout_scan wakeup if we already have
3941 * some waiters because vm_pageout_scan checks for waiters
3942 * before it returns and does so behind the vm_page_queue_free_lock,
3943 * which we own when we bump the waiter counts.
3944 */
3945
3946 if (vps_dynamic_priority_enabled == TRUE) {
3947 /*
3948 * We are waking up vm_pageout_scan here. If it needs
3949 * the vm_page_queue_free_lock before we unlock it
3950 * we'll end up just blocking and incur an extra
3951 * context switch. Could be a perf. issue.
3952 */
3953
3954 if (need_wakeup) {
3955 thread_wakeup((event_t)&vm_page_free_wanted);
3956 }
3957
3958 /*
3959 * LD: This event is going to get recorded every time because
3960 * we don't get back THREAD_WAITING from lck_mtx_sleep_with_inheritor.
3961 * We just block in that routine.
3962 */
3963 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,
3964 vm_page_free_wanted_privileged,
3965 vm_page_free_wanted,
3966 #if CONFIG_SECLUDED_MEMORY
3967 vm_page_free_wanted_secluded,
3968 #else /* CONFIG_SECLUDED_MEMORY */
3969 0,
3970 #endif /* CONFIG_SECLUDED_MEMORY */
3971 0);
3972 wait_result = lck_mtx_sleep_with_inheritor(&vm_page_queue_free_lock,
3973 LCK_SLEEP_UNLOCK,
3974 wait_event,
3975 vm_pageout_scan_thread,
3976 interruptible,
3977 0);
3978 } else {
3979 wait_result = assert_wait(wait_event, interruptible);
3980
3981 vm_free_page_unlock();
3982
3983 if (need_wakeup) {
3984 thread_wakeup((event_t)&vm_page_free_wanted);
3985 }
3986
3987 if (wait_result == THREAD_WAITING) {
3988 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,
3989 vm_page_free_wanted_privileged,
3990 vm_page_free_wanted,
3991 #if CONFIG_SECLUDED_MEMORY
3992 vm_page_free_wanted_secluded,
3993 #else /* CONFIG_SECLUDED_MEMORY */
3994 0,
3995 #endif /* CONFIG_SECLUDED_MEMORY */
3996 0);
3997 wait_result = thread_block(THREAD_CONTINUE_NULL);
3998 VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block,
3999 VM_PAGE_WAIT_BLOCK, DBG_FUNC_END, 0, 0, 0, 0);
4000 }
4001 }
4002
4003 return (wait_result == THREAD_AWAKENED) || (wait_result == THREAD_NOT_WAITING);
4004 }
4005
4006 /*
4007 * vm_page_alloc:
4008 *
4009 * Allocate and return a memory cell associated
4010 * with this VM object/offset pair.
4011 *
4012 * Object must be locked.
4013 */
4014
4015 vm_page_t
vm_page_alloc(vm_object_t object,vm_object_offset_t offset)4016 vm_page_alloc(
4017 vm_object_t object,
4018 vm_object_offset_t offset)
4019 {
4020 vm_page_t mem;
4021 int grab_options;
4022
4023 vm_object_lock_assert_exclusive(object);
4024 grab_options = 0;
4025 #if CONFIG_SECLUDED_MEMORY
4026 if (object->can_grab_secluded) {
4027 grab_options |= VM_PAGE_GRAB_SECLUDED;
4028 }
4029 #endif /* CONFIG_SECLUDED_MEMORY */
4030 mem = vm_page_grab_options(grab_options);
4031 if (mem == VM_PAGE_NULL) {
4032 return VM_PAGE_NULL;
4033 }
4034
4035 vm_page_insert(mem, object, offset);
4036
4037 return mem;
4038 }
4039
4040 /*
4041 * vm_page_free_prepare:
4042 *
4043 * Removes page from any queue it may be on
4044 * and disassociates it from its VM object.
4045 *
4046 * Object and page queues must be locked prior to entry.
4047 */
4048 static void
vm_page_free_prepare(vm_page_t mem)4049 vm_page_free_prepare(
4050 vm_page_t mem)
4051 {
4052 vm_page_free_prepare_queues(mem);
4053 vm_page_free_prepare_object(mem, TRUE);
4054 }
4055
4056
4057 void
vm_page_free_prepare_queues(vm_page_t mem)4058 vm_page_free_prepare_queues(
4059 vm_page_t mem)
4060 {
4061 vm_object_t m_object;
4062
4063 VM_PAGE_CHECK(mem);
4064
4065 assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
4066 assert(!mem->vmp_cleaning);
4067 m_object = VM_PAGE_OBJECT(mem);
4068
4069 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4070 if (m_object) {
4071 vm_object_lock_assert_exclusive(m_object);
4072 }
4073 if (mem->vmp_laundry) {
4074 /*
4075 * We may have to free a page while it's being laundered
4076 * if we lost its pager (due to a forced unmount, for example).
4077 * We need to call vm_pageout_steal_laundry() before removing
4078 * the page from its VM object, so that we can remove it
4079 * from its pageout queue and adjust the laundry accounting
4080 */
4081 vm_pageout_steal_laundry(mem, TRUE);
4082 }
4083
4084 vm_page_queues_remove(mem, TRUE);
4085
4086 if (VM_PAGE_WIRED(mem)) {
4087 assert(mem->vmp_wire_count > 0);
4088
4089 if (m_object) {
4090 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
4091 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
4092 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
4093
4094 assert(m_object->resident_page_count >=
4095 m_object->wired_page_count);
4096
4097 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
4098 OSAddAtomic(+1, &vm_page_purgeable_count);
4099 assert(vm_page_purgeable_wired_count > 0);
4100 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
4101 }
4102 if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
4103 m_object->purgable == VM_PURGABLE_EMPTY) &&
4104 m_object->vo_owner != TASK_NULL) {
4105 task_t owner;
4106 int ledger_idx_volatile;
4107 int ledger_idx_nonvolatile;
4108 int ledger_idx_volatile_compressed;
4109 int ledger_idx_nonvolatile_compressed;
4110 boolean_t do_footprint;
4111
4112 owner = VM_OBJECT_OWNER(m_object);
4113 vm_object_ledger_tag_ledgers(
4114 m_object,
4115 &ledger_idx_volatile,
4116 &ledger_idx_nonvolatile,
4117 &ledger_idx_volatile_compressed,
4118 &ledger_idx_nonvolatile_compressed,
4119 &do_footprint);
4120 /*
4121 * While wired, this page was accounted
4122 * as "non-volatile" but it should now
4123 * be accounted as "volatile".
4124 */
4125 /* one less "non-volatile"... */
4126 ledger_debit(owner->ledger,
4127 ledger_idx_nonvolatile,
4128 PAGE_SIZE);
4129 if (do_footprint) {
4130 /* ... and "phys_footprint" */
4131 ledger_debit(owner->ledger,
4132 task_ledgers.phys_footprint,
4133 PAGE_SIZE);
4134 }
4135 /* one more "volatile" */
4136 ledger_credit(owner->ledger,
4137 ledger_idx_volatile,
4138 PAGE_SIZE);
4139 }
4140 }
4141 if (!mem->vmp_private && !mem->vmp_fictitious) {
4142 vm_page_wire_count--;
4143 }
4144
4145 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
4146 mem->vmp_wire_count = 0;
4147 assert(!mem->vmp_gobbled);
4148 } else if (mem->vmp_gobbled) {
4149 if (!mem->vmp_private && !mem->vmp_fictitious) {
4150 vm_page_wire_count--;
4151 }
4152 vm_page_gobble_count--;
4153 }
4154 }
4155
4156
4157 void
vm_page_free_prepare_object(vm_page_t mem,boolean_t remove_from_hash)4158 vm_page_free_prepare_object(
4159 vm_page_t mem,
4160 boolean_t remove_from_hash)
4161 {
4162 if (mem->vmp_tabled) {
4163 vm_page_remove(mem, remove_from_hash); /* clears tabled, object, offset */
4164 }
4165 PAGE_WAKEUP(mem); /* clears wanted */
4166
4167 if (mem->vmp_private) {
4168 mem->vmp_private = FALSE;
4169 mem->vmp_fictitious = TRUE;
4170 VM_PAGE_SET_PHYS_PAGE(mem, vm_page_fictitious_addr);
4171 }
4172 if (!mem->vmp_fictitious) {
4173 assert(mem->vmp_pageq.next == 0);
4174 assert(mem->vmp_pageq.prev == 0);
4175 assert(mem->vmp_listq.next == 0);
4176 assert(mem->vmp_listq.prev == 0);
4177 assert(mem->vmp_specialq.next == 0);
4178 assert(mem->vmp_specialq.prev == 0);
4179 assert(mem->vmp_next_m == 0);
4180 ASSERT_PMAP_FREE(mem);
4181 {
4182 vm_page_init(mem, VM_PAGE_GET_PHYS_PAGE(mem), mem->vmp_lopage);
4183 }
4184 }
4185 }
4186
4187
4188 /*
4189 * vm_page_free:
4190 *
4191 * Returns the given page to the free list,
4192 * disassociating it with any VM object.
4193 *
4194 * Object and page queues must be locked prior to entry.
4195 */
4196 void
vm_page_free(vm_page_t mem)4197 vm_page_free(
4198 vm_page_t mem)
4199 {
4200 vm_page_free_prepare(mem);
4201
4202 if (mem->vmp_fictitious) {
4203 vm_page_release_fictitious(mem);
4204 } else {
4205 vm_page_release(mem, TRUE); /* page queues are locked */
4206 }
4207 }
4208
4209
4210 void
vm_page_free_unlocked(vm_page_t mem,boolean_t remove_from_hash)4211 vm_page_free_unlocked(
4212 vm_page_t mem,
4213 boolean_t remove_from_hash)
4214 {
4215 vm_page_lockspin_queues();
4216 vm_page_free_prepare_queues(mem);
4217 vm_page_unlock_queues();
4218
4219 vm_page_free_prepare_object(mem, remove_from_hash);
4220
4221 if (mem->vmp_fictitious) {
4222 vm_page_release_fictitious(mem);
4223 } else {
4224 vm_page_release(mem, FALSE); /* page queues are not locked */
4225 }
4226 }
4227
4228
4229 /*
4230 * Free a list of pages. The list can be up to several hundred pages,
4231 * as blocked up by vm_pageout_scan().
4232 * The big win is not having to take the free list lock once
4233 * per page.
4234 *
4235 * The VM page queues lock (vm_page_queue_lock) should NOT be held.
4236 * The VM page free queues lock (vm_page_queue_free_lock) should NOT be held.
4237 */
4238 void
vm_page_free_list(vm_page_t freeq,boolean_t prepare_object)4239 vm_page_free_list(
4240 vm_page_t freeq,
4241 boolean_t prepare_object)
4242 {
4243 vm_page_t mem;
4244 vm_page_t nxt;
4245 vm_page_t local_freeq;
4246 int pg_count;
4247
4248 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
4249 LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_NOTOWNED);
4250
4251 while (freeq) {
4252 pg_count = 0;
4253 local_freeq = VM_PAGE_NULL;
4254 mem = freeq;
4255
4256 /*
4257 * break up the processing into smaller chunks so
4258 * that we can 'pipeline' the pages onto the
4259 * free list w/o introducing too much
4260 * contention on the global free queue lock
4261 */
4262 while (mem && pg_count < 64) {
4263 assert((mem->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
4264 (mem->vmp_q_state == VM_PAGE_IS_WIRED));
4265 assert(mem->vmp_specialq.next == 0 &&
4266 mem->vmp_specialq.prev == 0);
4267 /*
4268 * &&
4269 * mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
4270 */
4271 nxt = mem->vmp_snext;
4272 mem->vmp_snext = NULL;
4273 assert(mem->vmp_pageq.prev == 0);
4274
4275 if (vm_page_free_verify && !mem->vmp_fictitious && !mem->vmp_private) {
4276 ASSERT_PMAP_FREE(mem);
4277 }
4278 if (prepare_object == TRUE) {
4279 vm_page_free_prepare_object(mem, TRUE);
4280 }
4281
4282 if (!mem->vmp_fictitious) {
4283 assert(mem->vmp_busy);
4284
4285 if ((mem->vmp_lopage == TRUE || vm_lopage_refill == TRUE) &&
4286 vm_lopage_free_count < vm_lopage_free_limit &&
4287 VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {
4288 vm_page_release(mem, FALSE); /* page queues are not locked */
4289 #if CONFIG_SECLUDED_MEMORY
4290 } else if (vm_page_secluded_count < vm_page_secluded_target &&
4291 num_tasks_can_use_secluded_mem == 0) {
4292 vm_page_release(mem,
4293 FALSE); /* page queues are not locked */
4294 #endif /* CONFIG_SECLUDED_MEMORY */
4295 } else {
4296 /*
4297 * IMPORTANT: we can't set the page "free" here
4298 * because that would make the page eligible for
4299 * a physically-contiguous allocation (see
4300 * vm_page_find_contiguous()) right away (we don't
4301 * hold the vm_page_queue_free lock). That would
4302 * cause trouble because the page is not actually
4303 * in the free queue yet...
4304 */
4305 mem->vmp_snext = local_freeq;
4306 local_freeq = mem;
4307 pg_count++;
4308
4309 pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
4310 }
4311 } else {
4312 assert(VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_fictitious_addr ||
4313 VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_guard_addr);
4314 vm_page_release_fictitious(mem);
4315 }
4316 mem = nxt;
4317 }
4318 freeq = mem;
4319
4320 if ((mem = local_freeq)) {
4321 unsigned int avail_free_count;
4322 unsigned int need_wakeup = 0;
4323 unsigned int need_priv_wakeup = 0;
4324 #if CONFIG_SECLUDED_MEMORY
4325 unsigned int need_wakeup_secluded = 0;
4326 #endif /* CONFIG_SECLUDED_MEMORY */
4327 event_t priv_wakeup_event, secluded_wakeup_event, normal_wakeup_event;
4328 boolean_t priv_wakeup_all, secluded_wakeup_all, normal_wakeup_all;
4329
4330 vm_free_page_lock_spin();
4331
4332 while (mem) {
4333 int color;
4334
4335 nxt = mem->vmp_snext;
4336
4337 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
4338 assert(mem->vmp_busy);
4339 mem->vmp_lopage = FALSE;
4340 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
4341
4342 color = VM_PAGE_GET_COLOR(mem);
4343 #if defined(__x86_64__)
4344 vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);
4345 #else
4346 vm_page_queue_enter(&vm_page_queue_free[color].qhead,
4347 mem, vmp_pageq);
4348 #endif
4349 mem = nxt;
4350 }
4351 vm_pageout_vminfo.vm_page_pages_freed += pg_count;
4352 vm_page_free_count += pg_count;
4353 avail_free_count = vm_page_free_count;
4354
4355 VM_DEBUG_CONSTANT_EVENT(vm_page_release, VM_PAGE_RELEASE, DBG_FUNC_NONE, pg_count, 0, 0, 0);
4356
4357 if (vm_page_free_wanted_privileged > 0 && avail_free_count > 0) {
4358 if (avail_free_count < vm_page_free_wanted_privileged) {
4359 need_priv_wakeup = avail_free_count;
4360 vm_page_free_wanted_privileged -= avail_free_count;
4361 avail_free_count = 0;
4362 } else {
4363 need_priv_wakeup = vm_page_free_wanted_privileged;
4364 avail_free_count -= vm_page_free_wanted_privileged;
4365 vm_page_free_wanted_privileged = 0;
4366 }
4367 }
4368 #if CONFIG_SECLUDED_MEMORY
4369 if (vm_page_free_wanted_secluded > 0 &&
4370 avail_free_count > vm_page_free_reserved) {
4371 unsigned int available_pages;
4372 available_pages = (avail_free_count -
4373 vm_page_free_reserved);
4374 if (available_pages <
4375 vm_page_free_wanted_secluded) {
4376 need_wakeup_secluded = available_pages;
4377 vm_page_free_wanted_secluded -=
4378 available_pages;
4379 avail_free_count -= available_pages;
4380 } else {
4381 need_wakeup_secluded =
4382 vm_page_free_wanted_secluded;
4383 avail_free_count -=
4384 vm_page_free_wanted_secluded;
4385 vm_page_free_wanted_secluded = 0;
4386 }
4387 }
4388 #endif /* CONFIG_SECLUDED_MEMORY */
4389 if (vm_page_free_wanted > 0 && avail_free_count > vm_page_free_reserved) {
4390 unsigned int available_pages;
4391
4392 available_pages = avail_free_count - vm_page_free_reserved;
4393
4394 if (available_pages >= vm_page_free_wanted) {
4395 need_wakeup = vm_page_free_wanted;
4396 vm_page_free_wanted = 0;
4397 } else {
4398 need_wakeup = available_pages;
4399 vm_page_free_wanted -= available_pages;
4400 }
4401 }
4402 vm_free_page_unlock();
4403
4404 priv_wakeup_event = NULL;
4405 secluded_wakeup_event = NULL;
4406 normal_wakeup_event = NULL;
4407
4408 priv_wakeup_all = FALSE;
4409 secluded_wakeup_all = FALSE;
4410 normal_wakeup_all = FALSE;
4411
4412
4413 if (need_priv_wakeup != 0) {
4414 /*
4415 * There shouldn't be that many VM-privileged threads,
4416 * so let's wake them all up, even if we don't quite
4417 * have enough pages to satisfy them all.
4418 */
4419 priv_wakeup_event = (event_t)&vm_page_free_wanted_privileged;
4420 priv_wakeup_all = TRUE;
4421 }
4422 #if CONFIG_SECLUDED_MEMORY
4423 if (need_wakeup_secluded != 0 &&
4424 vm_page_free_wanted_secluded == 0) {
4425 secluded_wakeup_event = (event_t)&vm_page_free_wanted_secluded;
4426 secluded_wakeup_all = TRUE;
4427 need_wakeup_secluded = 0;
4428 } else {
4429 secluded_wakeup_event = (event_t)&vm_page_free_wanted_secluded;
4430 }
4431 #endif /* CONFIG_SECLUDED_MEMORY */
4432 if (need_wakeup != 0 && vm_page_free_wanted == 0) {
4433 /*
4434 * We don't expect to have any more waiters
4435 * after this, so let's wake them all up at
4436 * once.
4437 */
4438 normal_wakeup_event = (event_t) &vm_page_free_count;
4439 normal_wakeup_all = TRUE;
4440 need_wakeup = 0;
4441 } else {
4442 normal_wakeup_event = (event_t) &vm_page_free_count;
4443 }
4444
4445 if (priv_wakeup_event ||
4446 #if CONFIG_SECLUDED_MEMORY
4447 secluded_wakeup_event ||
4448 #endif /* CONFIG_SECLUDED_MEMORY */
4449 normal_wakeup_event) {
4450 if (vps_dynamic_priority_enabled == TRUE) {
4451 thread_t thread_woken = NULL;
4452
4453 if (priv_wakeup_all == TRUE) {
4454 wakeup_all_with_inheritor(priv_wakeup_event, THREAD_AWAKENED);
4455 }
4456
4457 #if CONFIG_SECLUDED_MEMORY
4458 if (secluded_wakeup_all == TRUE) {
4459 wakeup_all_with_inheritor(secluded_wakeup_event, THREAD_AWAKENED);
4460 }
4461
4462 while (need_wakeup_secluded-- != 0) {
4463 /*
4464 * Wake up one waiter per page we just released.
4465 */
4466 wakeup_one_with_inheritor(secluded_wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
4467 thread_deallocate(thread_woken);
4468 }
4469 #endif /* CONFIG_SECLUDED_MEMORY */
4470
4471 if (normal_wakeup_all == TRUE) {
4472 wakeup_all_with_inheritor(normal_wakeup_event, THREAD_AWAKENED);
4473 }
4474
4475 while (need_wakeup-- != 0) {
4476 /*
4477 * Wake up one waiter per page we just released.
4478 */
4479 wakeup_one_with_inheritor(normal_wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
4480 thread_deallocate(thread_woken);
4481 }
4482 } else {
4483 /*
4484 * Non-priority-aware wakeups.
4485 */
4486
4487 if (priv_wakeup_all == TRUE) {
4488 thread_wakeup(priv_wakeup_event);
4489 }
4490
4491 #if CONFIG_SECLUDED_MEMORY
4492 if (secluded_wakeup_all == TRUE) {
4493 thread_wakeup(secluded_wakeup_event);
4494 }
4495
4496 while (need_wakeup_secluded-- != 0) {
4497 /*
4498 * Wake up one waiter per page we just released.
4499 */
4500 thread_wakeup_one(secluded_wakeup_event);
4501 }
4502
4503 #endif /* CONFIG_SECLUDED_MEMORY */
4504 if (normal_wakeup_all == TRUE) {
4505 thread_wakeup(normal_wakeup_event);
4506 }
4507
4508 while (need_wakeup-- != 0) {
4509 /*
4510 * Wake up one waiter per page we just released.
4511 */
4512 thread_wakeup_one(normal_wakeup_event);
4513 }
4514 }
4515 }
4516
4517 VM_CHECK_MEMORYSTATUS;
4518 }
4519 }
4520 }
4521
4522
4523 /*
4524 * vm_page_wire:
4525 *
4526 * Mark this page as wired down by yet
4527 * another map, removing it from paging queues
4528 * as necessary.
4529 *
4530 * The page's object and the page queues must be locked.
4531 */
4532
4533
4534 void
vm_page_wire(vm_page_t mem,vm_tag_t tag,boolean_t check_memorystatus)4535 vm_page_wire(
4536 vm_page_t mem,
4537 vm_tag_t tag,
4538 boolean_t check_memorystatus)
4539 {
4540 vm_object_t m_object;
4541
4542 m_object = VM_PAGE_OBJECT(mem);
4543
4544 // dbgLog(current_thread(), mem->vmp_offset, m_object, 1); /* (TEST/DEBUG) */
4545
4546 VM_PAGE_CHECK(mem);
4547 if (m_object) {
4548 vm_object_lock_assert_exclusive(m_object);
4549 } else {
4550 /*
4551 * In theory, the page should be in an object before it
4552 * gets wired, since we need to hold the object lock
4553 * to update some fields in the page structure.
4554 * However, some code (i386 pmap, for example) might want
4555 * to wire a page before it gets inserted into an object.
4556 * That's somewhat OK, as long as nobody else can get to
4557 * that page and update it at the same time.
4558 */
4559 }
4560 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4561 if (!VM_PAGE_WIRED(mem)) {
4562 if (mem->vmp_laundry) {
4563 vm_pageout_steal_laundry(mem, TRUE);
4564 }
4565
4566 vm_page_queues_remove(mem, TRUE);
4567
4568 assert(mem->vmp_wire_count == 0);
4569 mem->vmp_q_state = VM_PAGE_IS_WIRED;
4570
4571 if (m_object) {
4572 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
4573 VM_OBJECT_WIRED_PAGE_ADD(m_object, mem);
4574 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, tag);
4575
4576 assert(m_object->resident_page_count >=
4577 m_object->wired_page_count);
4578 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
4579 assert(vm_page_purgeable_count > 0);
4580 OSAddAtomic(-1, &vm_page_purgeable_count);
4581 OSAddAtomic(1, &vm_page_purgeable_wired_count);
4582 }
4583 if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
4584 m_object->purgable == VM_PURGABLE_EMPTY) &&
4585 m_object->vo_owner != TASK_NULL) {
4586 task_t owner;
4587 int ledger_idx_volatile;
4588 int ledger_idx_nonvolatile;
4589 int ledger_idx_volatile_compressed;
4590 int ledger_idx_nonvolatile_compressed;
4591 boolean_t do_footprint;
4592
4593 owner = VM_OBJECT_OWNER(m_object);
4594 vm_object_ledger_tag_ledgers(
4595 m_object,
4596 &ledger_idx_volatile,
4597 &ledger_idx_nonvolatile,
4598 &ledger_idx_volatile_compressed,
4599 &ledger_idx_nonvolatile_compressed,
4600 &do_footprint);
4601 /* less volatile bytes */
4602 ledger_debit(owner->ledger,
4603 ledger_idx_volatile,
4604 PAGE_SIZE);
4605 /* more not-quite-volatile bytes */
4606 ledger_credit(owner->ledger,
4607 ledger_idx_nonvolatile,
4608 PAGE_SIZE);
4609 if (do_footprint) {
4610 /* more footprint */
4611 ledger_credit(owner->ledger,
4612 task_ledgers.phys_footprint,
4613 PAGE_SIZE);
4614 }
4615 }
4616 if (m_object->all_reusable) {
4617 /*
4618 * Wired pages are not counted as "re-usable"
4619 * in "all_reusable" VM objects, so nothing
4620 * to do here.
4621 */
4622 } else if (mem->vmp_reusable) {
4623 /*
4624 * This page is not "re-usable" when it's
4625 * wired, so adjust its state and the
4626 * accounting.
4627 */
4628 vm_object_reuse_pages(m_object,
4629 mem->vmp_offset,
4630 mem->vmp_offset + PAGE_SIZE_64,
4631 FALSE);
4632 }
4633 }
4634 assert(!mem->vmp_reusable);
4635
4636 if (!mem->vmp_private && !mem->vmp_fictitious && !mem->vmp_gobbled) {
4637 vm_page_wire_count++;
4638 }
4639 if (mem->vmp_gobbled) {
4640 vm_page_gobble_count--;
4641 }
4642 mem->vmp_gobbled = FALSE;
4643
4644 if (check_memorystatus == TRUE) {
4645 VM_CHECK_MEMORYSTATUS;
4646 }
4647 }
4648 assert(!mem->vmp_gobbled);
4649 assert(mem->vmp_q_state == VM_PAGE_IS_WIRED);
4650 mem->vmp_wire_count++;
4651 if (__improbable(mem->vmp_wire_count == 0)) {
4652 panic("vm_page_wire(%p): wire_count overflow", mem);
4653 }
4654 VM_PAGE_CHECK(mem);
4655 }
4656
4657 /*
4658 * vm_page_unwire:
4659 *
4660 * Release one wiring of this page, potentially
4661 * enabling it to be paged again.
4662 *
4663 * The page's object and the page queues must be locked.
4664 */
4665 void
vm_page_unwire(vm_page_t mem,boolean_t queueit)4666 vm_page_unwire(
4667 vm_page_t mem,
4668 boolean_t queueit)
4669 {
4670 vm_object_t m_object;
4671
4672 m_object = VM_PAGE_OBJECT(mem);
4673
4674 // dbgLog(current_thread(), mem->vmp_offset, m_object, 0); /* (TEST/DEBUG) */
4675
4676 VM_PAGE_CHECK(mem);
4677 assert(VM_PAGE_WIRED(mem));
4678 assert(mem->vmp_wire_count > 0);
4679 assert(!mem->vmp_gobbled);
4680 assert(m_object != VM_OBJECT_NULL);
4681 vm_object_lock_assert_exclusive(m_object);
4682 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4683 if (--mem->vmp_wire_count == 0) {
4684 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
4685
4686 VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
4687 VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
4688 VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
4689 if (!mem->vmp_private && !mem->vmp_fictitious) {
4690 vm_page_wire_count--;
4691 }
4692
4693 assert(m_object->resident_page_count >=
4694 m_object->wired_page_count);
4695 if (m_object->purgable == VM_PURGABLE_VOLATILE) {
4696 OSAddAtomic(+1, &vm_page_purgeable_count);
4697 assert(vm_page_purgeable_wired_count > 0);
4698 OSAddAtomic(-1, &vm_page_purgeable_wired_count);
4699 }
4700 if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
4701 m_object->purgable == VM_PURGABLE_EMPTY) &&
4702 m_object->vo_owner != TASK_NULL) {
4703 task_t owner;
4704 int ledger_idx_volatile;
4705 int ledger_idx_nonvolatile;
4706 int ledger_idx_volatile_compressed;
4707 int ledger_idx_nonvolatile_compressed;
4708 boolean_t do_footprint;
4709
4710 owner = VM_OBJECT_OWNER(m_object);
4711 vm_object_ledger_tag_ledgers(
4712 m_object,
4713 &ledger_idx_volatile,
4714 &ledger_idx_nonvolatile,
4715 &ledger_idx_volatile_compressed,
4716 &ledger_idx_nonvolatile_compressed,
4717 &do_footprint);
4718 /* more volatile bytes */
4719 ledger_credit(owner->ledger,
4720 ledger_idx_volatile,
4721 PAGE_SIZE);
4722 /* less not-quite-volatile bytes */
4723 ledger_debit(owner->ledger,
4724 ledger_idx_nonvolatile,
4725 PAGE_SIZE);
4726 if (do_footprint) {
4727 /* less footprint */
4728 ledger_debit(owner->ledger,
4729 task_ledgers.phys_footprint,
4730 PAGE_SIZE);
4731 }
4732 }
4733 assert(m_object != kernel_object);
4734 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
4735
4736 if (queueit == TRUE) {
4737 if (m_object->purgable == VM_PURGABLE_EMPTY) {
4738 vm_page_deactivate(mem);
4739 } else {
4740 vm_page_activate(mem);
4741 }
4742 }
4743
4744 VM_CHECK_MEMORYSTATUS;
4745 }
4746 VM_PAGE_CHECK(mem);
4747 }
4748
4749 /*
4750 * vm_page_deactivate:
4751 *
4752 * Returns the given page to the inactive list,
4753 * indicating that no physical maps have access
4754 * to this page. [Used by the physical mapping system.]
4755 *
4756 * The page queues must be locked.
4757 */
4758 void
vm_page_deactivate(vm_page_t m)4759 vm_page_deactivate(
4760 vm_page_t m)
4761 {
4762 vm_page_deactivate_internal(m, TRUE);
4763 }
4764
4765
4766 void
vm_page_deactivate_internal(vm_page_t m,boolean_t clear_hw_reference)4767 vm_page_deactivate_internal(
4768 vm_page_t m,
4769 boolean_t clear_hw_reference)
4770 {
4771 vm_object_t m_object;
4772
4773 m_object = VM_PAGE_OBJECT(m);
4774
4775 VM_PAGE_CHECK(m);
4776 assert(m_object != kernel_object);
4777 assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
4778
4779 // dbgLog(VM_PAGE_GET_PHYS_PAGE(m), vm_page_free_count, vm_page_wire_count, 6); /* (TEST/DEBUG) */
4780 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4781 /*
4782 * This page is no longer very interesting. If it was
4783 * interesting (active or inactive/referenced), then we
4784 * clear the reference bit and (re)enter it in the
4785 * inactive queue. Note wired pages should not have
4786 * their reference bit cleared.
4787 */
4788 assert( !(m->vmp_absent && !m->vmp_unusual));
4789
4790 if (m->vmp_gobbled) { /* can this happen? */
4791 assert( !VM_PAGE_WIRED(m));
4792
4793 if (!m->vmp_private && !m->vmp_fictitious) {
4794 vm_page_wire_count--;
4795 }
4796 vm_page_gobble_count--;
4797 m->vmp_gobbled = FALSE;
4798 }
4799 /*
4800 * if this page is currently on the pageout queue, we can't do the
4801 * vm_page_queues_remove (which doesn't handle the pageout queue case)
4802 * and we can't remove it manually since we would need the object lock
4803 * (which is not required here) to decrement the activity_in_progress
4804 * reference which is held on the object while the page is in the pageout queue...
4805 * just let the normal laundry processing proceed
4806 */
4807 if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
4808 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
4809 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
4810 VM_PAGE_WIRED(m)) {
4811 return;
4812 }
4813 if (!m->vmp_absent && clear_hw_reference == TRUE) {
4814 pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
4815 }
4816
4817 m->vmp_reference = FALSE;
4818 m->vmp_no_cache = FALSE;
4819
4820 if (!VM_PAGE_INACTIVE(m)) {
4821 vm_page_queues_remove(m, FALSE);
4822
4823 if (!VM_DYNAMIC_PAGING_ENABLED() &&
4824 m->vmp_dirty && m_object->internal &&
4825 (m_object->purgable == VM_PURGABLE_DENY ||
4826 m_object->purgable == VM_PURGABLE_NONVOLATILE ||
4827 m_object->purgable == VM_PURGABLE_VOLATILE)) {
4828 vm_page_check_pageable_safe(m);
4829 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
4830 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
4831 vm_page_throttled_count++;
4832 } else {
4833 if (m_object->named && m_object->ref_count == 1) {
4834 vm_page_speculate(m, FALSE);
4835 #if DEVELOPMENT || DEBUG
4836 vm_page_speculative_recreated++;
4837 #endif
4838 } else {
4839 vm_page_enqueue_inactive(m, FALSE);
4840 }
4841 }
4842 }
4843 }
4844
4845 /*
4846 * vm_page_enqueue_cleaned
4847 *
4848 * Put the page on the cleaned queue, mark it cleaned, etc.
4849 * Being on the cleaned queue (and having m->clean_queue set)
4850 * does ** NOT ** guarantee that the page is clean!
4851 *
4852 * Call with the queues lock held.
4853 */
4854
4855 void
vm_page_enqueue_cleaned(vm_page_t m)4856 vm_page_enqueue_cleaned(vm_page_t m)
4857 {
4858 vm_object_t m_object;
4859
4860 m_object = VM_PAGE_OBJECT(m);
4861
4862 assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
4863 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4864 assert( !(m->vmp_absent && !m->vmp_unusual));
4865
4866 if (VM_PAGE_WIRED(m)) {
4867 return;
4868 }
4869
4870 if (m->vmp_gobbled) {
4871 if (!m->vmp_private && !m->vmp_fictitious) {
4872 vm_page_wire_count--;
4873 }
4874 vm_page_gobble_count--;
4875 m->vmp_gobbled = FALSE;
4876 }
4877 /*
4878 * if this page is currently on the pageout queue, we can't do the
4879 * vm_page_queues_remove (which doesn't handle the pageout queue case)
4880 * and we can't remove it manually since we would need the object lock
4881 * (which is not required here) to decrement the activity_in_progress
4882 * reference which is held on the object while the page is in the pageout queue...
4883 * just let the normal laundry processing proceed
4884 */
4885 if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
4886 (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
4887 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
4888 return;
4889 }
4890 vm_page_queues_remove(m, FALSE);
4891
4892 vm_page_check_pageable_safe(m);
4893 vm_page_queue_enter(&vm_page_queue_cleaned, m, vmp_pageq);
4894 m->vmp_q_state = VM_PAGE_ON_INACTIVE_CLEANED_Q;
4895 vm_page_cleaned_count++;
4896
4897 vm_page_inactive_count++;
4898 if (m_object->internal) {
4899 vm_page_pageable_internal_count++;
4900 } else {
4901 vm_page_pageable_external_count++;
4902 }
4903 vm_page_add_to_specialq(m, TRUE);
4904 VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
4905 }
4906
4907 /*
4908 * vm_page_activate:
4909 *
4910 * Put the specified page on the active list (if appropriate).
4911 *
4912 * The page queues must be locked.
4913 */
4914
4915 void
vm_page_activate(vm_page_t m)4916 vm_page_activate(
4917 vm_page_t m)
4918 {
4919 vm_object_t m_object;
4920
4921 m_object = VM_PAGE_OBJECT(m);
4922
4923 VM_PAGE_CHECK(m);
4924 #ifdef FIXME_4778297
4925 assert(m_object != kernel_object);
4926 #endif
4927 assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
4928 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
4929 assert( !(m->vmp_absent && !m->vmp_unusual));
4930
4931 if (m->vmp_gobbled) {
4932 assert( !VM_PAGE_WIRED(m));
4933 if (!m->vmp_private && !m->vmp_fictitious) {
4934 vm_page_wire_count--;
4935 }
4936 vm_page_gobble_count--;
4937 m->vmp_gobbled = FALSE;
4938 }
4939 /*
4940 * if this page is currently on the pageout queue, we can't do the
4941 * vm_page_queues_remove (which doesn't handle the pageout queue case)
4942 * and we can't remove it manually since we would need the object lock
4943 * (which is not required here) to decrement the activity_in_progress
4944 * reference which is held on the object while the page is in the pageout queue...
4945 * just let the normal laundry processing proceed
4946 */
4947 if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
4948 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
4949 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
4950 return;
4951 }
4952
4953 #if DEBUG
4954 if (m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q) {
4955 panic("vm_page_activate: already active");
4956 }
4957 #endif
4958
4959 if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
4960 DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
4961 DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL);
4962 }
4963
4964 /*
4965 * A freshly activated page should be promoted in the donation queue.
4966 * So we remove it here while preserving its hint and we will enqueue
4967 * it again in vm_page_enqueue_active.
4968 */
4969 vm_page_queues_remove(m, ((m->vmp_on_specialq == VM_PAGE_SPECIAL_Q_DONATE) ? TRUE : FALSE));
4970
4971 if (!VM_PAGE_WIRED(m)) {
4972 vm_page_check_pageable_safe(m);
4973 if (!VM_DYNAMIC_PAGING_ENABLED() &&
4974 m->vmp_dirty && m_object->internal &&
4975 (m_object->purgable == VM_PURGABLE_DENY ||
4976 m_object->purgable == VM_PURGABLE_NONVOLATILE ||
4977 m_object->purgable == VM_PURGABLE_VOLATILE)) {
4978 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
4979 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
4980 vm_page_throttled_count++;
4981 } else {
4982 #if CONFIG_SECLUDED_MEMORY
4983 if (secluded_for_filecache &&
4984 vm_page_secluded_target != 0 &&
4985 num_tasks_can_use_secluded_mem == 0 &&
4986 m_object->eligible_for_secluded) {
4987 vm_page_queue_enter(&vm_page_queue_secluded, m, vmp_pageq);
4988 m->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
4989 vm_page_secluded_count++;
4990 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
4991 vm_page_secluded_count_inuse++;
4992 assert(!m_object->internal);
4993 // vm_page_pageable_external_count++;
4994 } else
4995 #endif /* CONFIG_SECLUDED_MEMORY */
4996 vm_page_enqueue_active(m, FALSE);
4997 }
4998 m->vmp_reference = TRUE;
4999 m->vmp_no_cache = FALSE;
5000 }
5001 VM_PAGE_CHECK(m);
5002 }
5003
5004
5005 /*
5006 * vm_page_speculate:
5007 *
5008 * Put the specified page on the speculative list (if appropriate).
5009 *
5010 * The page queues must be locked.
5011 */
5012 void
vm_page_speculate(vm_page_t m,boolean_t new)5013 vm_page_speculate(
5014 vm_page_t m,
5015 boolean_t new)
5016 {
5017 struct vm_speculative_age_q *aq;
5018 vm_object_t m_object;
5019
5020 m_object = VM_PAGE_OBJECT(m);
5021
5022 VM_PAGE_CHECK(m);
5023 vm_page_check_pageable_safe(m);
5024
5025 assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
5026 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5027 assert( !(m->vmp_absent && !m->vmp_unusual));
5028 assert(m_object->internal == FALSE);
5029
5030 /*
5031 * if this page is currently on the pageout queue, we can't do the
5032 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5033 * and we can't remove it manually since we would need the object lock
5034 * (which is not required here) to decrement the activity_in_progress
5035 * reference which is held on the object while the page is in the pageout queue...
5036 * just let the normal laundry processing proceed
5037 */
5038 if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
5039 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
5040 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
5041 return;
5042 }
5043
5044 vm_page_queues_remove(m, FALSE);
5045
5046 if (!VM_PAGE_WIRED(m)) {
5047 mach_timespec_t ts;
5048 clock_sec_t sec;
5049 clock_nsec_t nsec;
5050
5051 clock_get_system_nanotime(&sec, &nsec);
5052 ts.tv_sec = (unsigned int) sec;
5053 ts.tv_nsec = nsec;
5054
5055 if (vm_page_speculative_count == 0) {
5056 speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5057 speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5058
5059 aq = &vm_page_queue_speculative[speculative_age_index];
5060
5061 /*
5062 * set the timer to begin a new group
5063 */
5064 aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
5065 aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
5066
5067 ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
5068 } else {
5069 aq = &vm_page_queue_speculative[speculative_age_index];
5070
5071 if (CMP_MACH_TIMESPEC(&ts, &aq->age_ts) >= 0) {
5072 speculative_age_index++;
5073
5074 if (speculative_age_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
5075 speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5076 }
5077 if (speculative_age_index == speculative_steal_index) {
5078 speculative_steal_index = speculative_age_index + 1;
5079
5080 if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
5081 speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
5082 }
5083 }
5084 aq = &vm_page_queue_speculative[speculative_age_index];
5085
5086 if (!vm_page_queue_empty(&aq->age_q)) {
5087 vm_page_speculate_ageit(aq);
5088 }
5089
5090 aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
5091 aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
5092
5093 ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
5094 }
5095 }
5096 vm_page_enqueue_tail(&aq->age_q, &m->vmp_pageq);
5097 m->vmp_q_state = VM_PAGE_ON_SPECULATIVE_Q;
5098 vm_page_speculative_count++;
5099 vm_page_pageable_external_count++;
5100
5101 if (new == TRUE) {
5102 vm_object_lock_assert_exclusive(m_object);
5103
5104 m_object->pages_created++;
5105 #if DEVELOPMENT || DEBUG
5106 vm_page_speculative_created++;
5107 #endif
5108 }
5109 }
5110 VM_PAGE_CHECK(m);
5111 }
5112
5113
5114 /*
5115 * move pages from the specified aging bin to
5116 * the speculative bin that pageout_scan claims from
5117 *
5118 * The page queues must be locked.
5119 */
5120 void
vm_page_speculate_ageit(struct vm_speculative_age_q * aq)5121 vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
5122 {
5123 struct vm_speculative_age_q *sq;
5124 vm_page_t t;
5125
5126 sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
5127
5128 if (vm_page_queue_empty(&sq->age_q)) {
5129 sq->age_q.next = aq->age_q.next;
5130 sq->age_q.prev = aq->age_q.prev;
5131
5132 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.next);
5133 t->vmp_pageq.prev = VM_PAGE_PACK_PTR(&sq->age_q);
5134
5135 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
5136 t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
5137 } else {
5138 t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
5139 t->vmp_pageq.next = aq->age_q.next;
5140
5141 t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.next);
5142 t->vmp_pageq.prev = sq->age_q.prev;
5143
5144 t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.prev);
5145 t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
5146
5147 sq->age_q.prev = aq->age_q.prev;
5148 }
5149 vm_page_queue_init(&aq->age_q);
5150 }
5151
5152
5153 void
vm_page_lru(vm_page_t m)5154 vm_page_lru(
5155 vm_page_t m)
5156 {
5157 VM_PAGE_CHECK(m);
5158 assert(VM_PAGE_OBJECT(m) != kernel_object);
5159 assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
5160
5161 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
5162
5163 if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q) {
5164 /*
5165 * we don't need to do all the other work that
5166 * vm_page_queues_remove and vm_page_enqueue_inactive
5167 * bring along for the ride
5168 */
5169 assert(!m->vmp_laundry);
5170 assert(!m->vmp_private);
5171
5172 m->vmp_no_cache = FALSE;
5173
5174 vm_page_queue_remove(&vm_page_queue_inactive, m, vmp_pageq);
5175 vm_page_queue_enter(&vm_page_queue_inactive, m, vmp_pageq);
5176
5177 return;
5178 }
5179 /*
5180 * if this page is currently on the pageout queue, we can't do the
5181 * vm_page_queues_remove (which doesn't handle the pageout queue case)
5182 * and we can't remove it manually since we would need the object lock
5183 * (which is not required here) to decrement the activity_in_progress
5184 * reference which is held on the object while the page is in the pageout queue...
5185 * just let the normal laundry processing proceed
5186 */
5187 if (m->vmp_laundry || m->vmp_private ||
5188 (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
5189 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
5190 VM_PAGE_WIRED(m)) {
5191 return;
5192 }
5193
5194 m->vmp_no_cache = FALSE;
5195
5196 vm_page_queues_remove(m, FALSE);
5197
5198 vm_page_enqueue_inactive(m, FALSE);
5199 }
5200
5201
5202 void
vm_page_reactivate_all_throttled(void)5203 vm_page_reactivate_all_throttled(void)
5204 {
5205 vm_page_t first_throttled, last_throttled;
5206 vm_page_t first_active;
5207 vm_page_t m;
5208 int extra_active_count;
5209 int extra_internal_count, extra_external_count;
5210 vm_object_t m_object;
5211
5212 if (!VM_DYNAMIC_PAGING_ENABLED()) {
5213 return;
5214 }
5215
5216 extra_active_count = 0;
5217 extra_internal_count = 0;
5218 extra_external_count = 0;
5219 vm_page_lock_queues();
5220 if (!vm_page_queue_empty(&vm_page_queue_throttled)) {
5221 /*
5222 * Switch "throttled" pages to "active".
5223 */
5224 vm_page_queue_iterate(&vm_page_queue_throttled, m, vmp_pageq) {
5225 VM_PAGE_CHECK(m);
5226 assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
5227
5228 m_object = VM_PAGE_OBJECT(m);
5229
5230 extra_active_count++;
5231 if (m_object->internal) {
5232 extra_internal_count++;
5233 } else {
5234 extra_external_count++;
5235 }
5236
5237 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
5238 VM_PAGE_CHECK(m);
5239 vm_page_add_to_specialq(m, FALSE);
5240 }
5241
5242 /*
5243 * Transfer the entire throttled queue to a regular LRU page queues.
5244 * We insert it at the head of the active queue, so that these pages
5245 * get re-evaluated by the LRU algorithm first, since they've been
5246 * completely out of it until now.
5247 */
5248 first_throttled = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
5249 last_throttled = (vm_page_t) vm_page_queue_last(&vm_page_queue_throttled);
5250 first_active = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
5251 if (vm_page_queue_empty(&vm_page_queue_active)) {
5252 vm_page_queue_active.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_throttled);
5253 } else {
5254 first_active->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_throttled);
5255 }
5256 vm_page_queue_active.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_throttled);
5257 first_throttled->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(&vm_page_queue_active);
5258 last_throttled->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_active);
5259
5260 #if DEBUG
5261 printf("reactivated %d throttled pages\n", vm_page_throttled_count);
5262 #endif
5263 vm_page_queue_init(&vm_page_queue_throttled);
5264 /*
5265 * Adjust the global page counts.
5266 */
5267 vm_page_active_count += extra_active_count;
5268 vm_page_pageable_internal_count += extra_internal_count;
5269 vm_page_pageable_external_count += extra_external_count;
5270 vm_page_throttled_count = 0;
5271 }
5272 assert(vm_page_throttled_count == 0);
5273 assert(vm_page_queue_empty(&vm_page_queue_throttled));
5274 vm_page_unlock_queues();
5275 }
5276
5277
5278 /*
5279 * move pages from the indicated local queue to the global active queue
5280 * its ok to fail if we're below the hard limit and force == FALSE
5281 * the nolocks == TRUE case is to allow this function to be run on
5282 * the hibernate path
5283 */
5284
5285 void
vm_page_reactivate_local(uint32_t lid,boolean_t force,boolean_t nolocks)5286 vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
5287 {
5288 struct vpl *lq;
5289 vm_page_t first_local, last_local;
5290 vm_page_t first_active;
5291 vm_page_t m;
5292 uint32_t count = 0;
5293
5294 if (vm_page_local_q == NULL) {
5295 return;
5296 }
5297
5298 lq = zpercpu_get_cpu(vm_page_local_q, lid);
5299
5300 if (nolocks == FALSE) {
5301 if (lq->vpl_count < vm_page_local_q_hard_limit && force == FALSE) {
5302 if (!vm_page_trylockspin_queues()) {
5303 return;
5304 }
5305 } else {
5306 vm_page_lockspin_queues();
5307 }
5308
5309 VPL_LOCK(&lq->vpl_lock);
5310 }
5311 if (lq->vpl_count) {
5312 /*
5313 * Switch "local" pages to "active".
5314 */
5315 assert(!vm_page_queue_empty(&lq->vpl_queue));
5316
5317 vm_page_queue_iterate(&lq->vpl_queue, m, vmp_pageq) {
5318 VM_PAGE_CHECK(m);
5319 vm_page_check_pageable_safe(m);
5320 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q);
5321 assert(!m->vmp_fictitious);
5322
5323 if (m->vmp_local_id != lid) {
5324 panic("vm_page_reactivate_local: found vm_page_t(%p) with wrong cpuid", m);
5325 }
5326
5327 m->vmp_local_id = 0;
5328 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
5329 VM_PAGE_CHECK(m);
5330 vm_page_add_to_specialq(m, FALSE);
5331 count++;
5332 }
5333 if (count != lq->vpl_count) {
5334 panic("vm_page_reactivate_local: count = %d, vm_page_local_count = %d", count, lq->vpl_count);
5335 }
5336
5337 /*
5338 * Transfer the entire local queue to a regular LRU page queues.
5339 */
5340 first_local = (vm_page_t) vm_page_queue_first(&lq->vpl_queue);
5341 last_local = (vm_page_t) vm_page_queue_last(&lq->vpl_queue);
5342 first_active = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
5343
5344 if (vm_page_queue_empty(&vm_page_queue_active)) {
5345 vm_page_queue_active.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
5346 } else {
5347 first_active->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
5348 }
5349 vm_page_queue_active.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
5350 first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(&vm_page_queue_active);
5351 last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_active);
5352
5353 vm_page_queue_init(&lq->vpl_queue);
5354 /*
5355 * Adjust the global page counts.
5356 */
5357 vm_page_active_count += lq->vpl_count;
5358 vm_page_pageable_internal_count += lq->vpl_internal_count;
5359 vm_page_pageable_external_count += lq->vpl_external_count;
5360 lq->vpl_count = 0;
5361 lq->vpl_internal_count = 0;
5362 lq->vpl_external_count = 0;
5363 }
5364 assert(vm_page_queue_empty(&lq->vpl_queue));
5365
5366 if (nolocks == FALSE) {
5367 VPL_UNLOCK(&lq->vpl_lock);
5368
5369 vm_page_balance_inactive(count / 4);
5370 vm_page_unlock_queues();
5371 }
5372 }
5373
5374 /*
5375 * vm_page_part_zero_fill:
5376 *
5377 * Zero-fill a part of the page.
5378 */
5379 #define PMAP_ZERO_PART_PAGE_IMPLEMENTED
5380 void
vm_page_part_zero_fill(vm_page_t m,vm_offset_t m_pa,vm_size_t len)5381 vm_page_part_zero_fill(
5382 vm_page_t m,
5383 vm_offset_t m_pa,
5384 vm_size_t len)
5385 {
5386 #if 0
5387 /*
5388 * we don't hold the page queue lock
5389 * so this check isn't safe to make
5390 */
5391 VM_PAGE_CHECK(m);
5392 #endif
5393
5394 #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED
5395 pmap_zero_part_page(VM_PAGE_GET_PHYS_PAGE(m), m_pa, len);
5396 #else
5397 vm_page_t tmp;
5398 while (1) {
5399 tmp = vm_page_grab();
5400 if (tmp == VM_PAGE_NULL) {
5401 vm_page_wait(THREAD_UNINT);
5402 continue;
5403 }
5404 break;
5405 }
5406 vm_page_zero_fill(tmp);
5407 if (m_pa != 0) {
5408 vm_page_part_copy(m, 0, tmp, 0, m_pa);
5409 }
5410 if ((m_pa + len) < PAGE_SIZE) {
5411 vm_page_part_copy(m, m_pa + len, tmp,
5412 m_pa + len, PAGE_SIZE - (m_pa + len));
5413 }
5414 vm_page_copy(tmp, m);
5415 VM_PAGE_FREE(tmp);
5416 #endif
5417 }
5418
5419 /*
5420 * vm_page_zero_fill:
5421 *
5422 * Zero-fill the specified page.
5423 */
5424 void
vm_page_zero_fill(vm_page_t m)5425 vm_page_zero_fill(
5426 vm_page_t m)
5427 {
5428 #if 0
5429 /*
5430 * we don't hold the page queue lock
5431 * so this check isn't safe to make
5432 */
5433 VM_PAGE_CHECK(m);
5434 #endif
5435
5436 // dbgTrace(0xAEAEAEAE, VM_PAGE_GET_PHYS_PAGE(m), 0); /* (BRINGUP) */
5437 pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(m));
5438 }
5439
5440 /*
5441 * vm_page_part_copy:
5442 *
5443 * copy part of one page to another
5444 */
5445
5446 void
vm_page_part_copy(vm_page_t src_m,vm_offset_t src_pa,vm_page_t dst_m,vm_offset_t dst_pa,vm_size_t len)5447 vm_page_part_copy(
5448 vm_page_t src_m,
5449 vm_offset_t src_pa,
5450 vm_page_t dst_m,
5451 vm_offset_t dst_pa,
5452 vm_size_t len)
5453 {
5454 #if 0
5455 /*
5456 * we don't hold the page queue lock
5457 * so this check isn't safe to make
5458 */
5459 VM_PAGE_CHECK(src_m);
5460 VM_PAGE_CHECK(dst_m);
5461 #endif
5462 pmap_copy_part_page(VM_PAGE_GET_PHYS_PAGE(src_m), src_pa,
5463 VM_PAGE_GET_PHYS_PAGE(dst_m), dst_pa, len);
5464 }
5465
5466 /*
5467 * vm_page_copy:
5468 *
5469 * Copy one page to another
5470 */
5471
5472 int vm_page_copy_cs_validations = 0;
5473 int vm_page_copy_cs_tainted = 0;
5474
5475 void
vm_page_copy(vm_page_t src_m,vm_page_t dest_m)5476 vm_page_copy(
5477 vm_page_t src_m,
5478 vm_page_t dest_m)
5479 {
5480 vm_object_t src_m_object;
5481
5482 src_m_object = VM_PAGE_OBJECT(src_m);
5483
5484 #if 0
5485 /*
5486 * we don't hold the page queue lock
5487 * so this check isn't safe to make
5488 */
5489 VM_PAGE_CHECK(src_m);
5490 VM_PAGE_CHECK(dest_m);
5491 #endif
5492 vm_object_lock_assert_held(src_m_object);
5493
5494 if (src_m_object != VM_OBJECT_NULL &&
5495 src_m_object->code_signed) {
5496 /*
5497 * We're copying a page from a code-signed object.
5498 * Whoever ends up mapping the copy page might care about
5499 * the original page's integrity, so let's validate the
5500 * source page now.
5501 */
5502 vm_page_copy_cs_validations++;
5503 vm_page_validate_cs(src_m, PAGE_SIZE, 0);
5504 #if DEVELOPMENT || DEBUG
5505 DTRACE_VM4(codesigned_copy,
5506 vm_object_t, src_m_object,
5507 vm_object_offset_t, src_m->vmp_offset,
5508 int, src_m->vmp_cs_validated,
5509 int, src_m->vmp_cs_tainted);
5510 #endif /* DEVELOPMENT || DEBUG */
5511 }
5512
5513 /*
5514 * Propagate the cs_tainted bit to the copy page. Do not propagate
5515 * the cs_validated bit.
5516 */
5517 dest_m->vmp_cs_tainted = src_m->vmp_cs_tainted;
5518 dest_m->vmp_cs_nx = src_m->vmp_cs_nx;
5519 if (dest_m->vmp_cs_tainted) {
5520 vm_page_copy_cs_tainted++;
5521 }
5522 dest_m->vmp_error = VMP_ERROR_GET(src_m); /* sliding src_m might have failed... */
5523 pmap_copy_page(VM_PAGE_GET_PHYS_PAGE(src_m), VM_PAGE_GET_PHYS_PAGE(dest_m));
5524 }
5525
5526 #if MACH_ASSERT
5527 static void
_vm_page_print(vm_page_t p)5528 _vm_page_print(
5529 vm_page_t p)
5530 {
5531 printf("vm_page %p: \n", p);
5532 printf(" pageq: next=%p prev=%p\n",
5533 (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.next),
5534 (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev));
5535 printf(" listq: next=%p prev=%p\n",
5536 (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.next)),
5537 (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.prev)));
5538 printf(" next=%p\n", (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m)));
5539 printf(" object=%p offset=0x%llx\n", VM_PAGE_OBJECT(p), p->vmp_offset);
5540 printf(" wire_count=%u\n", p->vmp_wire_count);
5541 printf(" q_state=%u\n", p->vmp_q_state);
5542
5543 printf(" %slaundry, %sref, %sgobbled, %sprivate\n",
5544 (p->vmp_laundry ? "" : "!"),
5545 (p->vmp_reference ? "" : "!"),
5546 (p->vmp_gobbled ? "" : "!"),
5547 (p->vmp_private ? "" : "!"));
5548 printf(" %sbusy, %swanted, %stabled, %sfictitious, %spmapped, %swpmapped\n",
5549 (p->vmp_busy ? "" : "!"),
5550 (p->vmp_wanted ? "" : "!"),
5551 (p->vmp_tabled ? "" : "!"),
5552 (p->vmp_fictitious ? "" : "!"),
5553 (p->vmp_pmapped ? "" : "!"),
5554 (p->vmp_wpmapped ? "" : "!"));
5555 printf(" %sfree_when_done, %sabsent, %serror, %sdirty, %scleaning, %sprecious, %sclustered\n",
5556 (p->vmp_free_when_done ? "" : "!"),
5557 (p->vmp_absent ? "" : "!"),
5558 (VMP_ERROR_GET(p) ? "" : "!"),
5559 (p->vmp_dirty ? "" : "!"),
5560 (p->vmp_cleaning ? "" : "!"),
5561 (p->vmp_precious ? "" : "!"),
5562 (p->vmp_clustered ? "" : "!"));
5563 printf(" %soverwriting, %srestart, %sunusual\n",
5564 (p->vmp_overwriting ? "" : "!"),
5565 (p->vmp_restart ? "" : "!"),
5566 (p->vmp_unusual ? "" : "!"));
5567 printf(" cs_validated=%d, cs_tainted=%d, cs_nx=%d, %sno_cache\n",
5568 p->vmp_cs_validated,
5569 p->vmp_cs_tainted,
5570 p->vmp_cs_nx,
5571 (p->vmp_no_cache ? "" : "!"));
5572
5573 printf("phys_page=0x%x\n", VM_PAGE_GET_PHYS_PAGE(p));
5574 }
5575
5576 /*
5577 * Check that the list of pages is ordered by
5578 * ascending physical address and has no holes.
5579 */
5580 static int
vm_page_verify_contiguous(vm_page_t pages,unsigned int npages)5581 vm_page_verify_contiguous(
5582 vm_page_t pages,
5583 unsigned int npages)
5584 {
5585 vm_page_t m;
5586 unsigned int page_count;
5587 vm_offset_t prev_addr;
5588
5589 prev_addr = VM_PAGE_GET_PHYS_PAGE(pages);
5590 page_count = 1;
5591 for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
5592 if (VM_PAGE_GET_PHYS_PAGE(m) != prev_addr + 1) {
5593 printf("m %p prev_addr 0x%lx, current addr 0x%x\n",
5594 m, (long)prev_addr, VM_PAGE_GET_PHYS_PAGE(m));
5595 printf("pages %p page_count %d npages %d\n", pages, page_count, npages);
5596 panic("vm_page_verify_contiguous: not contiguous!");
5597 }
5598 prev_addr = VM_PAGE_GET_PHYS_PAGE(m);
5599 ++page_count;
5600 }
5601 if (page_count != npages) {
5602 printf("pages %p actual count 0x%x but requested 0x%x\n",
5603 pages, page_count, npages);
5604 panic("vm_page_verify_contiguous: count error");
5605 }
5606 return 1;
5607 }
5608
5609
5610 /*
5611 * Check the free lists for proper length etc.
5612 */
5613 static boolean_t vm_page_verify_this_free_list_enabled = FALSE;
5614 static unsigned int
vm_page_verify_free_list(vm_page_queue_head_t * vm_page_queue,unsigned int color,vm_page_t look_for_page,boolean_t expect_page)5615 vm_page_verify_free_list(
5616 vm_page_queue_head_t *vm_page_queue,
5617 unsigned int color,
5618 vm_page_t look_for_page,
5619 boolean_t expect_page)
5620 {
5621 unsigned int npages;
5622 vm_page_t m;
5623 vm_page_t prev_m;
5624 boolean_t found_page;
5625
5626 if (!vm_page_verify_this_free_list_enabled) {
5627 return 0;
5628 }
5629
5630 found_page = FALSE;
5631 npages = 0;
5632 prev_m = (vm_page_t)((uintptr_t)vm_page_queue);
5633
5634 vm_page_queue_iterate(vm_page_queue, m, vmp_pageq) {
5635 if (m == look_for_page) {
5636 found_page = TRUE;
5637 }
5638 if ((vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev) != prev_m) {
5639 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p corrupted prev ptr %p instead of %p",
5640 color, npages, m, (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev), prev_m);
5641 }
5642 if (!m->vmp_busy) {
5643 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p not busy",
5644 color, npages, m);
5645 }
5646 if (color != (unsigned int) -1) {
5647 if (VM_PAGE_GET_COLOR(m) != color) {
5648 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p wrong color %u instead of %u",
5649 color, npages, m, VM_PAGE_GET_COLOR(m), color);
5650 }
5651 if (m->vmp_q_state != VM_PAGE_ON_FREE_Q) {
5652 panic("vm_page_verify_free_list(color=%u, npages=%u): page %p - expecting q_state == VM_PAGE_ON_FREE_Q, found %d",
5653 color, npages, m, m->vmp_q_state);
5654 }
5655 } else {
5656 if (m->vmp_q_state != VM_PAGE_ON_FREE_LOCAL_Q) {
5657 panic("vm_page_verify_free_list(npages=%u): local page %p - expecting q_state == VM_PAGE_ON_FREE_LOCAL_Q, found %d",
5658 npages, m, m->vmp_q_state);
5659 }
5660 }
5661 ++npages;
5662 prev_m = m;
5663 }
5664 if (look_for_page != VM_PAGE_NULL) {
5665 unsigned int other_color;
5666
5667 if (expect_page && !found_page) {
5668 printf("vm_page_verify_free_list(color=%u, npages=%u): page %p not found phys=%u\n",
5669 color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
5670 _vm_page_print(look_for_page);
5671 for (other_color = 0;
5672 other_color < vm_colors;
5673 other_color++) {
5674 if (other_color == color) {
5675 continue;
5676 }
5677 vm_page_verify_free_list(&vm_page_queue_free[other_color].qhead,
5678 other_color, look_for_page, FALSE);
5679 }
5680 if (color == (unsigned int) -1) {
5681 vm_page_verify_free_list(&vm_lopage_queue_free,
5682 (unsigned int) -1, look_for_page, FALSE);
5683 }
5684 panic("vm_page_verify_free_list(color=%u)", color);
5685 }
5686 if (!expect_page && found_page) {
5687 printf("vm_page_verify_free_list(color=%u, npages=%u): page %p found phys=%u\n",
5688 color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
5689 }
5690 }
5691 return npages;
5692 }
5693
5694 static boolean_t vm_page_verify_all_free_lists_enabled = FALSE;
5695 static void
vm_page_verify_free_lists(void)5696 vm_page_verify_free_lists( void )
5697 {
5698 unsigned int color, npages, nlopages;
5699 boolean_t toggle = TRUE;
5700
5701 if (!vm_page_verify_all_free_lists_enabled) {
5702 return;
5703 }
5704
5705 npages = 0;
5706
5707 vm_free_page_lock();
5708
5709 if (vm_page_verify_this_free_list_enabled == TRUE) {
5710 /*
5711 * This variable has been set globally for extra checking of
5712 * each free list Q. Since we didn't set it, we don't own it
5713 * and we shouldn't toggle it.
5714 */
5715 toggle = FALSE;
5716 }
5717
5718 if (toggle == TRUE) {
5719 vm_page_verify_this_free_list_enabled = TRUE;
5720 }
5721
5722 for (color = 0; color < vm_colors; color++) {
5723 npages += vm_page_verify_free_list(&vm_page_queue_free[color].qhead,
5724 color, VM_PAGE_NULL, FALSE);
5725 }
5726 nlopages = vm_page_verify_free_list(&vm_lopage_queue_free,
5727 (unsigned int) -1,
5728 VM_PAGE_NULL, FALSE);
5729 if (npages != vm_page_free_count || nlopages != vm_lopage_free_count) {
5730 panic("vm_page_verify_free_lists: "
5731 "npages %u free_count %d nlopages %u lo_free_count %u",
5732 npages, vm_page_free_count, nlopages, vm_lopage_free_count);
5733 }
5734
5735 if (toggle == TRUE) {
5736 vm_page_verify_this_free_list_enabled = FALSE;
5737 }
5738
5739 vm_free_page_unlock();
5740 }
5741
5742 #endif /* MACH_ASSERT */
5743
5744
5745 extern boolean_t(*volatile consider_buffer_cache_collect)(int);
5746
5747 /*
5748 * CONTIGUOUS PAGE ALLOCATION
5749 *
5750 * Find a region large enough to contain at least n pages
5751 * of contiguous physical memory.
5752 *
5753 * This is done by traversing the vm_page_t array in a linear fashion
5754 * we assume that the vm_page_t array has the avaiable physical pages in an
5755 * ordered, ascending list... this is currently true of all our implementations
5756 * and must remain so... there can be 'holes' in the array... we also can
5757 * no longer tolerate the vm_page_t's in the list being 'freed' and reclaimed
5758 * which use to happen via 'vm_page_convert'... that function was no longer
5759 * being called and was removed...
5760 *
5761 * The basic flow consists of stabilizing some of the interesting state of
5762 * a vm_page_t behind the vm_page_queue and vm_page_free locks... we start our
5763 * sweep at the beginning of the array looking for pages that meet our criterea
5764 * for a 'stealable' page... currently we are pretty conservative... if the page
5765 * meets this criterea and is physically contiguous to the previous page in the 'run'
5766 * we keep developing it. If we hit a page that doesn't fit, we reset our state
5767 * and start to develop a new run... if at this point we've already considered
5768 * at least MAX_CONSIDERED_BEFORE_YIELD pages, we'll drop the 2 locks we hold,
5769 * and mutex_pause (which will yield the processor), to keep the latency low w/r
5770 * to other threads trying to acquire free pages (or move pages from q to q),
5771 * and then continue from the spot we left off... we only make 1 pass through the
5772 * array. Once we have a 'run' that is long enough, we'll go into the loop which
5773 * which steals the pages from the queues they're currently on... pages on the free
5774 * queue can be stolen directly... pages that are on any of the other queues
5775 * must be removed from the object they are tabled on... this requires taking the
5776 * object lock... we do this as a 'try' to prevent deadlocks... if the 'try' fails
5777 * or if the state of the page behind the vm_object lock is no longer viable, we'll
5778 * dump the pages we've currently stolen back to the free list, and pick up our
5779 * scan from the point where we aborted the 'current' run.
5780 *
5781 *
5782 * Requirements:
5783 * - neither vm_page_queue nor vm_free_list lock can be held on entry
5784 *
5785 * Returns a pointer to a list of gobbled/wired pages or VM_PAGE_NULL.
5786 *
5787 * Algorithm:
5788 */
5789
5790 #define MAX_CONSIDERED_BEFORE_YIELD 1000
5791
5792
5793 #define RESET_STATE_OF_RUN() \
5794 MACRO_BEGIN \
5795 prevcontaddr = -2; \
5796 start_pnum = -1; \
5797 free_considered = 0; \
5798 substitute_needed = 0; \
5799 npages = 0; \
5800 MACRO_END
5801
5802 /*
5803 * Can we steal in-use (i.e. not free) pages when searching for
5804 * physically-contiguous pages ?
5805 */
5806 #define VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL 1
5807
5808 static unsigned int vm_page_find_contiguous_last_idx = 0, vm_page_lomem_find_contiguous_last_idx = 0;
5809 #if DEBUG
5810 int vm_page_find_contig_debug = 0;
5811 #endif
5812
5813 static vm_page_t
vm_page_find_contiguous(unsigned int contig_pages,ppnum_t max_pnum,ppnum_t pnum_mask,boolean_t wire,int flags)5814 vm_page_find_contiguous(
5815 unsigned int contig_pages,
5816 ppnum_t max_pnum,
5817 ppnum_t pnum_mask,
5818 boolean_t wire,
5819 int flags)
5820 {
5821 vm_page_t m = NULL;
5822 ppnum_t prevcontaddr = 0;
5823 ppnum_t start_pnum = 0;
5824 unsigned int npages = 0, considered = 0, scanned = 0;
5825 unsigned int page_idx = 0, start_idx = 0, last_idx = 0, orig_last_idx = 0;
5826 unsigned int idx_last_contig_page_found = 0;
5827 int free_considered = 0, free_available = 0;
5828 int substitute_needed = 0;
5829 int zone_gc_called = 0;
5830 boolean_t wrapped;
5831 kern_return_t kr;
5832 #if DEBUG
5833 clock_sec_t tv_start_sec = 0, tv_end_sec = 0;
5834 clock_usec_t tv_start_usec = 0, tv_end_usec = 0;
5835 #endif
5836
5837 int yielded = 0;
5838 int dumped_run = 0;
5839 int stolen_pages = 0;
5840 int compressed_pages = 0;
5841
5842
5843 if (contig_pages == 0) {
5844 return VM_PAGE_NULL;
5845 }
5846
5847 full_scan_again:
5848
5849 #if MACH_ASSERT
5850 vm_page_verify_free_lists();
5851 #endif
5852 #if DEBUG
5853 clock_get_system_microtime(&tv_start_sec, &tv_start_usec);
5854 #endif
5855 PAGE_REPLACEMENT_ALLOWED(TRUE);
5856
5857 /*
5858 * If there are still delayed pages, try to free up some that match.
5859 */
5860 if (__improbable(vm_delayed_count != 0 && contig_pages != 0)) {
5861 vm_free_delayed_pages_contig(contig_pages, max_pnum, pnum_mask);
5862 }
5863
5864 vm_page_lock_queues();
5865 vm_free_page_lock();
5866
5867 RESET_STATE_OF_RUN();
5868
5869 scanned = 0;
5870 considered = 0;
5871 free_available = vm_page_free_count - vm_page_free_reserved;
5872
5873 wrapped = FALSE;
5874
5875 if (flags & KMA_LOMEM) {
5876 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx;
5877 } else {
5878 idx_last_contig_page_found = vm_page_find_contiguous_last_idx;
5879 }
5880
5881 orig_last_idx = idx_last_contig_page_found;
5882 last_idx = orig_last_idx;
5883
5884 for (page_idx = last_idx, start_idx = last_idx;
5885 npages < contig_pages && page_idx < vm_pages_count;
5886 page_idx++) {
5887 retry:
5888 if (wrapped &&
5889 npages == 0 &&
5890 page_idx >= orig_last_idx) {
5891 /*
5892 * We're back where we started and we haven't
5893 * found any suitable contiguous range. Let's
5894 * give up.
5895 */
5896 break;
5897 }
5898 scanned++;
5899 m = &vm_pages[page_idx];
5900
5901 assert(!m->vmp_fictitious);
5902 assert(!m->vmp_private);
5903
5904 if (max_pnum && VM_PAGE_GET_PHYS_PAGE(m) > max_pnum) {
5905 /* no more low pages... */
5906 break;
5907 }
5908 if (!npages & ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0)) {
5909 /*
5910 * not aligned
5911 */
5912 RESET_STATE_OF_RUN();
5913 } else if (VM_PAGE_WIRED(m) || m->vmp_gobbled ||
5914 m->vmp_laundry || m->vmp_wanted ||
5915 m->vmp_cleaning || m->vmp_overwriting || m->vmp_free_when_done) {
5916 /*
5917 * page is in a transient state
5918 * or a state we don't want to deal
5919 * with, so don't consider it which
5920 * means starting a new run
5921 */
5922 RESET_STATE_OF_RUN();
5923 } else if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
5924 (m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q) ||
5925 (m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q) ||
5926 (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
5927 /*
5928 * page needs to be on one of our queues (other then the pageout or special free queues)
5929 * or it needs to belong to the compressor pool (which is now indicated
5930 * by vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR and falls out
5931 * from the check for VM_PAGE_NOT_ON_Q)
5932 * in order for it to be stable behind the
5933 * locks we hold at this point...
5934 * if not, don't consider it which
5935 * means starting a new run
5936 */
5937 RESET_STATE_OF_RUN();
5938 } else if ((m->vmp_q_state != VM_PAGE_ON_FREE_Q) && (!m->vmp_tabled || m->vmp_busy)) {
5939 /*
5940 * pages on the free list are always 'busy'
5941 * so we couldn't test for 'busy' in the check
5942 * for the transient states... pages that are
5943 * 'free' are never 'tabled', so we also couldn't
5944 * test for 'tabled'. So we check here to make
5945 * sure that a non-free page is not busy and is
5946 * tabled on an object...
5947 * if not, don't consider it which
5948 * means starting a new run
5949 */
5950 RESET_STATE_OF_RUN();
5951 } else {
5952 if (VM_PAGE_GET_PHYS_PAGE(m) != prevcontaddr + 1) {
5953 if ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0) {
5954 RESET_STATE_OF_RUN();
5955 goto did_consider;
5956 } else {
5957 npages = 1;
5958 start_idx = page_idx;
5959 start_pnum = VM_PAGE_GET_PHYS_PAGE(m);
5960 }
5961 } else {
5962 npages++;
5963 }
5964 prevcontaddr = VM_PAGE_GET_PHYS_PAGE(m);
5965
5966 VM_PAGE_CHECK(m);
5967 if (m->vmp_q_state == VM_PAGE_ON_FREE_Q) {
5968 free_considered++;
5969 } else {
5970 /*
5971 * This page is not free.
5972 * If we can't steal used pages,
5973 * we have to give up this run
5974 * and keep looking.
5975 * Otherwise, we might need to
5976 * move the contents of this page
5977 * into a substitute page.
5978 */
5979 #if VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL
5980 if (m->vmp_pmapped || m->vmp_dirty || m->vmp_precious) {
5981 substitute_needed++;
5982 }
5983 #else
5984 RESET_STATE_OF_RUN();
5985 #endif
5986 }
5987
5988 if ((free_considered + substitute_needed) > free_available) {
5989 /*
5990 * if we let this run continue
5991 * we will end up dropping the vm_page_free_count
5992 * below the reserve limit... we need to abort
5993 * this run, but we can at least re-consider this
5994 * page... thus the jump back to 'retry'
5995 */
5996 RESET_STATE_OF_RUN();
5997
5998 if (free_available && considered <= MAX_CONSIDERED_BEFORE_YIELD) {
5999 considered++;
6000 goto retry;
6001 }
6002 /*
6003 * free_available == 0
6004 * so can't consider any free pages... if
6005 * we went to retry in this case, we'd
6006 * get stuck looking at the same page
6007 * w/o making any forward progress
6008 * we also want to take this path if we've already
6009 * reached our limit that controls the lock latency
6010 */
6011 }
6012 }
6013 did_consider:
6014 if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) {
6015 PAGE_REPLACEMENT_ALLOWED(FALSE);
6016
6017 vm_free_page_unlock();
6018 vm_page_unlock_queues();
6019
6020 mutex_pause(0);
6021
6022 PAGE_REPLACEMENT_ALLOWED(TRUE);
6023
6024 vm_page_lock_queues();
6025 vm_free_page_lock();
6026
6027 RESET_STATE_OF_RUN();
6028 /*
6029 * reset our free page limit since we
6030 * dropped the lock protecting the vm_page_free_queue
6031 */
6032 free_available = vm_page_free_count - vm_page_free_reserved;
6033 considered = 0;
6034
6035 yielded++;
6036
6037 goto retry;
6038 }
6039 considered++;
6040 }
6041 m = VM_PAGE_NULL;
6042
6043 if (npages != contig_pages) {
6044 if (!wrapped) {
6045 /*
6046 * We didn't find a contiguous range but we didn't
6047 * start from the very first page.
6048 * Start again from the very first page.
6049 */
6050 RESET_STATE_OF_RUN();
6051 if (flags & KMA_LOMEM) {
6052 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx = 0;
6053 } else {
6054 idx_last_contig_page_found = vm_page_find_contiguous_last_idx = 0;
6055 }
6056 last_idx = 0;
6057 page_idx = last_idx;
6058 wrapped = TRUE;
6059 goto retry;
6060 }
6061 vm_free_page_unlock();
6062 } else {
6063 vm_page_t m1;
6064 vm_page_t m2;
6065 unsigned int cur_idx;
6066 unsigned int tmp_start_idx;
6067 vm_object_t locked_object = VM_OBJECT_NULL;
6068 boolean_t abort_run = FALSE;
6069
6070 assert(page_idx - start_idx == contig_pages);
6071
6072 tmp_start_idx = start_idx;
6073
6074 /*
6075 * first pass through to pull the free pages
6076 * off of the free queue so that in case we
6077 * need substitute pages, we won't grab any
6078 * of the free pages in the run... we'll clear
6079 * the 'free' bit in the 2nd pass, and even in
6080 * an abort_run case, we'll collect all of the
6081 * free pages in this run and return them to the free list
6082 */
6083 while (start_idx < page_idx) {
6084 m1 = &vm_pages[start_idx++];
6085
6086 #if !VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL
6087 assert(m1->vmp_q_state == VM_PAGE_ON_FREE_Q);
6088 #endif
6089
6090 if (m1->vmp_q_state == VM_PAGE_ON_FREE_Q) {
6091 unsigned int color;
6092
6093 color = VM_PAGE_GET_COLOR(m1);
6094 #if MACH_ASSERT
6095 vm_page_verify_free_list(&vm_page_queue_free[color].qhead, color, m1, TRUE);
6096 #endif
6097 vm_page_queue_remove(&vm_page_queue_free[color].qhead, m1, vmp_pageq);
6098
6099 VM_PAGE_ZERO_PAGEQ_ENTRY(m1);
6100 #if MACH_ASSERT
6101 vm_page_verify_free_list(&vm_page_queue_free[color].qhead, color, VM_PAGE_NULL, FALSE);
6102 #endif
6103 /*
6104 * Clear the "free" bit so that this page
6105 * does not get considered for another
6106 * concurrent physically-contiguous allocation.
6107 */
6108 m1->vmp_q_state = VM_PAGE_NOT_ON_Q;
6109 assert(m1->vmp_busy);
6110
6111 vm_page_free_count--;
6112 }
6113 }
6114 if (flags & KMA_LOMEM) {
6115 vm_page_lomem_find_contiguous_last_idx = page_idx;
6116 } else {
6117 vm_page_find_contiguous_last_idx = page_idx;
6118 }
6119
6120 /*
6121 * we can drop the free queue lock at this point since
6122 * we've pulled any 'free' candidates off of the list
6123 * we need it dropped so that we can do a vm_page_grab
6124 * when substituing for pmapped/dirty pages
6125 */
6126 vm_free_page_unlock();
6127
6128 start_idx = tmp_start_idx;
6129 cur_idx = page_idx - 1;
6130
6131 while (start_idx++ < page_idx) {
6132 /*
6133 * must go through the list from back to front
6134 * so that the page list is created in the
6135 * correct order - low -> high phys addresses
6136 */
6137 m1 = &vm_pages[cur_idx--];
6138
6139 if (m1->vmp_object == 0) {
6140 /*
6141 * page has already been removed from
6142 * the free list in the 1st pass
6143 */
6144 assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
6145 assert(m1->vmp_offset == (vm_object_offset_t) -1);
6146 assert(m1->vmp_busy);
6147 assert(!m1->vmp_wanted);
6148 assert(!m1->vmp_laundry);
6149 } else {
6150 vm_object_t object;
6151 int refmod;
6152 boolean_t disconnected, reusable;
6153
6154 if (abort_run == TRUE) {
6155 continue;
6156 }
6157
6158 assert(m1->vmp_q_state != VM_PAGE_NOT_ON_Q);
6159
6160 object = VM_PAGE_OBJECT(m1);
6161
6162 if (object != locked_object) {
6163 if (locked_object) {
6164 vm_object_unlock(locked_object);
6165 locked_object = VM_OBJECT_NULL;
6166 }
6167 if (vm_object_lock_try(object)) {
6168 locked_object = object;
6169 }
6170 }
6171 if (locked_object == VM_OBJECT_NULL ||
6172 (VM_PAGE_WIRED(m1) || m1->vmp_gobbled ||
6173 m1->vmp_laundry || m1->vmp_wanted ||
6174 m1->vmp_cleaning || m1->vmp_overwriting || m1->vmp_free_when_done || m1->vmp_busy) ||
6175 (m1->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
6176 if (locked_object) {
6177 vm_object_unlock(locked_object);
6178 locked_object = VM_OBJECT_NULL;
6179 }
6180 tmp_start_idx = cur_idx;
6181 abort_run = TRUE;
6182 continue;
6183 }
6184
6185 disconnected = FALSE;
6186 reusable = FALSE;
6187
6188 if ((m1->vmp_reusable ||
6189 object->all_reusable) &&
6190 (m1->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) &&
6191 !m1->vmp_dirty &&
6192 !m1->vmp_reference) {
6193 /* reusable page... */
6194 refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));
6195 disconnected = TRUE;
6196 if (refmod == 0) {
6197 /*
6198 * ... not reused: can steal
6199 * without relocating contents.
6200 */
6201 reusable = TRUE;
6202 }
6203 }
6204
6205 if ((m1->vmp_pmapped &&
6206 !reusable) ||
6207 m1->vmp_dirty ||
6208 m1->vmp_precious) {
6209 vm_object_offset_t offset;
6210
6211 m2 = vm_page_grab_options(VM_PAGE_GRAB_Q_LOCK_HELD);
6212
6213 if (m2 == VM_PAGE_NULL) {
6214 if (locked_object) {
6215 vm_object_unlock(locked_object);
6216 locked_object = VM_OBJECT_NULL;
6217 }
6218 tmp_start_idx = cur_idx;
6219 abort_run = TRUE;
6220 continue;
6221 }
6222 if (!disconnected) {
6223 if (m1->vmp_pmapped) {
6224 refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));
6225 } else {
6226 refmod = 0;
6227 }
6228 }
6229
6230 /* copy the page's contents */
6231 pmap_copy_page(VM_PAGE_GET_PHYS_PAGE(m1), VM_PAGE_GET_PHYS_PAGE(m2));
6232 /* copy the page's state */
6233 assert(!VM_PAGE_WIRED(m1));
6234 assert(m1->vmp_q_state != VM_PAGE_ON_FREE_Q);
6235 assert(m1->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q);
6236 assert(!m1->vmp_laundry);
6237 m2->vmp_reference = m1->vmp_reference;
6238 assert(!m1->vmp_gobbled);
6239 assert(!m1->vmp_private);
6240 m2->vmp_no_cache = m1->vmp_no_cache;
6241 m2->vmp_xpmapped = 0;
6242 assert(!m1->vmp_busy);
6243 assert(!m1->vmp_wanted);
6244 assert(!m1->vmp_fictitious);
6245 m2->vmp_pmapped = m1->vmp_pmapped; /* should flush cache ? */
6246 m2->vmp_wpmapped = m1->vmp_wpmapped;
6247 assert(!m1->vmp_free_when_done);
6248 m2->vmp_absent = m1->vmp_absent;
6249 m2->vmp_error = VMP_ERROR_GET(m1);
6250 m2->vmp_dirty = m1->vmp_dirty;
6251 assert(!m1->vmp_cleaning);
6252 m2->vmp_precious = m1->vmp_precious;
6253 m2->vmp_clustered = m1->vmp_clustered;
6254 assert(!m1->vmp_overwriting);
6255 m2->vmp_restart = m1->vmp_restart;
6256 m2->vmp_unusual = m1->vmp_unusual;
6257 m2->vmp_cs_validated = m1->vmp_cs_validated;
6258 m2->vmp_cs_tainted = m1->vmp_cs_tainted;
6259 m2->vmp_cs_nx = m1->vmp_cs_nx;
6260
6261 /*
6262 * If m1 had really been reusable,
6263 * we would have just stolen it, so
6264 * let's not propagate it's "reusable"
6265 * bit and assert that m2 is not
6266 * marked as "reusable".
6267 */
6268 // m2->vmp_reusable = m1->vmp_reusable;
6269 assert(!m2->vmp_reusable);
6270
6271 // assert(!m1->vmp_lopage);
6272
6273 if (m1->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6274 m2->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR;
6275 /*
6276 * We just grabbed m2 up above and so it isn't
6277 * going to be on any special Q as yet and so
6278 * we don't need to 'remove' it from the special
6279 * queues. Just resetting the state should be enough.
6280 */
6281 m2->vmp_on_specialq = VM_PAGE_SPECIAL_Q_EMPTY;
6282 }
6283
6284 /*
6285 * page may need to be flushed if
6286 * it is marshalled into a UPL
6287 * that is going to be used by a device
6288 * that doesn't support coherency
6289 */
6290 m2->vmp_written_by_kernel = TRUE;
6291
6292 /*
6293 * make sure we clear the ref/mod state
6294 * from the pmap layer... else we risk
6295 * inheriting state from the last time
6296 * this page was used...
6297 */
6298 pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m2), VM_MEM_MODIFIED | VM_MEM_REFERENCED);
6299
6300 if (refmod & VM_MEM_REFERENCED) {
6301 m2->vmp_reference = TRUE;
6302 }
6303 if (refmod & VM_MEM_MODIFIED) {
6304 SET_PAGE_DIRTY(m2, TRUE);
6305 }
6306 offset = m1->vmp_offset;
6307
6308 /*
6309 * completely cleans up the state
6310 * of the page so that it is ready
6311 * to be put onto the free list, or
6312 * for this purpose it looks like it
6313 * just came off of the free list
6314 */
6315 vm_page_free_prepare(m1);
6316
6317 /*
6318 * now put the substitute page
6319 * on the object
6320 */
6321 vm_page_insert_internal(m2, locked_object, offset, VM_KERN_MEMORY_NONE, TRUE, TRUE, FALSE, FALSE, NULL);
6322
6323 if (m2->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6324 m2->vmp_pmapped = TRUE;
6325 m2->vmp_wpmapped = TRUE;
6326
6327 PMAP_ENTER(kernel_pmap, (vm_map_offset_t)m2->vmp_offset, m2,
6328 VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, kr);
6329
6330 assert(kr == KERN_SUCCESS);
6331
6332 compressed_pages++;
6333 } else {
6334 if (m2->vmp_reference) {
6335 vm_page_activate(m2);
6336 } else {
6337 vm_page_deactivate(m2);
6338 }
6339 }
6340 PAGE_WAKEUP_DONE(m2);
6341 } else {
6342 assert(m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
6343
6344 /*
6345 * completely cleans up the state
6346 * of the page so that it is ready
6347 * to be put onto the free list, or
6348 * for this purpose it looks like it
6349 * just came off of the free list
6350 */
6351 vm_page_free_prepare(m1);
6352 }
6353
6354 stolen_pages++;
6355 }
6356 if (m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR) {
6357 /*
6358 * The Q state is preserved on m1 because vm_page_queues_remove doesn't
6359 * change it for pages marked as used-by-compressor.
6360 */
6361 vm_page_assign_special_state(m1, VM_PAGE_SPECIAL_Q_BG);
6362 }
6363 VM_PAGE_ZERO_PAGEQ_ENTRY(m1);
6364 m1->vmp_snext = m;
6365 m = m1;
6366 }
6367 if (locked_object) {
6368 vm_object_unlock(locked_object);
6369 locked_object = VM_OBJECT_NULL;
6370 }
6371
6372 if (abort_run == TRUE) {
6373 /*
6374 * want the index of the last
6375 * page in this run that was
6376 * successfully 'stolen', so back
6377 * it up 1 for the auto-decrement on use
6378 * and 1 more to bump back over this page
6379 */
6380 page_idx = tmp_start_idx + 2;
6381 if (page_idx >= vm_pages_count) {
6382 if (wrapped) {
6383 if (m != VM_PAGE_NULL) {
6384 vm_page_unlock_queues();
6385 vm_page_free_list(m, FALSE);
6386 vm_page_lock_queues();
6387 m = VM_PAGE_NULL;
6388 }
6389 dumped_run++;
6390 goto done_scanning;
6391 }
6392 page_idx = last_idx = 0;
6393 wrapped = TRUE;
6394 }
6395 abort_run = FALSE;
6396
6397 /*
6398 * We didn't find a contiguous range but we didn't
6399 * start from the very first page.
6400 * Start again from the very first page.
6401 */
6402 RESET_STATE_OF_RUN();
6403
6404 if (flags & KMA_LOMEM) {
6405 idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx = page_idx;
6406 } else {
6407 idx_last_contig_page_found = vm_page_find_contiguous_last_idx = page_idx;
6408 }
6409
6410 last_idx = page_idx;
6411
6412 if (m != VM_PAGE_NULL) {
6413 vm_page_unlock_queues();
6414 vm_page_free_list(m, FALSE);
6415 vm_page_lock_queues();
6416 m = VM_PAGE_NULL;
6417 }
6418 dumped_run++;
6419
6420 vm_free_page_lock();
6421 /*
6422 * reset our free page limit since we
6423 * dropped the lock protecting the vm_page_free_queue
6424 */
6425 free_available = vm_page_free_count - vm_page_free_reserved;
6426 goto retry;
6427 }
6428
6429 for (m1 = m; m1 != VM_PAGE_NULL; m1 = NEXT_PAGE(m1)) {
6430 assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
6431 assert(m1->vmp_wire_count == 0);
6432
6433 if (wire == TRUE) {
6434 m1->vmp_wire_count++;
6435 m1->vmp_q_state = VM_PAGE_IS_WIRED;
6436 } else {
6437 m1->vmp_gobbled = TRUE;
6438 }
6439 }
6440 if (wire == FALSE) {
6441 vm_page_gobble_count += npages;
6442 }
6443
6444 /*
6445 * gobbled pages are also counted as wired pages
6446 */
6447 vm_page_wire_count += npages;
6448
6449 assert(vm_page_verify_contiguous(m, npages));
6450 }
6451 done_scanning:
6452 PAGE_REPLACEMENT_ALLOWED(FALSE);
6453
6454 vm_page_unlock_queues();
6455
6456 #if DEBUG
6457 clock_get_system_microtime(&tv_end_sec, &tv_end_usec);
6458
6459 tv_end_sec -= tv_start_sec;
6460 if (tv_end_usec < tv_start_usec) {
6461 tv_end_sec--;
6462 tv_end_usec += 1000000;
6463 }
6464 tv_end_usec -= tv_start_usec;
6465 if (tv_end_usec >= 1000000) {
6466 tv_end_sec++;
6467 tv_end_sec -= 1000000;
6468 }
6469 if (vm_page_find_contig_debug) {
6470 printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds... started at %d... scanned %d pages... yielded %d times... dumped run %d times... stole %d pages... stole %d compressed pages\n",
6471 __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
6472 (long)tv_end_sec, tv_end_usec, orig_last_idx,
6473 scanned, yielded, dumped_run, stolen_pages, compressed_pages);
6474 }
6475
6476 #endif
6477 #if MACH_ASSERT
6478 vm_page_verify_free_lists();
6479 #endif
6480 if (m == NULL && zone_gc_called < 2) {
6481 printf("%s(num=%d,low=%d): found %d pages at 0x%llx...scanned %d pages... yielded %d times... dumped run %d times... stole %d pages... stole %d compressed pages... wired count is %d\n",
6482 __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
6483 scanned, yielded, dumped_run, stolen_pages, compressed_pages, vm_page_wire_count);
6484
6485 if (consider_buffer_cache_collect != NULL) {
6486 (void)(*consider_buffer_cache_collect)(1);
6487 }
6488
6489 zone_gc(zone_gc_called ? ZONE_GC_DRAIN : ZONE_GC_TRIM);
6490
6491 zone_gc_called++;
6492
6493 printf("vm_page_find_contiguous: zone_gc called... wired count is %d\n", vm_page_wire_count);
6494 goto full_scan_again;
6495 }
6496
6497 return m;
6498 }
6499
6500 /*
6501 * Allocate a list of contiguous, wired pages.
6502 */
6503 kern_return_t
cpm_allocate(vm_size_t size,vm_page_t * list,ppnum_t max_pnum,ppnum_t pnum_mask,boolean_t wire,int flags)6504 cpm_allocate(
6505 vm_size_t size,
6506 vm_page_t *list,
6507 ppnum_t max_pnum,
6508 ppnum_t pnum_mask,
6509 boolean_t wire,
6510 int flags)
6511 {
6512 vm_page_t pages;
6513 unsigned int npages;
6514
6515 if (size % PAGE_SIZE != 0) {
6516 return KERN_INVALID_ARGUMENT;
6517 }
6518
6519 npages = (unsigned int) (size / PAGE_SIZE);
6520 if (npages != size / PAGE_SIZE) {
6521 /* 32-bit overflow */
6522 return KERN_INVALID_ARGUMENT;
6523 }
6524
6525 /*
6526 * Obtain a pointer to a subset of the free
6527 * list large enough to satisfy the request;
6528 * the region will be physically contiguous.
6529 */
6530 pages = vm_page_find_contiguous(npages, max_pnum, pnum_mask, wire, flags);
6531
6532 if (pages == VM_PAGE_NULL) {
6533 return KERN_NO_SPACE;
6534 }
6535 /*
6536 * determine need for wakeups
6537 */
6538 if (vm_page_free_count < vm_page_free_min) {
6539 vm_free_page_lock();
6540 if (vm_pageout_running == FALSE) {
6541 vm_free_page_unlock();
6542 thread_wakeup((event_t) &vm_page_free_wanted);
6543 } else {
6544 vm_free_page_unlock();
6545 }
6546 }
6547
6548 VM_CHECK_MEMORYSTATUS;
6549
6550 /*
6551 * The CPM pages should now be available and
6552 * ordered by ascending physical address.
6553 */
6554 assert(vm_page_verify_contiguous(pages, npages));
6555
6556 *list = pages;
6557 return KERN_SUCCESS;
6558 }
6559
6560
6561 unsigned int vm_max_delayed_work_limit = DEFAULT_DELAYED_WORK_LIMIT;
6562
6563 /*
6564 * when working on a 'run' of pages, it is necessary to hold
6565 * the vm_page_queue_lock (a hot global lock) for certain operations
6566 * on the page... however, the majority of the work can be done
6567 * while merely holding the object lock... in fact there are certain
6568 * collections of pages that don't require any work brokered by the
6569 * vm_page_queue_lock... to mitigate the time spent behind the global
6570 * lock, go to a 2 pass algorithm... collect pages up to DELAYED_WORK_LIMIT
6571 * while doing all of the work that doesn't require the vm_page_queue_lock...
6572 * then call vm_page_do_delayed_work to acquire the vm_page_queue_lock and do the
6573 * necessary work for each page... we will grab the busy bit on the page
6574 * if it's not already held so that vm_page_do_delayed_work can drop the object lock
6575 * if it can't immediately take the vm_page_queue_lock in order to compete
6576 * for the locks in the same order that vm_pageout_scan takes them.
6577 * the operation names are modeled after the names of the routines that
6578 * need to be called in order to make the changes very obvious in the
6579 * original loop
6580 */
6581
6582 void
vm_page_do_delayed_work(vm_object_t object,vm_tag_t tag,struct vm_page_delayed_work * dwp,int dw_count)6583 vm_page_do_delayed_work(
6584 vm_object_t object,
6585 vm_tag_t tag,
6586 struct vm_page_delayed_work *dwp,
6587 int dw_count)
6588 {
6589 int j;
6590 vm_page_t m;
6591 vm_page_t local_free_q = VM_PAGE_NULL;
6592
6593 /*
6594 * pageout_scan takes the vm_page_lock_queues first
6595 * then tries for the object lock... to avoid what
6596 * is effectively a lock inversion, we'll go to the
6597 * trouble of taking them in that same order... otherwise
6598 * if this object contains the majority of the pages resident
6599 * in the UBC (or a small set of large objects actively being
6600 * worked on contain the majority of the pages), we could
6601 * cause the pageout_scan thread to 'starve' in its attempt
6602 * to find pages to move to the free queue, since it has to
6603 * successfully acquire the object lock of any candidate page
6604 * before it can steal/clean it.
6605 */
6606 if (!vm_page_trylockspin_queues()) {
6607 vm_object_unlock(object);
6608
6609 /*
6610 * "Turnstile enabled vm_pageout_scan" can be runnable
6611 * for a very long time without getting on a core.
6612 * If this is a higher priority thread it could be
6613 * waiting here for a very long time respecting the fact
6614 * that pageout_scan would like its object after VPS does
6615 * a mutex_pause(0).
6616 * So we cap the number of yields in the vm_object_lock_avoid()
6617 * case to a single mutex_pause(0) which will give vm_pageout_scan
6618 * 10us to run and grab the object if needed.
6619 */
6620 vm_page_lockspin_queues();
6621
6622 for (j = 0;; j++) {
6623 if ((!vm_object_lock_avoid(object) ||
6624 (vps_dynamic_priority_enabled && (j > 0))) &&
6625 _vm_object_lock_try(object)) {
6626 break;
6627 }
6628 vm_page_unlock_queues();
6629 mutex_pause(j);
6630 vm_page_lockspin_queues();
6631 }
6632 }
6633 for (j = 0; j < dw_count; j++, dwp++) {
6634 m = dwp->dw_m;
6635
6636 if (dwp->dw_mask & DW_vm_pageout_throttle_up) {
6637 vm_pageout_throttle_up(m);
6638 }
6639 #if CONFIG_PHANTOM_CACHE
6640 if (dwp->dw_mask & DW_vm_phantom_cache_update) {
6641 vm_phantom_cache_update(m);
6642 }
6643 #endif
6644 if (dwp->dw_mask & DW_vm_page_wire) {
6645 vm_page_wire(m, tag, FALSE);
6646 } else if (dwp->dw_mask & DW_vm_page_unwire) {
6647 boolean_t queueit;
6648
6649 queueit = (dwp->dw_mask & (DW_vm_page_free | DW_vm_page_deactivate_internal)) ? FALSE : TRUE;
6650
6651 vm_page_unwire(m, queueit);
6652 }
6653 if (dwp->dw_mask & DW_vm_page_free) {
6654 vm_page_free_prepare_queues(m);
6655
6656 assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
6657 /*
6658 * Add this page to our list of reclaimed pages,
6659 * to be freed later.
6660 */
6661 m->vmp_snext = local_free_q;
6662 local_free_q = m;
6663 } else {
6664 if (dwp->dw_mask & DW_vm_page_deactivate_internal) {
6665 vm_page_deactivate_internal(m, FALSE);
6666 } else if (dwp->dw_mask & DW_vm_page_activate) {
6667 if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
6668 vm_page_activate(m);
6669 }
6670 } else if (dwp->dw_mask & DW_vm_page_speculate) {
6671 vm_page_speculate(m, TRUE);
6672 } else if (dwp->dw_mask & DW_enqueue_cleaned) {
6673 /*
6674 * if we didn't hold the object lock and did this,
6675 * we might disconnect the page, then someone might
6676 * soft fault it back in, then we would put it on the
6677 * cleaned queue, and so we would have a referenced (maybe even dirty)
6678 * page on that queue, which we don't want
6679 */
6680 int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
6681
6682 if ((refmod_state & VM_MEM_REFERENCED)) {
6683 /*
6684 * this page has been touched since it got cleaned; let's activate it
6685 * if it hasn't already been
6686 */
6687 VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
6688 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
6689
6690 if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
6691 vm_page_activate(m);
6692 }
6693 } else {
6694 m->vmp_reference = FALSE;
6695 vm_page_enqueue_cleaned(m);
6696 }
6697 } else if (dwp->dw_mask & DW_vm_page_lru) {
6698 vm_page_lru(m);
6699 } else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) {
6700 if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {
6701 vm_page_queues_remove(m, TRUE);
6702 }
6703 }
6704 if (dwp->dw_mask & DW_set_reference) {
6705 m->vmp_reference = TRUE;
6706 } else if (dwp->dw_mask & DW_clear_reference) {
6707 m->vmp_reference = FALSE;
6708 }
6709
6710 if (dwp->dw_mask & DW_move_page) {
6711 if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {
6712 vm_page_queues_remove(m, FALSE);
6713
6714 assert(VM_PAGE_OBJECT(m) != kernel_object);
6715
6716 vm_page_enqueue_inactive(m, FALSE);
6717 }
6718 }
6719 if (dwp->dw_mask & DW_clear_busy) {
6720 m->vmp_busy = FALSE;
6721 }
6722
6723 if (dwp->dw_mask & DW_PAGE_WAKEUP) {
6724 PAGE_WAKEUP(m);
6725 }
6726 }
6727 }
6728 vm_page_unlock_queues();
6729
6730 if (local_free_q) {
6731 vm_page_free_list(local_free_q, TRUE);
6732 }
6733
6734 VM_CHECK_MEMORYSTATUS;
6735 }
6736
6737 __abortlike
6738 static void
__vm_page_alloc_list_failed_panic(vm_size_t page_count,kma_flags_t flags,kern_return_t kr)6739 __vm_page_alloc_list_failed_panic(
6740 vm_size_t page_count,
6741 kma_flags_t flags,
6742 kern_return_t kr)
6743 {
6744 panic("vm_page_alloc_list(%zd, 0x%x) failed unexpectedly with %d",
6745 (size_t)page_count, flags, kr);
6746 }
6747
6748 kern_return_t
vm_page_alloc_list(vm_size_t page_count,kma_flags_t flags,vm_page_t * list)6749 vm_page_alloc_list(
6750 vm_size_t page_count,
6751 kma_flags_t flags,
6752 vm_page_t *list)
6753 {
6754 vm_page_t page_list = VM_PAGE_NULL;
6755 vm_page_t mem;
6756 kern_return_t kr = KERN_SUCCESS;
6757 int page_grab_count = 0;
6758 #if DEVELOPMENT || DEBUG
6759 task_t task;
6760 #endif /* DEVELOPMENT || DEBUG */
6761
6762 for (vm_size_t i = 0; i < page_count; i++) {
6763 for (;;) {
6764 if (flags & KMA_LOMEM) {
6765 mem = vm_page_grablo();
6766 } else {
6767 mem = vm_page_grab();
6768 }
6769
6770 if (mem != VM_PAGE_NULL) {
6771 break;
6772 }
6773
6774 if (flags & KMA_NOPAGEWAIT) {
6775 kr = KERN_RESOURCE_SHORTAGE;
6776 goto out;
6777 }
6778 if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) {
6779 kr = KERN_RESOURCE_SHORTAGE;
6780 goto out;
6781 }
6782
6783 /* VM privileged threads should have waited in vm_page_grab() and not get here. */
6784 assert(!(current_thread()->options & TH_OPT_VMPRIV));
6785
6786 if ((flags & KMA_NOFAIL) == 0) {
6787 uint64_t unavailable = ptoa_64(vm_page_wire_count + vm_page_free_target);
6788 if (unavailable > max_mem || ptoa_64(page_count) > (max_mem - unavailable)) {
6789 kr = KERN_RESOURCE_SHORTAGE;
6790 goto out;
6791 }
6792 }
6793 VM_PAGE_WAIT();
6794 }
6795
6796 page_grab_count++;
6797 mem->vmp_snext = page_list;
6798 page_list = mem;
6799 }
6800
6801 if ((KMA_ZERO | KMA_NOENCRYPT) & flags) {
6802 for (mem = page_list; mem; mem = mem->vmp_snext) {
6803 vm_page_zero_fill(mem);
6804 }
6805 }
6806
6807 out:
6808 #if DEBUG || DEVELOPMENT
6809 task = current_task_early();
6810 if (task != NULL) {
6811 ledger_credit(task->ledger, task_ledgers.pages_grabbed_kern, page_grab_count);
6812 }
6813 #endif
6814
6815 if (kr == KERN_SUCCESS) {
6816 *list = page_list;
6817 } else if (flags & KMA_NOFAIL) {
6818 __vm_page_alloc_list_failed_panic(page_count, flags, kr);
6819 } else {
6820 vm_page_free_list(page_list, FALSE);
6821 }
6822
6823 return kr;
6824 }
6825
6826 void
vm_page_set_offset(vm_page_t page,vm_object_offset_t offset)6827 vm_page_set_offset(vm_page_t page, vm_object_offset_t offset)
6828 {
6829 page->vmp_offset = offset;
6830 }
6831
6832 vm_page_t
vm_page_get_next(vm_page_t page)6833 vm_page_get_next(vm_page_t page)
6834 {
6835 return page->vmp_snext;
6836 }
6837
6838 vm_object_offset_t
vm_page_get_offset(vm_page_t page)6839 vm_page_get_offset(vm_page_t page)
6840 {
6841 return page->vmp_offset;
6842 }
6843
6844 ppnum_t
vm_page_get_phys_page(vm_page_t page)6845 vm_page_get_phys_page(vm_page_t page)
6846 {
6847 return VM_PAGE_GET_PHYS_PAGE(page);
6848 }
6849
6850
6851 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6852
6853 #if HIBERNATION
6854
6855 static vm_page_t hibernate_gobble_queue;
6856
6857 static int hibernate_drain_pageout_queue(struct vm_pageout_queue *);
6858 static int hibernate_flush_dirty_pages(int);
6859 static int hibernate_flush_queue(vm_page_queue_head_t *, int);
6860
6861 void hibernate_flush_wait(void);
6862 void hibernate_mark_in_progress(void);
6863 void hibernate_clear_in_progress(void);
6864
6865 void hibernate_free_range(int, int);
6866 void hibernate_hash_insert_page(vm_page_t);
6867 uint32_t hibernate_mark_as_unneeded(addr64_t, addr64_t, hibernate_page_list_t *, hibernate_page_list_t *);
6868 uint32_t hibernate_teardown_vm_structs(hibernate_page_list_t *, hibernate_page_list_t *);
6869 ppnum_t hibernate_lookup_paddr(unsigned int);
6870
6871 struct hibernate_statistics {
6872 int hibernate_considered;
6873 int hibernate_reentered_on_q;
6874 int hibernate_found_dirty;
6875 int hibernate_skipped_cleaning;
6876 int hibernate_skipped_transient;
6877 int hibernate_skipped_precious;
6878 int hibernate_skipped_external;
6879 int hibernate_queue_nolock;
6880 int hibernate_queue_paused;
6881 int hibernate_throttled;
6882 int hibernate_throttle_timeout;
6883 int hibernate_drained;
6884 int hibernate_drain_timeout;
6885 int cd_lock_failed;
6886 int cd_found_precious;
6887 int cd_found_wired;
6888 int cd_found_busy;
6889 int cd_found_unusual;
6890 int cd_found_cleaning;
6891 int cd_found_laundry;
6892 int cd_found_dirty;
6893 int cd_found_xpmapped;
6894 int cd_skipped_xpmapped;
6895 int cd_local_free;
6896 int cd_total_free;
6897 int cd_vm_page_wire_count;
6898 int cd_vm_struct_pages_unneeded;
6899 int cd_pages;
6900 int cd_discarded;
6901 int cd_count_wire;
6902 } hibernate_stats;
6903
6904
6905 /*
6906 * clamp the number of 'xpmapped' pages we'll sweep into the hibernation image
6907 * so that we don't overrun the estimated image size, which would
6908 * result in a hibernation failure.
6909 *
6910 * We use a size value instead of pages because we don't want to take up more space
6911 * on disk if the system has a 16K page size vs 4K. Also, we are not guaranteed
6912 * to have that additional space available.
6913 *
6914 * Since this was set at 40000 pages on X86 we are going to use 160MB as our
6915 * xpmapped size.
6916 */
6917 #define HIBERNATE_XPMAPPED_LIMIT ((160 * 1024 * 1024ULL) / PAGE_SIZE)
6918
6919
6920 static int
hibernate_drain_pageout_queue(struct vm_pageout_queue * q)6921 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
6922 {
6923 wait_result_t wait_result;
6924
6925 vm_page_lock_queues();
6926
6927 while (!vm_page_queue_empty(&q->pgo_pending)) {
6928 q->pgo_draining = TRUE;
6929
6930 assert_wait_timeout((event_t) (&q->pgo_laundry + 1), THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);
6931
6932 vm_page_unlock_queues();
6933
6934 wait_result = thread_block(THREAD_CONTINUE_NULL);
6935
6936 if (wait_result == THREAD_TIMED_OUT && !vm_page_queue_empty(&q->pgo_pending)) {
6937 hibernate_stats.hibernate_drain_timeout++;
6938
6939 if (q == &vm_pageout_queue_external) {
6940 return 0;
6941 }
6942
6943 return 1;
6944 }
6945 vm_page_lock_queues();
6946
6947 hibernate_stats.hibernate_drained++;
6948 }
6949 vm_page_unlock_queues();
6950
6951 return 0;
6952 }
6953
6954
6955 boolean_t hibernate_skip_external = FALSE;
6956
6957 static int
hibernate_flush_queue(vm_page_queue_head_t * q,int qcount)6958 hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
6959 {
6960 vm_page_t m;
6961 vm_object_t l_object = NULL;
6962 vm_object_t m_object = NULL;
6963 int refmod_state = 0;
6964 int try_failed_count = 0;
6965 int retval = 0;
6966 int current_run = 0;
6967 struct vm_pageout_queue *iq;
6968 struct vm_pageout_queue *eq;
6969 struct vm_pageout_queue *tq;
6970
6971 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START,
6972 VM_KERNEL_UNSLIDE_OR_PERM(q), qcount);
6973
6974 iq = &vm_pageout_queue_internal;
6975 eq = &vm_pageout_queue_external;
6976
6977 vm_page_lock_queues();
6978
6979 while (qcount && !vm_page_queue_empty(q)) {
6980 if (current_run++ == 1000) {
6981 if (hibernate_should_abort()) {
6982 retval = 1;
6983 break;
6984 }
6985 current_run = 0;
6986 }
6987
6988 m = (vm_page_t) vm_page_queue_first(q);
6989 m_object = VM_PAGE_OBJECT(m);
6990
6991 /*
6992 * check to see if we currently are working
6993 * with the same object... if so, we've
6994 * already got the lock
6995 */
6996 if (m_object != l_object) {
6997 /*
6998 * the object associated with candidate page is
6999 * different from the one we were just working
7000 * with... dump the lock if we still own it
7001 */
7002 if (l_object != NULL) {
7003 vm_object_unlock(l_object);
7004 l_object = NULL;
7005 }
7006 /*
7007 * Try to lock object; since we've alread got the
7008 * page queues lock, we can only 'try' for this one.
7009 * if the 'try' fails, we need to do a mutex_pause
7010 * to allow the owner of the object lock a chance to
7011 * run...
7012 */
7013 if (!vm_object_lock_try_scan(m_object)) {
7014 if (try_failed_count > 20) {
7015 hibernate_stats.hibernate_queue_nolock++;
7016
7017 goto reenter_pg_on_q;
7018 }
7019
7020 vm_page_unlock_queues();
7021 mutex_pause(try_failed_count++);
7022 vm_page_lock_queues();
7023
7024 hibernate_stats.hibernate_queue_paused++;
7025 continue;
7026 } else {
7027 l_object = m_object;
7028 }
7029 }
7030 if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || VMP_ERROR_GET(m)) {
7031 /*
7032 * page is not to be cleaned
7033 * put it back on the head of its queue
7034 */
7035 if (m->vmp_cleaning) {
7036 hibernate_stats.hibernate_skipped_cleaning++;
7037 } else {
7038 hibernate_stats.hibernate_skipped_transient++;
7039 }
7040
7041 goto reenter_pg_on_q;
7042 }
7043 if (m_object->copy == VM_OBJECT_NULL) {
7044 if (m_object->purgable == VM_PURGABLE_VOLATILE || m_object->purgable == VM_PURGABLE_EMPTY) {
7045 /*
7046 * let the normal hibernate image path
7047 * deal with these
7048 */
7049 goto reenter_pg_on_q;
7050 }
7051 }
7052 if (!m->vmp_dirty && m->vmp_pmapped) {
7053 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
7054
7055 if ((refmod_state & VM_MEM_MODIFIED)) {
7056 SET_PAGE_DIRTY(m, FALSE);
7057 }
7058 } else {
7059 refmod_state = 0;
7060 }
7061
7062 if (!m->vmp_dirty) {
7063 /*
7064 * page is not to be cleaned
7065 * put it back on the head of its queue
7066 */
7067 if (m->vmp_precious) {
7068 hibernate_stats.hibernate_skipped_precious++;
7069 }
7070
7071 goto reenter_pg_on_q;
7072 }
7073
7074 if (hibernate_skip_external == TRUE && !m_object->internal) {
7075 hibernate_stats.hibernate_skipped_external++;
7076
7077 goto reenter_pg_on_q;
7078 }
7079 tq = NULL;
7080
7081 if (m_object->internal) {
7082 if (VM_PAGE_Q_THROTTLED(iq)) {
7083 tq = iq;
7084 }
7085 } else if (VM_PAGE_Q_THROTTLED(eq)) {
7086 tq = eq;
7087 }
7088
7089 if (tq != NULL) {
7090 wait_result_t wait_result;
7091 int wait_count = 5;
7092
7093 if (l_object != NULL) {
7094 vm_object_unlock(l_object);
7095 l_object = NULL;
7096 }
7097
7098 while (retval == 0) {
7099 tq->pgo_throttled = TRUE;
7100
7101 assert_wait_timeout((event_t) &tq->pgo_laundry, THREAD_INTERRUPTIBLE, 1000, 1000 * NSEC_PER_USEC);
7102
7103 vm_page_unlock_queues();
7104
7105 wait_result = thread_block(THREAD_CONTINUE_NULL);
7106
7107 vm_page_lock_queues();
7108
7109 if (wait_result != THREAD_TIMED_OUT) {
7110 break;
7111 }
7112 if (!VM_PAGE_Q_THROTTLED(tq)) {
7113 break;
7114 }
7115
7116 if (hibernate_should_abort()) {
7117 retval = 1;
7118 }
7119
7120 if (--wait_count == 0) {
7121 hibernate_stats.hibernate_throttle_timeout++;
7122
7123 if (tq == eq) {
7124 hibernate_skip_external = TRUE;
7125 break;
7126 }
7127 retval = 1;
7128 }
7129 }
7130 if (retval) {
7131 break;
7132 }
7133
7134 hibernate_stats.hibernate_throttled++;
7135
7136 continue;
7137 }
7138 /*
7139 * we've already factored out pages in the laundry which
7140 * means this page can't be on the pageout queue so it's
7141 * safe to do the vm_page_queues_remove
7142 */
7143 vm_page_queues_remove(m, TRUE);
7144
7145 if (m_object->internal == TRUE) {
7146 pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), PMAP_OPTIONS_COMPRESSOR, NULL);
7147 }
7148
7149 vm_pageout_cluster(m);
7150
7151 hibernate_stats.hibernate_found_dirty++;
7152
7153 goto next_pg;
7154
7155 reenter_pg_on_q:
7156 vm_page_queue_remove(q, m, vmp_pageq);
7157 vm_page_queue_enter(q, m, vmp_pageq);
7158
7159 hibernate_stats.hibernate_reentered_on_q++;
7160 next_pg:
7161 hibernate_stats.hibernate_considered++;
7162
7163 qcount--;
7164 try_failed_count = 0;
7165 }
7166 if (l_object != NULL) {
7167 vm_object_unlock(l_object);
7168 l_object = NULL;
7169 }
7170
7171 vm_page_unlock_queues();
7172
7173 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_END, hibernate_stats.hibernate_found_dirty, retval, 0, 0, 0);
7174
7175 return retval;
7176 }
7177
7178
7179 static int
hibernate_flush_dirty_pages(int pass)7180 hibernate_flush_dirty_pages(int pass)
7181 {
7182 struct vm_speculative_age_q *aq;
7183 uint32_t i;
7184
7185 if (vm_page_local_q) {
7186 zpercpu_foreach_cpu(lid) {
7187 vm_page_reactivate_local(lid, TRUE, FALSE);
7188 }
7189 }
7190
7191 for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
7192 int qcount;
7193 vm_page_t m;
7194
7195 aq = &vm_page_queue_speculative[i];
7196
7197 if (vm_page_queue_empty(&aq->age_q)) {
7198 continue;
7199 }
7200 qcount = 0;
7201
7202 vm_page_lockspin_queues();
7203
7204 vm_page_queue_iterate(&aq->age_q, m, vmp_pageq) {
7205 qcount++;
7206 }
7207 vm_page_unlock_queues();
7208
7209 if (qcount) {
7210 if (hibernate_flush_queue(&aq->age_q, qcount)) {
7211 return 1;
7212 }
7213 }
7214 }
7215 if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_page_anonymous_count - vm_page_cleaned_count)) {
7216 return 1;
7217 }
7218 /* XXX FBDP TODO: flush secluded queue */
7219 if (hibernate_flush_queue(&vm_page_queue_anonymous, vm_page_anonymous_count)) {
7220 return 1;
7221 }
7222 if (hibernate_flush_queue(&vm_page_queue_cleaned, vm_page_cleaned_count)) {
7223 return 1;
7224 }
7225 if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
7226 return 1;
7227 }
7228
7229 if (pass == 1) {
7230 vm_compressor_record_warmup_start();
7231 }
7232
7233 if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) {
7234 if (pass == 1) {
7235 vm_compressor_record_warmup_end();
7236 }
7237 return 1;
7238 }
7239 if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
7240 if (pass == 1) {
7241 vm_compressor_record_warmup_end();
7242 }
7243 return 1;
7244 }
7245 if (pass == 1) {
7246 vm_compressor_record_warmup_end();
7247 }
7248
7249 if (hibernate_skip_external == FALSE && hibernate_drain_pageout_queue(&vm_pageout_queue_external)) {
7250 return 1;
7251 }
7252
7253 return 0;
7254 }
7255
7256
7257 void
hibernate_reset_stats()7258 hibernate_reset_stats()
7259 {
7260 bzero(&hibernate_stats, sizeof(struct hibernate_statistics));
7261 }
7262
7263
7264 int
hibernate_flush_memory()7265 hibernate_flush_memory()
7266 {
7267 int retval;
7268
7269 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
7270
7271 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_START, vm_page_free_count, 0, 0, 0, 0);
7272
7273 hibernate_cleaning_in_progress = TRUE;
7274 hibernate_skip_external = FALSE;
7275
7276 if ((retval = hibernate_flush_dirty_pages(1)) == 0) {
7277 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_START, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
7278
7279 vm_compressor_flush();
7280
7281 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_END, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
7282
7283 if (consider_buffer_cache_collect != NULL) {
7284 unsigned int orig_wire_count;
7285
7286 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, 0, 0, 0, 0, 0);
7287 orig_wire_count = vm_page_wire_count;
7288
7289 (void)(*consider_buffer_cache_collect)(1);
7290 zone_gc(ZONE_GC_DRAIN);
7291
7292 HIBLOG("hibernate_flush_memory: buffer_cache_gc freed up %d wired pages\n", orig_wire_count - vm_page_wire_count);
7293
7294 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_END, orig_wire_count - vm_page_wire_count, 0, 0, 0, 0);
7295 }
7296 }
7297 hibernate_cleaning_in_progress = FALSE;
7298
7299 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_END, vm_page_free_count, hibernate_stats.hibernate_found_dirty, retval, 0, 0);
7300
7301 if (retval) {
7302 HIBLOG("hibernate_flush_memory() failed to finish - vm_page_compressor_count(%d)\n", VM_PAGE_COMPRESSOR_COUNT);
7303 }
7304
7305
7306 HIBPRINT("hibernate_flush_memory() considered(%d) reentered_on_q(%d) found_dirty(%d)\n",
7307 hibernate_stats.hibernate_considered,
7308 hibernate_stats.hibernate_reentered_on_q,
7309 hibernate_stats.hibernate_found_dirty);
7310 HIBPRINT(" skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) skipped_external(%d) queue_nolock(%d)\n",
7311 hibernate_stats.hibernate_skipped_cleaning,
7312 hibernate_stats.hibernate_skipped_transient,
7313 hibernate_stats.hibernate_skipped_precious,
7314 hibernate_stats.hibernate_skipped_external,
7315 hibernate_stats.hibernate_queue_nolock);
7316 HIBPRINT(" queue_paused(%d) throttled(%d) throttle_timeout(%d) drained(%d) drain_timeout(%d)\n",
7317 hibernate_stats.hibernate_queue_paused,
7318 hibernate_stats.hibernate_throttled,
7319 hibernate_stats.hibernate_throttle_timeout,
7320 hibernate_stats.hibernate_drained,
7321 hibernate_stats.hibernate_drain_timeout);
7322
7323 return retval;
7324 }
7325
7326
7327 static void
hibernate_page_list_zero(hibernate_page_list_t * list)7328 hibernate_page_list_zero(hibernate_page_list_t *list)
7329 {
7330 uint32_t bank;
7331 hibernate_bitmap_t * bitmap;
7332
7333 bitmap = &list->bank_bitmap[0];
7334 for (bank = 0; bank < list->bank_count; bank++) {
7335 uint32_t last_bit;
7336
7337 bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2);
7338 // set out-of-bound bits at end of bitmap.
7339 last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
7340 if (last_bit) {
7341 bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
7342 }
7343
7344 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
7345 }
7346 }
7347
7348 void
hibernate_free_gobble_pages(void)7349 hibernate_free_gobble_pages(void)
7350 {
7351 vm_page_t m, next;
7352 uint32_t count = 0;
7353
7354 m = (vm_page_t) hibernate_gobble_queue;
7355 while (m) {
7356 next = m->vmp_snext;
7357 vm_page_free(m);
7358 count++;
7359 m = next;
7360 }
7361 hibernate_gobble_queue = VM_PAGE_NULL;
7362
7363 if (count) {
7364 HIBLOG("Freed %d pages\n", count);
7365 }
7366 }
7367
7368 static boolean_t
hibernate_consider_discard(vm_page_t m,boolean_t preflight)7369 hibernate_consider_discard(vm_page_t m, boolean_t preflight)
7370 {
7371 vm_object_t object = NULL;
7372 int refmod_state;
7373 boolean_t discard = FALSE;
7374
7375 do{
7376 if (m->vmp_private) {
7377 panic("hibernate_consider_discard: private");
7378 }
7379
7380 object = VM_PAGE_OBJECT(m);
7381
7382 if (!vm_object_lock_try(object)) {
7383 object = NULL;
7384 if (!preflight) {
7385 hibernate_stats.cd_lock_failed++;
7386 }
7387 break;
7388 }
7389 if (VM_PAGE_WIRED(m)) {
7390 if (!preflight) {
7391 hibernate_stats.cd_found_wired++;
7392 }
7393 break;
7394 }
7395 if (m->vmp_precious) {
7396 if (!preflight) {
7397 hibernate_stats.cd_found_precious++;
7398 }
7399 break;
7400 }
7401 if (m->vmp_busy || !object->alive) {
7402 /*
7403 * Somebody is playing with this page.
7404 */
7405 if (!preflight) {
7406 hibernate_stats.cd_found_busy++;
7407 }
7408 break;
7409 }
7410 if (m->vmp_absent || m->vmp_unusual || VMP_ERROR_GET(m)) {
7411 /*
7412 * If it's unusual in anyway, ignore it
7413 */
7414 if (!preflight) {
7415 hibernate_stats.cd_found_unusual++;
7416 }
7417 break;
7418 }
7419 if (m->vmp_cleaning) {
7420 if (!preflight) {
7421 hibernate_stats.cd_found_cleaning++;
7422 }
7423 break;
7424 }
7425 if (m->vmp_laundry) {
7426 if (!preflight) {
7427 hibernate_stats.cd_found_laundry++;
7428 }
7429 break;
7430 }
7431 if (!m->vmp_dirty) {
7432 refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
7433
7434 if (refmod_state & VM_MEM_REFERENCED) {
7435 m->vmp_reference = TRUE;
7436 }
7437 if (refmod_state & VM_MEM_MODIFIED) {
7438 SET_PAGE_DIRTY(m, FALSE);
7439 }
7440 }
7441
7442 /*
7443 * If it's clean or purgeable we can discard the page on wakeup.
7444 */
7445 discard = (!m->vmp_dirty)
7446 || (VM_PURGABLE_VOLATILE == object->purgable)
7447 || (VM_PURGABLE_EMPTY == object->purgable);
7448
7449
7450 if (discard == FALSE) {
7451 if (!preflight) {
7452 hibernate_stats.cd_found_dirty++;
7453 }
7454 } else if (m->vmp_xpmapped && m->vmp_reference && !object->internal) {
7455 if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) {
7456 if (!preflight) {
7457 hibernate_stats.cd_found_xpmapped++;
7458 }
7459 discard = FALSE;
7460 } else {
7461 if (!preflight) {
7462 hibernate_stats.cd_skipped_xpmapped++;
7463 }
7464 }
7465 }
7466 }while (FALSE);
7467
7468 if (object) {
7469 vm_object_unlock(object);
7470 }
7471
7472 return discard;
7473 }
7474
7475
7476 static void
hibernate_discard_page(vm_page_t m)7477 hibernate_discard_page(vm_page_t m)
7478 {
7479 vm_object_t m_object;
7480
7481 if (m->vmp_absent || m->vmp_unusual || VMP_ERROR_GET(m)) {
7482 /*
7483 * If it's unusual in anyway, ignore
7484 */
7485 return;
7486 }
7487
7488 m_object = VM_PAGE_OBJECT(m);
7489
7490 #if MACH_ASSERT || DEBUG
7491 if (!vm_object_lock_try(m_object)) {
7492 panic("hibernate_discard_page(%p) !vm_object_lock_try", m);
7493 }
7494 #else
7495 /* No need to lock page queue for token delete, hibernate_vm_unlock()
7496 * makes sure these locks are uncontended before sleep */
7497 #endif /* MACH_ASSERT || DEBUG */
7498
7499 if (m->vmp_pmapped == TRUE) {
7500 __unused int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
7501 }
7502
7503 if (m->vmp_laundry) {
7504 panic("hibernate_discard_page(%p) laundry", m);
7505 }
7506 if (m->vmp_private) {
7507 panic("hibernate_discard_page(%p) private", m);
7508 }
7509 if (m->vmp_fictitious) {
7510 panic("hibernate_discard_page(%p) fictitious", m);
7511 }
7512
7513 if (VM_PURGABLE_VOLATILE == m_object->purgable) {
7514 /* object should be on a queue */
7515 assert((m_object->objq.next != NULL) && (m_object->objq.prev != NULL));
7516 purgeable_q_t old_queue = vm_purgeable_object_remove(m_object);
7517 assert(old_queue);
7518 if (m_object->purgeable_when_ripe) {
7519 vm_purgeable_token_delete_first(old_queue);
7520 }
7521 vm_object_lock_assert_exclusive(m_object);
7522 m_object->purgable = VM_PURGABLE_EMPTY;
7523
7524 /*
7525 * Purgeable ledgers: pages of VOLATILE and EMPTY objects are
7526 * accounted in the "volatile" ledger, so no change here.
7527 * We have to update vm_page_purgeable_count, though, since we're
7528 * effectively purging this object.
7529 */
7530 unsigned int delta;
7531 assert(m_object->resident_page_count >= m_object->wired_page_count);
7532 delta = (m_object->resident_page_count - m_object->wired_page_count);
7533 assert(vm_page_purgeable_count >= delta);
7534 assert(delta > 0);
7535 OSAddAtomic(-delta, (SInt32 *)&vm_page_purgeable_count);
7536 }
7537
7538 vm_page_free(m);
7539
7540 #if MACH_ASSERT || DEBUG
7541 vm_object_unlock(m_object);
7542 #endif /* MACH_ASSERT || DEBUG */
7543 }
7544
7545 /*
7546 * Grab locks for hibernate_page_list_setall()
7547 */
7548 void
hibernate_vm_lock_queues(void)7549 hibernate_vm_lock_queues(void)
7550 {
7551 vm_object_lock(compressor_object);
7552 vm_page_lock_queues();
7553 vm_free_page_lock();
7554 lck_mtx_lock(&vm_purgeable_queue_lock);
7555
7556 if (vm_page_local_q) {
7557 zpercpu_foreach(lq, vm_page_local_q) {
7558 VPL_LOCK(&lq->vpl_lock);
7559 }
7560 }
7561 }
7562
7563 void
hibernate_vm_unlock_queues(void)7564 hibernate_vm_unlock_queues(void)
7565 {
7566 if (vm_page_local_q) {
7567 zpercpu_foreach(lq, vm_page_local_q) {
7568 VPL_UNLOCK(&lq->vpl_lock);
7569 }
7570 }
7571 lck_mtx_unlock(&vm_purgeable_queue_lock);
7572 vm_free_page_unlock();
7573 vm_page_unlock_queues();
7574 vm_object_unlock(compressor_object);
7575 }
7576
7577 /*
7578 * Bits zero in the bitmaps => page needs to be saved. All pages default to be saved,
7579 * pages known to VM to not need saving are subtracted.
7580 * Wired pages to be saved are present in page_list_wired, pageable in page_list.
7581 */
7582
7583 void
hibernate_page_list_setall(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired,hibernate_page_list_t * page_list_pal,boolean_t preflight,boolean_t will_discard,uint32_t * pagesOut)7584 hibernate_page_list_setall(hibernate_page_list_t * page_list,
7585 hibernate_page_list_t * page_list_wired,
7586 hibernate_page_list_t * page_list_pal,
7587 boolean_t preflight,
7588 boolean_t will_discard,
7589 uint32_t * pagesOut)
7590 {
7591 uint64_t start, end, nsec;
7592 vm_page_t m;
7593 vm_page_t next;
7594 uint32_t pages = page_list->page_count;
7595 uint32_t count_anonymous = 0, count_throttled = 0, count_compressor = 0;
7596 uint32_t count_inactive = 0, count_active = 0, count_speculative = 0, count_cleaned = 0;
7597 uint32_t count_wire = pages;
7598 uint32_t count_discard_active = 0;
7599 uint32_t count_discard_inactive = 0;
7600 uint32_t count_retired = 0;
7601 uint32_t count_discard_cleaned = 0;
7602 uint32_t count_discard_purgeable = 0;
7603 uint32_t count_discard_speculative = 0;
7604 uint32_t count_discard_vm_struct_pages = 0;
7605 uint32_t i;
7606 uint32_t bank;
7607 hibernate_bitmap_t * bitmap;
7608 hibernate_bitmap_t * bitmap_wired;
7609 boolean_t discard_all;
7610 boolean_t discard = FALSE;
7611
7612 HIBLOG("hibernate_page_list_setall(preflight %d) start\n", preflight);
7613
7614 if (preflight) {
7615 page_list = NULL;
7616 page_list_wired = NULL;
7617 page_list_pal = NULL;
7618 discard_all = FALSE;
7619 } else {
7620 discard_all = will_discard;
7621 }
7622
7623 #if MACH_ASSERT || DEBUG
7624 if (!preflight) {
7625 assert(hibernate_vm_locks_are_safe());
7626 vm_page_lock_queues();
7627 if (vm_page_local_q) {
7628 zpercpu_foreach(lq, vm_page_local_q) {
7629 VPL_LOCK(&lq->vpl_lock);
7630 }
7631 }
7632 }
7633 #endif /* MACH_ASSERT || DEBUG */
7634
7635
7636 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_START, count_wire, 0, 0, 0, 0);
7637
7638 clock_get_uptime(&start);
7639
7640 if (!preflight) {
7641 hibernate_page_list_zero(page_list);
7642 hibernate_page_list_zero(page_list_wired);
7643 hibernate_page_list_zero(page_list_pal);
7644
7645 hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count;
7646 hibernate_stats.cd_pages = pages;
7647 }
7648
7649 if (vm_page_local_q) {
7650 zpercpu_foreach_cpu(lid) {
7651 vm_page_reactivate_local(lid, TRUE, !preflight);
7652 }
7653 }
7654
7655 if (preflight) {
7656 vm_object_lock(compressor_object);
7657 vm_page_lock_queues();
7658 vm_free_page_lock();
7659 }
7660
7661 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
7662
7663 hibernation_vmqueues_inspection = TRUE;
7664
7665 m = (vm_page_t) hibernate_gobble_queue;
7666 while (m) {
7667 pages--;
7668 count_wire--;
7669 if (!preflight) {
7670 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7671 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7672 }
7673 m = m->vmp_snext;
7674 }
7675
7676 if (!preflight) {
7677 percpu_foreach(free_pages_head, free_pages) {
7678 for (m = *free_pages_head; m; m = m->vmp_snext) {
7679 assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
7680
7681 pages--;
7682 count_wire--;
7683 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7684 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7685
7686 hibernate_stats.cd_local_free++;
7687 hibernate_stats.cd_total_free++;
7688 }
7689 }
7690 }
7691
7692 for (i = 0; i < vm_colors; i++) {
7693 vm_page_queue_iterate(&vm_page_queue_free[i].qhead, m, vmp_pageq) {
7694 assert(m->vmp_q_state == VM_PAGE_ON_FREE_Q);
7695
7696 pages--;
7697 count_wire--;
7698 if (!preflight) {
7699 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7700 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7701
7702 hibernate_stats.cd_total_free++;
7703 }
7704 }
7705 }
7706
7707 vm_page_queue_iterate(&vm_lopage_queue_free, m, vmp_pageq) {
7708 assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);
7709
7710 pages--;
7711 count_wire--;
7712 if (!preflight) {
7713 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7714 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7715
7716 hibernate_stats.cd_total_free++;
7717 }
7718 }
7719
7720 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
7721 while (m && !vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t)m)) {
7722 assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
7723
7724 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7725 discard = FALSE;
7726 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode)
7727 && hibernate_consider_discard(m, preflight)) {
7728 if (!preflight) {
7729 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7730 }
7731 count_discard_inactive++;
7732 discard = discard_all;
7733 } else {
7734 count_throttled++;
7735 }
7736 count_wire--;
7737 if (!preflight) {
7738 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7739 }
7740
7741 if (discard) {
7742 hibernate_discard_page(m);
7743 }
7744 m = next;
7745 }
7746
7747 m = (vm_page_t)vm_page_queue_first(&vm_page_queue_anonymous);
7748 while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
7749 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
7750
7751 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7752 discard = FALSE;
7753 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
7754 hibernate_consider_discard(m, preflight)) {
7755 if (!preflight) {
7756 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7757 }
7758 if (m->vmp_dirty) {
7759 count_discard_purgeable++;
7760 } else {
7761 count_discard_inactive++;
7762 }
7763 discard = discard_all;
7764 } else {
7765 count_anonymous++;
7766 }
7767 count_wire--;
7768 if (!preflight) {
7769 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7770 }
7771 if (discard) {
7772 hibernate_discard_page(m);
7773 }
7774 m = next;
7775 }
7776
7777 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
7778 while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
7779 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
7780
7781 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7782 discard = FALSE;
7783 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
7784 hibernate_consider_discard(m, preflight)) {
7785 if (!preflight) {
7786 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7787 }
7788 if (m->vmp_dirty) {
7789 count_discard_purgeable++;
7790 } else {
7791 count_discard_cleaned++;
7792 }
7793 discard = discard_all;
7794 } else {
7795 count_cleaned++;
7796 }
7797 count_wire--;
7798 if (!preflight) {
7799 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7800 }
7801 if (discard) {
7802 hibernate_discard_page(m);
7803 }
7804 m = next;
7805 }
7806
7807 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
7808 while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
7809 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
7810
7811 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7812 discard = FALSE;
7813 if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) &&
7814 hibernate_consider_discard(m, preflight)) {
7815 if (!preflight) {
7816 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7817 }
7818 if (m->vmp_dirty) {
7819 count_discard_purgeable++;
7820 } else {
7821 count_discard_active++;
7822 }
7823 discard = discard_all;
7824 } else {
7825 count_active++;
7826 }
7827 count_wire--;
7828 if (!preflight) {
7829 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7830 }
7831 if (discard) {
7832 hibernate_discard_page(m);
7833 }
7834 m = next;
7835 }
7836
7837 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
7838 while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
7839 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
7840
7841 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7842 discard = FALSE;
7843 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
7844 hibernate_consider_discard(m, preflight)) {
7845 if (!preflight) {
7846 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7847 }
7848 if (m->vmp_dirty) {
7849 count_discard_purgeable++;
7850 } else {
7851 count_discard_inactive++;
7852 }
7853 discard = discard_all;
7854 } else {
7855 count_inactive++;
7856 }
7857 count_wire--;
7858 if (!preflight) {
7859 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7860 }
7861 if (discard) {
7862 hibernate_discard_page(m);
7863 }
7864 m = next;
7865 }
7866 /* XXX FBDP TODO: secluded queue */
7867
7868 for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
7869 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
7870 while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
7871 assertf(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q,
7872 "Bad page: %p (0x%x:0x%x) on queue %d has state: %d (Discard: %d, Preflight: %d)",
7873 m, m->vmp_pageq.next, m->vmp_pageq.prev, i, m->vmp_q_state, discard, preflight);
7874
7875 next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
7876 discard = FALSE;
7877 if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
7878 hibernate_consider_discard(m, preflight)) {
7879 if (!preflight) {
7880 hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7881 }
7882 count_discard_speculative++;
7883 discard = discard_all;
7884 } else {
7885 count_speculative++;
7886 }
7887 count_wire--;
7888 if (!preflight) {
7889 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7890 }
7891 if (discard) {
7892 hibernate_discard_page(m);
7893 }
7894 m = next;
7895 }
7896 }
7897
7898 vm_page_queue_iterate(&compressor_object->memq, m, vmp_listq) {
7899 assert(m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR);
7900
7901 count_compressor++;
7902 count_wire--;
7903 if (!preflight) {
7904 hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
7905 }
7906 }
7907
7908
7909 if (preflight == FALSE && discard_all == TRUE) {
7910 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START);
7911
7912 HIBLOG("hibernate_teardown started\n");
7913 count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired);
7914 HIBLOG("hibernate_teardown completed - discarded %d\n", count_discard_vm_struct_pages);
7915
7916 pages -= count_discard_vm_struct_pages;
7917 count_wire -= count_discard_vm_struct_pages;
7918
7919 hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages;
7920
7921 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_END);
7922 }
7923
7924 if (!preflight) {
7925 // pull wired from hibernate_bitmap
7926 bitmap = &page_list->bank_bitmap[0];
7927 bitmap_wired = &page_list_wired->bank_bitmap[0];
7928 for (bank = 0; bank < page_list->bank_count; bank++) {
7929 for (i = 0; i < bitmap->bitmapwords; i++) {
7930 bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
7931 }
7932 bitmap = (hibernate_bitmap_t *)&bitmap->bitmap[bitmap->bitmapwords];
7933 bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
7934 }
7935 }
7936
7937 // machine dependent adjustments
7938 hibernate_page_list_setall_machine(page_list, page_list_wired, preflight, &pages);
7939
7940 if (!preflight) {
7941 hibernate_stats.cd_count_wire = count_wire;
7942 hibernate_stats.cd_discarded = count_discard_active + count_discard_inactive + count_discard_purgeable +
7943 count_discard_speculative + count_discard_cleaned + count_discard_vm_struct_pages;
7944 }
7945
7946 clock_get_uptime(&end);
7947 absolutetime_to_nanoseconds(end - start, &nsec);
7948 HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
7949
7950 HIBLOG("pages %d, wire %d, act %d, inact %d, cleaned %d spec %d, zf %d, throt %d, compr %d, xpmapped %d\n %s discard act %d inact %d purgeable %d spec %d cleaned %d retired %d\n",
7951 pages, count_wire, count_active, count_inactive, count_cleaned, count_speculative, count_anonymous, count_throttled, count_compressor, hibernate_stats.cd_found_xpmapped,
7952 discard_all ? "did" : "could",
7953 count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned, count_retired);
7954
7955 if (hibernate_stats.cd_skipped_xpmapped) {
7956 HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n", hibernate_stats.cd_skipped_xpmapped);
7957 }
7958
7959 *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned - count_retired;
7960
7961 if (preflight && will_discard) {
7962 *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active;
7963 /*
7964 * We try to keep max HIBERNATE_XPMAPPED_LIMIT pages around in the hibernation image
7965 * even if these are clean and so we need to size the hibernation image accordingly.
7966 *
7967 * NB: We have to assume all HIBERNATE_XPMAPPED_LIMIT pages might show up because 'dirty'
7968 * xpmapped pages aren't distinguishable from other 'dirty' pages in preflight. So we might
7969 * only see part of the xpmapped pages if we look at 'cd_found_xpmapped' which solely tracks
7970 * clean xpmapped pages.
7971 *
7972 * Since these pages are all cleaned by the time we are in the post-preflight phase, we might
7973 * see a much larger number in 'cd_found_xpmapped' now than we did in the preflight phase
7974 */
7975 *pagesOut += HIBERNATE_XPMAPPED_LIMIT;
7976 }
7977
7978 hibernation_vmqueues_inspection = FALSE;
7979
7980 #if MACH_ASSERT || DEBUG
7981 if (!preflight) {
7982 if (vm_page_local_q) {
7983 zpercpu_foreach(lq, vm_page_local_q) {
7984 VPL_UNLOCK(&lq->vpl_lock);
7985 }
7986 }
7987 vm_page_unlock_queues();
7988 }
7989 #endif /* MACH_ASSERT || DEBUG */
7990
7991 if (preflight) {
7992 vm_free_page_unlock();
7993 vm_page_unlock_queues();
7994 vm_object_unlock(compressor_object);
7995 }
7996
7997 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_END, count_wire, *pagesOut, 0, 0, 0);
7998 }
7999
8000 void
hibernate_page_list_discard(hibernate_page_list_t * page_list)8001 hibernate_page_list_discard(hibernate_page_list_t * page_list)
8002 {
8003 uint64_t start, end, nsec;
8004 vm_page_t m;
8005 vm_page_t next;
8006 uint32_t i;
8007 uint32_t count_discard_active = 0;
8008 uint32_t count_discard_inactive = 0;
8009 uint32_t count_discard_purgeable = 0;
8010 uint32_t count_discard_cleaned = 0;
8011 uint32_t count_discard_speculative = 0;
8012
8013
8014 #if MACH_ASSERT || DEBUG
8015 vm_page_lock_queues();
8016 if (vm_page_local_q) {
8017 zpercpu_foreach(lq, vm_page_local_q) {
8018 VPL_LOCK(&lq->vpl_lock);
8019 }
8020 }
8021 #endif /* MACH_ASSERT || DEBUG */
8022
8023 clock_get_uptime(&start);
8024
8025 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
8026 while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
8027 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
8028
8029 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8030 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
8031 if (m->vmp_dirty) {
8032 count_discard_purgeable++;
8033 } else {
8034 count_discard_inactive++;
8035 }
8036 hibernate_discard_page(m);
8037 }
8038 m = next;
8039 }
8040
8041 for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
8042 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
8043 while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
8044 assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
8045
8046 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8047 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
8048 count_discard_speculative++;
8049 hibernate_discard_page(m);
8050 }
8051 m = next;
8052 }
8053 }
8054
8055 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
8056 while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
8057 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
8058
8059 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8060 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
8061 if (m->vmp_dirty) {
8062 count_discard_purgeable++;
8063 } else {
8064 count_discard_inactive++;
8065 }
8066 hibernate_discard_page(m);
8067 }
8068 m = next;
8069 }
8070 /* XXX FBDP TODO: secluded queue */
8071
8072 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
8073 while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
8074 assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
8075
8076 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8077 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
8078 if (m->vmp_dirty) {
8079 count_discard_purgeable++;
8080 } else {
8081 count_discard_active++;
8082 }
8083 hibernate_discard_page(m);
8084 }
8085 m = next;
8086 }
8087
8088 m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
8089 while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
8090 assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
8091
8092 next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
8093 if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
8094 if (m->vmp_dirty) {
8095 count_discard_purgeable++;
8096 } else {
8097 count_discard_cleaned++;
8098 }
8099 hibernate_discard_page(m);
8100 }
8101 m = next;
8102 }
8103
8104 #if MACH_ASSERT || DEBUG
8105 if (vm_page_local_q) {
8106 zpercpu_foreach(lq, vm_page_local_q) {
8107 VPL_UNLOCK(&lq->vpl_lock);
8108 }
8109 }
8110 vm_page_unlock_queues();
8111 #endif /* MACH_ASSERT || DEBUG */
8112
8113 clock_get_uptime(&end);
8114 absolutetime_to_nanoseconds(end - start, &nsec);
8115 HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d cleaned %d\n",
8116 nsec / 1000000ULL,
8117 count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
8118 }
8119
8120 boolean_t hibernate_paddr_map_inited = FALSE;
8121 unsigned int hibernate_teardown_last_valid_compact_indx = -1;
8122 vm_page_t hibernate_rebuild_hash_list = NULL;
8123
8124 unsigned int hibernate_teardown_found_tabled_pages = 0;
8125 unsigned int hibernate_teardown_found_created_pages = 0;
8126 unsigned int hibernate_teardown_found_free_pages = 0;
8127 unsigned int hibernate_teardown_vm_page_free_count;
8128
8129
8130 struct ppnum_mapping {
8131 struct ppnum_mapping *ppnm_next;
8132 ppnum_t ppnm_base_paddr;
8133 unsigned int ppnm_sindx;
8134 unsigned int ppnm_eindx;
8135 };
8136
8137 struct ppnum_mapping *ppnm_head;
8138 struct ppnum_mapping *ppnm_last_found = NULL;
8139
8140
8141 void
hibernate_create_paddr_map(void)8142 hibernate_create_paddr_map(void)
8143 {
8144 unsigned int i;
8145 ppnum_t next_ppnum_in_run = 0;
8146 struct ppnum_mapping *ppnm = NULL;
8147
8148 if (hibernate_paddr_map_inited == FALSE) {
8149 for (i = 0; i < vm_pages_count; i++) {
8150 if (ppnm) {
8151 ppnm->ppnm_eindx = i;
8152 }
8153
8154 if (ppnm == NULL || VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]) != next_ppnum_in_run) {
8155 ppnm = zalloc_permanent_type(struct ppnum_mapping);
8156
8157 ppnm->ppnm_next = ppnm_head;
8158 ppnm_head = ppnm;
8159
8160 ppnm->ppnm_sindx = i;
8161 ppnm->ppnm_base_paddr = VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]);
8162 }
8163 next_ppnum_in_run = VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]) + 1;
8164 }
8165 ppnm->ppnm_eindx = vm_pages_count;
8166
8167 hibernate_paddr_map_inited = TRUE;
8168 }
8169 }
8170
8171 ppnum_t
hibernate_lookup_paddr(unsigned int indx)8172 hibernate_lookup_paddr(unsigned int indx)
8173 {
8174 struct ppnum_mapping *ppnm = NULL;
8175
8176 ppnm = ppnm_last_found;
8177
8178 if (ppnm) {
8179 if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
8180 goto done;
8181 }
8182 }
8183 for (ppnm = ppnm_head; ppnm; ppnm = ppnm->ppnm_next) {
8184 if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
8185 ppnm_last_found = ppnm;
8186 break;
8187 }
8188 }
8189 if (ppnm == NULL) {
8190 panic("hibernate_lookup_paddr of %d failed", indx);
8191 }
8192 done:
8193 return ppnm->ppnm_base_paddr + (indx - ppnm->ppnm_sindx);
8194 }
8195
8196
8197 uint32_t
hibernate_mark_as_unneeded(addr64_t saddr,addr64_t eaddr,hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired)8198 hibernate_mark_as_unneeded(addr64_t saddr, addr64_t eaddr, hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
8199 {
8200 addr64_t saddr_aligned;
8201 addr64_t eaddr_aligned;
8202 addr64_t addr;
8203 ppnum_t paddr;
8204 unsigned int mark_as_unneeded_pages = 0;
8205
8206 saddr_aligned = (saddr + PAGE_MASK_64) & ~PAGE_MASK_64;
8207 eaddr_aligned = eaddr & ~PAGE_MASK_64;
8208
8209 for (addr = saddr_aligned; addr < eaddr_aligned; addr += PAGE_SIZE_64) {
8210 paddr = pmap_find_phys(kernel_pmap, addr);
8211
8212 assert(paddr);
8213
8214 hibernate_page_bitset(page_list, TRUE, paddr);
8215 hibernate_page_bitset(page_list_wired, TRUE, paddr);
8216
8217 mark_as_unneeded_pages++;
8218 }
8219 return mark_as_unneeded_pages;
8220 }
8221
8222
8223 void
hibernate_hash_insert_page(vm_page_t mem)8224 hibernate_hash_insert_page(vm_page_t mem)
8225 {
8226 vm_page_bucket_t *bucket;
8227 int hash_id;
8228 vm_object_t m_object;
8229
8230 m_object = VM_PAGE_OBJECT(mem);
8231
8232 assert(mem->vmp_hashed);
8233 assert(m_object);
8234 assert(mem->vmp_offset != (vm_object_offset_t) -1);
8235
8236 /*
8237 * Insert it into the object_object/offset hash table
8238 */
8239 hash_id = vm_page_hash(m_object, mem->vmp_offset);
8240 bucket = &vm_page_buckets[hash_id];
8241
8242 mem->vmp_next_m = bucket->page_list;
8243 bucket->page_list = VM_PAGE_PACK_PTR(mem);
8244 }
8245
8246
8247 void
hibernate_free_range(int sindx,int eindx)8248 hibernate_free_range(int sindx, int eindx)
8249 {
8250 vm_page_t mem;
8251 unsigned int color;
8252
8253 while (sindx < eindx) {
8254 mem = &vm_pages[sindx];
8255
8256 vm_page_init(mem, hibernate_lookup_paddr(sindx), FALSE);
8257
8258 mem->vmp_lopage = FALSE;
8259 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
8260
8261 color = VM_PAGE_GET_COLOR(mem);
8262 #if defined(__x86_64__)
8263 vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);
8264 #else
8265 vm_page_queue_enter(&vm_page_queue_free[color].qhead, mem, vmp_pageq);
8266 #endif
8267 vm_page_free_count++;
8268
8269 sindx++;
8270 }
8271 }
8272
8273 void
hibernate_rebuild_vm_structs(void)8274 hibernate_rebuild_vm_structs(void)
8275 {
8276 int i, cindx, sindx, eindx;
8277 vm_page_t mem, tmem, mem_next;
8278 AbsoluteTime startTime, endTime;
8279 uint64_t nsec;
8280
8281 if (hibernate_rebuild_needed == FALSE) {
8282 return;
8283 }
8284
8285 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START);
8286 HIBLOG("hibernate_rebuild started\n");
8287
8288 clock_get_uptime(&startTime);
8289
8290 pal_hib_rebuild_pmap_structs();
8291
8292 bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t));
8293 eindx = vm_pages_count;
8294
8295 /*
8296 * Mark all the vm_pages[] that have not been initialized yet as being
8297 * transient. This is needed to ensure that buddy page search is corrrect.
8298 * Without this random data in these vm_pages[] can trip the buddy search
8299 */
8300 for (i = hibernate_teardown_last_valid_compact_indx + 1; i < eindx; ++i) {
8301 vm_pages[i].vmp_q_state = VM_PAGE_NOT_ON_Q;
8302 }
8303
8304 for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) {
8305 mem = &vm_pages[cindx];
8306 assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
8307 /*
8308 * hibernate_teardown_vm_structs leaves the location where
8309 * this vm_page_t must be located in "next".
8310 */
8311 tmem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
8312 mem->vmp_next_m = VM_PAGE_PACK_PTR(NULL);
8313
8314 sindx = (int)(tmem - &vm_pages[0]);
8315
8316 if (mem != tmem) {
8317 /*
8318 * this vm_page_t was moved by hibernate_teardown_vm_structs,
8319 * so move it back to its real location
8320 */
8321 *tmem = *mem;
8322 mem = tmem;
8323 }
8324 if (mem->vmp_hashed) {
8325 hibernate_hash_insert_page(mem);
8326 }
8327 /*
8328 * the 'hole' between this vm_page_t and the previous
8329 * vm_page_t we moved needs to be initialized as
8330 * a range of free vm_page_t's
8331 */
8332 hibernate_free_range(sindx + 1, eindx);
8333
8334 eindx = sindx;
8335 }
8336 if (sindx) {
8337 hibernate_free_range(0, sindx);
8338 }
8339
8340 assert(vm_page_free_count == hibernate_teardown_vm_page_free_count);
8341
8342 /*
8343 * process the list of vm_page_t's that were entered in the hash,
8344 * but were not located in the vm_pages arrary... these are
8345 * vm_page_t's that were created on the fly (i.e. fictitious)
8346 */
8347 for (mem = hibernate_rebuild_hash_list; mem; mem = mem_next) {
8348 mem_next = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
8349
8350 mem->vmp_next_m = 0;
8351 hibernate_hash_insert_page(mem);
8352 }
8353 hibernate_rebuild_hash_list = NULL;
8354
8355 clock_get_uptime(&endTime);
8356 SUB_ABSOLUTETIME(&endTime, &startTime);
8357 absolutetime_to_nanoseconds(endTime, &nsec);
8358
8359 HIBLOG("hibernate_rebuild completed - took %qd msecs\n", nsec / 1000000ULL);
8360
8361 hibernate_rebuild_needed = FALSE;
8362
8363 KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END);
8364 }
8365
8366 uint32_t
hibernate_teardown_vm_structs(hibernate_page_list_t * page_list,hibernate_page_list_t * page_list_wired)8367 hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
8368 {
8369 unsigned int i;
8370 unsigned int compact_target_indx;
8371 vm_page_t mem, mem_next;
8372 vm_page_bucket_t *bucket;
8373 unsigned int mark_as_unneeded_pages = 0;
8374 unsigned int unneeded_vm_page_bucket_pages = 0;
8375 unsigned int unneeded_vm_pages_pages = 0;
8376 unsigned int unneeded_pmap_pages = 0;
8377 addr64_t start_of_unneeded = 0;
8378 addr64_t end_of_unneeded = 0;
8379
8380
8381 if (hibernate_should_abort()) {
8382 return 0;
8383 }
8384
8385 hibernate_rebuild_needed = TRUE;
8386
8387 HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, active_pages %d, inactive_pages %d, speculative_pages %d, cleaned_pages %d, compressor_pages %d\n",
8388 vm_page_wire_count, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count,
8389 vm_page_cleaned_count, compressor_object->resident_page_count);
8390
8391 for (i = 0; i < vm_page_bucket_count; i++) {
8392 bucket = &vm_page_buckets[i];
8393
8394 for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)); mem != VM_PAGE_NULL; mem = mem_next) {
8395 assert(mem->vmp_hashed);
8396
8397 mem_next = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m));
8398
8399 if (mem < &vm_pages[0] || mem >= &vm_pages[vm_pages_count]) {
8400 mem->vmp_next_m = VM_PAGE_PACK_PTR(hibernate_rebuild_hash_list);
8401 hibernate_rebuild_hash_list = mem;
8402 }
8403 }
8404 }
8405 unneeded_vm_page_bucket_pages = hibernate_mark_as_unneeded((addr64_t)&vm_page_buckets[0], (addr64_t)&vm_page_buckets[vm_page_bucket_count], page_list, page_list_wired);
8406 mark_as_unneeded_pages += unneeded_vm_page_bucket_pages;
8407
8408 hibernate_teardown_vm_page_free_count = vm_page_free_count;
8409
8410 compact_target_indx = 0;
8411
8412 for (i = 0; i < vm_pages_count; i++) {
8413 mem = &vm_pages[i];
8414
8415 if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
8416 unsigned int color;
8417
8418 assert(mem->vmp_busy);
8419 assert(!mem->vmp_lopage);
8420
8421 color = VM_PAGE_GET_COLOR(mem);
8422
8423 vm_page_queue_remove(&vm_page_queue_free[color].qhead, mem, vmp_pageq);
8424
8425 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
8426
8427 vm_page_free_count--;
8428
8429 hibernate_teardown_found_free_pages++;
8430
8431 if (vm_pages[compact_target_indx].vmp_q_state != VM_PAGE_ON_FREE_Q) {
8432 compact_target_indx = i;
8433 }
8434 } else {
8435 /*
8436 * record this vm_page_t's original location
8437 * we need this even if it doesn't get moved
8438 * as an indicator to the rebuild function that
8439 * we don't have to move it
8440 */
8441 mem->vmp_next_m = VM_PAGE_PACK_PTR(mem);
8442
8443 if (vm_pages[compact_target_indx].vmp_q_state == VM_PAGE_ON_FREE_Q) {
8444 /*
8445 * we've got a hole to fill, so
8446 * move this vm_page_t to it's new home
8447 */
8448 vm_pages[compact_target_indx] = *mem;
8449 mem->vmp_q_state = VM_PAGE_ON_FREE_Q;
8450
8451 hibernate_teardown_last_valid_compact_indx = compact_target_indx;
8452 compact_target_indx++;
8453 } else {
8454 hibernate_teardown_last_valid_compact_indx = i;
8455 }
8456 }
8457 }
8458 unneeded_vm_pages_pages = hibernate_mark_as_unneeded((addr64_t)&vm_pages[hibernate_teardown_last_valid_compact_indx + 1],
8459 (addr64_t)&vm_pages[vm_pages_count - 1], page_list, page_list_wired);
8460 mark_as_unneeded_pages += unneeded_vm_pages_pages;
8461
8462 pal_hib_teardown_pmap_structs(&start_of_unneeded, &end_of_unneeded);
8463
8464 if (start_of_unneeded) {
8465 unneeded_pmap_pages = hibernate_mark_as_unneeded(start_of_unneeded, end_of_unneeded, page_list, page_list_wired);
8466 mark_as_unneeded_pages += unneeded_pmap_pages;
8467 }
8468 HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n", unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages);
8469
8470 return mark_as_unneeded_pages;
8471 }
8472
8473
8474 #endif /* HIBERNATION */
8475
8476 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
8477
8478 #include <mach_vm_debug.h>
8479 #if MACH_VM_DEBUG
8480
8481 #include <mach_debug/hash_info.h>
8482 #include <vm/vm_debug.h>
8483
8484 /*
8485 * Routine: vm_page_info
8486 * Purpose:
8487 * Return information about the global VP table.
8488 * Fills the buffer with as much information as possible
8489 * and returns the desired size of the buffer.
8490 * Conditions:
8491 * Nothing locked. The caller should provide
8492 * possibly-pageable memory.
8493 */
8494
8495 unsigned int
vm_page_info(hash_info_bucket_t * info,unsigned int count)8496 vm_page_info(
8497 hash_info_bucket_t *info,
8498 unsigned int count)
8499 {
8500 unsigned int i;
8501 lck_spin_t *bucket_lock;
8502
8503 if (vm_page_bucket_count < count) {
8504 count = vm_page_bucket_count;
8505 }
8506
8507 for (i = 0; i < count; i++) {
8508 vm_page_bucket_t *bucket = &vm_page_buckets[i];
8509 unsigned int bucket_count = 0;
8510 vm_page_t m;
8511
8512 bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
8513 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
8514
8515 for (m = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
8516 m != VM_PAGE_NULL;
8517 m = (vm_page_t)(VM_PAGE_UNPACK_PTR(m->vmp_next_m))) {
8518 bucket_count++;
8519 }
8520
8521 lck_spin_unlock(bucket_lock);
8522
8523 /* don't touch pageable memory while holding locks */
8524 info[i].hib_count = bucket_count;
8525 }
8526
8527 return vm_page_bucket_count;
8528 }
8529 #endif /* MACH_VM_DEBUG */
8530
8531 #if VM_PAGE_BUCKETS_CHECK
8532 void
vm_page_buckets_check(void)8533 vm_page_buckets_check(void)
8534 {
8535 unsigned int i;
8536 vm_page_t p;
8537 unsigned int p_hash;
8538 vm_page_bucket_t *bucket;
8539 lck_spin_t *bucket_lock;
8540
8541 if (!vm_page_buckets_check_ready) {
8542 return;
8543 }
8544
8545 #if HIBERNATION
8546 if (hibernate_rebuild_needed ||
8547 hibernate_rebuild_hash_list) {
8548 panic("BUCKET_CHECK: hibernation in progress: "
8549 "rebuild_needed=%d rebuild_hash_list=%p\n",
8550 hibernate_rebuild_needed,
8551 hibernate_rebuild_hash_list);
8552 }
8553 #endif /* HIBERNATION */
8554
8555 #if VM_PAGE_FAKE_BUCKETS
8556 char *cp;
8557 for (cp = (char *) vm_page_fake_buckets_start;
8558 cp < (char *) vm_page_fake_buckets_end;
8559 cp++) {
8560 if (*cp != 0x5a) {
8561 panic("BUCKET_CHECK: corruption at %p in fake buckets "
8562 "[0x%llx:0x%llx]\n",
8563 cp,
8564 (uint64_t) vm_page_fake_buckets_start,
8565 (uint64_t) vm_page_fake_buckets_end);
8566 }
8567 }
8568 #endif /* VM_PAGE_FAKE_BUCKETS */
8569
8570 for (i = 0; i < vm_page_bucket_count; i++) {
8571 vm_object_t p_object;
8572
8573 bucket = &vm_page_buckets[i];
8574 if (!bucket->page_list) {
8575 continue;
8576 }
8577
8578 bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
8579 lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);
8580 p = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
8581
8582 while (p != VM_PAGE_NULL) {
8583 p_object = VM_PAGE_OBJECT(p);
8584
8585 if (!p->vmp_hashed) {
8586 panic("BUCKET_CHECK: page %p (%p,0x%llx) "
8587 "hash %d in bucket %d at %p "
8588 "is not hashed\n",
8589 p, p_object, p->vmp_offset,
8590 p_hash, i, bucket);
8591 }
8592 p_hash = vm_page_hash(p_object, p->vmp_offset);
8593 if (p_hash != i) {
8594 panic("BUCKET_CHECK: corruption in bucket %d "
8595 "at %p: page %p object %p offset 0x%llx "
8596 "hash %d\n",
8597 i, bucket, p, p_object, p->vmp_offset,
8598 p_hash);
8599 }
8600 p = (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m));
8601 }
8602 lck_spin_unlock(bucket_lock);
8603 }
8604
8605 // printf("BUCKET_CHECK: checked buckets\n");
8606 }
8607 #endif /* VM_PAGE_BUCKETS_CHECK */
8608
8609 /*
8610 * 'vm_fault_enter' will place newly created pages (zero-fill and COW) onto the
8611 * local queues if they exist... its the only spot in the system where we add pages
8612 * to those queues... once on those queues, those pages can only move to one of the
8613 * global page queues or the free queues... they NEVER move from local q to local q.
8614 * the 'local' state is stable when vm_page_queues_remove is called since we're behind
8615 * the global vm_page_queue_lock at this point... we still need to take the local lock
8616 * in case this operation is being run on a different CPU then the local queue's identity,
8617 * but we don't have to worry about the page moving to a global queue or becoming wired
8618 * while we're grabbing the local lock since those operations would require the global
8619 * vm_page_queue_lock to be held, and we already own it.
8620 *
8621 * this is why its safe to utilze the wire_count field in the vm_page_t as the local_id...
8622 * 'wired' and local are ALWAYS mutually exclusive conditions.
8623 */
8624
8625 void
vm_page_queues_remove(vm_page_t mem,boolean_t remove_from_specialq)8626 vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_specialq)
8627 {
8628 boolean_t was_pageable = TRUE;
8629 vm_object_t m_object;
8630
8631 m_object = VM_PAGE_OBJECT(mem);
8632
8633 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
8634
8635 if (mem->vmp_q_state == VM_PAGE_NOT_ON_Q) {
8636 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
8637 if (remove_from_specialq == TRUE) {
8638 vm_page_remove_from_specialq(mem);
8639 }
8640 /*if (mem->vmp_on_specialq != VM_PAGE_SPECIAL_Q_EMPTY) {
8641 * assert(mem->vmp_specialq.next != 0);
8642 * assert(mem->vmp_specialq.prev != 0);
8643 * } else {*/
8644 if (mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY) {
8645 assert(mem->vmp_specialq.next == 0);
8646 assert(mem->vmp_specialq.prev == 0);
8647 }
8648 return;
8649 }
8650
8651 if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
8652 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
8653 assert(mem->vmp_specialq.next == 0 &&
8654 mem->vmp_specialq.prev == 0 &&
8655 mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
8656 return;
8657 }
8658 if (mem->vmp_q_state == VM_PAGE_IS_WIRED) {
8659 /*
8660 * might put these guys on a list for debugging purposes
8661 * if we do, we'll need to remove this assert
8662 */
8663 assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
8664 assert(mem->vmp_specialq.next == 0 &&
8665 mem->vmp_specialq.prev == 0);
8666 /*
8667 * Recall that vmp_on_specialq also means a request to put
8668 * it on the special Q. So we don't want to reset that bit
8669 * just because a wiring request came in. We might want to
8670 * put it on the special queue post-unwiring.
8671 *
8672 * &&
8673 * mem->vmp_on_specialq == VM_PAGE_SPECIAL_Q_EMPTY);
8674 */
8675 return;
8676 }
8677
8678 assert(m_object != compressor_object);
8679 assert(m_object != kernel_object);
8680 assert(!mem->vmp_fictitious);
8681
8682 switch (mem->vmp_q_state) {
8683 case VM_PAGE_ON_ACTIVE_LOCAL_Q:
8684 {
8685 struct vpl *lq;
8686
8687 lq = zpercpu_get_cpu(vm_page_local_q, mem->vmp_local_id);
8688 VPL_LOCK(&lq->vpl_lock);
8689 vm_page_queue_remove(&lq->vpl_queue, mem, vmp_pageq);
8690 mem->vmp_local_id = 0;
8691 lq->vpl_count--;
8692 if (m_object->internal) {
8693 lq->vpl_internal_count--;
8694 } else {
8695 lq->vpl_external_count--;
8696 }
8697 VPL_UNLOCK(&lq->vpl_lock);
8698 was_pageable = FALSE;
8699 break;
8700 }
8701 case VM_PAGE_ON_ACTIVE_Q:
8702 {
8703 vm_page_queue_remove(&vm_page_queue_active, mem, vmp_pageq);
8704 vm_page_active_count--;
8705 break;
8706 }
8707
8708 case VM_PAGE_ON_INACTIVE_INTERNAL_Q:
8709 {
8710 assert(m_object->internal == TRUE);
8711
8712 vm_page_inactive_count--;
8713 vm_page_queue_remove(&vm_page_queue_anonymous, mem, vmp_pageq);
8714 vm_page_anonymous_count--;
8715
8716 vm_purgeable_q_advance_all();
8717 vm_page_balance_inactive(3);
8718 break;
8719 }
8720
8721 case VM_PAGE_ON_INACTIVE_EXTERNAL_Q:
8722 {
8723 assert(m_object->internal == FALSE);
8724
8725 vm_page_inactive_count--;
8726 vm_page_queue_remove(&vm_page_queue_inactive, mem, vmp_pageq);
8727 vm_purgeable_q_advance_all();
8728 vm_page_balance_inactive(3);
8729 break;
8730 }
8731
8732 case VM_PAGE_ON_INACTIVE_CLEANED_Q:
8733 {
8734 assert(m_object->internal == FALSE);
8735
8736 vm_page_inactive_count--;
8737 vm_page_queue_remove(&vm_page_queue_cleaned, mem, vmp_pageq);
8738 vm_page_cleaned_count--;
8739 vm_page_balance_inactive(3);
8740 break;
8741 }
8742
8743 case VM_PAGE_ON_THROTTLED_Q:
8744 {
8745 assert(m_object->internal == TRUE);
8746
8747 vm_page_queue_remove(&vm_page_queue_throttled, mem, vmp_pageq);
8748 vm_page_throttled_count--;
8749 was_pageable = FALSE;
8750 break;
8751 }
8752
8753 case VM_PAGE_ON_SPECULATIVE_Q:
8754 {
8755 assert(m_object->internal == FALSE);
8756
8757 vm_page_remque(&mem->vmp_pageq);
8758 vm_page_speculative_count--;
8759 vm_page_balance_inactive(3);
8760 break;
8761 }
8762
8763 #if CONFIG_SECLUDED_MEMORY
8764 case VM_PAGE_ON_SECLUDED_Q:
8765 {
8766 vm_page_queue_remove(&vm_page_queue_secluded, mem, vmp_pageq);
8767 vm_page_secluded_count--;
8768 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
8769 if (m_object == VM_OBJECT_NULL) {
8770 vm_page_secluded_count_free--;
8771 was_pageable = FALSE;
8772 } else {
8773 assert(!m_object->internal);
8774 vm_page_secluded_count_inuse--;
8775 was_pageable = FALSE;
8776 // was_pageable = TRUE;
8777 }
8778 break;
8779 }
8780 #endif /* CONFIG_SECLUDED_MEMORY */
8781
8782 default:
8783 {
8784 /*
8785 * if (mem->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)
8786 * NOTE: vm_page_queues_remove does not deal with removing pages from the pageout queue...
8787 * the caller is responsible for determing if the page is on that queue, and if so, must
8788 * either first remove it (it needs both the page queues lock and the object lock to do
8789 * this via vm_pageout_steal_laundry), or avoid the call to vm_page_queues_remove
8790 *
8791 * we also don't expect to encounter VM_PAGE_ON_FREE_Q, VM_PAGE_ON_FREE_LOCAL_Q, VM_PAGE_ON_FREE_LOPAGE_Q
8792 * or any of the undefined states
8793 */
8794 panic("vm_page_queues_remove - bad page q_state (%p, %d)", mem, mem->vmp_q_state);
8795 break;
8796 }
8797 }
8798 VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
8799 mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
8800
8801 if (remove_from_specialq == TRUE) {
8802 vm_page_remove_from_specialq(mem);
8803 }
8804 if (was_pageable) {
8805 if (m_object->internal) {
8806 vm_page_pageable_internal_count--;
8807 } else {
8808 vm_page_pageable_external_count--;
8809 }
8810 }
8811 }
8812
8813 void
vm_page_remove_internal(vm_page_t page)8814 vm_page_remove_internal(vm_page_t page)
8815 {
8816 vm_object_t __object = VM_PAGE_OBJECT(page);
8817 if (page == __object->memq_hint) {
8818 vm_page_t __new_hint;
8819 vm_page_queue_entry_t __qe;
8820 __qe = (vm_page_queue_entry_t)vm_page_queue_next(&page->vmp_listq);
8821 if (vm_page_queue_end(&__object->memq, __qe)) {
8822 __qe = (vm_page_queue_entry_t)vm_page_queue_prev(&page->vmp_listq);
8823 if (vm_page_queue_end(&__object->memq, __qe)) {
8824 __qe = NULL;
8825 }
8826 }
8827 __new_hint = (vm_page_t)((uintptr_t) __qe);
8828 __object->memq_hint = __new_hint;
8829 }
8830 vm_page_queue_remove(&__object->memq, page, vmp_listq);
8831 #if CONFIG_SECLUDED_MEMORY
8832 if (__object->eligible_for_secluded) {
8833 vm_page_secluded.eligible_for_secluded--;
8834 }
8835 #endif /* CONFIG_SECLUDED_MEMORY */
8836 }
8837
8838 void
vm_page_enqueue_inactive(vm_page_t mem,boolean_t first)8839 vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
8840 {
8841 vm_object_t m_object;
8842
8843 m_object = VM_PAGE_OBJECT(mem);
8844
8845 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
8846 assert(!mem->vmp_fictitious);
8847 assert(!mem->vmp_laundry);
8848 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
8849 vm_page_check_pageable_safe(mem);
8850
8851 if (m_object->internal) {
8852 mem->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
8853
8854 if (first == TRUE) {
8855 vm_page_queue_enter_first(&vm_page_queue_anonymous, mem, vmp_pageq);
8856 } else {
8857 vm_page_queue_enter(&vm_page_queue_anonymous, mem, vmp_pageq);
8858 }
8859
8860 vm_page_anonymous_count++;
8861 vm_page_pageable_internal_count++;
8862 } else {
8863 mem->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
8864
8865 if (first == TRUE) {
8866 vm_page_queue_enter_first(&vm_page_queue_inactive, mem, vmp_pageq);
8867 } else {
8868 vm_page_queue_enter(&vm_page_queue_inactive, mem, vmp_pageq);
8869 }
8870
8871 vm_page_pageable_external_count++;
8872 }
8873 vm_page_inactive_count++;
8874 token_new_pagecount++;
8875
8876 vm_page_add_to_specialq(mem, FALSE);
8877 }
8878
8879 void
vm_page_enqueue_active(vm_page_t mem,boolean_t first)8880 vm_page_enqueue_active(vm_page_t mem, boolean_t first)
8881 {
8882 vm_object_t m_object;
8883
8884 m_object = VM_PAGE_OBJECT(mem);
8885
8886 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
8887 assert(!mem->vmp_fictitious);
8888 assert(!mem->vmp_laundry);
8889 assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
8890 vm_page_check_pageable_safe(mem);
8891
8892 mem->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
8893 if (first == TRUE) {
8894 vm_page_queue_enter_first(&vm_page_queue_active, mem, vmp_pageq);
8895 } else {
8896 vm_page_queue_enter(&vm_page_queue_active, mem, vmp_pageq);
8897 }
8898 vm_page_active_count++;
8899
8900 if (m_object->internal) {
8901 vm_page_pageable_internal_count++;
8902 } else {
8903 vm_page_pageable_external_count++;
8904 }
8905
8906 vm_page_add_to_specialq(mem, FALSE);
8907 vm_page_balance_inactive(3);
8908 }
8909
8910 /*
8911 * Pages from special kernel objects shouldn't
8912 * be placed on pageable queues.
8913 */
8914 void
vm_page_check_pageable_safe(vm_page_t page)8915 vm_page_check_pageable_safe(vm_page_t page)
8916 {
8917 vm_object_t page_object;
8918
8919 page_object = VM_PAGE_OBJECT(page);
8920
8921 if (page_object == kernel_object) {
8922 panic("vm_page_check_pageable_safe: trying to add page"
8923 "from kernel object (%p) to pageable queue", kernel_object);
8924 }
8925
8926 if (page_object == compressor_object) {
8927 panic("vm_page_check_pageable_safe: trying to add page"
8928 "from compressor object (%p) to pageable queue", compressor_object);
8929 }
8930 }
8931
8932 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
8933 * wired page diagnose
8934 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
8935
8936 #include <libkern/OSKextLibPrivate.h>
8937
8938 #define KA_SIZE(namelen, subtotalscount) \
8939 (sizeof(struct vm_allocation_site) + (namelen) + 1 + ((subtotalscount) * sizeof(struct vm_allocation_total)))
8940
8941 #define KA_NAME(alloc) \
8942 ((char *)(&(alloc)->subtotals[(alloc->subtotalscount)]))
8943
8944 #define KA_NAME_LEN(alloc) \
8945 (VM_TAG_NAME_LEN_MAX & (alloc->flags >> VM_TAG_NAME_LEN_SHIFT))
8946
8947 vm_tag_t
vm_tag_bt(void)8948 vm_tag_bt(void)
8949 {
8950 uintptr_t* frameptr;
8951 uintptr_t* frameptr_next;
8952 uintptr_t retaddr;
8953 uintptr_t kstackb, kstackt;
8954 const vm_allocation_site_t * site;
8955 thread_t cthread;
8956 kern_allocation_name_t name;
8957
8958 cthread = current_thread();
8959 if (__improbable(cthread == NULL)) {
8960 return VM_KERN_MEMORY_OSFMK;
8961 }
8962
8963 if ((name = thread_get_kernel_state(cthread)->allocation_name)) {
8964 if (!name->tag) {
8965 vm_tag_alloc(name);
8966 }
8967 return name->tag;
8968 }
8969
8970 kstackb = cthread->kernel_stack;
8971 kstackt = kstackb + kernel_stack_size;
8972
8973 /* Load stack frame pointer (EBP on x86) into frameptr */
8974 frameptr = __builtin_frame_address(0);
8975 site = NULL;
8976 while (frameptr != NULL) {
8977 /* Verify thread stack bounds */
8978 if (((uintptr_t)(frameptr + 2) > kstackt) || ((uintptr_t)frameptr < kstackb)) {
8979 break;
8980 }
8981
8982 /* Next frame pointer is pointed to by the previous one */
8983 frameptr_next = (uintptr_t*) *frameptr;
8984
8985 /* Pull return address from one spot above the frame pointer */
8986 retaddr = *(frameptr + 1);
8987
8988 #if defined(HAS_APPLE_PAC)
8989 retaddr = (uintptr_t) ptrauth_strip((void *)retaddr, ptrauth_key_return_address);
8990 #endif
8991
8992 if (((retaddr < vm_kernel_builtinkmod_text_end) && (retaddr >= vm_kernel_builtinkmod_text))
8993 || (retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top)) {
8994 site = OSKextGetAllocationSiteForCaller(retaddr);
8995 break;
8996 }
8997 frameptr = frameptr_next;
8998 }
8999
9000 return site ? site->tag : VM_KERN_MEMORY_NONE;
9001 }
9002
9003 static uint64_t free_tag_bits[VM_MAX_TAG_VALUE / 64];
9004
9005 void
vm_tag_alloc_locked(vm_allocation_site_t * site,vm_allocation_site_t ** releasesiteP)9006 vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP)
9007 {
9008 vm_tag_t tag;
9009 uint64_t avail;
9010 uint32_t idx;
9011 vm_allocation_site_t * prev;
9012
9013 if (site->tag) {
9014 return;
9015 }
9016
9017 idx = 0;
9018 while (TRUE) {
9019 avail = free_tag_bits[idx];
9020 if (avail) {
9021 tag = (vm_tag_t)__builtin_clzll(avail);
9022 avail &= ~(1ULL << (63 - tag));
9023 free_tag_bits[idx] = avail;
9024 tag += (idx << 6);
9025 break;
9026 }
9027 idx++;
9028 if (idx >= ARRAY_COUNT(free_tag_bits)) {
9029 for (idx = 0; idx < ARRAY_COUNT(vm_allocation_sites); idx++) {
9030 prev = vm_allocation_sites[idx];
9031 if (!prev) {
9032 continue;
9033 }
9034 if (!KA_NAME_LEN(prev)) {
9035 continue;
9036 }
9037 if (!prev->tag) {
9038 continue;
9039 }
9040 if (prev->total) {
9041 continue;
9042 }
9043 if (1 != prev->refcount) {
9044 continue;
9045 }
9046
9047 assert(idx == prev->tag);
9048 tag = (vm_tag_t)idx;
9049 prev->tag = VM_KERN_MEMORY_NONE;
9050 *releasesiteP = prev;
9051 break;
9052 }
9053 if (idx >= ARRAY_COUNT(vm_allocation_sites)) {
9054 tag = VM_KERN_MEMORY_ANY;
9055 }
9056 break;
9057 }
9058 }
9059 site->tag = tag;
9060
9061 OSAddAtomic16(1, &site->refcount);
9062
9063 if (VM_KERN_MEMORY_ANY != tag) {
9064 vm_allocation_sites[tag] = site;
9065 }
9066
9067 if (tag > vm_allocation_tag_highest) {
9068 vm_allocation_tag_highest = tag;
9069 }
9070 }
9071
9072 static void
vm_tag_free_locked(vm_tag_t tag)9073 vm_tag_free_locked(vm_tag_t tag)
9074 {
9075 uint64_t avail;
9076 uint32_t idx;
9077 uint64_t bit;
9078
9079 if (VM_KERN_MEMORY_ANY == tag) {
9080 return;
9081 }
9082
9083 idx = (tag >> 6);
9084 avail = free_tag_bits[idx];
9085 tag &= 63;
9086 bit = (1ULL << (63 - tag));
9087 assert(!(avail & bit));
9088 free_tag_bits[idx] = (avail | bit);
9089 }
9090
9091 static void
vm_tag_init(void)9092 vm_tag_init(void)
9093 {
9094 vm_tag_t tag;
9095 for (tag = VM_KERN_MEMORY_FIRST_DYNAMIC; tag < VM_KERN_MEMORY_ANY; tag++) {
9096 vm_tag_free_locked(tag);
9097 }
9098
9099 for (tag = VM_KERN_MEMORY_ANY + 1; tag < VM_MAX_TAG_VALUE; tag++) {
9100 vm_tag_free_locked(tag);
9101 }
9102 }
9103
9104 vm_tag_t
vm_tag_alloc(vm_allocation_site_t * site)9105 vm_tag_alloc(vm_allocation_site_t * site)
9106 {
9107 vm_allocation_site_t * releasesite;
9108
9109 if (!site->tag) {
9110 releasesite = NULL;
9111 lck_ticket_lock(&vm_allocation_sites_lock, LCK_GRP_NULL);
9112 vm_tag_alloc_locked(site, &releasesite);
9113 lck_ticket_unlock(&vm_allocation_sites_lock);
9114 if (releasesite) {
9115 kern_allocation_name_release(releasesite);
9116 }
9117 }
9118
9119 return site->tag;
9120 }
9121
9122 void
vm_tag_update_size(vm_tag_t tag,int64_t delta)9123 vm_tag_update_size(vm_tag_t tag, int64_t delta)
9124 {
9125 vm_allocation_site_t * allocation;
9126 uint64_t value;
9127
9128 assert(VM_KERN_MEMORY_NONE != tag);
9129 assert(tag < VM_MAX_TAG_VALUE);
9130
9131 allocation = vm_allocation_sites[tag];
9132 assert(allocation);
9133
9134 value = os_atomic_add(&allocation->total, delta, relaxed);
9135 if (delta < 0) {
9136 assertf(value + (uint64_t)-delta > value,
9137 "tag %d, site %p", tag, allocation);
9138 }
9139
9140 #if DEBUG || DEVELOPMENT
9141 if (value > allocation->peak) {
9142 os_atomic_max(&allocation->peak, value, relaxed);
9143 }
9144 #endif /* DEBUG || DEVELOPMENT */
9145
9146 if (tag < VM_KERN_MEMORY_FIRST_DYNAMIC) {
9147 return;
9148 }
9149
9150 if (value == (uint64_t)delta && !allocation->tag) {
9151 vm_tag_alloc(allocation);
9152 }
9153 }
9154
9155 uint64_t
vm_tag_get_size(vm_tag_t tag)9156 vm_tag_get_size(vm_tag_t tag)
9157 {
9158 vm_allocation_site_t * allocation;
9159
9160 assert(VM_KERN_MEMORY_NONE != tag);
9161 assert(tag < VM_MAX_TAG_VALUE);
9162
9163 allocation = vm_allocation_sites[tag];
9164 return allocation ? os_atomic_load(&allocation->total, relaxed) : 0;
9165 }
9166
9167 void
kern_allocation_update_size(kern_allocation_name_t allocation,int64_t delta)9168 kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta)
9169 {
9170 uint64_t prior;
9171
9172 if (delta < 0) {
9173 assertf(allocation->total >= ((uint64_t)-delta), "name %p", allocation);
9174 }
9175 prior = OSAddAtomic64(delta, &allocation->total);
9176
9177 #if DEBUG || DEVELOPMENT
9178
9179 uint64_t new, peak;
9180 new = prior + delta;
9181 do{
9182 peak = allocation->peak;
9183 if (new <= peak) {
9184 break;
9185 }
9186 }while (!OSCompareAndSwap64(peak, new, &allocation->peak));
9187
9188 #endif /* DEBUG || DEVELOPMENT */
9189
9190 if (!prior && !allocation->tag) {
9191 vm_tag_alloc(allocation);
9192 }
9193 }
9194
9195 #if VM_TAG_SIZECLASSES
9196
9197 void
vm_allocation_zones_init(void)9198 vm_allocation_zones_init(void)
9199 {
9200 vm_offset_t addr;
9201 vm_size_t size;
9202
9203 const vm_tag_t early_tags[] = {
9204 VM_KERN_MEMORY_DIAG,
9205 VM_KERN_MEMORY_KALLOC,
9206 VM_KERN_MEMORY_KALLOC_DATA,
9207 VM_KERN_MEMORY_KALLOC_TYPE,
9208 VM_KERN_MEMORY_LIBKERN,
9209 VM_KERN_MEMORY_OSFMK,
9210 VM_KERN_MEMORY_RECOUNT,
9211 };
9212
9213 size = VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *)
9214 + ARRAY_COUNT(early_tags) * VM_TAG_SIZECLASSES * sizeof(vm_allocation_zone_total_t);
9215
9216 kmem_alloc(kernel_map, &addr, round_page(size),
9217 KMA_NOFAIL | KMA_KOBJECT | KMA_ZERO | KMA_PERMANENT,
9218 VM_KERN_MEMORY_DIAG);
9219
9220 vm_allocation_zone_totals = (vm_allocation_zone_total_t **) addr;
9221 addr += VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *);
9222
9223 // prepopulate early tag ranges so allocations
9224 // in vm_tag_update_zone_size() and early boot won't recurse
9225 for (size_t i = 0; i < ARRAY_COUNT(early_tags); i++) {
9226 vm_allocation_zone_totals[early_tags[i]] = (vm_allocation_zone_total_t *)addr;
9227 addr += VM_TAG_SIZECLASSES * sizeof(vm_allocation_zone_total_t);
9228 }
9229 }
9230
9231 __attribute__((noinline))
9232 static vm_tag_t
vm_tag_zone_stats_alloc(vm_tag_t tag,zalloc_flags_t flags)9233 vm_tag_zone_stats_alloc(vm_tag_t tag, zalloc_flags_t flags)
9234 {
9235 vm_allocation_zone_total_t *stats;
9236 vm_size_t size = sizeof(*stats) * VM_TAG_SIZECLASSES;
9237
9238 flags = Z_VM_TAG(Z_ZERO | flags, VM_KERN_MEMORY_DIAG);
9239 stats = kalloc_data(size, flags);
9240 if (!stats) {
9241 return VM_KERN_MEMORY_NONE;
9242 }
9243 if (!os_atomic_cmpxchg(&vm_allocation_zone_totals[tag], NULL, stats, release)) {
9244 kfree_data(stats, size);
9245 }
9246 return tag;
9247 }
9248
9249 vm_tag_t
vm_tag_will_update_zone(vm_tag_t tag,uint32_t zidx,uint32_t zflags)9250 vm_tag_will_update_zone(vm_tag_t tag, uint32_t zidx, uint32_t zflags)
9251 {
9252 assert(VM_KERN_MEMORY_NONE != tag);
9253 assert(tag < VM_MAX_TAG_VALUE);
9254
9255 if (zidx >= VM_TAG_SIZECLASSES) {
9256 return VM_KERN_MEMORY_NONE;
9257 }
9258
9259 if (__probable(vm_allocation_zone_totals[tag])) {
9260 return tag;
9261 }
9262 return vm_tag_zone_stats_alloc(tag, zflags);
9263 }
9264
9265 void
vm_tag_update_zone_size(vm_tag_t tag,uint32_t zidx,long delta)9266 vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, long delta)
9267 {
9268 vm_allocation_zone_total_t *stats;
9269 vm_size_t value;
9270
9271 assert(VM_KERN_MEMORY_NONE != tag);
9272 assert(tag < VM_MAX_TAG_VALUE);
9273
9274 if (zidx >= VM_TAG_SIZECLASSES) {
9275 return;
9276 }
9277
9278 stats = vm_allocation_zone_totals[tag];
9279 assert(stats);
9280 stats += zidx;
9281
9282 value = os_atomic_add(&stats->vazt_total, delta, relaxed);
9283 if (delta < 0) {
9284 assertf((long)value >= 0, "zidx %d, tag %d, %p", zidx, tag, stats);
9285 return;
9286 } else if (os_atomic_load(&stats->vazt_peak, relaxed) < value) {
9287 os_atomic_max(&stats->vazt_peak, value, relaxed);
9288 }
9289 }
9290
9291 #endif /* VM_TAG_SIZECLASSES */
9292
9293 void
kern_allocation_update_subtotal(kern_allocation_name_t allocation,uint32_t subtag,int64_t delta)9294 kern_allocation_update_subtotal(kern_allocation_name_t allocation, uint32_t subtag, int64_t delta)
9295 {
9296 kern_allocation_name_t other;
9297 struct vm_allocation_total * total;
9298 uint32_t subidx;
9299
9300 subidx = 0;
9301 assert(VM_KERN_MEMORY_NONE != subtag);
9302 lck_ticket_lock(&vm_allocation_sites_lock, LCK_GRP_NULL);
9303 for (; subidx < allocation->subtotalscount; subidx++) {
9304 if (VM_KERN_MEMORY_NONE == allocation->subtotals[subidx].tag) {
9305 allocation->subtotals[subidx].tag = (vm_tag_t)subtag;
9306 break;
9307 }
9308 if (subtag == allocation->subtotals[subidx].tag) {
9309 break;
9310 }
9311 }
9312 lck_ticket_unlock(&vm_allocation_sites_lock);
9313 assert(subidx < allocation->subtotalscount);
9314 if (subidx >= allocation->subtotalscount) {
9315 return;
9316 }
9317
9318 total = &allocation->subtotals[subidx];
9319 other = vm_allocation_sites[subtag];
9320 assert(other);
9321
9322 if (delta < 0) {
9323 assertf(total->total >= ((uint64_t)-delta), "name %p", allocation);
9324 assertf(other->mapped >= ((uint64_t)-delta), "other %p", other);
9325 }
9326 OSAddAtomic64(delta, &other->mapped);
9327 OSAddAtomic64(delta, &total->total);
9328 }
9329
9330 const char *
kern_allocation_get_name(kern_allocation_name_t allocation)9331 kern_allocation_get_name(kern_allocation_name_t allocation)
9332 {
9333 return KA_NAME(allocation);
9334 }
9335
9336 kern_allocation_name_t
kern_allocation_name_allocate(const char * name,uint16_t subtotalscount)9337 kern_allocation_name_allocate(const char * name, uint16_t subtotalscount)
9338 {
9339 kern_allocation_name_t allocation;
9340 uint16_t namelen;
9341
9342 namelen = (uint16_t)strnlen(name, MACH_MEMORY_INFO_NAME_MAX_LEN - 1);
9343
9344 allocation = kalloc_data(KA_SIZE(namelen, subtotalscount), Z_WAITOK | Z_ZERO);
9345 allocation->refcount = 1;
9346 allocation->subtotalscount = subtotalscount;
9347 allocation->flags = (uint16_t)(namelen << VM_TAG_NAME_LEN_SHIFT);
9348 strlcpy(KA_NAME(allocation), name, namelen + 1);
9349
9350 vm_tag_alloc(allocation);
9351 return allocation;
9352 }
9353
9354 void
kern_allocation_name_release(kern_allocation_name_t allocation)9355 kern_allocation_name_release(kern_allocation_name_t allocation)
9356 {
9357 assert(allocation->refcount > 0);
9358 if (1 == OSAddAtomic16(-1, &allocation->refcount)) {
9359 kfree_data(allocation,
9360 KA_SIZE(KA_NAME_LEN(allocation), allocation->subtotalscount));
9361 }
9362 }
9363
9364 vm_tag_t
kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation)9365 kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation)
9366 {
9367 return vm_tag_alloc(allocation);
9368 }
9369
9370 #if !VM_TAG_ACTIVE_UPDATE
9371 static void
vm_page_count_object(mach_memory_info_t * info,unsigned int __unused num_info,vm_object_t object)9372 vm_page_count_object(mach_memory_info_t * info, unsigned int __unused num_info, vm_object_t object)
9373 {
9374 if (!object->wired_page_count) {
9375 return;
9376 }
9377 if (object != kernel_object) {
9378 assert(object->wire_tag < num_info);
9379 info[object->wire_tag].size += ptoa_64(object->wired_page_count);
9380 }
9381 }
9382
9383 typedef void (*vm_page_iterate_proc)(mach_memory_info_t * info,
9384 unsigned int num_info, vm_object_t object);
9385
9386 static void
vm_page_iterate_purgeable_objects(mach_memory_info_t * info,unsigned int num_info,vm_page_iterate_proc proc,purgeable_q_t queue,int group)9387 vm_page_iterate_purgeable_objects(mach_memory_info_t * info, unsigned int num_info,
9388 vm_page_iterate_proc proc, purgeable_q_t queue,
9389 int group)
9390 {
9391 vm_object_t object;
9392
9393 for (object = (vm_object_t) queue_first(&queue->objq[group]);
9394 !queue_end(&queue->objq[group], (queue_entry_t) object);
9395 object = (vm_object_t) queue_next(&object->objq)) {
9396 proc(info, num_info, object);
9397 }
9398 }
9399
9400 static void
vm_page_iterate_objects(mach_memory_info_t * info,unsigned int num_info,vm_page_iterate_proc proc)9401 vm_page_iterate_objects(mach_memory_info_t * info, unsigned int num_info,
9402 vm_page_iterate_proc proc)
9403 {
9404 vm_object_t object;
9405
9406 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket);
9407 queue_iterate(&vm_objects_wired,
9408 object,
9409 vm_object_t,
9410 wired_objq)
9411 {
9412 proc(info, num_info, object);
9413 }
9414 lck_spin_unlock(&vm_objects_wired_lock);
9415 }
9416 #endif /* ! VM_TAG_ACTIVE_UPDATE */
9417
9418 static uint64_t
process_account(mach_memory_info_t * info,unsigned int num_info,uint64_t zones_collectable_bytes,boolean_t iterated)9419 process_account(mach_memory_info_t * info, unsigned int num_info,
9420 uint64_t zones_collectable_bytes, boolean_t iterated)
9421 {
9422 size_t namelen;
9423 unsigned int idx, count, nextinfo;
9424 vm_allocation_site_t * site;
9425 lck_ticket_lock(&vm_allocation_sites_lock, LCK_GRP_NULL);
9426
9427 for (idx = 0; idx <= vm_allocation_tag_highest; idx++) {
9428 site = vm_allocation_sites[idx];
9429 if (!site) {
9430 continue;
9431 }
9432 info[idx].mapped = site->mapped;
9433 info[idx].tag = site->tag;
9434 if (!iterated) {
9435 info[idx].size = site->total;
9436 #if DEBUG || DEVELOPMENT
9437 info[idx].peak = site->peak;
9438 #endif /* DEBUG || DEVELOPMENT */
9439 } else {
9440 if (!site->subtotalscount && (site->total != info[idx].size)) {
9441 printf("tag mismatch[%d] 0x%qx, iter 0x%qx\n", idx, site->total, info[idx].size);
9442 info[idx].size = site->total;
9443 }
9444 }
9445 info[idx].flags |= VM_KERN_SITE_WIRED;
9446 if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC) {
9447 info[idx].site = idx;
9448 info[idx].flags |= VM_KERN_SITE_TAG;
9449 if (VM_KERN_MEMORY_ZONE == idx) {
9450 info[idx].flags |= VM_KERN_SITE_HIDE;
9451 info[idx].flags &= ~VM_KERN_SITE_WIRED;
9452 info[idx].collectable_bytes = zones_collectable_bytes;
9453 }
9454 } else if ((namelen = (VM_TAG_NAME_LEN_MAX & (site->flags >> VM_TAG_NAME_LEN_SHIFT)))) {
9455 info[idx].site = 0;
9456 info[idx].flags |= VM_KERN_SITE_NAMED;
9457 if (namelen > sizeof(info[idx].name)) {
9458 namelen = sizeof(info[idx].name);
9459 }
9460 strncpy(&info[idx].name[0], KA_NAME(site), namelen);
9461 } else if (VM_TAG_KMOD & site->flags) {
9462 info[idx].site = OSKextGetKmodIDForSite(site, NULL, 0);
9463 info[idx].flags |= VM_KERN_SITE_KMOD;
9464 } else {
9465 info[idx].site = VM_KERNEL_UNSLIDE(site);
9466 info[idx].flags |= VM_KERN_SITE_KERNEL;
9467 }
9468 }
9469
9470 nextinfo = (vm_allocation_tag_highest + 1);
9471 count = nextinfo;
9472 if (count >= num_info) {
9473 count = num_info;
9474 }
9475
9476 for (idx = 0; idx < count; idx++) {
9477 site = vm_allocation_sites[idx];
9478 if (!site) {
9479 continue;
9480 }
9481 #if VM_TAG_SIZECLASSES
9482 vm_allocation_zone_total_t * zone;
9483 unsigned int zidx;
9484
9485 if (vm_allocation_zone_totals
9486 && (zone = vm_allocation_zone_totals[idx])
9487 && (nextinfo < num_info)) {
9488 for (zidx = 0; zidx < VM_TAG_SIZECLASSES; zidx++) {
9489 if (!zone[zidx].vazt_peak) {
9490 continue;
9491 }
9492 info[nextinfo] = info[idx];
9493 info[nextinfo].zone = (uint16_t)zone_index_from_tag_index(zidx);
9494 info[nextinfo].flags &= ~VM_KERN_SITE_WIRED;
9495 info[nextinfo].flags |= VM_KERN_SITE_ZONE;
9496 info[nextinfo].flags |= VM_KERN_SITE_KALLOC;
9497 info[nextinfo].size = zone[zidx].vazt_total;
9498 info[nextinfo].peak = zone[zidx].vazt_peak;
9499 info[nextinfo].mapped = 0;
9500 nextinfo++;
9501 }
9502 }
9503 #endif /* VM_TAG_SIZECLASSES */
9504 if (site->subtotalscount) {
9505 uint64_t mapped, mapcost, take;
9506 uint32_t sub;
9507 vm_tag_t alloctag;
9508
9509 info[idx].size = site->total;
9510 mapped = info[idx].size;
9511 info[idx].mapped = mapped;
9512 mapcost = 0;
9513 for (sub = 0; sub < site->subtotalscount; sub++) {
9514 alloctag = site->subtotals[sub].tag;
9515 assert(alloctag < num_info);
9516 if (info[alloctag].name[0]) {
9517 continue;
9518 }
9519 take = site->subtotals[sub].total;
9520 if (take > info[alloctag].size) {
9521 take = info[alloctag].size;
9522 }
9523 if (take > mapped) {
9524 take = mapped;
9525 }
9526 info[alloctag].mapped -= take;
9527 info[alloctag].size -= take;
9528 mapped -= take;
9529 mapcost += take;
9530 }
9531 info[idx].size = mapcost;
9532 }
9533 }
9534 lck_ticket_unlock(&vm_allocation_sites_lock);
9535
9536 return 0;
9537 }
9538
9539 uint32_t
vm_page_diagnose_estimate(void)9540 vm_page_diagnose_estimate(void)
9541 {
9542 vm_allocation_site_t * site;
9543 uint32_t count = zone_view_count;
9544 uint32_t idx;
9545
9546 lck_ticket_lock(&vm_allocation_sites_lock, LCK_GRP_NULL);
9547 for (idx = 0; idx < VM_MAX_TAG_VALUE; idx++) {
9548 site = vm_allocation_sites[idx];
9549 if (!site) {
9550 continue;
9551 }
9552 count++;
9553 #if VM_TAG_SIZECLASSES
9554 if (vm_allocation_zone_totals) {
9555 vm_allocation_zone_total_t * zone;
9556 zone = vm_allocation_zone_totals[idx];
9557 if (!zone) {
9558 continue;
9559 }
9560 for (uint32_t zidx = 0; zidx < VM_TAG_SIZECLASSES; zidx++) {
9561 count += (zone[zidx].vazt_peak != 0);
9562 }
9563 }
9564 #endif
9565 }
9566 lck_ticket_unlock(&vm_allocation_sites_lock);
9567
9568 /* some slop for new tags created */
9569 count += 8;
9570 count += VM_KERN_COUNTER_COUNT;
9571
9572 return count;
9573 }
9574
9575 static void
vm_page_diagnose_zone_stats(mach_memory_info_t * info,zone_stats_t zstats,bool percpu)9576 vm_page_diagnose_zone_stats(mach_memory_info_t *info, zone_stats_t zstats,
9577 bool percpu)
9578 {
9579 zpercpu_foreach(zs, zstats) {
9580 info->size += zs->zs_mem_allocated - zs->zs_mem_freed;
9581 }
9582 if (percpu) {
9583 info->size *= zpercpu_count();
9584 }
9585 info->flags |= VM_KERN_SITE_NAMED | VM_KERN_SITE_ZONE_VIEW;
9586 }
9587
9588 static void
vm_page_diagnose_zone(mach_memory_info_t * info,zone_t z)9589 vm_page_diagnose_zone(mach_memory_info_t *info, zone_t z)
9590 {
9591 vm_page_diagnose_zone_stats(info, z->z_stats, z->z_percpu);
9592 snprintf(info->name, sizeof(info->name),
9593 "%s%s[raw]", zone_heap_name(z), z->z_name);
9594 }
9595
9596 static int
vm_page_diagnose_heap(mach_memory_info_t * info,kalloc_heap_t kheap)9597 vm_page_diagnose_heap(mach_memory_info_t *info, kalloc_heap_t kheap)
9598 {
9599 struct kheap_zones *zones = kheap->kh_zones;
9600 int i = 0;
9601
9602 for (; i < zones->max_k_zone; i++) {
9603 vm_page_diagnose_zone(info + i, zones->k_zone[i]);
9604 }
9605
9606 for (kalloc_heap_t kh = zones->views; kh; kh = kh->kh_next, i++) {
9607 vm_page_diagnose_zone_stats(info + i, kh->kh_stats, false);
9608 snprintf(info[i].name, sizeof(info[i].name),
9609 "%skalloc[%s]", kheap->kh_name, kh->kh_name);
9610 }
9611
9612 return i;
9613 }
9614
9615 static int
vm_page_diagnose_kt_heaps(mach_memory_info_t * info)9616 vm_page_diagnose_kt_heaps(mach_memory_info_t *info)
9617 {
9618 uint32_t idx = 0;
9619 vm_page_diagnose_zone_stats(info + idx, KHEAP_KT_VAR->kh_stats, false);
9620 snprintf(info[idx].name, sizeof(info[idx].name),
9621 "%s[raw]", KHEAP_KT_VAR->kh_name);
9622 idx++;
9623
9624 for (uint32_t i = 0; i < KT_VAR_MAX_HEAPS; i++) {
9625 struct kt_heap_zones heap = kalloc_type_heap_array[i];
9626
9627 for (kalloc_type_var_view_t ktv = heap.views; ktv;
9628 ktv = (kalloc_type_var_view_t) ktv->kt_next) {
9629 if (ktv->kt_stats && ktv->kt_stats != KHEAP_KT_VAR->kh_stats) {
9630 vm_page_diagnose_zone_stats(info + idx, ktv->kt_stats, false);
9631 snprintf(info[idx].name, sizeof(info[idx].name),
9632 "%s[%s]", KHEAP_KT_VAR->kh_name, ktv->kt_name);
9633 idx++;
9634 }
9635 }
9636 }
9637
9638 return idx;
9639 }
9640
9641 kern_return_t
vm_page_diagnose(mach_memory_info_t * info,unsigned int num_info,uint64_t zones_collectable_bytes)9642 vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes)
9643 {
9644 uint64_t wired_size;
9645 uint64_t wired_managed_size;
9646 uint64_t wired_reserved_size;
9647 boolean_t iterate;
9648 mach_memory_info_t * counts;
9649 uint32_t i;
9650
9651 bzero(info, num_info * sizeof(mach_memory_info_t));
9652
9653 if (!vm_page_wire_count_initial) {
9654 return KERN_ABORTED;
9655 }
9656
9657 #if !XNU_TARGET_OS_OSX
9658 wired_size = ptoa_64(vm_page_wire_count);
9659 wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count);
9660 #else /* !XNU_TARGET_OS_OSX */
9661 wired_size = ptoa_64(vm_page_wire_count + vm_lopage_free_count + vm_page_throttled_count);
9662 wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count + vm_page_throttled_count);
9663 #endif /* !XNU_TARGET_OS_OSX */
9664 wired_managed_size = ptoa_64(vm_page_wire_count - vm_page_wire_count_initial);
9665
9666 wired_size += booter_size;
9667
9668 assert(num_info >= VM_KERN_COUNTER_COUNT);
9669 num_info -= VM_KERN_COUNTER_COUNT;
9670 counts = &info[num_info];
9671
9672 #define SET_COUNT(xcount, xsize, xflags) \
9673 counts[xcount].tag = VM_MAX_TAG_VALUE + xcount; \
9674 counts[xcount].site = (xcount); \
9675 counts[xcount].size = (xsize); \
9676 counts[xcount].mapped = (xsize); \
9677 counts[xcount].flags = VM_KERN_SITE_COUNTER | xflags;
9678
9679 SET_COUNT(VM_KERN_COUNT_MANAGED, ptoa_64(vm_page_pages), 0);
9680 SET_COUNT(VM_KERN_COUNT_WIRED, wired_size, 0);
9681 SET_COUNT(VM_KERN_COUNT_WIRED_MANAGED, wired_managed_size, 0);
9682 SET_COUNT(VM_KERN_COUNT_RESERVED, wired_reserved_size, VM_KERN_SITE_WIRED);
9683 SET_COUNT(VM_KERN_COUNT_STOLEN, ptoa_64(vm_page_stolen_count), VM_KERN_SITE_WIRED);
9684 SET_COUNT(VM_KERN_COUNT_LOPAGE, ptoa_64(vm_lopage_free_count), VM_KERN_SITE_WIRED);
9685 SET_COUNT(VM_KERN_COUNT_WIRED_BOOT, ptoa_64(vm_page_wire_count_on_boot), 0);
9686 SET_COUNT(VM_KERN_COUNT_BOOT_STOLEN, booter_size, VM_KERN_SITE_WIRED);
9687 SET_COUNT(VM_KERN_COUNT_WIRED_STATIC_KERNELCACHE, ptoa_64(vm_page_kernelcache_count), 0);
9688
9689 #define SET_MAP(xcount, xsize, xfree, xlargest) \
9690 counts[xcount].site = (xcount); \
9691 counts[xcount].size = (xsize); \
9692 counts[xcount].mapped = (xsize); \
9693 counts[xcount].free = (xfree); \
9694 counts[xcount].largest = (xlargest); \
9695 counts[xcount].flags = VM_KERN_SITE_COUNTER;
9696
9697 vm_map_size_t map_size, map_free, map_largest;
9698
9699 vm_map_sizes(kernel_map, &map_size, &map_free, &map_largest);
9700 SET_MAP(VM_KERN_COUNT_MAP_KERNEL, map_size, map_free, map_largest);
9701
9702 zone_map_sizes(&map_size, &map_free, &map_largest);
9703 SET_MAP(VM_KERN_COUNT_MAP_ZONE, map_size, map_free, map_largest);
9704
9705 assert(num_info >= zone_view_count);
9706 num_info -= zone_view_count;
9707 counts = &info[num_info];
9708 i = 0;
9709
9710 i += vm_page_diagnose_heap(counts + i, KHEAP_DEFAULT);
9711 if (KHEAP_DATA_BUFFERS->kh_heap_id == KHEAP_ID_DATA_BUFFERS) {
9712 i += vm_page_diagnose_heap(counts + i, KHEAP_DATA_BUFFERS);
9713 }
9714 if (KHEAP_KT_VAR->kh_heap_id == KHEAP_ID_KT_VAR) {
9715 i += vm_page_diagnose_kt_heaps(counts + i);
9716 }
9717 assert(i <= zone_view_count);
9718
9719 zone_index_foreach(zidx) {
9720 zone_t z = &zone_array[zidx];
9721 zone_security_flags_t zsflags = zone_security_array[zidx];
9722 zone_view_t zv = z->z_views;
9723
9724 if (zv == NULL) {
9725 continue;
9726 }
9727
9728 zone_stats_t zv_stats_head = z->z_stats;
9729 bool has_raw_view = false;
9730
9731 for (; zv; zv = zv->zv_next) {
9732 /*
9733 * kalloc_types that allocate from the same zone are linked
9734 * as views. Only print the ones that have their own stats.
9735 */
9736 if (zv->zv_stats == zv_stats_head) {
9737 continue;
9738 }
9739 has_raw_view = true;
9740 vm_page_diagnose_zone_stats(counts + i, zv->zv_stats,
9741 z->z_percpu);
9742 snprintf(counts[i].name, sizeof(counts[i].name), "%s%s[%s]",
9743 zone_heap_name(z), z->z_name, zv->zv_name);
9744 i++;
9745 assert(i <= zone_view_count);
9746 }
9747
9748 /*
9749 * Print raw views for non kalloc or kalloc_type zones
9750 */
9751 bool kalloc_type = zsflags.z_kalloc_type;
9752 if ((zsflags.z_kheap_id == KHEAP_ID_NONE && !kalloc_type) ||
9753 (kalloc_type && has_raw_view)) {
9754 vm_page_diagnose_zone(counts + i, z);
9755 i++;
9756 assert(i <= zone_view_count);
9757 }
9758 }
9759
9760 iterate = !VM_TAG_ACTIVE_UPDATE;
9761 if (iterate) {
9762 enum { kMaxKernelDepth = 1 };
9763 vm_map_t maps[kMaxKernelDepth];
9764 vm_map_entry_t entries[kMaxKernelDepth];
9765 vm_map_t map;
9766 vm_map_entry_t entry;
9767 vm_object_offset_t offset;
9768 vm_page_t page;
9769 int stackIdx, count;
9770
9771 #if !VM_TAG_ACTIVE_UPDATE
9772 vm_page_iterate_objects(info, num_info, &vm_page_count_object);
9773 #endif /* ! VM_TAG_ACTIVE_UPDATE */
9774
9775 map = kernel_map;
9776 stackIdx = 0;
9777 while (map) {
9778 vm_map_lock(map);
9779 for (entry = map->hdr.links.next; map; entry = entry->links.next) {
9780 if (entry->is_sub_map) {
9781 assert(stackIdx < kMaxKernelDepth);
9782 maps[stackIdx] = map;
9783 entries[stackIdx] = entry;
9784 stackIdx++;
9785 map = VME_SUBMAP(entry);
9786 entry = NULL;
9787 break;
9788 }
9789 if (VME_OBJECT(entry) == kernel_object) {
9790 count = 0;
9791 vm_object_lock(VME_OBJECT(entry));
9792 for (offset = entry->links.start; offset < entry->links.end; offset += page_size) {
9793 page = vm_page_lookup(VME_OBJECT(entry), offset);
9794 if (page && VM_PAGE_WIRED(page)) {
9795 count++;
9796 }
9797 }
9798 vm_object_unlock(VME_OBJECT(entry));
9799
9800 if (count) {
9801 assert(VME_ALIAS(entry) != VM_KERN_MEMORY_NONE);
9802 assert(VME_ALIAS(entry) < num_info);
9803 info[VME_ALIAS(entry)].size += ptoa_64(count);
9804 }
9805 }
9806 while (map && (entry == vm_map_last_entry(map))) {
9807 vm_map_unlock(map);
9808 if (!stackIdx) {
9809 map = NULL;
9810 } else {
9811 --stackIdx;
9812 map = maps[stackIdx];
9813 entry = entries[stackIdx];
9814 }
9815 }
9816 }
9817 }
9818 }
9819
9820 process_account(info, num_info, zones_collectable_bytes, iterate);
9821
9822 return KERN_SUCCESS;
9823 }
9824
9825 #if DEBUG || DEVELOPMENT
9826
9827 kern_return_t
vm_kern_allocation_info(uintptr_t addr,vm_size_t * size,vm_tag_t * tag,vm_size_t * zone_size)9828 vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size)
9829 {
9830 kern_return_t ret;
9831 vm_size_t zsize;
9832 vm_map_t map;
9833 vm_map_entry_t entry;
9834
9835 zsize = zone_element_info((void *) addr, tag);
9836 if (zsize) {
9837 *zone_size = *size = zsize;
9838 return KERN_SUCCESS;
9839 }
9840
9841 *zone_size = 0;
9842 ret = KERN_INVALID_ADDRESS;
9843 for (map = kernel_map; map;) {
9844 vm_map_lock(map);
9845 if (!vm_map_lookup_entry_allow_pgz(map, addr, &entry)) {
9846 break;
9847 }
9848 if (entry->is_sub_map) {
9849 if (map != kernel_map) {
9850 break;
9851 }
9852 map = VME_SUBMAP(entry);
9853 continue;
9854 }
9855 if (entry->vme_start != addr) {
9856 break;
9857 }
9858 *tag = (vm_tag_t)VME_ALIAS(entry);
9859 *size = (entry->vme_end - addr);
9860 ret = KERN_SUCCESS;
9861 break;
9862 }
9863 if (map != kernel_map) {
9864 vm_map_unlock(map);
9865 }
9866 vm_map_unlock(kernel_map);
9867
9868 return ret;
9869 }
9870
9871 #endif /* DEBUG || DEVELOPMENT */
9872
9873 uint32_t
vm_tag_get_kext(vm_tag_t tag,char * name,vm_size_t namelen)9874 vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen)
9875 {
9876 vm_allocation_site_t * site;
9877 uint32_t kmodId;
9878
9879 kmodId = 0;
9880 lck_ticket_lock(&vm_allocation_sites_lock, LCK_GRP_NULL);
9881 if ((site = vm_allocation_sites[tag])) {
9882 if (VM_TAG_KMOD & site->flags) {
9883 kmodId = OSKextGetKmodIDForSite(site, name, namelen);
9884 }
9885 }
9886 lck_ticket_unlock(&vm_allocation_sites_lock);
9887
9888 return kmodId;
9889 }
9890
9891
9892 #if CONFIG_SECLUDED_MEMORY
9893 /*
9894 * Note that there's no locking around other accesses to vm_page_secluded_target.
9895 * That should be OK, since these are the only place where it can be changed after
9896 * initialization. Other users (like vm_pageout) may see the wrong value briefly,
9897 * but will eventually get the correct value. This brief mismatch is OK as pageout
9898 * and page freeing will auto-adjust the vm_page_secluded_count to match the target
9899 * over time.
9900 */
9901 unsigned int vm_page_secluded_suppress_cnt = 0;
9902 unsigned int vm_page_secluded_save_target;
9903
9904 LCK_GRP_DECLARE(secluded_suppress_slock_grp, "secluded_suppress_slock");
9905 LCK_SPIN_DECLARE(secluded_suppress_slock, &secluded_suppress_slock_grp);
9906
9907 void
start_secluded_suppression(task_t task)9908 start_secluded_suppression(task_t task)
9909 {
9910 if (task->task_suppressed_secluded) {
9911 return;
9912 }
9913 lck_spin_lock(&secluded_suppress_slock);
9914 if (!task->task_suppressed_secluded && vm_page_secluded_suppress_cnt++ == 0) {
9915 task->task_suppressed_secluded = TRUE;
9916 vm_page_secluded_save_target = vm_page_secluded_target;
9917 vm_page_secluded_target = 0;
9918 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
9919 }
9920 lck_spin_unlock(&secluded_suppress_slock);
9921 }
9922
9923 void
stop_secluded_suppression(task_t task)9924 stop_secluded_suppression(task_t task)
9925 {
9926 lck_spin_lock(&secluded_suppress_slock);
9927 if (task->task_suppressed_secluded && --vm_page_secluded_suppress_cnt == 0) {
9928 task->task_suppressed_secluded = FALSE;
9929 vm_page_secluded_target = vm_page_secluded_save_target;
9930 VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
9931 }
9932 lck_spin_unlock(&secluded_suppress_slock);
9933 }
9934
9935 #endif /* CONFIG_SECLUDED_MEMORY */
9936
9937 /*
9938 * Move the list of retired pages on the vm_page_queue_retired to
9939 * their final resting place on retired_pages_object.
9940 */
9941 void
vm_retire_boot_pages(void)9942 vm_retire_boot_pages(void)
9943 {
9944 }
9945
9946 /*
9947 * This holds the reported physical address if an ECC error leads to a panic.
9948 * SMC will store it in PMU SRAM under the 'sECC' key.
9949 */
9950 uint64_t ecc_panic_physical_address = 0;
9951
9952