1 /*
2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66 #ifndef _VM_VM_PAGE_H_
67 #define _VM_VM_PAGE_H_
68
69 #include <debug.h>
70 #include <vm/vm_options.h>
71 #include <vm/vm_protos.h>
72 #include <mach/boolean.h>
73 #include <mach/vm_prot.h>
74 #include <mach/vm_param.h>
75 #include <mach/memory_object_types.h> /* for VMP_CS_BITS... */
76
77
78 #if defined(__LP64__)
79
80 /*
81 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
82 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
83 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
84 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
85 * pointers from the 2 ends of these spaces
86 */
87 typedef uint32_t vm_page_packed_t;
88
89 struct vm_page_packed_queue_entry {
90 vm_page_packed_t next; /* next element */
91 vm_page_packed_t prev; /* previous element */
92 };
93
94 typedef struct vm_page_packed_queue_entry *vm_page_queue_t;
95 typedef struct vm_page_packed_queue_entry vm_page_queue_head_t;
96 typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t;
97 typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t;
98
99 typedef vm_page_packed_t vm_page_object_t;
100
101 #else // __LP64__
102
103 /*
104 * we can't do the packing trick on 32 bit architectures
105 * so just turn the macros into noops.
106 */
107 typedef struct vm_page *vm_page_packed_t;
108
109 #define vm_page_queue_t queue_t
110 #define vm_page_queue_head_t queue_head_t
111 #define vm_page_queue_chain_t queue_chain_t
112 #define vm_page_queue_entry_t queue_entry_t
113
114 #define vm_page_object_t vm_object_t
115 #endif // __LP64__
116
117
118 #include <vm/vm_object.h>
119 #include <kern/queue.h>
120 #include <kern/locks.h>
121
122 #include <kern/macro_help.h>
123 #include <libkern/OSAtomic.h>
124
125
126
127 #define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count)
128
129 /*
130 * Management of resident (logical) pages.
131 *
132 * A small structure is kept for each resident
133 * page, indexed by page number. Each structure
134 * is an element of several lists:
135 *
136 * A hash table bucket used to quickly
137 * perform object/offset lookups
138 *
139 * A list of all pages for a given object,
140 * so they can be quickly deactivated at
141 * time of deallocation.
142 *
143 * An ordered list of pages due for pageout.
144 *
145 * In addition, the structure contains the object
146 * and offset to which this page belongs (for pageout),
147 * and sundry status bits.
148 *
149 * Fields in this structure are locked either by the lock on the
150 * object that the page belongs to (O) or by the lock on the page
151 * queues (P). [Some fields require that both locks be held to
152 * change that field; holding either lock is sufficient to read.]
153 */
154
155 #define VM_PAGE_NULL ((vm_page_t) 0)
156
157 extern char vm_page_inactive_states[];
158 extern char vm_page_pageable_states[];
159 extern char vm_page_non_speculative_pageable_states[];
160 extern char vm_page_active_or_inactive_states[];
161
162
163 #define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vmp_q_state])
164 #define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vmp_q_state])
165 #define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vmp_q_state])
166 #define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vmp_q_state])
167
168
169 #define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */
170 #define VM_PAGE_IS_WIRED 1 /* page is currently wired */
171 #define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */
172 #define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */
173 #define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */
174 #define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */
175 #define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */
176 #define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */
177 #define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */
178 #define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */
179 #define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */
180 #define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */
181 #define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */
182 #define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */
183 #define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */
184 #define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */
185
186 #define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1)
187
188
189 /*
190 * The structure itself. See the block comment above for what (O) and (P) mean.
191 */
192 #define vmp_pageq vmp_q_un.vmp_q_pageq
193 #define vmp_snext vmp_q_un.vmp_q_snext
194
195 struct vm_page {
196 union {
197 vm_page_queue_chain_t vmp_q_pageq; /* queue info for FIFO queue or free list (P) */
198 struct vm_page *vmp_q_snext;
199 } vmp_q_un;
200
201 vm_page_queue_chain_t vmp_listq; /* all pages in same object (O) */
202
203 vm_page_queue_chain_t vmp_specialq; /* anonymous pages in the special queues (P) */
204 vm_object_offset_t vmp_offset; /* offset into that object (O,P) */
205
206 vm_page_object_t vmp_object; /* which object am I in (O&P) */
207
208 /*
209 * The following word of flags used to be protected by the "page queues" lock.
210 * That's no longer true and what lock, if any, is needed may depend on the
211 * value of vmp_q_state.
212 *
213 * We use 'vmp_wire_count' to store the local queue id if local queues are enabled.
214 * See the comments at 'vm_page_queues_remove' as to why this is safe to do.
215 */
216 #define VM_PAGE_SPECIAL_Q_EMPTY (0)
217 #define VM_PAGE_SPECIAL_Q_BG (1)
218 #define VM_PAGE_SPECIAL_Q_DONATE (2)
219 #define VM_PAGE_SPECIAL_Q_FG (3)
220 #define vmp_local_id vmp_wire_count
221 unsigned int vmp_wire_count:16, /* how many wired down maps use me? (O&P) */
222 vmp_q_state:4, /* which q is the page on (P) */
223 vmp_on_specialq:2,
224 vmp_gobbled:1, /* page used internally (P) */
225 vmp_laundry:1, /* page is being cleaned now (P)*/
226 vmp_no_cache:1, /* page is not to be cached and should */
227 /* be reused ahead of other pages (P) */
228 vmp_private:1, /* Page should not be returned to the free list (P) */
229 vmp_reference:1, /* page has been used (P) */
230 vmp_lopage:1,
231 vmp_realtime:1, /* page used by realtime thread */
232 #if !CONFIG_TRACK_UNMODIFIED_ANON_PAGES
233 vmp_unused_page_bits:3;
234 #else /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
235 vmp_unmodified_ro:1, /* Tracks if an anonymous page is modified after a decompression (O&P).*/
236 vmp_unused_page_bits:2;
237 #endif /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
238
239 /*
240 * MUST keep the 2 32 bit words used as bit fields
241 * separated since the compiler has a nasty habit
242 * of using 64 bit loads and stores on them as
243 * if they were a single 64 bit field... since
244 * they are protected by 2 different locks, this
245 * is a real problem
246 */
247 vm_page_packed_t vmp_next_m; /* VP bucket link (O) */
248
249 /*
250 * The following word of flags is protected by the "VM object" lock.
251 *
252 * IMPORTANT: the "vmp_pmapped", "vmp_xpmapped" and "vmp_clustered" bits can be modified while holding the
253 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
254 * This is done in vm_fault_enter() and the CONSUME_CLUSTERED macro.
255 * It's also ok to modify them behind just the VM object "exclusive" lock.
256 */
257 unsigned int vmp_busy:1, /* page is in transit (O) */
258 vmp_wanted:1, /* someone is waiting for page (O) */
259 vmp_tabled:1, /* page is in VP table (O) */
260 vmp_hashed:1, /* page is in vm_page_buckets[] (O) + the bucket lock */
261 vmp_fictitious:1, /* Physical page doesn't exist (O) */
262 vmp_clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
263 vmp_pmapped:1, /* page has at some time been entered into a pmap (O) or */
264 /* (O-shared AND pmap_page) */
265 vmp_xpmapped:1, /* page has been entered with execute permission (O) or */
266 /* (O-shared AND pmap_page) */
267 vmp_wpmapped:1, /* page has been entered at some point into a pmap for write (O) */
268 vmp_free_when_done:1, /* page is to be freed once cleaning is completed (O) */
269 vmp_absent:1, /* Data has been requested, but is not yet available (O) */
270 vmp_error:1, /* Data manager was unable to provide data due to error (O) */
271 vmp_dirty:1, /* Page must be cleaned (O) */
272 vmp_cleaning:1, /* Page clean has begun (O) */
273 vmp_precious:1, /* Page is precious; data must be returned even if clean (O) */
274 vmp_overwriting:1, /* Request to unlock has been made without having data. (O) */
275 /* [See vm_fault_page_overwrite] */
276 vmp_restart:1, /* Page was pushed higher in shadow chain by copy_call-related pagers */
277 /* start again at top of chain */
278 vmp_unusual:1, /* Page is absent, error, restart or page locked */
279 vmp_cs_validated:VMP_CS_BITS, /* code-signing: page was checked */
280 vmp_cs_tainted:VMP_CS_BITS, /* code-signing: page is tainted */
281 vmp_cs_nx:VMP_CS_BITS, /* code-signing: page is nx */
282 vmp_reusable:1,
283 vmp_written_by_kernel:1; /* page was written by kernel (i.e. decompressed) */
284
285 #if !defined(__arm64__)
286 ppnum_t vmp_phys_page; /* Physical page number of the page */
287 #endif
288 };
289
290 typedef struct vm_page *vm_page_t;
291 extern vm_page_t vm_pages;
292 extern vm_page_t vm_page_array_beginning_addr;
293 extern vm_page_t vm_page_array_ending_addr;
294
295 static inline int
VMP_CS_FOR_OFFSET(vm_map_offset_t fault_phys_offset)296 VMP_CS_FOR_OFFSET(
297 vm_map_offset_t fault_phys_offset)
298 {
299 assertf(fault_phys_offset < PAGE_SIZE &&
300 !(fault_phys_offset & FOURK_PAGE_MASK),
301 "offset 0x%llx\n", (uint64_t)fault_phys_offset);
302 return 1 << (fault_phys_offset >> FOURK_PAGE_SHIFT);
303 }
304 static inline bool
VMP_CS_VALIDATED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)305 VMP_CS_VALIDATED(
306 vm_page_t p,
307 vm_map_size_t fault_page_size,
308 vm_map_offset_t fault_phys_offset)
309 {
310 assertf(fault_page_size <= PAGE_SIZE,
311 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
312 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
313 if (fault_page_size == PAGE_SIZE) {
314 return p->vmp_cs_validated == VMP_CS_ALL_TRUE;
315 }
316 return p->vmp_cs_validated & VMP_CS_FOR_OFFSET(fault_phys_offset);
317 }
318 static inline bool
VMP_CS_TAINTED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)319 VMP_CS_TAINTED(
320 vm_page_t p,
321 vm_map_size_t fault_page_size,
322 vm_map_offset_t fault_phys_offset)
323 {
324 assertf(fault_page_size <= PAGE_SIZE,
325 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
326 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
327 if (fault_page_size == PAGE_SIZE) {
328 return p->vmp_cs_tainted != VMP_CS_ALL_FALSE;
329 }
330 return p->vmp_cs_tainted & VMP_CS_FOR_OFFSET(fault_phys_offset);
331 }
332 static inline bool
VMP_CS_NX(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)333 VMP_CS_NX(
334 vm_page_t p,
335 vm_map_size_t fault_page_size,
336 vm_map_offset_t fault_phys_offset)
337 {
338 assertf(fault_page_size <= PAGE_SIZE,
339 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
340 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
341 if (fault_page_size == PAGE_SIZE) {
342 return p->vmp_cs_nx != VMP_CS_ALL_FALSE;
343 }
344 return p->vmp_cs_nx & VMP_CS_FOR_OFFSET(fault_phys_offset);
345 }
346 static inline void
VMP_CS_SET_VALIDATED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)347 VMP_CS_SET_VALIDATED(
348 vm_page_t p,
349 vm_map_size_t fault_page_size,
350 vm_map_offset_t fault_phys_offset,
351 boolean_t value)
352 {
353 assertf(fault_page_size <= PAGE_SIZE,
354 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
355 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
356 if (value) {
357 if (fault_page_size == PAGE_SIZE) {
358 p->vmp_cs_validated = VMP_CS_ALL_TRUE;
359 }
360 p->vmp_cs_validated |= VMP_CS_FOR_OFFSET(fault_phys_offset);
361 } else {
362 if (fault_page_size == PAGE_SIZE) {
363 p->vmp_cs_validated = VMP_CS_ALL_FALSE;
364 }
365 p->vmp_cs_validated &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
366 }
367 }
368 static inline void
VMP_CS_SET_TAINTED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)369 VMP_CS_SET_TAINTED(
370 vm_page_t p,
371 vm_map_size_t fault_page_size,
372 vm_map_offset_t fault_phys_offset,
373 boolean_t value)
374 {
375 assertf(fault_page_size <= PAGE_SIZE,
376 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
377 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
378 if (value) {
379 if (fault_page_size == PAGE_SIZE) {
380 p->vmp_cs_tainted = VMP_CS_ALL_TRUE;
381 }
382 p->vmp_cs_tainted |= VMP_CS_FOR_OFFSET(fault_phys_offset);
383 } else {
384 if (fault_page_size == PAGE_SIZE) {
385 p->vmp_cs_tainted = VMP_CS_ALL_FALSE;
386 }
387 p->vmp_cs_tainted &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
388 }
389 }
390 static inline void
VMP_CS_SET_NX(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)391 VMP_CS_SET_NX(
392 vm_page_t p,
393 vm_map_size_t fault_page_size,
394 vm_map_offset_t fault_phys_offset,
395 boolean_t value)
396 {
397 assertf(fault_page_size <= PAGE_SIZE,
398 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
399 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
400 if (value) {
401 if (fault_page_size == PAGE_SIZE) {
402 p->vmp_cs_nx = VMP_CS_ALL_TRUE;
403 }
404 p->vmp_cs_nx |= VMP_CS_FOR_OFFSET(fault_phys_offset);
405 } else {
406 if (fault_page_size == PAGE_SIZE) {
407 p->vmp_cs_nx = VMP_CS_ALL_FALSE;
408 }
409 p->vmp_cs_nx &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
410 }
411 }
412
413
414 #if defined(__arm64__)
415
416 extern unsigned int vm_first_phys_ppnum;
417
418 struct vm_page_with_ppnum {
419 struct vm_page vm_page_wo_ppnum;
420
421 ppnum_t vmp_phys_page;
422 };
423 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
424
425
426 static inline ppnum_t
VM_PAGE_GET_PHYS_PAGE(vm_page_t m)427 VM_PAGE_GET_PHYS_PAGE(vm_page_t m)
428 {
429 if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) {
430 return (ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum);
431 } else {
432 return ((vm_page_with_ppnum_t)m)->vmp_phys_page;
433 }
434 }
435
436 #define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \
437 MACRO_BEGIN \
438 if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \
439 ((vm_page_with_ppnum_t)(m))->vmp_phys_page = ppnum; \
440 assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \
441 MACRO_END
442
443 #define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask)
444
445 #else /* defined(__arm64__) */
446
447
448 struct vm_page_with_ppnum {
449 struct vm_page vm_page_with_ppnum;
450 };
451 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
452
453
454 #define VM_PAGE_GET_PHYS_PAGE(page) (page)->vmp_phys_page
455 #define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
456 MACRO_BEGIN \
457 (page)->vmp_phys_page = ppnum; \
458 MACRO_END
459
460 #define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift)
461 #define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask)
462
463 #endif /* defined(__arm64__) */
464
465
466
467 #if defined(__LP64__)
468 /*
469 * Parameters for pointer packing
470 *
471 *
472 * VM Pages pointers might point to:
473 *
474 * 1. VM_PAGE_PACKED_ALIGNED aligned kernel globals,
475 *
476 * 2. VM_PAGE_PACKED_ALIGNED aligned heap allocated vm pages
477 *
478 * 3. entries in the vm_pages array (whose entries aren't VM_PAGE_PACKED_ALIGNED
479 * aligned).
480 *
481 *
482 * The current scheme uses 31 bits of storage and 6 bits of shift using the
483 * VM_PACK_POINTER() scheme for (1-2), and packs (3) as an index within the
484 * vm_pages array, setting the top bit (VM_PAGE_PACKED_FROM_ARRAY).
485 *
486 * This scheme gives us a reach of 128G from VM_MIN_KERNEL_AND_KEXT_ADDRESS.
487 */
488 #define VM_VPLQ_ALIGNMENT 128
489 #define VM_PAGE_PACKED_PTR_ALIGNMENT 64 /* must be a power of 2 */
490 #define VM_PAGE_PACKED_ALIGNED __attribute__((aligned(VM_PAGE_PACKED_PTR_ALIGNMENT)))
491 #define VM_PAGE_PACKED_PTR_BITS 31
492 #define VM_PAGE_PACKED_PTR_SHIFT 6
493 #define VM_PAGE_PACKED_PTR_BASE ((uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS)
494
495 #define VM_PAGE_PACKED_FROM_ARRAY 0x80000000
496
497 static inline vm_page_packed_t
vm_page_pack_ptr(uintptr_t p)498 vm_page_pack_ptr(uintptr_t p)
499 {
500 if (p >= (uintptr_t)vm_page_array_beginning_addr &&
501 p < (uintptr_t)vm_page_array_ending_addr) {
502 ptrdiff_t diff = (vm_page_t)p - vm_page_array_beginning_addr;
503 assert((vm_page_t)p == &vm_pages[diff]);
504 return (vm_page_packed_t)(diff | VM_PAGE_PACKED_FROM_ARRAY);
505 }
506
507 VM_ASSERT_POINTER_PACKABLE(p, VM_PAGE_PACKED_PTR);
508 vm_offset_t packed = VM_PACK_POINTER(p, VM_PAGE_PACKED_PTR);
509 return CAST_DOWN_EXPLICIT(vm_page_packed_t, packed);
510 }
511
512
513 static inline uintptr_t
vm_page_unpack_ptr(uintptr_t p)514 vm_page_unpack_ptr(uintptr_t p)
515 {
516 extern unsigned int vm_pages_count;
517
518 if (p >= VM_PAGE_PACKED_FROM_ARRAY) {
519 p &= ~VM_PAGE_PACKED_FROM_ARRAY;
520 assert(p < (uintptr_t)vm_pages_count);
521 return (uintptr_t)&vm_pages[p];
522 }
523
524 return VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR);
525 }
526
527
528 #define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
529 #define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
530
531 #define VM_OBJECT_PACK(o) ((vm_page_object_t)VM_PACK_POINTER((uintptr_t)(o), VM_PAGE_PACKED_PTR))
532 #define VM_OBJECT_UNPACK(p) ((vm_object_t)VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR))
533
534 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
535 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
536
537
538 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
539 MACRO_BEGIN \
540 (p)->vmp_snext = 0; \
541 MACRO_END
542
543
544 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
545
546
547 static __inline__ void
vm_page_enqueue_tail(vm_page_queue_t que,vm_page_queue_entry_t elt)548 vm_page_enqueue_tail(
549 vm_page_queue_t que,
550 vm_page_queue_entry_t elt)
551 {
552 vm_page_queue_entry_t old_tail;
553
554 old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev);
555 elt->next = VM_PAGE_PACK_PTR(que);
556 elt->prev = que->prev;
557 que->prev = old_tail->next = VM_PAGE_PACK_PTR(elt);
558 }
559
560
561 static __inline__ void
vm_page_remque(vm_page_queue_entry_t elt)562 vm_page_remque(
563 vm_page_queue_entry_t elt)
564 {
565 vm_page_queue_entry_t next;
566 vm_page_queue_entry_t prev;
567 vm_page_packed_t next_pck = elt->next;
568 vm_page_packed_t prev_pck = elt->prev;
569
570 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(next_pck);
571
572 /* next may equal prev (and the queue head) if elt was the only element */
573 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev_pck);
574
575 next->prev = prev_pck;
576 prev->next = next_pck;
577
578 elt->next = 0;
579 elt->prev = 0;
580 }
581
582
583 /*
584 * Macro: vm_page_queue_init
585 * Function:
586 * Initialize the given queue.
587 * Header:
588 * void vm_page_queue_init(q)
589 * vm_page_queue_t q; \* MODIFIED *\
590 */
591 #define vm_page_queue_init(q) \
592 MACRO_BEGIN \
593 VM_ASSERT_POINTER_PACKABLE((vm_offset_t)(q), VM_PAGE_PACKED_PTR); \
594 (q)->next = VM_PAGE_PACK_PTR(q); \
595 (q)->prev = VM_PAGE_PACK_PTR(q); \
596 MACRO_END
597
598
599 /*
600 * Macro: vm_page_queue_enter
601 * Function:
602 * Insert a new element at the tail of the vm_page queue.
603 * Header:
604 * void vm_page_queue_enter(q, elt, field)
605 * queue_t q;
606 * vm_page_t elt;
607 * <field> is the list field in vm_page_t
608 *
609 * This macro's arguments have to match the generic "queue_enter()" macro which is
610 * what is used for this on 32 bit kernels.
611 */
612 #define vm_page_queue_enter(head, elt, field) \
613 MACRO_BEGIN \
614 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
615 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
616 vm_page_packed_t __pck_prev = (head)->prev; \
617 \
618 if (__pck_head == __pck_prev) { \
619 (head)->next = __pck_elt; \
620 } else { \
621 vm_page_t __prev; \
622 __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
623 __prev->field.next = __pck_elt; \
624 } \
625 (elt)->field.prev = __pck_prev; \
626 (elt)->field.next = __pck_head; \
627 (head)->prev = __pck_elt; \
628 MACRO_END
629
630
631 #if defined(__x86_64__)
632 /*
633 * These are helper macros for vm_page_queue_enter_clump to assist
634 * with conditional compilation (release / debug / development)
635 */
636 #if DEVELOPMENT || DEBUG
637
638 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field) \
639 MACRO_BEGIN \
640 if (__prev != NULL) { \
641 assert(__p == (vm_page_t)VM_PAGE_UNPACK_PTR(__prev->next)); \
642 assert(__prev == (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__p->field.prev)); \
643 } \
644 MACRO_END
645
646 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next) \
647 MACRO_BEGIN \
648 unsigned int __i; \
649 vm_page_queue_entry_t __tmp; \
650 for (__i = 0, __tmp = __first; __i < __n_free; __i++) { \
651 __tmp = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__tmp->next); \
652 } \
653 assert(__tmp == __last_next); \
654 MACRO_END
655
656 #define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++
657 #define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++
658 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free
659
660 #else
661
662 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field)
663 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next)
664 #define __DEBUG_STAT_INCREMENT_INRANGE
665 #define __DEBUG_STAT_INCREMENT_INSERTS
666 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free)
667
668 #endif /* if DEVELOPMENT || DEBUG */
669
670 /*
671 * Insert a new page into a free queue and clump pages within the same 16K boundary together
672 */
673 static inline void
vm_page_queue_enter_clump(vm_page_queue_t head,vm_page_t elt)674 vm_page_queue_enter_clump(
675 vm_page_queue_t head,
676 vm_page_t elt)
677 {
678 vm_page_queue_entry_t first = NULL; /* first page in the clump */
679 vm_page_queue_entry_t last = NULL; /* last page in the clump */
680 vm_page_queue_entry_t prev = NULL;
681 vm_page_queue_entry_t next;
682 uint_t n_free = 1;
683 extern unsigned int vm_pages_count;
684 extern unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
685 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
686
687 /*
688 * If elt is part of the vm_pages[] array, find its neighboring buddies in the array.
689 */
690 if (vm_page_array_beginning_addr <= elt && elt < &vm_pages[vm_pages_count]) {
691 vm_page_t p;
692 uint_t i;
693 uint_t n;
694 ppnum_t clump_num;
695
696 first = last = (vm_page_queue_entry_t)elt;
697 clump_num = VM_PAGE_GET_CLUMP(elt);
698 n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask;
699
700 /*
701 * Check for preceeding vm_pages[] entries in the same chunk
702 */
703 for (i = 0, p = elt - 1; i < n && vm_page_array_beginning_addr <= p; i++, p--) {
704 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
705 if (prev == NULL) {
706 prev = (vm_page_queue_entry_t)p;
707 }
708 first = (vm_page_queue_entry_t)p;
709 n_free++;
710 }
711 }
712
713 /*
714 * Check the following vm_pages[] entries in the same chunk
715 */
716 for (i = n + 1, p = elt + 1; i < vm_clump_size && p < &vm_pages[vm_pages_count]; i++, p++) {
717 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
718 if (last == (vm_page_queue_entry_t)elt) { /* first one only */
719 __DEBUG_CHECK_BUDDIES(prev, p, vmp_pageq);
720 }
721
722 if (prev == NULL) {
723 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev);
724 }
725 last = (vm_page_queue_entry_t)p;
726 n_free++;
727 }
728 }
729 __DEBUG_STAT_INCREMENT_INRANGE;
730 }
731
732 /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */
733 if (prev == NULL) {
734 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->prev);
735 }
736
737 /* insert the element */
738 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev->next);
739 elt->vmp_pageq.next = prev->next;
740 elt->vmp_pageq.prev = next->prev;
741 prev->next = next->prev = VM_PAGE_PACK_PTR(elt);
742 __DEBUG_STAT_INCREMENT_INSERTS;
743
744 /*
745 * Check if clump needs to be promoted to head.
746 */
747 if (n_free >= vm_clump_promote_threshold && n_free > 1) {
748 vm_page_queue_entry_t first_prev;
749
750 first_prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(first->prev);
751
752 /* If not at head already */
753 if (first_prev != head) {
754 vm_page_queue_entry_t last_next;
755 vm_page_queue_entry_t head_next;
756
757 last_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(last->next);
758
759 /* verify that the links within the clump are consistent */
760 __DEBUG_VERIFY_LINKS(first, n_free, last_next);
761
762 /* promote clump to head */
763 first_prev->next = last->next;
764 last_next->prev = first->prev;
765 first->prev = VM_PAGE_PACK_PTR(head);
766 last->next = head->next;
767
768 head_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->next);
769 head_next->prev = VM_PAGE_PACK_PTR(last);
770 head->next = VM_PAGE_PACK_PTR(first);
771 __DEBUG_STAT_INCREMENT_PROMOTES(n_free);
772 }
773 }
774 }
775 #endif
776
777 /*
778 * Macro: vm_page_queue_enter_first
779 * Function:
780 * Insert a new element at the head of the vm_page queue.
781 * Header:
782 * void queue_enter_first(q, elt, , field)
783 * queue_t q;
784 * vm_page_t elt;
785 * <field> is the linkage field in vm_page
786 *
787 * This macro's arguments have to match the generic "queue_enter_first()" macro which is
788 * what is used for this on 32 bit kernels.
789 */
790 #define vm_page_queue_enter_first(head, elt, field) \
791 MACRO_BEGIN \
792 vm_page_packed_t __pck_next = (head)->next; \
793 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
794 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
795 \
796 if (__pck_head == __pck_next) { \
797 (head)->prev = __pck_elt; \
798 } else { \
799 vm_page_t __next; \
800 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
801 __next->field.prev = __pck_elt; \
802 } \
803 \
804 (elt)->field.next = __pck_next; \
805 (elt)->field.prev = __pck_head; \
806 (head)->next = __pck_elt; \
807 MACRO_END
808
809
810 /*
811 * Macro: vm_page_queue_remove
812 * Function:
813 * Remove an arbitrary page from a vm_page queue.
814 * Header:
815 * void vm_page_queue_remove(q, qe, field)
816 * arguments as in vm_page_queue_enter
817 *
818 * This macro's arguments have to match the generic "queue_enter()" macro which is
819 * what is used for this on 32 bit kernels.
820 */
821 #define vm_page_queue_remove(head, elt, field) \
822 MACRO_BEGIN \
823 vm_page_packed_t __pck_next = (elt)->field.next; \
824 vm_page_packed_t __pck_prev = (elt)->field.prev; \
825 vm_page_t __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
826 vm_page_t __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
827 \
828 if ((void *)(head) == (void *)__next) { \
829 (head)->prev = __pck_prev; \
830 } else { \
831 __next->field.prev = __pck_prev; \
832 } \
833 \
834 if ((void *)(head) == (void *)__prev) { \
835 (head)->next = __pck_next; \
836 } else { \
837 __prev->field.next = __pck_next; \
838 } \
839 \
840 (elt)->field.next = 0; \
841 (elt)->field.prev = 0; \
842 MACRO_END
843
844
845 /*
846 * Macro: vm_page_queue_remove_first
847 *
848 * Function:
849 * Remove and return the entry at the head of a vm_page queue.
850 *
851 * Header:
852 * vm_page_queue_remove_first(head, entry, field)
853 * N.B. entry is returned by reference
854 *
855 * This macro's arguments have to match the generic "queue_remove_first()" macro which is
856 * what is used for this on 32 bit kernels.
857 */
858 #define vm_page_queue_remove_first(head, entry, field) \
859 MACRO_BEGIN \
860 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
861 vm_page_packed_t __pck_next; \
862 vm_page_t __next; \
863 \
864 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
865 __pck_next = (entry)->field.next; \
866 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
867 \
868 if (__pck_head == __pck_next) { \
869 (head)->prev = __pck_head; \
870 } else { \
871 __next->field.prev = __pck_head; \
872 } \
873 \
874 (head)->next = __pck_next; \
875 (entry)->field.next = 0; \
876 (entry)->field.prev = 0; \
877 MACRO_END
878
879
880 #if defined(__x86_64__)
881 /*
882 * Macro: vm_page_queue_remove_first_with_clump
883 * Function:
884 * Remove and return the entry at the head of the free queue
885 * end is set to 1 to indicate that we just returned the last page in a clump
886 *
887 * Header:
888 * vm_page_queue_remove_first_with_clump(head, entry, end)
889 * entry is returned by reference
890 * end is returned by reference
891 */
892 #define vm_page_queue_remove_first_with_clump(head, entry, end) \
893 MACRO_BEGIN \
894 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
895 vm_page_packed_t __pck_next; \
896 vm_page_t __next; \
897 \
898 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
899 __pck_next = (entry)->vmp_pageq.next; \
900 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
901 \
902 (end) = 0; \
903 if (__pck_head == __pck_next) { \
904 (head)->prev = __pck_head; \
905 (end) = 1; \
906 } else { \
907 __next->vmp_pageq.prev = __pck_head; \
908 if (VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(__next)) { \
909 (end) = 1; \
910 } \
911 } \
912 \
913 (head)->next = __pck_next; \
914 (entry)->vmp_pageq.next = 0; \
915 (entry)->vmp_pageq.prev = 0; \
916 MACRO_END
917 #endif
918
919 /*
920 * Macro: vm_page_queue_end
921 * Function:
922 * Tests whether a new entry is really the end of
923 * the queue.
924 * Header:
925 * boolean_t vm_page_queue_end(q, qe)
926 * vm_page_queue_t q;
927 * vm_page_queue_entry_t qe;
928 */
929 #define vm_page_queue_end(q, qe) ((q) == (qe))
930
931
932 /*
933 * Macro: vm_page_queue_empty
934 * Function:
935 * Tests whether a queue is empty.
936 * Header:
937 * boolean_t vm_page_queue_empty(q)
938 * vm_page_queue_t q;
939 */
940 #define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
941
942
943
944 /*
945 * Macro: vm_page_queue_first
946 * Function:
947 * Returns the first entry in the queue,
948 * Header:
949 * uintpr_t vm_page_queue_first(q)
950 * vm_page_queue_t q; \* IN *\
951 */
952 #define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
953
954
955
956 /*
957 * Macro: vm_page_queue_last
958 * Function:
959 * Returns the last entry in the queue.
960 * Header:
961 * vm_page_queue_entry_t queue_last(q)
962 * queue_t q; \* IN *\
963 */
964 #define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
965
966
967
968 /*
969 * Macro: vm_page_queue_next
970 * Function:
971 * Returns the entry after an item in the queue.
972 * Header:
973 * uintpr_t vm_page_queue_next(qc)
974 * vm_page_queue_t qc;
975 */
976 #define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next))
977
978
979
980 /*
981 * Macro: vm_page_queue_prev
982 * Function:
983 * Returns the entry before an item in the queue.
984 * Header:
985 * uinptr_t vm_page_queue_prev(qc)
986 * vm_page_queue_t qc;
987 */
988 #define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev))
989
990
991
992 /*
993 * Macro: vm_page_queue_iterate
994 * Function:
995 * iterate over each item in a vm_page queue.
996 * Generates a 'for' loop, setting elt to
997 * each item in turn (by reference).
998 * Header:
999 * vm_page_queue_iterate(q, elt, field)
1000 * queue_t q;
1001 * vm_page_t elt;
1002 * <field> is the chain field in vm_page_t
1003 */
1004 #define vm_page_queue_iterate(head, elt, field) \
1005 for ((elt) = (vm_page_t)vm_page_queue_first(head); \
1006 !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \
1007 (elt) = (vm_page_t)vm_page_queue_next(&(elt)->field)) \
1008
1009 #else // LP64
1010
1011 #define VM_VPLQ_ALIGNMENT 128
1012 #define VM_PAGE_PACKED_PTR_ALIGNMENT sizeof(vm_offset_t)
1013 #define VM_PAGE_PACKED_ALIGNED
1014 #define VM_PAGE_PACKED_PTR_BITS 32
1015 #define VM_PAGE_PACKED_PTR_SHIFT 0
1016 #define VM_PAGE_PACKED_PTR_BASE 0
1017
1018 #define VM_PAGE_PACKED_FROM_ARRAY 0
1019
1020 #define VM_PAGE_PACK_PTR(p) (p)
1021 #define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p))
1022
1023 #define VM_OBJECT_PACK(o) ((vm_page_object_t)(o))
1024 #define VM_OBJECT_UNPACK(p) ((vm_object_t)(p))
1025
1026 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
1027 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
1028
1029
1030 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
1031 MACRO_BEGIN \
1032 (p)->vmp_pageq.next = 0; \
1033 (p)->vmp_pageq.prev = 0; \
1034 MACRO_END
1035
1036 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p))
1037
1038 #define vm_page_remque remque
1039 #define vm_page_enqueue_tail enqueue_tail
1040 #define vm_page_queue_init queue_init
1041 #define vm_page_queue_enter(h, e, f) queue_enter(h, e, vm_page_t, f)
1042 #define vm_page_queue_enter_first(h, e, f) queue_enter_first(h, e, vm_page_t, f)
1043 #define vm_page_queue_remove(h, e, f) queue_remove(h, e, vm_page_t, f)
1044 #define vm_page_queue_remove_first(h, e, f) queue_remove_first(h, e, vm_page_t, f)
1045 #define vm_page_queue_end queue_end
1046 #define vm_page_queue_empty queue_empty
1047 #define vm_page_queue_first queue_first
1048 #define vm_page_queue_last queue_last
1049 #define vm_page_queue_next queue_next
1050 #define vm_page_queue_prev queue_prev
1051 #define vm_page_queue_iterate(h, e, f) queue_iterate(h, e, vm_page_t, f)
1052
1053 #endif // __LP64__
1054
1055
1056
1057 /*
1058 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
1059 * represents a set of aging bins that are 'protected'...
1060 *
1061 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
1062 * not yet been 'claimed' but have been aged out of the protective bins
1063 * this occurs in vm_page_speculate when it advances to the next bin
1064 * and discovers that it is still occupied... at that point, all of the
1065 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
1066 * in that bin are all guaranteed to have reached at least the maximum age
1067 * we allow for a protected page... they can be older if there is no
1068 * memory pressure to pull them from the bin, or there are no new speculative pages
1069 * being generated to push them out.
1070 * this list is the one that vm_pageout_scan will prefer when looking
1071 * for pages to move to the underweight free list
1072 *
1073 * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
1074 * defines the amount of time a speculative page is normally
1075 * allowed to live in the 'protected' state (i.e. not available
1076 * to be stolen if vm_pageout_scan is running and looking for
1077 * pages)... however, if the total number of speculative pages
1078 * in the protected state exceeds our limit (defined in vm_pageout.c)
1079 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
1080 * vm_pageout_scan is allowed to steal pages from the protected
1081 * bucket even if they are underage.
1082 *
1083 * vm_pageout_scan is also allowed to pull pages from a protected
1084 * bin if the bin has reached the "age of consent" we've set
1085 */
1086 #define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
1087 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
1088 #define VM_PAGE_SPECULATIVE_AGED_Q 0
1089
1090 #define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
1091
1092 struct vm_speculative_age_q {
1093 /*
1094 * memory queue for speculative pages via clustered pageins
1095 */
1096 vm_page_queue_head_t age_q;
1097 mach_timespec_t age_ts;
1098 } VM_PAGE_PACKED_ALIGNED;
1099
1100
1101
1102 extern
1103 struct vm_speculative_age_q vm_page_queue_speculative[];
1104
1105 extern int speculative_steal_index;
1106 extern int speculative_age_index;
1107 extern unsigned int vm_page_speculative_q_age_ms;
1108
1109
1110 typedef struct vm_locks_array {
1111 char pad __attribute__ ((aligned(64)));
1112 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned(64)));
1113 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned(64)));
1114 char pad2 __attribute__ ((aligned(64)));
1115 } vm_locks_array_t;
1116
1117
1118 extern void vm_page_assign_special_state(vm_page_t mem, int mode);
1119 extern void vm_page_update_special_state(vm_page_t mem);
1120 extern void vm_page_add_to_specialq(vm_page_t mem, boolean_t first);
1121 extern void vm_page_remove_from_specialq(vm_page_t mem);
1122
1123 #define VM_PAGE_WIRED(m) ((m)->vmp_q_state == VM_PAGE_IS_WIRED)
1124 #define NEXT_PAGE(m) ((m)->vmp_snext)
1125 #define NEXT_PAGE_PTR(m) (&(m)->vmp_snext)
1126
1127 /*
1128 * XXX The unusual bit should not be necessary. Most of the bit
1129 * XXX fields above really want to be masks.
1130 */
1131
1132 /*
1133 * For debugging, this macro can be defined to perform
1134 * some useful check on a page structure.
1135 * INTENTIONALLY left as a no-op so that the
1136 * current call-sites can be left intact for future uses.
1137 */
1138
1139 #define VM_PAGE_CHECK(mem) \
1140 MACRO_BEGIN \
1141 MACRO_END
1142
1143 /* Page coloring:
1144 *
1145 * The free page list is actually n lists, one per color,
1146 * where the number of colors is a function of the machine's
1147 * cache geometry set at system initialization. To disable
1148 * coloring, set vm_colors to 1 and vm_color_mask to 0.
1149 * The boot-arg "colors" may be used to override vm_colors.
1150 * Note that there is little harm in having more colors than needed.
1151 */
1152
1153 #define MAX_COLORS 128
1154 #define DEFAULT_COLORS 32
1155
1156 extern
1157 unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
1158 extern
1159 unsigned int vm_color_mask; /* must be (vm_colors-1) */
1160 extern
1161 unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
1162
1163 /*
1164 * Wired memory is a very limited resource and we can't let users exhaust it
1165 * and deadlock the entire system. We enforce the following limits:
1166 *
1167 * vm_per_task_user_wire_limit
1168 * how much memory can be user-wired in one user task
1169 *
1170 * vm_global_user_wire_limit (default: same as vm_per_task_user_wire_limit)
1171 * how much memory can be user-wired in all user tasks
1172 *
1173 * These values are set to defaults based on the number of pages managed
1174 * by the VM system. They can be overriden via sysctls.
1175 * See kmem_set_user_wire_limits for details on the default values.
1176 *
1177 * Regardless of the amount of memory in the system, we never reserve
1178 * more than VM_NOT_USER_WIREABLE_MAX bytes as unlockable.
1179 */
1180 #if defined(__LP64__)
1181 #define VM_NOT_USER_WIREABLE_MAX (32ULL*1024*1024*1024) /* 32GB */
1182 #else
1183 #define VM_NOT_USER_WIREABLE_MAX (1UL*1024*1024*1024) /* 1GB */
1184 #endif /* __LP64__ */
1185 extern
1186 vm_map_size_t vm_per_task_user_wire_limit;
1187 extern
1188 vm_map_size_t vm_global_user_wire_limit;
1189 extern
1190 uint64_t vm_add_wire_count_over_global_limit;
1191 extern
1192 uint64_t vm_add_wire_count_over_user_limit;
1193
1194 /*
1195 * Each pageable resident page falls into one of three lists:
1196 *
1197 * free
1198 * Available for allocation now. The free list is
1199 * actually an array of lists, one per color.
1200 * inactive
1201 * Not referenced in any map, but still has an
1202 * object/offset-page mapping, and may be dirty.
1203 * This is the list of pages that should be
1204 * paged out next. There are actually two
1205 * inactive lists, one for pages brought in from
1206 * disk or other backing store, and another
1207 * for "zero-filled" pages. See vm_pageout_scan()
1208 * for the distinction and usage.
1209 * active
1210 * A list of pages which have been placed in
1211 * at least one physical map. This list is
1212 * ordered, in LRU-like fashion.
1213 */
1214
1215
1216 #define VPL_LOCK_SPIN 1
1217
1218 struct vpl {
1219 vm_page_queue_head_t vpl_queue;
1220 unsigned int vpl_count;
1221 unsigned int vpl_internal_count;
1222 unsigned int vpl_external_count;
1223 lck_spin_t vpl_lock;
1224 };
1225
1226 extern
1227 struct vpl * /* __zpercpu */ vm_page_local_q;
1228 extern
1229 unsigned int vm_page_local_q_soft_limit;
1230 extern
1231 unsigned int vm_page_local_q_hard_limit;
1232 extern
1233 vm_locks_array_t vm_page_locks;
1234
1235 extern
1236 vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */
1237 extern
1238 vm_page_queue_head_t vm_page_queue_active; /* active memory queue */
1239 extern
1240 vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
1241 #if CONFIG_SECLUDED_MEMORY
1242 extern
1243 vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */
1244 #endif /* CONFIG_SECLUDED_MEMORY */
1245 extern
1246 vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
1247 extern
1248 vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
1249 extern
1250 vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
1251
1252 extern
1253 queue_head_t vm_objects_wired;
1254 extern
1255 lck_spin_t vm_objects_wired_lock;
1256
1257 #define VM_PAGE_DONATE_DISABLED 0
1258 #define VM_PAGE_DONATE_ENABLED 1
1259 extern
1260 uint32_t vm_page_donate_mode;
1261 extern
1262 bool vm_page_donate_queue_ripe;
1263
1264 #define VM_PAGE_BACKGROUND_TARGET_MAX 50000
1265 #define VM_PAGE_BG_DISABLED 0
1266 #define VM_PAGE_BG_ENABLED 1
1267
1268 extern
1269 vm_page_queue_head_t vm_page_queue_background;
1270 extern
1271 uint64_t vm_page_background_promoted_count;
1272 extern
1273 uint32_t vm_page_background_count;
1274 extern
1275 uint32_t vm_page_background_target;
1276 extern
1277 uint32_t vm_page_background_internal_count;
1278 extern
1279 uint32_t vm_page_background_external_count;
1280 extern
1281 uint32_t vm_page_background_mode;
1282 extern
1283 uint32_t vm_page_background_exclude_external;
1284
1285 extern
1286 vm_page_queue_head_t vm_page_queue_donate;
1287 extern
1288 uint32_t vm_page_donate_count;
1289 extern
1290 uint32_t vm_page_donate_target_low;
1291 extern
1292 uint32_t vm_page_donate_target_high;
1293 #define VM_PAGE_DONATE_TARGET_LOWWATER (100)
1294 #define VM_PAGE_DONATE_TARGET_HIGHWATER ((unsigned int)(atop_64(max_mem) / 8))
1295
1296 extern
1297 vm_offset_t first_phys_addr; /* physical address for first_page */
1298 extern
1299 vm_offset_t last_phys_addr; /* physical address for last_page */
1300
1301 extern
1302 unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1303 extern
1304 unsigned int vm_page_active_count; /* How many pages are active? */
1305 extern
1306 unsigned int vm_page_inactive_count; /* How many pages are inactive? */
1307 extern
1308 unsigned int vm_page_kernelcache_count; /* How many pages are used for the kernelcache? */
1309 extern
1310 unsigned int vm_page_realtime_count; /* How many pages are used by realtime threads? */
1311 #if CONFIG_SECLUDED_MEMORY
1312 extern
1313 unsigned int vm_page_secluded_count; /* How many pages are secluded? */
1314 extern
1315 unsigned int vm_page_secluded_count_free; /* how many of them are free? */
1316 extern
1317 unsigned int vm_page_secluded_count_inuse; /* how many of them are in use? */
1318 /*
1319 * We keep filling the secluded pool with new eligible pages and
1320 * we can overshoot our target by a lot.
1321 * When there's memory pressure, vm_pageout_scan() will re-balance the queues,
1322 * pushing the extra secluded pages to the active or free queue.
1323 * Since these "over target" secluded pages are actually "available", jetsam
1324 * should consider them as such, so make them visible to jetsam via the
1325 * "vm_page_secluded_count_over_target" counter and update it whenever we
1326 * update vm_page_secluded_count or vm_page_secluded_target.
1327 */
1328 extern
1329 unsigned int vm_page_secluded_count_over_target;
1330 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1331 MACRO_BEGIN \
1332 if (vm_page_secluded_count > vm_page_secluded_target) { \
1333 vm_page_secluded_count_over_target = \
1334 (vm_page_secluded_count - vm_page_secluded_target); \
1335 } else { \
1336 vm_page_secluded_count_over_target = 0; \
1337 } \
1338 MACRO_END
1339 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() vm_page_secluded_count_over_target
1340 #else /* CONFIG_SECLUDED_MEMORY */
1341 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1342 MACRO_BEGIN \
1343 MACRO_END
1344 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() 0
1345 #endif /* CONFIG_SECLUDED_MEMORY */
1346 extern
1347 unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
1348 extern
1349 unsigned int vm_page_throttled_count;/* How many inactives are throttled */
1350 extern
1351 unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
1352 extern unsigned int vm_page_pageable_internal_count;
1353 extern unsigned int vm_page_pageable_external_count;
1354 extern
1355 unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
1356 extern
1357 unsigned int vm_page_external_count; /* How many pages are file-backed? */
1358 extern
1359 unsigned int vm_page_internal_count; /* How many pages are anonymous? */
1360 extern
1361 unsigned int vm_page_wire_count; /* How many pages are wired? */
1362 extern
1363 unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
1364 extern
1365 unsigned int vm_page_wire_count_on_boot; /* even earlier than _initial */
1366 extern
1367 unsigned int vm_page_free_target; /* How many do we want free? */
1368 extern
1369 unsigned int vm_page_free_min; /* When to wakeup pageout */
1370 extern
1371 unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
1372 extern
1373 unsigned int vm_page_inactive_target;/* How many do we want inactive? */
1374 #if CONFIG_SECLUDED_MEMORY
1375 extern
1376 unsigned int vm_page_secluded_target;/* How many do we want secluded? */
1377 #endif /* CONFIG_SECLUDED_MEMORY */
1378 extern
1379 unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
1380 extern
1381 unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1382 extern
1383 unsigned int vm_page_gobble_count;
1384 extern
1385 unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */
1386 extern
1387 unsigned int vm_page_kern_lpage_count; /* Count of large pages used in early boot */
1388
1389
1390 #if DEVELOPMENT || DEBUG
1391 extern
1392 unsigned int vm_page_speculative_used;
1393 #endif
1394
1395 extern
1396 unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
1397 extern
1398 unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
1399 extern
1400 uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1401
1402 extern unsigned int vm_page_free_wanted;
1403 /* how many threads are waiting for memory */
1404
1405 extern unsigned int vm_page_free_wanted_privileged;
1406 /* how many VM privileged threads are waiting for memory */
1407 #if CONFIG_SECLUDED_MEMORY
1408 extern unsigned int vm_page_free_wanted_secluded;
1409 /* how many threads are waiting for secluded memory */
1410 #endif /* CONFIG_SECLUDED_MEMORY */
1411
1412 extern const ppnum_t vm_page_fictitious_addr;
1413 /* (fake) phys_addr of fictitious pages */
1414
1415 extern const ppnum_t vm_page_guard_addr;
1416 /* (fake) phys_addr of guard pages */
1417
1418
1419 extern boolean_t vm_page_deactivate_hint;
1420
1421 extern int vm_compressor_mode;
1422
1423 /*
1424 * Defaults to true, so highest memory is used first.
1425 */
1426 extern boolean_t vm_himemory_mode;
1427
1428 extern boolean_t vm_lopage_needed;
1429 extern uint32_t vm_lopage_free_count;
1430 extern uint32_t vm_lopage_free_limit;
1431 extern uint32_t vm_lopage_lowater;
1432 extern boolean_t vm_lopage_refill;
1433 extern uint64_t max_valid_dma_address;
1434 extern ppnum_t max_valid_low_ppnum;
1435
1436 /*
1437 * Prototypes for functions exported by this module.
1438 */
1439 extern void vm_page_bootstrap(
1440 vm_offset_t *startp,
1441 vm_offset_t *endp);
1442
1443 extern void vm_page_init_local_q(unsigned int num_cpus);
1444
1445 extern void vm_page_create(
1446 ppnum_t start,
1447 ppnum_t end);
1448
1449 extern void vm_page_create_retired(
1450 ppnum_t pn);
1451
1452 extern boolean_t vm_page_created(
1453 vm_page_t page);
1454
1455 extern vm_page_t kdp_vm_page_lookup(
1456 vm_object_t object,
1457 vm_object_offset_t offset);
1458
1459 extern vm_page_t vm_page_lookup(
1460 vm_object_t object,
1461 vm_object_offset_t offset);
1462
1463 extern vm_page_t vm_page_grab_fictitious(boolean_t canwait);
1464
1465 extern vm_page_t vm_page_grab_guard(boolean_t canwait);
1466
1467 extern void vm_page_release_fictitious(
1468 vm_page_t page);
1469
1470 extern void vm_free_delayed_pages(void);
1471
1472 extern bool vm_pool_low(void);
1473
1474 extern vm_page_t vm_page_grab(void);
1475 extern vm_page_t vm_page_grab_options(int flags);
1476
1477 #define VM_PAGE_GRAB_OPTIONS_NONE 0x00000000
1478 #if CONFIG_SECLUDED_MEMORY
1479 #define VM_PAGE_GRAB_SECLUDED 0x00000001
1480 #endif /* CONFIG_SECLUDED_MEMORY */
1481 #define VM_PAGE_GRAB_Q_LOCK_HELD 0x00000002
1482
1483 extern vm_page_t vm_page_grablo(void);
1484
1485 extern void vm_page_release(
1486 vm_page_t page,
1487 boolean_t page_queues_locked);
1488
1489 extern boolean_t vm_page_wait(
1490 int interruptible );
1491
1492 extern vm_page_t vm_page_alloc(
1493 vm_object_t object,
1494 vm_object_offset_t offset);
1495
1496 extern void vm_page_init(
1497 vm_page_t page,
1498 ppnum_t phys_page,
1499 boolean_t lopage);
1500
1501 extern void vm_page_free(
1502 vm_page_t page);
1503
1504 extern void vm_page_free_unlocked(
1505 vm_page_t page,
1506 boolean_t remove_from_hash);
1507
1508 extern void vm_page_balance_inactive(
1509 int max_to_move);
1510
1511 extern void vm_page_activate(
1512 vm_page_t page);
1513
1514 extern void vm_page_deactivate(
1515 vm_page_t page);
1516
1517 extern void vm_page_deactivate_internal(
1518 vm_page_t page,
1519 boolean_t clear_hw_reference);
1520
1521 extern void vm_page_enqueue_cleaned(vm_page_t page);
1522
1523 extern void vm_page_lru(
1524 vm_page_t page);
1525
1526 extern void vm_page_speculate(
1527 vm_page_t page,
1528 boolean_t new);
1529
1530 extern void vm_page_speculate_ageit(
1531 struct vm_speculative_age_q *aq);
1532
1533 extern void vm_page_reactivate_all_throttled(void);
1534
1535 extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
1536
1537 extern void vm_page_rename(
1538 vm_page_t page,
1539 vm_object_t new_object,
1540 vm_object_offset_t new_offset);
1541
1542 extern void vm_page_insert(
1543 vm_page_t page,
1544 vm_object_t object,
1545 vm_object_offset_t offset);
1546
1547 extern void vm_page_insert_wired(
1548 vm_page_t page,
1549 vm_object_t object,
1550 vm_object_offset_t offset,
1551 vm_tag_t tag);
1552
1553 extern void vm_page_insert_internal(
1554 vm_page_t page,
1555 vm_object_t object,
1556 vm_object_offset_t offset,
1557 vm_tag_t tag,
1558 boolean_t queues_lock_held,
1559 boolean_t insert_in_hash,
1560 boolean_t batch_pmap_op,
1561 boolean_t delayed_accounting,
1562 uint64_t *delayed_ledger_update);
1563
1564 extern void vm_page_replace(
1565 vm_page_t mem,
1566 vm_object_t object,
1567 vm_object_offset_t offset);
1568
1569 extern void vm_page_remove(
1570 vm_page_t page,
1571 boolean_t remove_from_hash);
1572
1573 extern void vm_page_zero_fill(
1574 vm_page_t page);
1575
1576 extern void vm_page_part_zero_fill(
1577 vm_page_t m,
1578 vm_offset_t m_pa,
1579 vm_size_t len);
1580
1581 extern void vm_page_copy(
1582 vm_page_t src_page,
1583 vm_page_t dest_page);
1584
1585 extern void vm_page_part_copy(
1586 vm_page_t src_m,
1587 vm_offset_t src_pa,
1588 vm_page_t dst_m,
1589 vm_offset_t dst_pa,
1590 vm_size_t len);
1591
1592 extern void vm_page_wire(
1593 vm_page_t page,
1594 vm_tag_t tag,
1595 boolean_t check_memorystatus);
1596
1597 extern void vm_page_unwire(
1598 vm_page_t page,
1599 boolean_t queueit);
1600
1601 extern void vm_set_page_size(void);
1602
1603 extern void vm_page_gobble(
1604 vm_page_t page);
1605
1606 extern void vm_page_validate_cs(
1607 vm_page_t page,
1608 vm_map_size_t fault_page_size,
1609 vm_map_offset_t fault_phys_offset);
1610 extern void vm_page_validate_cs_mapped(
1611 vm_page_t page,
1612 vm_map_size_t fault_page_size,
1613 vm_map_offset_t fault_phys_offset,
1614 const void *kaddr);
1615 extern void vm_page_validate_cs_mapped_slow(
1616 vm_page_t page,
1617 const void *kaddr);
1618 extern void vm_page_validate_cs_mapped_chunk(
1619 vm_page_t page,
1620 const void *kaddr,
1621 vm_offset_t chunk_offset,
1622 vm_size_t chunk_size,
1623 boolean_t *validated,
1624 unsigned *tainted);
1625
1626 extern void vm_page_free_prepare_queues(
1627 vm_page_t page);
1628
1629 extern void vm_page_free_prepare_object(
1630 vm_page_t page,
1631 boolean_t remove_from_hash);
1632
1633 #if CONFIG_IOSCHED
1634 extern wait_result_t vm_page_sleep(
1635 vm_object_t object,
1636 vm_page_t m,
1637 int interruptible);
1638 #endif
1639
1640 extern void vm_pressure_response(void);
1641
1642 #if CONFIG_JETSAM
1643 extern void memorystatus_pages_update(unsigned int pages_avail);
1644
1645 #define VM_CHECK_MEMORYSTATUS do { \
1646 memorystatus_pages_update( \
1647 vm_page_pageable_external_count + \
1648 vm_page_free_count + \
1649 VM_PAGE_SECLUDED_COUNT_OVER_TARGET() + \
1650 (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \
1651 ); \
1652 } while(0)
1653
1654 #else /* CONFIG_JETSAM */
1655
1656 #if !XNU_TARGET_OS_OSX
1657
1658 #define VM_CHECK_MEMORYSTATUS do {} while(0)
1659
1660 #else /* !XNU_TARGET_OS_OSX */
1661
1662 #define VM_CHECK_MEMORYSTATUS vm_pressure_response()
1663
1664 #endif /* !XNU_TARGET_OS_OSX */
1665
1666 #endif /* CONFIG_JETSAM */
1667
1668 /*
1669 * Functions implemented as macros. m->vmp_wanted and m->vmp_busy are
1670 * protected by the object lock.
1671 */
1672
1673 #if !XNU_TARGET_OS_OSX
1674 #define SET_PAGE_DIRTY(m, set_pmap_modified) \
1675 MACRO_BEGIN \
1676 vm_page_t __page__ = (m); \
1677 if (__page__->vmp_pmapped == TRUE && \
1678 __page__->vmp_wpmapped == TRUE && \
1679 __page__->vmp_dirty == FALSE && \
1680 (set_pmap_modified)) { \
1681 pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \
1682 } \
1683 __page__->vmp_dirty = TRUE; \
1684 MACRO_END
1685 #else /* !XNU_TARGET_OS_OSX */
1686 #define SET_PAGE_DIRTY(m, set_pmap_modified) \
1687 MACRO_BEGIN \
1688 vm_page_t __page__ = (m); \
1689 __page__->vmp_dirty = TRUE; \
1690 MACRO_END
1691 #endif /* !XNU_TARGET_OS_OSX */
1692
1693 #define PAGE_ASSERT_WAIT(m, interruptible) \
1694 (((m)->vmp_wanted = TRUE), \
1695 assert_wait((event_t) (m), (interruptible)))
1696
1697 #if CONFIG_IOSCHED
1698 #define PAGE_SLEEP(o, m, interruptible) \
1699 vm_page_sleep(o, m, interruptible)
1700 #else
1701 #define PAGE_SLEEP(o, m, interruptible) \
1702 (((m)->vmp_wanted = TRUE), \
1703 thread_sleep_vm_object((o), (m), (interruptible)))
1704 #endif
1705
1706 #define PAGE_WAKEUP_DONE(m) \
1707 MACRO_BEGIN \
1708 (m)->vmp_busy = FALSE; \
1709 if ((m)->vmp_wanted) { \
1710 (m)->vmp_wanted = FALSE; \
1711 thread_wakeup((event_t) (m)); \
1712 } \
1713 MACRO_END
1714
1715 #define PAGE_WAKEUP(m) \
1716 MACRO_BEGIN \
1717 if ((m)->vmp_wanted) { \
1718 (m)->vmp_wanted = FALSE; \
1719 thread_wakeup((event_t) (m)); \
1720 } \
1721 MACRO_END
1722
1723 #define VM_PAGE_FREE(p) \
1724 MACRO_BEGIN \
1725 vm_page_free_unlocked(p, TRUE); \
1726 MACRO_END
1727
1728 #define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
1729
1730 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
1731 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
1732
1733 static inline void
vm_free_page_lock(void)1734 vm_free_page_lock(void)
1735 {
1736 lck_mtx_lock(&vm_page_queue_free_lock);
1737 }
1738
1739 static inline void
vm_free_page_lock_spin(void)1740 vm_free_page_lock_spin(void)
1741 {
1742 lck_mtx_lock_spin(&vm_page_queue_free_lock);
1743 }
1744
1745 static inline void
vm_free_page_unlock(void)1746 vm_free_page_unlock(void)
1747 {
1748 lck_mtx_unlock(&vm_page_queue_free_lock);
1749 }
1750
1751
1752 static inline void
vm_page_lock_queues(void)1753 vm_page_lock_queues(void)
1754 {
1755 lck_mtx_lock(&vm_page_queue_lock);
1756 }
1757
1758 static inline boolean_t
vm_page_trylock_queues(void)1759 vm_page_trylock_queues(void)
1760 {
1761 boolean_t ret;
1762 ret = lck_mtx_try_lock(&vm_page_queue_lock);
1763 return ret;
1764 }
1765
1766 static inline void
vm_page_unlock_queues(void)1767 vm_page_unlock_queues(void)
1768 {
1769 lck_mtx_unlock(&vm_page_queue_lock);
1770 }
1771
1772 static inline void
vm_page_lockspin_queues(void)1773 vm_page_lockspin_queues(void)
1774 {
1775 lck_mtx_lock_spin(&vm_page_queue_lock);
1776 }
1777
1778 static inline boolean_t
vm_page_trylockspin_queues(void)1779 vm_page_trylockspin_queues(void)
1780 {
1781 boolean_t ret;
1782 ret = lck_mtx_try_lock_spin(&vm_page_queue_lock);
1783 return ret;
1784 }
1785 #define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
1786
1787 #ifdef VPL_LOCK_SPIN
1788 extern lck_grp_t vm_page_lck_grp_local;
1789
1790 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
1791 #define VPL_LOCK(vpl) lck_spin_lock_grp(vpl, &vm_page_lck_grp_local)
1792 #define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
1793 #else
1794 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
1795 #define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
1796 #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
1797 #endif
1798
1799
1800 #if DEVELOPMENT || DEBUG
1801 #define VM_PAGE_SPECULATIVE_USED_ADD() \
1802 MACRO_BEGIN \
1803 OSAddAtomic(1, &vm_page_speculative_used); \
1804 MACRO_END
1805 #else
1806 #define VM_PAGE_SPECULATIVE_USED_ADD()
1807 #endif
1808
1809
1810 #define VM_PAGE_CONSUME_CLUSTERED(mem) \
1811 MACRO_BEGIN \
1812 ppnum_t __phys_page; \
1813 __phys_page = VM_PAGE_GET_PHYS_PAGE(mem); \
1814 pmap_lock_phys_page(__phys_page); \
1815 if (mem->vmp_clustered) { \
1816 vm_object_t o; \
1817 o = VM_PAGE_OBJECT(mem); \
1818 assert(o); \
1819 o->pages_used++; \
1820 mem->vmp_clustered = FALSE; \
1821 VM_PAGE_SPECULATIVE_USED_ADD(); \
1822 } \
1823 pmap_unlock_phys_page(__phys_page); \
1824 MACRO_END
1825
1826
1827 #define VM_PAGE_COUNT_AS_PAGEIN(mem) \
1828 MACRO_BEGIN \
1829 { \
1830 vm_object_t o; \
1831 o = VM_PAGE_OBJECT(mem); \
1832 DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
1833 counter_inc(¤t_task()->pageins); \
1834 if (o->internal) { \
1835 DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
1836 } else { \
1837 DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
1838 } \
1839 } \
1840 MACRO_END
1841
1842 /* adjust for stolen pages accounted elsewhere */
1843 #define VM_PAGE_MOVE_STOLEN(page_count) \
1844 MACRO_BEGIN \
1845 vm_page_stolen_count -= (page_count); \
1846 vm_page_wire_count_initial -= (page_count); \
1847 MACRO_END
1848
1849 extern kern_return_t pmap_enter_check(
1850 pmap_t pmap,
1851 vm_map_address_t virtual_address,
1852 vm_page_t page,
1853 vm_prot_t protection,
1854 vm_prot_t fault_type,
1855 unsigned int flags,
1856 boolean_t wired);
1857
1858 #define DW_vm_page_unwire 0x01
1859 #define DW_vm_page_wire 0x02
1860 #define DW_vm_page_free 0x04
1861 #define DW_vm_page_activate 0x08
1862 #define DW_vm_page_deactivate_internal 0x10
1863 #define DW_vm_page_speculate 0x20
1864 #define DW_vm_page_lru 0x40
1865 #define DW_vm_pageout_throttle_up 0x80
1866 #define DW_PAGE_WAKEUP 0x100
1867 #define DW_clear_busy 0x200
1868 #define DW_clear_reference 0x400
1869 #define DW_set_reference 0x800
1870 #define DW_move_page 0x1000
1871 #define DW_VM_PAGE_QUEUES_REMOVE 0x2000
1872 #define DW_enqueue_cleaned 0x4000
1873 #define DW_vm_phantom_cache_update 0x8000
1874
1875 struct vm_page_delayed_work {
1876 vm_page_t dw_m;
1877 int dw_mask;
1878 };
1879
1880 #define DEFAULT_DELAYED_WORK_LIMIT 32
1881
1882 struct vm_page_delayed_work_ctx {
1883 struct vm_page_delayed_work dwp[DEFAULT_DELAYED_WORK_LIMIT];
1884 thread_t delayed_owner;
1885 };
1886
1887 void vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count);
1888
1889 extern unsigned int vm_max_delayed_work_limit;
1890
1891 #define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
1892
1893 /*
1894 * vm_page_do_delayed_work may need to drop the object lock...
1895 * if it does, we need the pages it's looking at to
1896 * be held stable via the busy bit, so if busy isn't already
1897 * set, we need to set it and ask vm_page_do_delayed_work
1898 * to clear it and wakeup anyone that might have blocked on
1899 * it once we're done processing the page.
1900 */
1901
1902 #define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
1903 MACRO_BEGIN \
1904 if (mem->vmp_busy == FALSE) { \
1905 mem->vmp_busy = TRUE; \
1906 if ( !(dwp->dw_mask & DW_vm_page_free)) \
1907 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
1908 } \
1909 dwp->dw_m = mem; \
1910 dwp++; \
1911 dw_cnt++; \
1912 MACRO_END
1913
1914 extern vm_page_t vm_object_page_grab(vm_object_t);
1915
1916 #if VM_PAGE_BUCKETS_CHECK
1917 extern void vm_page_buckets_check(void);
1918 #endif /* VM_PAGE_BUCKETS_CHECK */
1919
1920 extern void vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_specialq);
1921 extern void vm_page_remove_internal(vm_page_t page);
1922 extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first);
1923 extern void vm_page_enqueue_active(vm_page_t mem, boolean_t first);
1924 extern void vm_page_check_pageable_safe(vm_page_t page);
1925
1926 #if CONFIG_SECLUDED_MEMORY
1927 extern uint64_t secluded_shutoff_trigger;
1928 extern uint64_t secluded_shutoff_headroom;
1929 extern void start_secluded_suppression(task_t);
1930 extern void stop_secluded_suppression(task_t);
1931 #endif /* CONFIG_SECLUDED_MEMORY */
1932
1933 extern void vm_retire_boot_pages(void);
1934
1935
1936 #define VMP_ERROR_GET(p) ((p)->vmp_error)
1937
1938
1939 #endif /* _VM_VM_PAGE_H_ */
1940