1 /*
2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66 #ifndef _VM_VM_PAGE_H_
67 #define _VM_VM_PAGE_H_
68
69 #include <debug.h>
70 #include <vm/vm_options.h>
71 #include <vm/vm_protos.h>
72 #include <mach/boolean.h>
73 #include <mach/vm_prot.h>
74 #include <mach/vm_param.h>
75 #include <mach/memory_object_types.h> /* for VMP_CS_BITS... */
76
77
78 #if defined(__LP64__)
79
80 /*
81 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
82 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
83 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
84 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
85 * pointers from the 2 ends of these spaces
86 */
87 typedef uint32_t vm_page_packed_t;
88
89 struct vm_page_packed_queue_entry {
90 vm_page_packed_t next; /* next element */
91 vm_page_packed_t prev; /* previous element */
92 };
93
94 typedef struct vm_page_packed_queue_entry *vm_page_queue_t;
95 typedef struct vm_page_packed_queue_entry vm_page_queue_head_t;
96 typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t;
97 typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t;
98
99 typedef vm_page_packed_t vm_page_object_t;
100
101 #else // __LP64__
102
103 /*
104 * we can't do the packing trick on 32 bit architectures
105 * so just turn the macros into noops.
106 */
107 typedef struct vm_page *vm_page_packed_t;
108
109 #define vm_page_queue_t queue_t
110 #define vm_page_queue_head_t queue_head_t
111 #define vm_page_queue_chain_t queue_chain_t
112 #define vm_page_queue_entry_t queue_entry_t
113
114 #define vm_page_object_t vm_object_t
115 #endif // __LP64__
116
117
118 #include <vm/vm_object_xnu.h>
119 #include <kern/queue.h>
120 #include <kern/locks.h>
121
122 #include <kern/macro_help.h>
123 #include <libkern/OSAtomic.h>
124
125
126 /* pages of compressed data */
127 #define VM_PAGE_COMPRESSOR_COUNT os_atomic_load(&compressor_object->resident_page_count, relaxed)
128
129 /*
130 * Management of resident (logical) pages.
131 *
132 * A small structure is kept for each resident
133 * page, indexed by page number. Each structure
134 * is an element of several lists:
135 *
136 * A hash table bucket used to quickly
137 * perform object/offset lookups
138 *
139 * A list of all pages for a given object,
140 * so they can be quickly deactivated at
141 * time of deallocation.
142 *
143 * An ordered list of pages due for pageout.
144 *
145 * In addition, the structure contains the object
146 * and offset to which this page belongs (for pageout),
147 * and sundry status bits.
148 *
149 * Fields in this structure are locked either by the lock on the
150 * object that the page belongs to (O) or by the lock on the page
151 * queues (P). [Some fields require that both locks be held to
152 * change that field; holding either lock is sufficient to read.]
153 */
154
155 #define VM_PAGE_NULL ((vm_page_t) 0)
156
157 extern char vm_page_inactive_states[];
158 extern char vm_page_pageable_states[];
159 extern char vm_page_non_speculative_pageable_states[];
160 extern char vm_page_active_or_inactive_states[];
161
162
163 #define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vmp_q_state])
164 #define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vmp_q_state])
165 #define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vmp_q_state])
166 #define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vmp_q_state])
167
168
169 #define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */
170 #define VM_PAGE_IS_WIRED 1 /* page is currently wired */
171 #define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */
172 #define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */
173 #define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */
174 #define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */
175 #define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */
176 #define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */
177 #define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */
178 #define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */
179 #define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */
180 #define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */
181 #define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */
182 #define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */
183 #define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */
184 #define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */
185
186 #define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1)
187
188
189 /*
190 * The structure itself. See the block comment above for what (O) and (P) mean.
191 */
192 #define vmp_pageq vmp_q_un.vmp_q_pageq
193 #define vmp_snext vmp_q_un.vmp_q_snext
194
195 struct vm_page {
196 union {
197 vm_page_queue_chain_t vmp_q_pageq; /* queue info for FIFO queue or free list (P) */
198 struct vm_page *vmp_q_snext;
199 } vmp_q_un;
200
201 vm_page_queue_chain_t vmp_listq; /* all pages in same object (O) */
202
203 vm_page_queue_chain_t vmp_specialq; /* anonymous pages in the special queues (P) */
204 vm_object_offset_t vmp_offset; /* offset into that object (O,P) */
205
206 vm_page_object_t vmp_object; /* which object am I in (O&P) */
207
208 /*
209 * The following word of flags used to be protected by the "page queues" lock.
210 * That's no longer true and what lock, if any, is needed may depend on the
211 * value of vmp_q_state.
212 *
213 * We use 'vmp_wire_count' to store the local queue id if local queues are enabled.
214 * See the comments at 'vm_page_queues_remove' as to why this is safe to do.
215 */
216 #define VM_PAGE_SPECIAL_Q_EMPTY (0)
217 #define VM_PAGE_SPECIAL_Q_BG (1)
218 #define VM_PAGE_SPECIAL_Q_DONATE (2)
219 #define VM_PAGE_SPECIAL_Q_FG (3)
220 #define vmp_local_id vmp_wire_count
221 unsigned int vmp_wire_count:16, /* how many wired down maps use me? (O&P) */
222 vmp_q_state:4, /* which q is the page on (P) */
223 vmp_on_specialq:2,
224 vmp_gobbled:1, /* page used internally (P) */
225 vmp_laundry:1, /* page is being cleaned now (P)*/
226 vmp_no_cache:1, /* page is not to be cached and should */
227 /* be reused ahead of other pages (P) */
228 vmp_private:1, /* Page should not be returned to the free list (P) */
229 vmp_reference:1, /* page has been used (P) */
230 vmp_lopage:1,
231 vmp_realtime:1, /* page used by realtime thread */
232 #if !CONFIG_TRACK_UNMODIFIED_ANON_PAGES
233 vmp_unused_page_bits:3;
234 #else /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
235 vmp_unmodified_ro:1, /* Tracks if an anonymous page is modified after a decompression (O&P).*/
236 vmp_unused_page_bits:2;
237 #endif /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
238
239 /*
240 * MUST keep the 2 32 bit words used as bit fields
241 * separated since the compiler has a nasty habit
242 * of using 64 bit loads and stores on them as
243 * if they were a single 64 bit field... since
244 * they are protected by 2 different locks, this
245 * is a real problem
246 */
247 vm_page_packed_t vmp_next_m; /* VP bucket link (O) */
248
249 /*
250 * The following word of flags is protected by the "VM object" lock.
251 *
252 * IMPORTANT: the "vmp_pmapped", "vmp_xpmapped" and "vmp_clustered" bits can be modified while holding the
253 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
254 * This is done in vm_fault_enter() and the CONSUME_CLUSTERED macro.
255 * It's also ok to modify them behind just the VM object "exclusive" lock.
256 */
257 unsigned int vmp_busy:1, /* page is in transit (O) */
258 vmp_wanted:1, /* someone is waiting for page (O) */
259 vmp_tabled:1, /* page is in VP table (O) */
260 vmp_hashed:1, /* page is in vm_page_buckets[] (O) + the bucket lock */
261 vmp_fictitious:1, /* Physical page doesn't exist (O) */
262 vmp_clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
263 vmp_pmapped:1, /* page has at some time been entered into a pmap (O) or */
264 /* (O-shared AND pmap_page) */
265 vmp_xpmapped:1, /* page has been entered with execute permission (O) or */
266 /* (O-shared AND pmap_page) */
267 vmp_wpmapped:1, /* page has been entered at some point into a pmap for write (O) */
268 vmp_free_when_done:1, /* page is to be freed once cleaning is completed (O) */
269 vmp_absent:1, /* Data has been requested, but is not yet available (O) */
270 vmp_error:1, /* Data manager was unable to provide data due to error (O) */
271 vmp_dirty:1, /* Page must be cleaned (O) */
272 vmp_cleaning:1, /* Page clean has begun (O) */
273 vmp_precious:1, /* Page is precious; data must be returned even if clean (O) */
274 vmp_overwriting:1, /* Request to unlock has been made without having data. (O) */
275 /* [See vm_fault_page_overwrite] */
276 vmp_restart:1, /* Page was pushed higher in shadow chain by copy_call-related pagers */
277 /* start again at top of chain */
278 vmp_unusual:1, /* Page is absent, error, restart or page locked */
279 vmp_cs_validated:VMP_CS_BITS, /* code-signing: page was checked */
280 vmp_cs_tainted:VMP_CS_BITS, /* code-signing: page is tainted */
281 vmp_cs_nx:VMP_CS_BITS, /* code-signing: page is nx */
282 vmp_reusable:1,
283 vmp_written_by_kernel:1; /* page was written by kernel (i.e. decompressed) */
284
285
286 #if !defined(__arm64__)
287 ppnum_t vmp_phys_page; /* Physical page number of the page */
288 #endif
289 };
290
291 extern vm_page_t vm_pages;
292 extern vm_page_t vm_page_array_beginning_addr;
293 extern vm_page_t vm_page_array_ending_addr;
294
295 #if defined(__arm64__)
296
297 extern unsigned int vm_first_phys_ppnum;
298
299 struct vm_page_with_ppnum {
300 struct vm_page vm_page_wo_ppnum;
301
302 ppnum_t vmp_phys_page;
303 };
304 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
305
306 static inline ppnum_t
VM_PAGE_GET_PHYS_PAGE(vm_page_t m)307 VM_PAGE_GET_PHYS_PAGE(vm_page_t m)
308 {
309 if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) { /* real pages in vm_pages array */
310 return (ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum);
311 } else {
312 return ((vm_page_with_ppnum_t)m)->vmp_phys_page; /* pages in vm_page_zone */
313 }
314 }
315
316 #define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \
317 MACRO_BEGIN \
318 if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \
319 ((vm_page_with_ppnum_t)(m))->vmp_phys_page = ppnum; \
320 assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \
321 MACRO_END
322
323 #define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask)
324
325 #else /* defined(__arm64__) */
326
327
328 struct vm_page_with_ppnum {
329 struct vm_page vm_page_with_ppnum;
330 };
331 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
332
333
334 #define VM_PAGE_GET_PHYS_PAGE(page) (page)->vmp_phys_page
335 #define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
336 MACRO_BEGIN \
337 (page)->vmp_phys_page = ppnum; \
338 MACRO_END
339
340 #define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift)
341 #define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask)
342
343 #endif /* defined(__arm64__) */
344
345
346
347 #if defined(__LP64__)
348 /*
349 * Parameters for pointer packing
350 *
351 *
352 * VM Pages pointers might point to:
353 *
354 * 1. VM_PAGE_PACKED_ALIGNED aligned kernel globals,
355 *
356 * 2. VM_PAGE_PACKED_ALIGNED aligned heap allocated vm pages
357 *
358 * 3. entries in the vm_pages array (whose entries aren't VM_PAGE_PACKED_ALIGNED
359 * aligned).
360 *
361 *
362 * The current scheme uses 31 bits of storage and 6 bits of shift using the
363 * VM_PACK_POINTER() scheme for (1-2), and packs (3) as an index within the
364 * vm_pages array, setting the top bit (VM_PAGE_PACKED_FROM_ARRAY).
365 *
366 * This scheme gives us a reach of 128G from VM_MIN_KERNEL_AND_KEXT_ADDRESS.
367 */
368 #define VM_VPLQ_ALIGNMENT 128
369 #define VM_PAGE_PACKED_PTR_ALIGNMENT 64 /* must be a power of 2 */
370 #define VM_PAGE_PACKED_ALIGNED __attribute__((aligned(VM_PAGE_PACKED_PTR_ALIGNMENT)))
371 #define VM_PAGE_PACKED_PTR_BITS 31
372 #define VM_PAGE_PACKED_PTR_SHIFT 6
373 #define VM_PAGE_PACKED_PTR_BASE ((uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS)
374
375 #define VM_PAGE_PACKED_FROM_ARRAY 0x80000000
376
377 static inline vm_page_packed_t
vm_page_pack_ptr(uintptr_t p)378 vm_page_pack_ptr(uintptr_t p)
379 {
380 if (p >= (uintptr_t)vm_page_array_beginning_addr &&
381 p < (uintptr_t)vm_page_array_ending_addr) {
382 ptrdiff_t diff = (vm_page_t)p - vm_page_array_beginning_addr;
383 assert((vm_page_t)p == &vm_pages[diff]);
384 return (vm_page_packed_t)(diff | VM_PAGE_PACKED_FROM_ARRAY);
385 }
386
387 VM_ASSERT_POINTER_PACKABLE(p, VM_PAGE_PACKED_PTR);
388 vm_offset_t packed = VM_PACK_POINTER(p, VM_PAGE_PACKED_PTR);
389 return CAST_DOWN_EXPLICIT(vm_page_packed_t, packed);
390 }
391
392
393 static inline uintptr_t
vm_page_unpack_ptr(uintptr_t p)394 vm_page_unpack_ptr(uintptr_t p)
395 {
396 extern unsigned int vm_pages_count;
397
398 if (p >= VM_PAGE_PACKED_FROM_ARRAY) {
399 p &= ~VM_PAGE_PACKED_FROM_ARRAY;
400 assert(p < (uintptr_t)vm_pages_count);
401 return (uintptr_t)&vm_pages[p];
402 }
403
404 return VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR);
405 }
406
407
408 #define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
409 #define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
410
411 #define VM_OBJECT_PACK(o) ((vm_page_object_t)VM_PACK_POINTER((uintptr_t)(o), VM_PAGE_PACKED_PTR))
412 #define VM_OBJECT_UNPACK(p) ((vm_object_t)VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR))
413
414 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
415 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
416
417
418 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
419 MACRO_BEGIN \
420 (p)->vmp_snext = 0; \
421 MACRO_END
422
423
424 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
425
426 /*
427 * Macro: vm_page_queue_init
428 * Function:
429 * Initialize the given queue.
430 * Header:
431 * void vm_page_queue_init(q)
432 * vm_page_queue_t q; \* MODIFIED *\
433 */
434 #define vm_page_queue_init(q) \
435 MACRO_BEGIN \
436 VM_ASSERT_POINTER_PACKABLE((vm_offset_t)(q), VM_PAGE_PACKED_PTR); \
437 (q)->next = VM_PAGE_PACK_PTR(q); \
438 (q)->prev = VM_PAGE_PACK_PTR(q); \
439 MACRO_END
440
441
442 /*
443 * Macro: vm_page_queue_enter
444 * Function:
445 * Insert a new element at the tail of the vm_page queue.
446 * Header:
447 * void vm_page_queue_enter(q, elt, field)
448 * queue_t q;
449 * vm_page_t elt;
450 * <field> is the list field in vm_page_t
451 *
452 * This macro's arguments have to match the generic "queue_enter()" macro which is
453 * what is used for this on 32 bit kernels.
454 */
455 #define vm_page_queue_enter(head, elt, field) \
456 MACRO_BEGIN \
457 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
458 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
459 vm_page_packed_t __pck_prev = (head)->prev; \
460 \
461 if (__pck_head == __pck_prev) { \
462 (head)->next = __pck_elt; \
463 } else { \
464 vm_page_t __prev; \
465 __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
466 __prev->field.next = __pck_elt; \
467 } \
468 (elt)->field.prev = __pck_prev; \
469 (elt)->field.next = __pck_head; \
470 (head)->prev = __pck_elt; \
471 MACRO_END
472
473
474 #if defined(__x86_64__)
475 /*
476 * These are helper macros for vm_page_queue_enter_clump to assist
477 * with conditional compilation (release / debug / development)
478 */
479 #if DEVELOPMENT || DEBUG
480
481 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field) \
482 MACRO_BEGIN \
483 if (__prev != NULL) { \
484 assert(__p == (vm_page_t)VM_PAGE_UNPACK_PTR(__prev->next)); \
485 assert(__prev == (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__p->field.prev)); \
486 } \
487 MACRO_END
488
489 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next) \
490 MACRO_BEGIN \
491 unsigned int __i; \
492 vm_page_queue_entry_t __tmp; \
493 for (__i = 0, __tmp = __first; __i < __n_free; __i++) { \
494 __tmp = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__tmp->next); \
495 } \
496 assert(__tmp == __last_next); \
497 MACRO_END
498
499 #define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++
500 #define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++
501 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free
502
503 #else
504
505 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field)
506 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next)
507 #define __DEBUG_STAT_INCREMENT_INRANGE
508 #define __DEBUG_STAT_INCREMENT_INSERTS
509 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free)
510
511 #endif /* if DEVELOPMENT || DEBUG */
512
513 #endif
514
515 /*
516 * Macro: vm_page_queue_enter_first
517 * Function:
518 * Insert a new element at the head of the vm_page queue.
519 * Header:
520 * void queue_enter_first(q, elt, , field)
521 * queue_t q;
522 * vm_page_t elt;
523 * <field> is the linkage field in vm_page
524 *
525 * This macro's arguments have to match the generic "queue_enter_first()" macro which is
526 * what is used for this on 32 bit kernels.
527 */
528 #define vm_page_queue_enter_first(head, elt, field) \
529 MACRO_BEGIN \
530 vm_page_packed_t __pck_next = (head)->next; \
531 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
532 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
533 \
534 if (__pck_head == __pck_next) { \
535 (head)->prev = __pck_elt; \
536 } else { \
537 vm_page_t __next; \
538 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
539 __next->field.prev = __pck_elt; \
540 } \
541 \
542 (elt)->field.next = __pck_next; \
543 (elt)->field.prev = __pck_head; \
544 (head)->next = __pck_elt; \
545 MACRO_END
546
547
548 /*
549 * Macro: vm_page_queue_remove
550 * Function:
551 * Remove an arbitrary page from a vm_page queue.
552 * Header:
553 * void vm_page_queue_remove(q, qe, field)
554 * arguments as in vm_page_queue_enter
555 *
556 * This macro's arguments have to match the generic "queue_enter()" macro which is
557 * what is used for this on 32 bit kernels.
558 */
559 #define vm_page_queue_remove(head, elt, field) \
560 MACRO_BEGIN \
561 vm_page_packed_t __pck_next = (elt)->field.next; \
562 vm_page_packed_t __pck_prev = (elt)->field.prev; \
563 vm_page_t __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
564 vm_page_t __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
565 \
566 if ((void *)(head) == (void *)__next) { \
567 (head)->prev = __pck_prev; \
568 } else { \
569 __next->field.prev = __pck_prev; \
570 } \
571 \
572 if ((void *)(head) == (void *)__prev) { \
573 (head)->next = __pck_next; \
574 } else { \
575 __prev->field.next = __pck_next; \
576 } \
577 \
578 (elt)->field.next = 0; \
579 (elt)->field.prev = 0; \
580 MACRO_END
581
582
583 /*
584 * Macro: vm_page_queue_remove_first
585 *
586 * Function:
587 * Remove and return the entry at the head of a vm_page queue.
588 *
589 * Header:
590 * vm_page_queue_remove_first(head, entry, field)
591 * N.B. entry is returned by reference
592 *
593 * This macro's arguments have to match the generic "queue_remove_first()" macro which is
594 * what is used for this on 32 bit kernels.
595 */
596 #define vm_page_queue_remove_first(head, entry, field) \
597 MACRO_BEGIN \
598 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
599 vm_page_packed_t __pck_next; \
600 vm_page_t __next; \
601 \
602 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
603 __pck_next = (entry)->field.next; \
604 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
605 \
606 if (__pck_head == __pck_next) { \
607 (head)->prev = __pck_head; \
608 } else { \
609 __next->field.prev = __pck_head; \
610 } \
611 \
612 (head)->next = __pck_next; \
613 (entry)->field.next = 0; \
614 (entry)->field.prev = 0; \
615 MACRO_END
616
617
618 #if defined(__x86_64__)
619 /*
620 * Macro: vm_page_queue_remove_first_with_clump
621 * Function:
622 * Remove and return the entry at the head of the free queue
623 * end is set to 1 to indicate that we just returned the last page in a clump
624 *
625 * Header:
626 * vm_page_queue_remove_first_with_clump(head, entry, end)
627 * entry is returned by reference
628 * end is returned by reference
629 */
630 #define vm_page_queue_remove_first_with_clump(head, entry, end) \
631 MACRO_BEGIN \
632 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
633 vm_page_packed_t __pck_next; \
634 vm_page_t __next; \
635 \
636 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
637 __pck_next = (entry)->vmp_pageq.next; \
638 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
639 \
640 (end) = 0; \
641 if (__pck_head == __pck_next) { \
642 (head)->prev = __pck_head; \
643 (end) = 1; \
644 } else { \
645 __next->vmp_pageq.prev = __pck_head; \
646 if (VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(__next)) { \
647 (end) = 1; \
648 } \
649 } \
650 \
651 (head)->next = __pck_next; \
652 (entry)->vmp_pageq.next = 0; \
653 (entry)->vmp_pageq.prev = 0; \
654 MACRO_END
655 #endif
656
657 /*
658 * Macro: vm_page_queue_end
659 * Function:
660 * Tests whether a new entry is really the end of
661 * the queue.
662 * Header:
663 * boolean_t vm_page_queue_end(q, qe)
664 * vm_page_queue_t q;
665 * vm_page_queue_entry_t qe;
666 */
667 #define vm_page_queue_end(q, qe) ((q) == (qe))
668
669
670 /*
671 * Macro: vm_page_queue_empty
672 * Function:
673 * Tests whether a queue is empty.
674 * Header:
675 * boolean_t vm_page_queue_empty(q)
676 * vm_page_queue_t q;
677 */
678 #define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
679
680
681
682 /*
683 * Macro: vm_page_queue_first
684 * Function:
685 * Returns the first entry in the queue,
686 * Header:
687 * uintpr_t vm_page_queue_first(q)
688 * vm_page_queue_t q; \* IN *\
689 */
690 #define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
691
692
693
694 /*
695 * Macro: vm_page_queue_last
696 * Function:
697 * Returns the last entry in the queue.
698 * Header:
699 * vm_page_queue_entry_t queue_last(q)
700 * queue_t q; \* IN *\
701 */
702 #define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
703
704
705
706 /*
707 * Macro: vm_page_queue_next
708 * Function:
709 * Returns the entry after an item in the queue.
710 * Header:
711 * uintpr_t vm_page_queue_next(qc)
712 * vm_page_queue_t qc;
713 */
714 #define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next))
715
716
717
718 /*
719 * Macro: vm_page_queue_prev
720 * Function:
721 * Returns the entry before an item in the queue.
722 * Header:
723 * uinptr_t vm_page_queue_prev(qc)
724 * vm_page_queue_t qc;
725 */
726 #define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev))
727
728
729
730 /*
731 * Macro: vm_page_queue_iterate
732 * Function:
733 * iterate over each item in a vm_page queue.
734 * Generates a 'for' loop, setting elt to
735 * each item in turn (by reference).
736 * Header:
737 * vm_page_queue_iterate(q, elt, field)
738 * queue_t q;
739 * vm_page_t elt;
740 * <field> is the chain field in vm_page_t
741 */
742 #define vm_page_queue_iterate(head, elt, field) \
743 for ((elt) = (vm_page_t)vm_page_queue_first(head); \
744 !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \
745 (elt) = (vm_page_t)vm_page_queue_next(&(elt)->field)) \
746
747 #else // LP64
748
749 #define VM_VPLQ_ALIGNMENT 128
750 #define VM_PAGE_PACKED_PTR_ALIGNMENT sizeof(vm_offset_t)
751 #define VM_PAGE_PACKED_ALIGNED
752 #define VM_PAGE_PACKED_PTR_BITS 32
753 #define VM_PAGE_PACKED_PTR_SHIFT 0
754 #define VM_PAGE_PACKED_PTR_BASE 0
755
756 #define VM_PAGE_PACKED_FROM_ARRAY 0
757
758 #define VM_PAGE_PACK_PTR(p) (p)
759 #define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p))
760
761 #define VM_OBJECT_PACK(o) ((vm_page_object_t)(o))
762 #define VM_OBJECT_UNPACK(p) ((vm_object_t)(p))
763
764 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
765 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
766
767
768 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
769 MACRO_BEGIN \
770 (p)->vmp_pageq.next = 0; \
771 (p)->vmp_pageq.prev = 0; \
772 MACRO_END
773
774 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p))
775
776 #define vm_page_remque remque
777 #define vm_page_enqueue_tail enqueue_tail
778 #define vm_page_queue_init queue_init
779 #define vm_page_queue_enter(h, e, f) queue_enter(h, e, vm_page_t, f)
780 #define vm_page_queue_enter_first(h, e, f) queue_enter_first(h, e, vm_page_t, f)
781 #define vm_page_queue_remove(h, e, f) queue_remove(h, e, vm_page_t, f)
782 #define vm_page_queue_remove_first(h, e, f) queue_remove_first(h, e, vm_page_t, f)
783 #define vm_page_queue_end queue_end
784 #define vm_page_queue_empty queue_empty
785 #define vm_page_queue_first queue_first
786 #define vm_page_queue_last queue_last
787 #define vm_page_queue_next queue_next
788 #define vm_page_queue_prev queue_prev
789 #define vm_page_queue_iterate(h, e, f) queue_iterate(h, e, vm_page_t, f)
790
791 #endif // __LP64__
792
793
794
795 /*
796 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through vm_page_max_speculative_age_q
797 * represents a set of aging bins that are 'protected'...
798 *
799 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
800 * not yet been 'claimed' but have been aged out of the protective bins
801 * this occurs in vm_page_speculate when it advances to the next bin
802 * and discovers that it is still occupied... at that point, all of the
803 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
804 * in that bin are all guaranteed to have reached at least the maximum age
805 * we allow for a protected page... they can be older if there is no
806 * memory pressure to pull them from the bin, or there are no new speculative pages
807 * being generated to push them out.
808 * this list is the one that vm_pageout_scan will prefer when looking
809 * for pages to move to the underweight free list
810 *
811 * vm_page_max_speculative_age_q * VM_PAGE_SPECULATIVE_Q_AGE_MS
812 * defines the amount of time a speculative page is normally
813 * allowed to live in the 'protected' state (i.e. not available
814 * to be stolen if vm_pageout_scan is running and looking for
815 * pages)... however, if the total number of speculative pages
816 * in the protected state exceeds our limit (defined in vm_pageout.c)
817 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
818 * vm_pageout_scan is allowed to steal pages from the protected
819 * bucket even if they are underage.
820 *
821 * vm_pageout_scan is also allowed to pull pages from a protected
822 * bin if the bin has reached the "age of consent" we've set
823 */
824 #define VM_PAGE_RESERVED_SPECULATIVE_AGE_Q 40
825 #define VM_PAGE_DEFAULT_MAX_SPECULATIVE_AGE_Q 10
826 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
827 #define VM_PAGE_SPECULATIVE_AGED_Q 0
828
829 #define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
830
831 struct vm_speculative_age_q {
832 /*
833 * memory queue for speculative pages via clustered pageins
834 */
835 vm_page_queue_head_t age_q;
836 mach_timespec_t age_ts;
837 } VM_PAGE_PACKED_ALIGNED;
838
839
840
841 extern
842 struct vm_speculative_age_q vm_page_queue_speculative[];
843
844 extern int speculative_steal_index;
845 extern int speculative_age_index;
846 extern unsigned int vm_page_speculative_q_age_ms;
847 extern unsigned int vm_page_max_speculative_age_q;
848
849
850 typedef struct vm_locks_array {
851 char pad __attribute__ ((aligned(64)));
852 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned(64)));
853 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned(64)));
854 char pad2 __attribute__ ((aligned(64)));
855 } vm_locks_array_t;
856
857
858 #define VM_PAGE_WIRED(m) ((m)->vmp_q_state == VM_PAGE_IS_WIRED)
859 #define NEXT_PAGE(m) ((m)->vmp_snext)
860 #define NEXT_PAGE_PTR(m) (&(m)->vmp_snext)
861
862 /*
863 * XXX The unusual bit should not be necessary. Most of the bit
864 * XXX fields above really want to be masks.
865 */
866
867 /*
868 * For debugging, this macro can be defined to perform
869 * some useful check on a page structure.
870 * INTENTIONALLY left as a no-op so that the
871 * current call-sites can be left intact for future uses.
872 */
873
874 #define VM_PAGE_CHECK(mem) \
875 MACRO_BEGIN \
876 MACRO_END
877
878 /* Page coloring:
879 *
880 * The free page list is actually n lists, one per color,
881 * where the number of colors is a function of the machine's
882 * cache geometry set at system initialization. To disable
883 * coloring, set vm_colors to 1 and vm_color_mask to 0.
884 * The boot-arg "colors" may be used to override vm_colors.
885 * Note that there is little harm in having more colors than needed.
886 */
887
888 #define MAX_COLORS 128
889 #define DEFAULT_COLORS 32
890
891 extern
892 unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
893 extern
894 unsigned int vm_color_mask; /* must be (vm_colors-1) */
895 extern
896 unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
897
898 /*
899 * Wired memory is a very limited resource and we can't let users exhaust it
900 * and deadlock the entire system. We enforce the following limits:
901 *
902 * vm_per_task_user_wire_limit
903 * how much memory can be user-wired in one user task
904 *
905 * vm_global_user_wire_limit (default: same as vm_per_task_user_wire_limit)
906 * how much memory can be user-wired in all user tasks
907 *
908 * These values are set to defaults based on the number of pages managed
909 * by the VM system. They can be overriden via sysctls.
910 * See kmem_set_user_wire_limits for details on the default values.
911 *
912 * Regardless of the amount of memory in the system, we never reserve
913 * more than VM_NOT_USER_WIREABLE_MAX bytes as unlockable.
914 */
915 #if defined(__LP64__)
916 #define VM_NOT_USER_WIREABLE_MAX (32ULL*1024*1024*1024) /* 32GB */
917 #else
918 #define VM_NOT_USER_WIREABLE_MAX (1UL*1024*1024*1024) /* 1GB */
919 #endif /* __LP64__ */
920 extern
921 vm_map_size_t vm_per_task_user_wire_limit;
922 extern
923 vm_map_size_t vm_global_user_wire_limit;
924 extern
925 uint64_t vm_add_wire_count_over_global_limit;
926 extern
927 uint64_t vm_add_wire_count_over_user_limit;
928
929 /*
930 * Each pageable resident page falls into one of three lists:
931 *
932 * free
933 * Available for allocation now. The free list is
934 * actually an array of lists, one per color.
935 * inactive
936 * Not referenced in any map, but still has an
937 * object/offset-page mapping, and may be dirty.
938 * This is the list of pages that should be
939 * paged out next. There are actually two
940 * inactive lists, one for pages brought in from
941 * disk or other backing store, and another
942 * for "zero-filled" pages. See vm_pageout_scan()
943 * for the distinction and usage.
944 * active
945 * A list of pages which have been placed in
946 * at least one physical map. This list is
947 * ordered, in LRU-like fashion.
948 */
949
950
951 #define VPL_LOCK_SPIN 1
952
953 struct vpl {
954 vm_page_queue_head_t vpl_queue;
955 unsigned int vpl_count;
956 unsigned int vpl_internal_count;
957 unsigned int vpl_external_count;
958 lck_spin_t vpl_lock;
959 };
960
961 extern
962 struct vpl * /* __zpercpu */ vm_page_local_q;
963 extern
964 unsigned int vm_page_local_q_soft_limit;
965 extern
966 unsigned int vm_page_local_q_hard_limit;
967 extern
968 vm_locks_array_t vm_page_locks;
969
970 extern
971 vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */
972 extern
973 vm_page_queue_head_t vm_page_queue_active; /* active memory queue */
974 extern
975 vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
976 #if CONFIG_SECLUDED_MEMORY
977 extern
978 vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */
979 #endif /* CONFIG_SECLUDED_MEMORY */
980 extern
981 vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
982 extern
983 vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
984 extern
985 vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
986
987 extern
988 queue_head_t vm_objects_wired;
989 extern
990 lck_spin_t vm_objects_wired_lock;
991
992 #define VM_PAGE_DONATE_DISABLED 0
993 #define VM_PAGE_DONATE_ENABLED 1
994 extern
995 uint32_t vm_page_donate_mode;
996 extern
997 bool vm_page_donate_queue_ripe;
998
999 #define VM_PAGE_BACKGROUND_TARGET_MAX 50000
1000 #define VM_PAGE_BG_DISABLED 0
1001 #define VM_PAGE_BG_ENABLED 1
1002
1003 extern
1004 vm_page_queue_head_t vm_page_queue_background;
1005 extern
1006 uint64_t vm_page_background_promoted_count;
1007 extern
1008 uint32_t vm_page_background_count;
1009 extern
1010 uint32_t vm_page_background_target;
1011 extern
1012 uint32_t vm_page_background_internal_count;
1013 extern
1014 uint32_t vm_page_background_external_count;
1015 extern
1016 uint32_t vm_page_background_mode;
1017 extern
1018 uint32_t vm_page_background_exclude_external;
1019
1020 extern
1021 vm_page_queue_head_t vm_page_queue_donate;
1022 extern
1023 uint32_t vm_page_donate_count;
1024 extern
1025 uint32_t vm_page_donate_target_low;
1026 extern
1027 uint32_t vm_page_donate_target_high;
1028 #define VM_PAGE_DONATE_TARGET_LOWWATER (100)
1029 #define VM_PAGE_DONATE_TARGET_HIGHWATER ((unsigned int)(atop_64(max_mem) / 8))
1030
1031 extern
1032 vm_offset_t first_phys_addr; /* physical address for first_page */
1033 extern
1034 vm_offset_t last_phys_addr; /* physical address for last_page */
1035
1036 extern
1037 unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1038 extern
1039 unsigned int vm_page_active_count; /* How many pages are active? */
1040 extern
1041 unsigned int vm_page_inactive_count; /* How many pages are inactive? */
1042 extern
1043 unsigned int vm_page_kernelcache_count; /* How many pages are used for the kernelcache? */
1044 extern
1045 unsigned int vm_page_realtime_count; /* How many pages are used by realtime threads? */
1046 #if CONFIG_SECLUDED_MEMORY
1047 extern
1048 unsigned int vm_page_secluded_count; /* How many pages are secluded? */
1049 extern
1050 unsigned int vm_page_secluded_count_free; /* how many of them are free? */
1051 extern
1052 unsigned int vm_page_secluded_count_inuse; /* how many of them are in use? */
1053 /*
1054 * We keep filling the secluded pool with new eligible pages and
1055 * we can overshoot our target by a lot.
1056 * When there's memory pressure, vm_pageout_scan() will re-balance the queues,
1057 * pushing the extra secluded pages to the active or free queue.
1058 * Since these "over target" secluded pages are actually "available", jetsam
1059 * should consider them as such, so make them visible to jetsam via the
1060 * "vm_page_secluded_count_over_target" counter and update it whenever we
1061 * update vm_page_secluded_count or vm_page_secluded_target.
1062 */
1063 extern
1064 unsigned int vm_page_secluded_count_over_target;
1065 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1066 MACRO_BEGIN \
1067 if (vm_page_secluded_count > vm_page_secluded_target) { \
1068 vm_page_secluded_count_over_target = \
1069 (vm_page_secluded_count - vm_page_secluded_target); \
1070 } else { \
1071 vm_page_secluded_count_over_target = 0; \
1072 } \
1073 MACRO_END
1074 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() vm_page_secluded_count_over_target
1075 #else /* CONFIG_SECLUDED_MEMORY */
1076 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1077 MACRO_BEGIN \
1078 MACRO_END
1079 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() 0
1080 #endif /* CONFIG_SECLUDED_MEMORY */
1081 extern
1082 unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
1083 extern
1084 unsigned int vm_page_throttled_count;/* How many inactives are throttled */
1085 extern
1086 unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
1087 extern unsigned int vm_page_pageable_internal_count;
1088 extern unsigned int vm_page_pageable_external_count;
1089 extern
1090 unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
1091 extern
1092 unsigned int vm_page_external_count; /* How many pages are file-backed? */
1093 extern
1094 unsigned int vm_page_internal_count; /* How many pages are anonymous? */
1095 extern
1096 unsigned int vm_page_wire_count; /* How many pages are wired? */
1097 extern
1098 unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
1099 extern
1100 unsigned int vm_page_wire_count_on_boot; /* even earlier than _initial */
1101 extern
1102 unsigned int vm_page_free_target; /* How many do we want free? */
1103 extern
1104 unsigned int vm_page_free_min; /* When to wakeup pageout */
1105 extern
1106 unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
1107 extern
1108 unsigned int vm_page_inactive_target;/* How many do we want inactive? */
1109 #if CONFIG_SECLUDED_MEMORY
1110 extern
1111 unsigned int vm_page_secluded_target;/* How many do we want secluded? */
1112 #endif /* CONFIG_SECLUDED_MEMORY */
1113 extern
1114 unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
1115 extern
1116 unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1117 extern
1118 unsigned int vm_page_gobble_count;
1119 extern
1120 unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */
1121 extern
1122 unsigned int vm_page_kern_lpage_count; /* Count of large pages used in early boot */
1123
1124
1125 #if DEVELOPMENT || DEBUG
1126 extern
1127 unsigned int vm_page_speculative_used;
1128 #endif
1129
1130 extern
1131 unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
1132 extern
1133 unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
1134 extern
1135 uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1136
1137 extern unsigned int vm_page_free_wanted;
1138 /* how many threads are waiting for memory */
1139
1140 extern unsigned int vm_page_free_wanted_privileged;
1141 /* how many VM privileged threads are waiting for memory */
1142 #if CONFIG_SECLUDED_MEMORY
1143 extern unsigned int vm_page_free_wanted_secluded;
1144 /* how many threads are waiting for secluded memory */
1145 #endif /* CONFIG_SECLUDED_MEMORY */
1146
1147 extern const ppnum_t vm_page_fictitious_addr;
1148 /* (fake) phys_addr of fictitious pages */
1149
1150 extern const ppnum_t vm_page_guard_addr;
1151 /* (fake) phys_addr of guard pages */
1152
1153
1154 extern boolean_t vm_page_deactivate_hint;
1155
1156 extern int vm_compressor_mode;
1157
1158 /*
1159 * Defaults to true, so highest memory is used first.
1160 */
1161 extern boolean_t vm_himemory_mode;
1162
1163 extern boolean_t vm_lopage_needed;
1164 extern uint32_t vm_lopage_free_count;
1165 extern uint32_t vm_lopage_free_limit;
1166 extern uint32_t vm_lopage_lowater;
1167 extern boolean_t vm_lopage_refill;
1168 extern uint64_t max_valid_dma_address;
1169 extern ppnum_t max_valid_low_ppnum;
1170
1171 /*
1172 * Prototypes for functions exported by this module.
1173 */
1174
1175 extern void vm_page_init_local_q(unsigned int num_cpus);
1176
1177 extern void vm_page_create(
1178 ppnum_t start,
1179 ppnum_t end);
1180
1181 extern void vm_page_create_retired(
1182 ppnum_t pn);
1183
1184 extern boolean_t vm_page_created(
1185 vm_page_t page);
1186
1187
1188 extern void vm_free_delayed_pages(void);
1189
1190 extern vm_page_t vm_page_alloc(
1191 vm_object_t object,
1192 vm_object_offset_t offset);
1193
1194 extern void vm_page_reactivate_all_throttled(void);
1195
1196 extern void vm_pressure_response(void);
1197
1198 #if CONFIG_JETSAM
1199 extern void memorystatus_pages_update(unsigned int pages_avail);
1200
1201 #define VM_CHECK_MEMORYSTATUS do { \
1202 memorystatus_pages_update( \
1203 vm_page_pageable_external_count + \
1204 vm_page_free_count + \
1205 VM_PAGE_SECLUDED_COUNT_OVER_TARGET() + \
1206 (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \
1207 ); \
1208 } while(0)
1209
1210 #else /* CONFIG_JETSAM */
1211
1212 #if !XNU_TARGET_OS_OSX
1213
1214 #define VM_CHECK_MEMORYSTATUS do {} while(0)
1215
1216 #else /* !XNU_TARGET_OS_OSX */
1217
1218 #define VM_CHECK_MEMORYSTATUS vm_pressure_response()
1219
1220 #endif /* !XNU_TARGET_OS_OSX */
1221
1222 #endif /* CONFIG_JETSAM */
1223
1224 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
1225 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
1226
1227 #ifdef MACH_KERNEL_PRIVATE
1228 static inline void
vm_page_lock_queues(void)1229 vm_page_lock_queues(void)
1230 {
1231 lck_mtx_lock(&vm_page_queue_lock);
1232 }
1233
1234 static inline boolean_t
vm_page_trylock_queues(void)1235 vm_page_trylock_queues(void)
1236 {
1237 boolean_t ret;
1238 ret = lck_mtx_try_lock(&vm_page_queue_lock);
1239 return ret;
1240 }
1241
1242 static inline void
vm_page_unlock_queues(void)1243 vm_page_unlock_queues(void)
1244 {
1245 lck_mtx_unlock(&vm_page_queue_lock);
1246 }
1247
1248 static inline void
vm_page_lockspin_queues(void)1249 vm_page_lockspin_queues(void)
1250 {
1251 lck_mtx_lock_spin(&vm_page_queue_lock);
1252 }
1253
1254 static inline boolean_t
vm_page_trylockspin_queues(void)1255 vm_page_trylockspin_queues(void)
1256 {
1257 boolean_t ret;
1258 ret = lck_mtx_try_lock_spin(&vm_page_queue_lock);
1259 return ret;
1260 }
1261
1262 extern void kdp_vm_page_sleep_find_owner(
1263 event64_t wait_event,
1264 thread_waitinfo_t *waitinfo);
1265
1266 #endif /* MACH_KERNEL_PRIVATE */
1267
1268 extern unsigned int vm_max_delayed_work_limit;
1269
1270 #if CONFIG_SECLUDED_MEMORY
1271 extern uint64_t secluded_shutoff_trigger;
1272 extern uint64_t secluded_shutoff_headroom;
1273 extern void start_secluded_suppression(task_t);
1274 extern void stop_secluded_suppression(task_t);
1275 #endif /* CONFIG_SECLUDED_MEMORY */
1276
1277 #endif /* _VM_VM_PAGE_H_ */
1278