1 /*
2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm/vm_page.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Resident memory system definitions.
64 */
65
66 #ifndef _VM_VM_PAGE_H_
67 #define _VM_VM_PAGE_H_
68
69 #include <debug.h>
70 #include <vm/vm_options.h>
71 #include <vm/vm_protos.h>
72 #include <mach/boolean.h>
73 #include <mach/vm_prot.h>
74 #include <mach/vm_param.h>
75 #include <mach/memory_object_types.h> /* for VMP_CS_BITS... */
76 #include <sys/kern_memorystatus_xnu.h>
77
78
79 #if defined(__LP64__)
80
81 /*
82 * in order to make the size of a vm_page_t 64 bytes (cache line size for both arm64 and x86_64)
83 * we'll keep the next_m pointer packed... as long as the kernel virtual space where we allocate
84 * vm_page_t's from doesn't span more then 256 Gbytes, we're safe. There are live tests in the
85 * vm_page_t array allocation and the zone init code to determine if we can safely pack and unpack
86 * pointers from the 2 ends of these spaces
87 */
88 typedef uint32_t vm_page_packed_t;
89
90 struct vm_page_packed_queue_entry {
91 vm_page_packed_t next; /* next element */
92 vm_page_packed_t prev; /* previous element */
93 };
94
95 typedef struct vm_page_packed_queue_entry *vm_page_queue_t;
96 typedef struct vm_page_packed_queue_entry vm_page_queue_head_t;
97 typedef struct vm_page_packed_queue_entry vm_page_queue_chain_t;
98 typedef struct vm_page_packed_queue_entry *vm_page_queue_entry_t;
99
100 typedef vm_page_packed_t vm_page_object_t;
101
102 #else // __LP64__
103
104 /*
105 * we can't do the packing trick on 32 bit architectures
106 * so just turn the macros into noops.
107 */
108 typedef struct vm_page *vm_page_packed_t;
109
110 #define vm_page_queue_t queue_t
111 #define vm_page_queue_head_t queue_head_t
112 #define vm_page_queue_chain_t queue_chain_t
113 #define vm_page_queue_entry_t queue_entry_t
114
115 #define vm_page_object_t vm_object_t
116 #endif // __LP64__
117
118
119 #include <vm/vm_object_xnu.h>
120 #include <kern/queue.h>
121 #include <kern/locks.h>
122
123 #include <kern/macro_help.h>
124 #include <libkern/OSAtomic.h>
125
126
127 /* pages of compressed data */
128 #define VM_PAGE_COMPRESSOR_COUNT os_atomic_load(&compressor_object->resident_page_count, relaxed)
129
130 /*
131 * Management of resident (logical) pages.
132 *
133 * A small structure is kept for each resident
134 * page, indexed by page number. Each structure
135 * is an element of several lists:
136 *
137 * A hash table bucket used to quickly
138 * perform object/offset lookups
139 *
140 * A list of all pages for a given object,
141 * so they can be quickly deactivated at
142 * time of deallocation.
143 *
144 * An ordered list of pages due for pageout.
145 *
146 * In addition, the structure contains the object
147 * and offset to which this page belongs (for pageout),
148 * and sundry status bits.
149 *
150 * Fields in this structure are locked either by the lock on the
151 * object that the page belongs to (O) or by the lock on the page
152 * queues (P). [Some fields require that both locks be held to
153 * change that field; holding either lock is sufficient to read.]
154 */
155
156 #define VM_PAGE_NULL ((vm_page_t) 0)
157
158 extern char vm_page_inactive_states[];
159 extern char vm_page_pageable_states[];
160 extern char vm_page_non_speculative_pageable_states[];
161 extern char vm_page_active_or_inactive_states[];
162
163
164 #define VM_PAGE_INACTIVE(m) (vm_page_inactive_states[m->vmp_q_state])
165 #define VM_PAGE_PAGEABLE(m) (vm_page_pageable_states[m->vmp_q_state])
166 #define VM_PAGE_NON_SPECULATIVE_PAGEABLE(m) (vm_page_non_speculative_pageable_states[m->vmp_q_state])
167 #define VM_PAGE_ACTIVE_OR_INACTIVE(m) (vm_page_active_or_inactive_states[m->vmp_q_state])
168
169
170 #define VM_PAGE_NOT_ON_Q 0 /* page is not present on any queue, nor is it wired... mainly a transient state */
171 #define VM_PAGE_IS_WIRED 1 /* page is currently wired */
172 #define VM_PAGE_USED_BY_COMPRESSOR 2 /* page is in use by the compressor to hold compressed data */
173 #define VM_PAGE_ON_FREE_Q 3 /* page is on the main free queue */
174 #define VM_PAGE_ON_FREE_LOCAL_Q 4 /* page is on one of the per-CPU free queues */
175 #define VM_PAGE_ON_FREE_LOPAGE_Q 5 /* page is on the lopage pool free list */
176 #define VM_PAGE_ON_THROTTLED_Q 6 /* page is on the throttled queue... we stash anonymous pages here when not paging */
177 #define VM_PAGE_ON_PAGEOUT_Q 7 /* page is on one of the pageout queues (internal/external) awaiting processing */
178 #define VM_PAGE_ON_SPECULATIVE_Q 8 /* page is on one of the speculative queues */
179 #define VM_PAGE_ON_ACTIVE_LOCAL_Q 9 /* page has recently been created and is being held in one of the per-CPU local queues */
180 #define VM_PAGE_ON_ACTIVE_Q 10 /* page is in global active queue */
181 #define VM_PAGE_ON_INACTIVE_INTERNAL_Q 11 /* page is on the inactive internal queue a.k.a. anonymous queue */
182 #define VM_PAGE_ON_INACTIVE_EXTERNAL_Q 12 /* page in on the inactive external queue a.k.a. file backed queue */
183 #define VM_PAGE_ON_INACTIVE_CLEANED_Q 13 /* page has been cleaned to a backing file and is ready to be stolen */
184 #define VM_PAGE_ON_SECLUDED_Q 14 /* page is on secluded queue */
185 #define VM_PAGE_Q_STATE_LAST_VALID_VALUE 14 /* we currently use 4 bits for the state... don't let this go beyond 15 */
186
187 #define VM_PAGE_Q_STATE_ARRAY_SIZE (VM_PAGE_Q_STATE_LAST_VALID_VALUE+1)
188
189
190 /*
191 * The structure itself. See the block comment above for what (O) and (P) mean.
192 */
193 #define vmp_pageq vmp_q_un.vmp_q_pageq
194 #define vmp_snext vmp_q_un.vmp_q_snext
195
196 struct vm_page {
197 union {
198 vm_page_queue_chain_t vmp_q_pageq; /* queue info for FIFO queue or free list (P) */
199 struct vm_page *vmp_q_snext;
200 } vmp_q_un;
201
202 vm_page_queue_chain_t vmp_listq; /* all pages in same object (O) */
203
204 vm_page_queue_chain_t vmp_specialq; /* anonymous pages in the special queues (P) */
205 vm_object_offset_t vmp_offset; /* offset into that object (O,P) */
206
207 vm_page_object_t vmp_object; /* which object am I in (O&P) */
208
209 /*
210 * The following word of flags used to be protected by the "page queues" lock.
211 * That's no longer true and what lock, if any, is needed may depend on the
212 * value of vmp_q_state.
213 *
214 * We use 'vmp_wire_count' to store the local queue id if local queues are enabled.
215 * See the comments at 'vm_page_queues_remove' as to why this is safe to do.
216 */
217 #define VM_PAGE_SPECIAL_Q_EMPTY (0)
218 #define VM_PAGE_SPECIAL_Q_BG (1)
219 #define VM_PAGE_SPECIAL_Q_DONATE (2)
220 #define VM_PAGE_SPECIAL_Q_FG (3)
221 #define vmp_local_id vmp_wire_count
222 unsigned int vmp_wire_count:16, /* how many wired down maps use me? (O&P) */
223 vmp_q_state:4, /* which q is the page on (P) */
224 vmp_on_specialq:2,
225 vmp_gobbled:1, /* page used internally (P) */
226 vmp_laundry:1, /* page is being cleaned now (P)*/
227 vmp_no_cache:1, /* page is not to be cached and should */
228 /* be reused ahead of other pages (P) */
229 vmp_private:1, /* Page should not be returned to the free list (P) */
230 vmp_reference:1, /* page has been used (P) */
231 vmp_lopage:1,
232 vmp_realtime:1, /* page used by realtime thread */
233 #if !CONFIG_TRACK_UNMODIFIED_ANON_PAGES
234 vmp_unused_page_bits:3;
235 #else /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
236 vmp_unmodified_ro:1, /* Tracks if an anonymous page is modified after a decompression (O&P).*/
237 vmp_unused_page_bits:2;
238 #endif /* ! CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
239
240 /*
241 * MUST keep the 2 32 bit words used as bit fields
242 * separated since the compiler has a nasty habit
243 * of using 64 bit loads and stores on them as
244 * if they were a single 64 bit field... since
245 * they are protected by 2 different locks, this
246 * is a real problem
247 */
248 vm_page_packed_t vmp_next_m; /* VP bucket link (O) */
249
250 /*
251 * The following word of flags is protected by the "VM object" lock.
252 *
253 * IMPORTANT: the "vmp_pmapped", "vmp_xpmapped" and "vmp_clustered" bits can be modified while holding the
254 * VM object "shared" lock + the page lock provided through the pmap_lock_phys_page function.
255 * This is done in vm_fault_enter() and the CONSUME_CLUSTERED macro.
256 * It's also ok to modify them behind just the VM object "exclusive" lock.
257 */
258 unsigned int vmp_busy:1, /* page is in transit (O) */
259 vmp_wanted:1, /* someone is waiting for page (O) */
260 vmp_tabled:1, /* page is in VP table (O) */
261 vmp_hashed:1, /* page is in vm_page_buckets[] (O) + the bucket lock */
262 vmp_fictitious:1, /* Physical page doesn't exist (O) */
263 vmp_clustered:1, /* page is not the faulted page (O) or (O-shared AND pmap_page) */
264 vmp_pmapped:1, /* page has at some time been entered into a pmap (O) or */
265 /* (O-shared AND pmap_page) */
266 vmp_xpmapped:1, /* page has been entered with execute permission (O) or */
267 /* (O-shared AND pmap_page) */
268 vmp_wpmapped:1, /* page has been entered at some point into a pmap for write (O) */
269 vmp_free_when_done:1, /* page is to be freed once cleaning is completed (O) */
270 vmp_absent:1, /* Data has been requested, but is not yet available (O) */
271 vmp_error:1, /* Data manager was unable to provide data due to error (O) */
272 vmp_dirty:1, /* Page must be cleaned (O) */
273 vmp_cleaning:1, /* Page clean has begun (O) */
274 vmp_precious:1, /* Page is precious; data must be returned even if clean (O) */
275 vmp_overwriting:1, /* Request to unlock has been made without having data. (O) */
276 /* [See vm_fault_page_overwrite] */
277 vmp_restart:1, /* Page was pushed higher in shadow chain by copy_call-related pagers */
278 /* start again at top of chain */
279 vmp_unusual:1, /* Page is absent, error, restart or page locked */
280 vmp_cs_validated:VMP_CS_BITS, /* code-signing: page was checked */
281 vmp_cs_tainted:VMP_CS_BITS, /* code-signing: page is tainted */
282 vmp_cs_nx:VMP_CS_BITS, /* code-signing: page is nx */
283 vmp_reusable:1,
284 vmp_written_by_kernel:1; /* page was written by kernel (i.e. decompressed) */
285
286
287 #if !defined(__arm64__)
288 ppnum_t vmp_phys_page; /* Physical page number of the page */
289 #endif
290 };
291
292 extern vm_page_t vm_pages;
293 extern vm_page_t vm_page_array_beginning_addr;
294 extern vm_page_t vm_page_array_ending_addr;
295
296 #if defined(__arm64__)
297
298 extern unsigned int vm_first_phys_ppnum;
299
300 struct vm_page_with_ppnum {
301 struct vm_page vm_page_wo_ppnum;
302
303 ppnum_t vmp_phys_page;
304 };
305 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
306
307 static inline ppnum_t
VM_PAGE_GET_PHYS_PAGE(vm_page_t m)308 VM_PAGE_GET_PHYS_PAGE(vm_page_t m)
309 {
310 if (m >= vm_page_array_beginning_addr && m < vm_page_array_ending_addr) { /* real pages in vm_pages array */
311 return (ppnum_t)((uintptr_t)(m - vm_page_array_beginning_addr) + vm_first_phys_ppnum);
312 } else {
313 return ((vm_page_with_ppnum_t)m)->vmp_phys_page; /* pages in vm_page_zone */
314 }
315 }
316
317 #define VM_PAGE_SET_PHYS_PAGE(m, ppnum) \
318 MACRO_BEGIN \
319 if ((m) < vm_page_array_beginning_addr || (m) >= vm_page_array_ending_addr) \
320 ((vm_page_with_ppnum_t)(m))->vmp_phys_page = ppnum; \
321 assert(ppnum == VM_PAGE_GET_PHYS_PAGE(m)); \
322 MACRO_END
323
324 #define VM_PAGE_GET_COLOR(m) (VM_PAGE_GET_PHYS_PAGE(m) & vm_color_mask)
325
326 #else /* defined(__arm64__) */
327
328
329 struct vm_page_with_ppnum {
330 struct vm_page vm_page_with_ppnum;
331 };
332 typedef struct vm_page_with_ppnum *vm_page_with_ppnum_t;
333
334
335 #define VM_PAGE_GET_PHYS_PAGE(page) (page)->vmp_phys_page
336 #define VM_PAGE_SET_PHYS_PAGE(page, ppnum) \
337 MACRO_BEGIN \
338 (page)->vmp_phys_page = ppnum; \
339 MACRO_END
340
341 #define VM_PAGE_GET_CLUMP(m) ((VM_PAGE_GET_PHYS_PAGE(m)) >> vm_clump_shift)
342 #define VM_PAGE_GET_COLOR(m) ((VM_PAGE_GET_CLUMP(m)) & vm_color_mask)
343
344 #endif /* defined(__arm64__) */
345
346
347
348 #if defined(__LP64__)
349 /*
350 * Parameters for pointer packing
351 *
352 *
353 * VM Pages pointers might point to:
354 *
355 * 1. VM_PAGE_PACKED_ALIGNED aligned kernel globals,
356 *
357 * 2. VM_PAGE_PACKED_ALIGNED aligned heap allocated vm pages
358 *
359 * 3. entries in the vm_pages array (whose entries aren't VM_PAGE_PACKED_ALIGNED
360 * aligned).
361 *
362 *
363 * The current scheme uses 31 bits of storage and 6 bits of shift using the
364 * VM_PACK_POINTER() scheme for (1-2), and packs (3) as an index within the
365 * vm_pages array, setting the top bit (VM_PAGE_PACKED_FROM_ARRAY).
366 *
367 * This scheme gives us a reach of 128G from VM_MIN_KERNEL_AND_KEXT_ADDRESS.
368 */
369 #define VM_VPLQ_ALIGNMENT 128
370 #define VM_PAGE_PACKED_PTR_ALIGNMENT 64 /* must be a power of 2 */
371 #define VM_PAGE_PACKED_ALIGNED __attribute__((aligned(VM_PAGE_PACKED_PTR_ALIGNMENT)))
372 #define VM_PAGE_PACKED_PTR_BITS 31
373 #define VM_PAGE_PACKED_PTR_SHIFT 6
374 #define VM_PAGE_PACKED_PTR_BASE ((uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS)
375
376 #define VM_PAGE_PACKED_FROM_ARRAY 0x80000000
377
378 static inline vm_page_packed_t
vm_page_pack_ptr(uintptr_t p)379 vm_page_pack_ptr(uintptr_t p)
380 {
381 if (p >= (uintptr_t)vm_page_array_beginning_addr &&
382 p < (uintptr_t)vm_page_array_ending_addr) {
383 ptrdiff_t diff = (vm_page_t)p - vm_page_array_beginning_addr;
384 assert((vm_page_t)p == &vm_pages[diff]);
385 return (vm_page_packed_t)(diff | VM_PAGE_PACKED_FROM_ARRAY);
386 }
387
388 VM_ASSERT_POINTER_PACKABLE(p, VM_PAGE_PACKED_PTR);
389 vm_offset_t packed = VM_PACK_POINTER(p, VM_PAGE_PACKED_PTR);
390 return CAST_DOWN_EXPLICIT(vm_page_packed_t, packed);
391 }
392
393
394 static inline uintptr_t
vm_page_unpack_ptr(uintptr_t p)395 vm_page_unpack_ptr(uintptr_t p)
396 {
397 extern unsigned int vm_pages_count;
398
399 if (p >= VM_PAGE_PACKED_FROM_ARRAY) {
400 p &= ~VM_PAGE_PACKED_FROM_ARRAY;
401 assert(p < (uintptr_t)vm_pages_count);
402 return (uintptr_t)&vm_pages[p];
403 }
404
405 return VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR);
406 }
407
408
409 #define VM_PAGE_PACK_PTR(p) vm_page_pack_ptr((uintptr_t)(p))
410 #define VM_PAGE_UNPACK_PTR(p) vm_page_unpack_ptr((uintptr_t)(p))
411
412 #define VM_OBJECT_PACK(o) ((vm_page_object_t)VM_PACK_POINTER((uintptr_t)(o), VM_PAGE_PACKED_PTR))
413 #define VM_OBJECT_UNPACK(p) ((vm_object_t)VM_UNPACK_POINTER(p, VM_PAGE_PACKED_PTR))
414
415 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
416 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
417
418
419 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
420 MACRO_BEGIN \
421 (p)->vmp_snext = 0; \
422 MACRO_END
423
424
425 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) VM_PAGE_PACK_PTR(p)
426
427 /*
428 * Macro: vm_page_queue_init
429 * Function:
430 * Initialize the given queue.
431 * Header:
432 * void vm_page_queue_init(q)
433 * vm_page_queue_t q; \* MODIFIED *\
434 */
435 #define vm_page_queue_init(q) \
436 MACRO_BEGIN \
437 VM_ASSERT_POINTER_PACKABLE((vm_offset_t)(q), VM_PAGE_PACKED_PTR); \
438 (q)->next = VM_PAGE_PACK_PTR(q); \
439 (q)->prev = VM_PAGE_PACK_PTR(q); \
440 MACRO_END
441
442
443 /*
444 * Macro: vm_page_queue_enter
445 * Function:
446 * Insert a new element at the tail of the vm_page queue.
447 * Header:
448 * void vm_page_queue_enter(q, elt, field)
449 * queue_t q;
450 * vm_page_t elt;
451 * <field> is the list field in vm_page_t
452 *
453 * This macro's arguments have to match the generic "queue_enter()" macro which is
454 * what is used for this on 32 bit kernels.
455 */
456 #define vm_page_queue_enter(head, elt, field) \
457 MACRO_BEGIN \
458 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
459 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
460 vm_page_packed_t __pck_prev = (head)->prev; \
461 \
462 if (__pck_head == __pck_prev) { \
463 (head)->next = __pck_elt; \
464 } else { \
465 vm_page_t __prev; \
466 __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
467 __prev->field.next = __pck_elt; \
468 } \
469 (elt)->field.prev = __pck_prev; \
470 (elt)->field.next = __pck_head; \
471 (head)->prev = __pck_elt; \
472 MACRO_END
473
474
475 #if defined(__x86_64__)
476 /*
477 * These are helper macros for vm_page_queue_enter_clump to assist
478 * with conditional compilation (release / debug / development)
479 */
480 #if DEVELOPMENT || DEBUG
481
482 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field) \
483 MACRO_BEGIN \
484 if (__prev != NULL) { \
485 assert(__p == (vm_page_t)VM_PAGE_UNPACK_PTR(__prev->next)); \
486 assert(__prev == (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__p->field.prev)); \
487 } \
488 MACRO_END
489
490 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next) \
491 MACRO_BEGIN \
492 unsigned int __i; \
493 vm_page_queue_entry_t __tmp; \
494 for (__i = 0, __tmp = __first; __i < __n_free; __i++) { \
495 __tmp = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(__tmp->next); \
496 } \
497 assert(__tmp == __last_next); \
498 MACRO_END
499
500 #define __DEBUG_STAT_INCREMENT_INRANGE vm_clump_inrange++
501 #define __DEBUG_STAT_INCREMENT_INSERTS vm_clump_inserts++
502 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free) vm_clump_promotes+=__n_free
503
504 #else
505
506 #define __DEBUG_CHECK_BUDDIES(__prev, __p, field)
507 #define __DEBUG_VERIFY_LINKS(__first, __n_free, __last_next)
508 #define __DEBUG_STAT_INCREMENT_INRANGE
509 #define __DEBUG_STAT_INCREMENT_INSERTS
510 #define __DEBUG_STAT_INCREMENT_PROMOTES(__n_free)
511
512 #endif /* if DEVELOPMENT || DEBUG */
513
514 #endif
515
516 /*
517 * Macro: vm_page_queue_enter_first
518 * Function:
519 * Insert a new element at the head of the vm_page queue.
520 * Header:
521 * void queue_enter_first(q, elt, , field)
522 * queue_t q;
523 * vm_page_t elt;
524 * <field> is the linkage field in vm_page
525 *
526 * This macro's arguments have to match the generic "queue_enter_first()" macro which is
527 * what is used for this on 32 bit kernels.
528 */
529 #define vm_page_queue_enter_first(head, elt, field) \
530 MACRO_BEGIN \
531 vm_page_packed_t __pck_next = (head)->next; \
532 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
533 vm_page_packed_t __pck_elt = VM_PAGE_PACK_PTR(elt); \
534 \
535 if (__pck_head == __pck_next) { \
536 (head)->prev = __pck_elt; \
537 } else { \
538 vm_page_t __next; \
539 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
540 __next->field.prev = __pck_elt; \
541 } \
542 \
543 (elt)->field.next = __pck_next; \
544 (elt)->field.prev = __pck_head; \
545 (head)->next = __pck_elt; \
546 MACRO_END
547
548
549 /*
550 * Macro: vm_page_queue_remove
551 * Function:
552 * Remove an arbitrary page from a vm_page queue.
553 * Header:
554 * void vm_page_queue_remove(q, qe, field)
555 * arguments as in vm_page_queue_enter
556 *
557 * This macro's arguments have to match the generic "queue_enter()" macro which is
558 * what is used for this on 32 bit kernels.
559 */
560 #define vm_page_queue_remove(head, elt, field) \
561 MACRO_BEGIN \
562 vm_page_packed_t __pck_next = (elt)->field.next; \
563 vm_page_packed_t __pck_prev = (elt)->field.prev; \
564 vm_page_t __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
565 vm_page_t __prev = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_prev); \
566 \
567 if ((void *)(head) == (void *)__next) { \
568 (head)->prev = __pck_prev; \
569 } else { \
570 __next->field.prev = __pck_prev; \
571 } \
572 \
573 if ((void *)(head) == (void *)__prev) { \
574 (head)->next = __pck_next; \
575 } else { \
576 __prev->field.next = __pck_next; \
577 } \
578 \
579 (elt)->field.next = 0; \
580 (elt)->field.prev = 0; \
581 MACRO_END
582
583
584 /*
585 * Macro: vm_page_queue_remove_first
586 *
587 * Function:
588 * Remove and return the entry at the head of a vm_page queue.
589 *
590 * Header:
591 * vm_page_queue_remove_first(head, entry, field)
592 * N.B. entry is returned by reference
593 *
594 * This macro's arguments have to match the generic "queue_remove_first()" macro which is
595 * what is used for this on 32 bit kernels.
596 */
597 #define vm_page_queue_remove_first(head, entry, field) \
598 MACRO_BEGIN \
599 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
600 vm_page_packed_t __pck_next; \
601 vm_page_t __next; \
602 \
603 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
604 __pck_next = (entry)->field.next; \
605 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
606 \
607 if (__pck_head == __pck_next) { \
608 (head)->prev = __pck_head; \
609 } else { \
610 __next->field.prev = __pck_head; \
611 } \
612 \
613 (head)->next = __pck_next; \
614 (entry)->field.next = 0; \
615 (entry)->field.prev = 0; \
616 MACRO_END
617
618
619 #if defined(__x86_64__)
620 /*
621 * Macro: vm_page_queue_remove_first_with_clump
622 * Function:
623 * Remove and return the entry at the head of the free queue
624 * end is set to 1 to indicate that we just returned the last page in a clump
625 *
626 * Header:
627 * vm_page_queue_remove_first_with_clump(head, entry, end)
628 * entry is returned by reference
629 * end is returned by reference
630 */
631 #define vm_page_queue_remove_first_with_clump(head, entry, end) \
632 MACRO_BEGIN \
633 vm_page_packed_t __pck_head = VM_PAGE_PACK_PTR(head); \
634 vm_page_packed_t __pck_next; \
635 vm_page_t __next; \
636 \
637 (entry) = (vm_page_t)VM_PAGE_UNPACK_PTR((head)->next); \
638 __pck_next = (entry)->vmp_pageq.next; \
639 __next = (vm_page_t)VM_PAGE_UNPACK_PTR(__pck_next); \
640 \
641 (end) = 0; \
642 if (__pck_head == __pck_next) { \
643 (head)->prev = __pck_head; \
644 (end) = 1; \
645 } else { \
646 __next->vmp_pageq.prev = __pck_head; \
647 if (VM_PAGE_GET_CLUMP(entry) != VM_PAGE_GET_CLUMP(__next)) { \
648 (end) = 1; \
649 } \
650 } \
651 \
652 (head)->next = __pck_next; \
653 (entry)->vmp_pageq.next = 0; \
654 (entry)->vmp_pageq.prev = 0; \
655 MACRO_END
656 #endif
657
658 /*
659 * Macro: vm_page_queue_end
660 * Function:
661 * Tests whether a new entry is really the end of
662 * the queue.
663 * Header:
664 * boolean_t vm_page_queue_end(q, qe)
665 * vm_page_queue_t q;
666 * vm_page_queue_entry_t qe;
667 */
668 #define vm_page_queue_end(q, qe) ((q) == (qe))
669
670
671 /*
672 * Macro: vm_page_queue_empty
673 * Function:
674 * Tests whether a queue is empty.
675 * Header:
676 * boolean_t vm_page_queue_empty(q)
677 * vm_page_queue_t q;
678 */
679 #define vm_page_queue_empty(q) vm_page_queue_end((q), ((vm_page_queue_entry_t)vm_page_queue_first(q)))
680
681
682
683 /*
684 * Macro: vm_page_queue_first
685 * Function:
686 * Returns the first entry in the queue,
687 * Header:
688 * uintpr_t vm_page_queue_first(q)
689 * vm_page_queue_t q; \* IN *\
690 */
691 #define vm_page_queue_first(q) (VM_PAGE_UNPACK_PTR((q)->next))
692
693
694
695 /*
696 * Macro: vm_page_queue_last
697 * Function:
698 * Returns the last entry in the queue.
699 * Header:
700 * vm_page_queue_entry_t queue_last(q)
701 * queue_t q; \* IN *\
702 */
703 #define vm_page_queue_last(q) (VM_PAGE_UNPACK_PTR((q)->prev))
704
705
706
707 /*
708 * Macro: vm_page_queue_next
709 * Function:
710 * Returns the entry after an item in the queue.
711 * Header:
712 * uintpr_t vm_page_queue_next(qc)
713 * vm_page_queue_t qc;
714 */
715 #define vm_page_queue_next(qc) (VM_PAGE_UNPACK_PTR((qc)->next))
716
717
718
719 /*
720 * Macro: vm_page_queue_prev
721 * Function:
722 * Returns the entry before an item in the queue.
723 * Header:
724 * uinptr_t vm_page_queue_prev(qc)
725 * vm_page_queue_t qc;
726 */
727 #define vm_page_queue_prev(qc) (VM_PAGE_UNPACK_PTR((qc)->prev))
728
729
730
731 /*
732 * Macro: vm_page_queue_iterate
733 * Function:
734 * iterate over each item in a vm_page queue.
735 * Generates a 'for' loop, setting elt to
736 * each item in turn (by reference).
737 * Header:
738 * vm_page_queue_iterate(q, elt, field)
739 * queue_t q;
740 * vm_page_t elt;
741 * <field> is the chain field in vm_page_t
742 */
743 #define vm_page_queue_iterate(head, elt, field) \
744 for ((elt) = (vm_page_t)vm_page_queue_first(head); \
745 !vm_page_queue_end((head), (vm_page_queue_entry_t)(elt)); \
746 (elt) = (vm_page_t)vm_page_queue_next(&(elt)->field)) \
747
748 #else // LP64
749
750 #define VM_VPLQ_ALIGNMENT 128
751 #define VM_PAGE_PACKED_PTR_ALIGNMENT sizeof(vm_offset_t)
752 #define VM_PAGE_PACKED_ALIGNED
753 #define VM_PAGE_PACKED_PTR_BITS 32
754 #define VM_PAGE_PACKED_PTR_SHIFT 0
755 #define VM_PAGE_PACKED_PTR_BASE 0
756
757 #define VM_PAGE_PACKED_FROM_ARRAY 0
758
759 #define VM_PAGE_PACK_PTR(p) (p)
760 #define VM_PAGE_UNPACK_PTR(p) ((uintptr_t)(p))
761
762 #define VM_OBJECT_PACK(o) ((vm_page_object_t)(o))
763 #define VM_OBJECT_UNPACK(p) ((vm_object_t)(p))
764
765 #define VM_PAGE_PACK_OBJECT(o) VM_OBJECT_PACK(o)
766 #define VM_PAGE_OBJECT(p) VM_OBJECT_UNPACK((p)->vmp_object)
767
768
769 #define VM_PAGE_ZERO_PAGEQ_ENTRY(p) \
770 MACRO_BEGIN \
771 (p)->vmp_pageq.next = 0; \
772 (p)->vmp_pageq.prev = 0; \
773 MACRO_END
774
775 #define VM_PAGE_CONVERT_TO_QUEUE_ENTRY(p) ((queue_entry_t)(p))
776
777 #define vm_page_remque remque
778 #define vm_page_enqueue_tail enqueue_tail
779 #define vm_page_queue_init queue_init
780 #define vm_page_queue_enter(h, e, f) queue_enter(h, e, vm_page_t, f)
781 #define vm_page_queue_enter_first(h, e, f) queue_enter_first(h, e, vm_page_t, f)
782 #define vm_page_queue_remove(h, e, f) queue_remove(h, e, vm_page_t, f)
783 #define vm_page_queue_remove_first(h, e, f) queue_remove_first(h, e, vm_page_t, f)
784 #define vm_page_queue_end queue_end
785 #define vm_page_queue_empty queue_empty
786 #define vm_page_queue_first queue_first
787 #define vm_page_queue_last queue_last
788 #define vm_page_queue_next queue_next
789 #define vm_page_queue_prev queue_prev
790 #define vm_page_queue_iterate(h, e, f) queue_iterate(h, e, vm_page_t, f)
791
792 #endif // __LP64__
793
794
795
796 /*
797 * VM_PAGE_MIN_SPECULATIVE_AGE_Q through vm_page_max_speculative_age_q
798 * represents a set of aging bins that are 'protected'...
799 *
800 * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
801 * not yet been 'claimed' but have been aged out of the protective bins
802 * this occurs in vm_page_speculate when it advances to the next bin
803 * and discovers that it is still occupied... at that point, all of the
804 * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q. the pages
805 * in that bin are all guaranteed to have reached at least the maximum age
806 * we allow for a protected page... they can be older if there is no
807 * memory pressure to pull them from the bin, or there are no new speculative pages
808 * being generated to push them out.
809 * this list is the one that vm_pageout_scan will prefer when looking
810 * for pages to move to the underweight free list
811 *
812 * vm_page_max_speculative_age_q * VM_PAGE_SPECULATIVE_Q_AGE_MS
813 * defines the amount of time a speculative page is normally
814 * allowed to live in the 'protected' state (i.e. not available
815 * to be stolen if vm_pageout_scan is running and looking for
816 * pages)... however, if the total number of speculative pages
817 * in the protected state exceeds our limit (defined in vm_pageout.c)
818 * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
819 * vm_pageout_scan is allowed to steal pages from the protected
820 * bucket even if they are underage.
821 *
822 * vm_pageout_scan is also allowed to pull pages from a protected
823 * bin if the bin has reached the "age of consent" we've set
824 */
825 #define VM_PAGE_RESERVED_SPECULATIVE_AGE_Q 40
826 #define VM_PAGE_DEFAULT_MAX_SPECULATIVE_AGE_Q 10
827 #define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
828 #define VM_PAGE_SPECULATIVE_AGED_Q 0
829
830 #define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
831
832 struct vm_speculative_age_q {
833 /*
834 * memory queue for speculative pages via clustered pageins
835 */
836 vm_page_queue_head_t age_q;
837 mach_timespec_t age_ts;
838 } VM_PAGE_PACKED_ALIGNED;
839
840
841
842 extern
843 struct vm_speculative_age_q vm_page_queue_speculative[];
844
845 extern int speculative_steal_index;
846 extern int speculative_age_index;
847 extern unsigned int vm_page_speculative_q_age_ms;
848 extern unsigned int vm_page_max_speculative_age_q;
849
850
851 typedef struct vm_locks_array {
852 char pad __attribute__ ((aligned(64)));
853 lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned(64)));
854 lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned(64)));
855 char pad2 __attribute__ ((aligned(64)));
856 } vm_locks_array_t;
857
858
859 #define VM_PAGE_WIRED(m) ((m)->vmp_q_state == VM_PAGE_IS_WIRED)
860 #define NEXT_PAGE(m) ((m)->vmp_snext)
861 #define NEXT_PAGE_PTR(m) (&(m)->vmp_snext)
862
863 /*
864 * XXX The unusual bit should not be necessary. Most of the bit
865 * XXX fields above really want to be masks.
866 */
867
868 /*
869 * For debugging, this macro can be defined to perform
870 * some useful check on a page structure.
871 * INTENTIONALLY left as a no-op so that the
872 * current call-sites can be left intact for future uses.
873 */
874
875 #define VM_PAGE_CHECK(mem) \
876 MACRO_BEGIN \
877 MACRO_END
878
879 /* Page coloring:
880 *
881 * The free page list is actually n lists, one per color,
882 * where the number of colors is a function of the machine's
883 * cache geometry set at system initialization. To disable
884 * coloring, set vm_colors to 1 and vm_color_mask to 0.
885 * The boot-arg "colors" may be used to override vm_colors.
886 * Note that there is little harm in having more colors than needed.
887 */
888
889 #define MAX_COLORS 128
890 #define DEFAULT_COLORS 32
891
892 extern
893 unsigned int vm_colors; /* must be in range 1..MAX_COLORS */
894 extern
895 unsigned int vm_color_mask; /* must be (vm_colors-1) */
896 extern
897 unsigned int vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
898
899 /*
900 * Wired memory is a very limited resource and we can't let users exhaust it
901 * and deadlock the entire system. We enforce the following limits:
902 *
903 * vm_per_task_user_wire_limit
904 * how much memory can be user-wired in one user task
905 *
906 * vm_global_user_wire_limit (default: same as vm_per_task_user_wire_limit)
907 * how much memory can be user-wired in all user tasks
908 *
909 * These values are set to defaults based on the number of pages managed
910 * by the VM system. They can be overriden via sysctls.
911 * See kmem_set_user_wire_limits for details on the default values.
912 *
913 * Regardless of the amount of memory in the system, we never reserve
914 * more than VM_NOT_USER_WIREABLE_MAX bytes as unlockable.
915 */
916 #if defined(__LP64__)
917 #define VM_NOT_USER_WIREABLE_MAX (32ULL*1024*1024*1024) /* 32GB */
918 #else
919 #define VM_NOT_USER_WIREABLE_MAX (1UL*1024*1024*1024) /* 1GB */
920 #endif /* __LP64__ */
921 extern
922 vm_map_size_t vm_per_task_user_wire_limit;
923 extern
924 vm_map_size_t vm_global_user_wire_limit;
925 extern
926 uint64_t vm_add_wire_count_over_global_limit;
927 extern
928 uint64_t vm_add_wire_count_over_user_limit;
929
930 /*
931 * Each pageable resident page falls into one of three lists:
932 *
933 * free
934 * Available for allocation now. The free list is
935 * actually an array of lists, one per color.
936 * inactive
937 * Not referenced in any map, but still has an
938 * object/offset-page mapping, and may be dirty.
939 * This is the list of pages that should be
940 * paged out next. There are actually two
941 * inactive lists, one for pages brought in from
942 * disk or other backing store, and another
943 * for "zero-filled" pages. See vm_pageout_scan()
944 * for the distinction and usage.
945 * active
946 * A list of pages which have been placed in
947 * at least one physical map. This list is
948 * ordered, in LRU-like fashion.
949 */
950
951
952 #define VPL_LOCK_SPIN 1
953
954 struct vpl {
955 vm_page_queue_head_t vpl_queue;
956 unsigned int vpl_count;
957 unsigned int vpl_internal_count;
958 unsigned int vpl_external_count;
959 lck_spin_t vpl_lock;
960 };
961
962 extern
963 struct vpl * /* __zpercpu */ vm_page_local_q;
964 extern
965 unsigned int vm_page_local_q_soft_limit;
966 extern
967 unsigned int vm_page_local_q_hard_limit;
968 extern
969 vm_locks_array_t vm_page_locks;
970
971 extern
972 vm_page_queue_head_t vm_lopage_queue_free; /* low memory free queue */
973 extern
974 vm_page_queue_head_t vm_page_queue_active; /* active memory queue */
975 extern
976 vm_page_queue_head_t vm_page_queue_inactive; /* inactive memory queue for normal pages */
977 #if CONFIG_SECLUDED_MEMORY
978 extern
979 vm_page_queue_head_t vm_page_queue_secluded; /* reclaimable pages secluded for Camera */
980 #endif /* CONFIG_SECLUDED_MEMORY */
981 extern
982 vm_page_queue_head_t vm_page_queue_cleaned; /* clean-queue inactive memory */
983 extern
984 vm_page_queue_head_t vm_page_queue_anonymous; /* inactive memory queue for anonymous pages */
985 extern
986 vm_page_queue_head_t vm_page_queue_throttled; /* memory queue for throttled pageout pages */
987
988 extern
989 queue_head_t vm_objects_wired;
990 extern
991 lck_spin_t vm_objects_wired_lock;
992
993 #define VM_PAGE_DONATE_DISABLED 0
994 #define VM_PAGE_DONATE_ENABLED 1
995 extern
996 uint32_t vm_page_donate_mode;
997 extern
998 bool vm_page_donate_queue_ripe;
999
1000 #define VM_PAGE_BACKGROUND_TARGET_MAX 50000
1001 #define VM_PAGE_BG_DISABLED 0
1002 #define VM_PAGE_BG_ENABLED 1
1003
1004 extern
1005 vm_page_queue_head_t vm_page_queue_background;
1006 extern
1007 uint64_t vm_page_background_promoted_count;
1008 extern
1009 uint32_t vm_page_background_count;
1010 extern
1011 uint32_t vm_page_background_target;
1012 extern
1013 uint32_t vm_page_background_internal_count;
1014 extern
1015 uint32_t vm_page_background_external_count;
1016 extern
1017 uint32_t vm_page_background_mode;
1018 extern
1019 uint32_t vm_page_background_exclude_external;
1020
1021 extern
1022 vm_page_queue_head_t vm_page_queue_donate;
1023 extern
1024 uint32_t vm_page_donate_count;
1025 extern
1026 uint32_t vm_page_donate_target_low;
1027 extern
1028 uint32_t vm_page_donate_target_high;
1029 #define VM_PAGE_DONATE_TARGET_LOWWATER (100)
1030 #define VM_PAGE_DONATE_TARGET_HIGHWATER ((unsigned int)(atop_64(max_mem) / 8))
1031
1032 extern
1033 vm_offset_t first_phys_addr; /* physical address for first_page */
1034 extern
1035 vm_offset_t last_phys_addr; /* physical address for last_page */
1036
1037 extern
1038 unsigned int vm_page_free_count; /* How many pages are free? (sum of all colors) */
1039 extern
1040 unsigned int vm_page_active_count; /* How many pages are active? */
1041 extern
1042 unsigned int vm_page_inactive_count; /* How many pages are inactive? */
1043 extern
1044 unsigned int vm_page_kernelcache_count; /* How many pages are used for the kernelcache? */
1045 extern
1046 unsigned int vm_page_realtime_count; /* How many pages are used by realtime threads? */
1047 #if CONFIG_SECLUDED_MEMORY
1048 extern
1049 unsigned int vm_page_secluded_count; /* How many pages are secluded? */
1050 extern
1051 unsigned int vm_page_secluded_count_free; /* how many of them are free? */
1052 extern
1053 unsigned int vm_page_secluded_count_inuse; /* how many of them are in use? */
1054 /*
1055 * We keep filling the secluded pool with new eligible pages and
1056 * we can overshoot our target by a lot.
1057 * When there's memory pressure, vm_pageout_scan() will re-balance the queues,
1058 * pushing the extra secluded pages to the active or free queue.
1059 * Since these "over target" secluded pages are actually "available", jetsam
1060 * should consider them as such, so make them visible to jetsam via the
1061 * "vm_page_secluded_count_over_target" counter and update it whenever we
1062 * update vm_page_secluded_count or vm_page_secluded_target.
1063 */
1064 extern
1065 unsigned int vm_page_secluded_count_over_target;
1066 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1067 MACRO_BEGIN \
1068 if (vm_page_secluded_count > vm_page_secluded_target) { \
1069 vm_page_secluded_count_over_target = \
1070 (vm_page_secluded_count - vm_page_secluded_target); \
1071 } else { \
1072 vm_page_secluded_count_over_target = 0; \
1073 } \
1074 MACRO_END
1075 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() vm_page_secluded_count_over_target
1076 #else /* CONFIG_SECLUDED_MEMORY */
1077 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE() \
1078 MACRO_BEGIN \
1079 MACRO_END
1080 #define VM_PAGE_SECLUDED_COUNT_OVER_TARGET() 0
1081 #endif /* CONFIG_SECLUDED_MEMORY */
1082 extern
1083 unsigned int vm_page_cleaned_count; /* How many pages are in the clean queue? */
1084 extern
1085 unsigned int vm_page_throttled_count;/* How many inactives are throttled */
1086 extern
1087 unsigned int vm_page_speculative_count; /* How many speculative pages are unclaimed? */
1088 extern unsigned int vm_page_pageable_internal_count;
1089 extern unsigned int vm_page_pageable_external_count;
1090 extern
1091 unsigned int vm_page_xpmapped_external_count; /* How many pages are mapped executable? */
1092 extern
1093 unsigned int vm_page_external_count; /* How many pages are file-backed? */
1094 extern
1095 unsigned int vm_page_internal_count; /* How many pages are anonymous? */
1096 extern
1097 unsigned int vm_page_wire_count; /* How many pages are wired? */
1098 extern
1099 unsigned int vm_page_wire_count_initial; /* How many pages wired at startup */
1100 extern
1101 unsigned int vm_page_wire_count_on_boot; /* even earlier than _initial */
1102 extern
1103 unsigned int vm_page_free_target; /* How many do we want free? */
1104 extern
1105 unsigned int vm_page_free_min; /* When to wakeup pageout */
1106 extern
1107 unsigned int vm_page_throttle_limit; /* When to throttle new page creation */
1108 extern
1109 unsigned int vm_page_inactive_target;/* How many do we want inactive? */
1110 #if CONFIG_SECLUDED_MEMORY
1111 extern
1112 unsigned int vm_page_secluded_target;/* How many do we want secluded? */
1113 #endif /* CONFIG_SECLUDED_MEMORY */
1114 extern
1115 unsigned int vm_page_anonymous_min; /* When it's ok to pre-clean */
1116 extern
1117 unsigned int vm_page_free_reserved; /* How many pages reserved to do pageout */
1118 extern
1119 unsigned int vm_page_gobble_count;
1120 extern
1121 unsigned int vm_page_stolen_count; /* Count of stolen pages not acccounted in zones */
1122 extern
1123 unsigned int vm_page_kern_lpage_count; /* Count of large pages used in early boot */
1124
1125
1126 #if DEVELOPMENT || DEBUG
1127 extern
1128 unsigned int vm_page_speculative_used;
1129 #endif
1130
1131 extern
1132 unsigned int vm_page_purgeable_count;/* How many pages are purgeable now ? */
1133 extern
1134 unsigned int vm_page_purgeable_wired_count;/* How many purgeable pages are wired now ? */
1135 extern
1136 uint64_t vm_page_purged_count; /* How many pages got purged so far ? */
1137
1138 extern unsigned int vm_page_free_wanted;
1139 /* how many threads are waiting for memory */
1140
1141 extern unsigned int vm_page_free_wanted_privileged;
1142 /* how many VM privileged threads are waiting for memory */
1143 #if CONFIG_SECLUDED_MEMORY
1144 extern unsigned int vm_page_free_wanted_secluded;
1145 /* how many threads are waiting for secluded memory */
1146 #endif /* CONFIG_SECLUDED_MEMORY */
1147
1148 extern const ppnum_t vm_page_fictitious_addr;
1149 /* (fake) phys_addr of fictitious pages */
1150
1151 extern const ppnum_t vm_page_guard_addr;
1152 /* (fake) phys_addr of guard pages */
1153
1154
1155 extern boolean_t vm_page_deactivate_hint;
1156
1157 extern int vm_compressor_mode;
1158
1159 /*
1160 * Defaults to true, so highest memory is used first.
1161 */
1162 extern boolean_t vm_himemory_mode;
1163
1164 extern boolean_t vm_lopage_needed;
1165 extern uint32_t vm_lopage_free_count;
1166 extern uint32_t vm_lopage_free_limit;
1167 extern uint32_t vm_lopage_lowater;
1168 extern boolean_t vm_lopage_refill;
1169 extern uint64_t max_valid_dma_address;
1170 extern ppnum_t max_valid_low_ppnum;
1171
1172 /*
1173 * Prototypes for functions exported by this module.
1174 */
1175
1176 extern void vm_page_init_local_q(unsigned int num_cpus);
1177
1178 extern void vm_page_create(
1179 ppnum_t start,
1180 ppnum_t end);
1181
1182 extern void vm_page_create_retired(
1183 ppnum_t pn);
1184
1185 extern boolean_t vm_page_created(
1186 vm_page_t page);
1187
1188
1189 extern void vm_free_delayed_pages(void);
1190
1191 extern vm_page_t vm_page_alloc(
1192 vm_object_t object,
1193 vm_object_offset_t offset);
1194
1195 extern void vm_page_reactivate_all_throttled(void);
1196
1197 extern void vm_pressure_response(void);
1198
1199 #define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
1200 #define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
1201
1202 #if CONFIG_JETSAM
1203
1204 #define VM_CHECK_MEMORYSTATUS \
1205 memorystatus_update_available_page_count( \
1206 vm_page_pageable_external_count + \
1207 vm_page_free_count + \
1208 VM_PAGE_SECLUDED_COUNT_OVER_TARGET() + \
1209 (VM_DYNAMIC_PAGING_ENABLED() ? 0 : vm_page_purgeable_count) \
1210 )
1211
1212 #else /* CONFIG_JETSAM */
1213
1214 #if !XNU_TARGET_OS_OSX
1215
1216 #define VM_CHECK_MEMORYSTATUS do {} while(0)
1217
1218 #else /* !XNU_TARGET_OS_OSX */
1219
1220 #define VM_CHECK_MEMORYSTATUS memorystatus_update_available_page_count(AVAILABLE_NON_COMPRESSED_MEMORY)
1221
1222 #endif /* !XNU_TARGET_OS_OSX */
1223
1224 #endif /* CONFIG_JETSAM */
1225
1226 #define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
1227 #define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
1228
1229 #ifdef MACH_KERNEL_PRIVATE
1230 static inline void
vm_page_lock_queues(void)1231 vm_page_lock_queues(void)
1232 {
1233 lck_mtx_lock(&vm_page_queue_lock);
1234 }
1235
1236 static inline boolean_t
vm_page_trylock_queues(void)1237 vm_page_trylock_queues(void)
1238 {
1239 boolean_t ret;
1240 ret = lck_mtx_try_lock(&vm_page_queue_lock);
1241 return ret;
1242 }
1243
1244 static inline void
vm_page_unlock_queues(void)1245 vm_page_unlock_queues(void)
1246 {
1247 lck_mtx_unlock(&vm_page_queue_lock);
1248 }
1249
1250 static inline void
vm_page_lockspin_queues(void)1251 vm_page_lockspin_queues(void)
1252 {
1253 lck_mtx_lock_spin(&vm_page_queue_lock);
1254 }
1255
1256 static inline boolean_t
vm_page_trylockspin_queues(void)1257 vm_page_trylockspin_queues(void)
1258 {
1259 boolean_t ret;
1260 ret = lck_mtx_try_lock_spin(&vm_page_queue_lock);
1261 return ret;
1262 }
1263
1264 extern void kdp_vm_page_sleep_find_owner(
1265 event64_t wait_event,
1266 thread_waitinfo_t *waitinfo);
1267
1268 #endif /* MACH_KERNEL_PRIVATE */
1269
1270 extern unsigned int vm_max_delayed_work_limit;
1271
1272 #if CONFIG_SECLUDED_MEMORY
1273 extern uint64_t secluded_shutoff_trigger;
1274 extern uint64_t secluded_shutoff_headroom;
1275 extern void start_secluded_suppression(task_t);
1276 extern void stop_secluded_suppression(task_t);
1277 #endif /* CONFIG_SECLUDED_MEMORY */
1278
1279 #endif /* _VM_VM_PAGE_H_ */
1280