1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _VM_VM_PAGE_INTERNAL_H_
30 #define _VM_VM_PAGE_INTERNAL_H_
31
32 #include <sys/cdefs.h>
33 #include <vm/vm_page.h>
34
35 __BEGIN_DECLS
36 #ifdef XNU_KERNEL_PRIVATE
37
38 PERCPU_DECL(unsigned int, start_color);
39
40 extern struct vm_page_free_queue vm_page_queue_free;
41
42 /*!
43 * @abstract
44 * Applies a signed delta to a VM counter that is not meant to ever overflow.
45 *
46 * @discussion
47 * This is not meant for counters counting "events", but for counters that
48 * maintain how many objects there is in a given state (free pages, ...).
49 *
50 * @param counter A pointer to a counter of any integer type.
51 * @param value The signed delta to apply.
52 * @returns The new value of the counter.
53 */
54 #define VM_COUNTER_DELTA(counter, value) ({ \
55 __auto_type __counter = (counter); \
56 release_assert(!os_add_overflow(*__counter, value, __counter)); \
57 *__counter; \
58 })
59 #define VM_COUNTER_ATOMIC_DELTA(counter, value) ({ \
60 __auto_type __value = (value); \
61 __auto_type __orig = os_atomic_add_orig(counter, __value, relaxed); \
62 release_assert(!os_add_overflow(__orig, __value, &__orig)); \
63 __orig + __value; \
64 })
65
66
67 /*!
68 * @abstract
69 * Applies an unsigned increment to a VM counter that is not meant to ever
70 * overflow.
71 *
72 * @discussion
73 * This is not meant for counters counting "events", but for counters that
74 * maintain how many objects there is in a given state (free pages, ...).
75 *
76 * @param counter A pointer to a counter of any integer type.
77 * @param value The unsigned value to add.
78 * @returns The new value of the counter.
79 */
80 #define VM_COUNTER_ADD(counter, value) ({ \
81 __auto_type __counter = (counter); \
82 release_assert(!os_add_overflow(*__counter, value, __counter)); \
83 *__counter; \
84 })
85 #define VM_COUNTER_ATOMIC_ADD(counter, value) ({ \
86 __auto_type __value = (value); \
87 __auto_type __orig = os_atomic_add_orig(counter, __value, relaxed); \
88 release_assert(!os_add_overflow(__orig, __value, &__orig)); \
89 __orig + __value; \
90 })
91
92 /*!
93 * @abstract
94 * Applies an unsigned decrement to a VM counter that is not meant to ever
95 * overflow.
96 *
97 * @discussion
98 * This is not meant for counters counting "events", but for counters that
99 * maintain how many objects there is in a given state (free pages, ...).
100 *
101 * @param counter A pointer to a counter of any integer type.
102 * @param value The unsigned value to substract.
103 * @returns The new value of the counter.
104 */
105 #define VM_COUNTER_SUB(counter, value) ({ \
106 __auto_type __counter = (counter); \
107 release_assert(!os_sub_overflow(*__counter, value, __counter)); \
108 *__counter; \
109 })
110 #define VM_COUNTER_ATOMIC_SUB(counter, value) ({ \
111 __auto_type __value = (value); \
112 __auto_type __orig = os_atomic_sub_orig(counter, __value, relaxed); \
113 release_assert(!os_sub_overflow(__orig, __value, &__orig)); \
114 __orig - __value; \
115 })
116
117
118 /*!
119 * @abstract
120 * Convenience wrapper to increment a VM counter.
121 *
122 * @discussion
123 * This is not meant for counters counting "events", but for counters that
124 * maintain how many objects there is in a given state (free pages, ...).
125 *
126 * @param counter A pointer to a counter of any integer type.
127 * @returns The new value of the counter.
128 */
129 #define VM_COUNTER_INC(counter) VM_COUNTER_ADD(counter, 1)
130 #define VM_COUNTER_ATOMIC_INC(counter) VM_COUNTER_ATOMIC_ADD(counter, 1)
131
132 /*!
133 * @abstract
134 * Convenience wrapper to decrement a VM counter.
135 *
136 * @discussion
137 * This is not meant for counters counting "events", but for counters that
138 * maintain how many objects there is in a given state (free pages, ...).
139 *
140 * @param counter A pointer to a counter of any integer type.
141 * @returns The new value of the counter.
142 */
143 #define VM_COUNTER_DEC(counter) VM_COUNTER_SUB(counter, 1)
144 #define VM_COUNTER_ATOMIC_DEC(counter) VM_COUNTER_ATOMIC_SUB(counter, 1)
145
146 static inline int
VMP_CS_FOR_OFFSET(vm_map_offset_t fault_phys_offset)147 VMP_CS_FOR_OFFSET(
148 vm_map_offset_t fault_phys_offset)
149 {
150 assertf(fault_phys_offset < PAGE_SIZE &&
151 !(fault_phys_offset & FOURK_PAGE_MASK),
152 "offset 0x%llx\n", (uint64_t)fault_phys_offset);
153 return 1 << (fault_phys_offset >> FOURK_PAGE_SHIFT);
154 }
155 static inline bool
VMP_CS_VALIDATED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)156 VMP_CS_VALIDATED(
157 vm_page_t p,
158 vm_map_size_t fault_page_size,
159 vm_map_offset_t fault_phys_offset)
160 {
161 assertf(fault_page_size <= PAGE_SIZE,
162 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
163 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
164 if (fault_page_size == PAGE_SIZE) {
165 return p->vmp_cs_validated == VMP_CS_ALL_TRUE;
166 }
167 return p->vmp_cs_validated & VMP_CS_FOR_OFFSET(fault_phys_offset);
168 }
169 static inline bool
VMP_CS_TAINTED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)170 VMP_CS_TAINTED(
171 vm_page_t p,
172 vm_map_size_t fault_page_size,
173 vm_map_offset_t fault_phys_offset)
174 {
175 assertf(fault_page_size <= PAGE_SIZE,
176 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
177 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
178 if (fault_page_size == PAGE_SIZE) {
179 return p->vmp_cs_tainted != VMP_CS_ALL_FALSE;
180 }
181 return p->vmp_cs_tainted & VMP_CS_FOR_OFFSET(fault_phys_offset);
182 }
183 static inline bool
VMP_CS_NX(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset)184 VMP_CS_NX(
185 vm_page_t p,
186 vm_map_size_t fault_page_size,
187 vm_map_offset_t fault_phys_offset)
188 {
189 assertf(fault_page_size <= PAGE_SIZE,
190 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
191 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
192 if (fault_page_size == PAGE_SIZE) {
193 return p->vmp_cs_nx != VMP_CS_ALL_FALSE;
194 }
195 return p->vmp_cs_nx & VMP_CS_FOR_OFFSET(fault_phys_offset);
196 }
197 static inline void
VMP_CS_SET_VALIDATED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)198 VMP_CS_SET_VALIDATED(
199 vm_page_t p,
200 vm_map_size_t fault_page_size,
201 vm_map_offset_t fault_phys_offset,
202 boolean_t value)
203 {
204 assertf(fault_page_size <= PAGE_SIZE,
205 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
206 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
207 if (value) {
208 if (fault_page_size == PAGE_SIZE) {
209 p->vmp_cs_validated = VMP_CS_ALL_TRUE;
210 }
211 p->vmp_cs_validated |= VMP_CS_FOR_OFFSET(fault_phys_offset);
212 } else {
213 if (fault_page_size == PAGE_SIZE) {
214 p->vmp_cs_validated = VMP_CS_ALL_FALSE;
215 }
216 p->vmp_cs_validated &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
217 }
218 }
219 static inline void
VMP_CS_SET_TAINTED(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)220 VMP_CS_SET_TAINTED(
221 vm_page_t p,
222 vm_map_size_t fault_page_size,
223 vm_map_offset_t fault_phys_offset,
224 boolean_t value)
225 {
226 assertf(fault_page_size <= PAGE_SIZE,
227 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
228 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
229 if (value) {
230 if (fault_page_size == PAGE_SIZE) {
231 p->vmp_cs_tainted = VMP_CS_ALL_TRUE;
232 }
233 p->vmp_cs_tainted |= VMP_CS_FOR_OFFSET(fault_phys_offset);
234 } else {
235 if (fault_page_size == PAGE_SIZE) {
236 p->vmp_cs_tainted = VMP_CS_ALL_FALSE;
237 }
238 p->vmp_cs_tainted &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
239 }
240 }
241 static inline void
VMP_CS_SET_NX(vm_page_t p,vm_map_size_t fault_page_size,vm_map_offset_t fault_phys_offset,boolean_t value)242 VMP_CS_SET_NX(
243 vm_page_t p,
244 vm_map_size_t fault_page_size,
245 vm_map_offset_t fault_phys_offset,
246 boolean_t value)
247 {
248 assertf(fault_page_size <= PAGE_SIZE,
249 "fault_page_size 0x%llx fault_phys_offset 0x%llx\n",
250 (uint64_t)fault_page_size, (uint64_t)fault_phys_offset);
251 if (value) {
252 if (fault_page_size == PAGE_SIZE) {
253 p->vmp_cs_nx = VMP_CS_ALL_TRUE;
254 }
255 p->vmp_cs_nx |= VMP_CS_FOR_OFFSET(fault_phys_offset);
256 } else {
257 if (fault_page_size == PAGE_SIZE) {
258 p->vmp_cs_nx = VMP_CS_ALL_FALSE;
259 }
260 p->vmp_cs_nx &= ~VMP_CS_FOR_OFFSET(fault_phys_offset);
261 }
262 }
263
264
265 #if defined(__LP64__)
266 static __inline__ void
vm_page_enqueue_tail(vm_page_queue_t que,vm_page_queue_entry_t elt)267 vm_page_enqueue_tail(
268 vm_page_queue_t que,
269 vm_page_queue_entry_t elt)
270 {
271 vm_page_queue_entry_t old_tail;
272
273 old_tail = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(que->prev);
274 elt->next = VM_PAGE_PACK_PTR(que);
275 elt->prev = que->prev;
276 que->prev = old_tail->next = VM_PAGE_PACK_PTR(elt);
277 }
278
279 static __inline__ void
vm_page_remque(vm_page_queue_entry_t elt)280 vm_page_remque(
281 vm_page_queue_entry_t elt)
282 {
283 vm_page_queue_entry_t next;
284 vm_page_queue_entry_t prev;
285 vm_page_packed_t next_pck = elt->next;
286 vm_page_packed_t prev_pck = elt->prev;
287
288 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(next_pck);
289
290 /* next may equal prev (and the queue head) if elt was the only element */
291 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev_pck);
292
293 next->prev = prev_pck;
294 prev->next = next_pck;
295
296 elt->next = 0;
297 elt->prev = 0;
298 }
299
300 #if defined(__x86_64__)
301 /*
302 * Insert a new page into a free queue and clump pages within the same 16K boundary together
303 */
304 static inline void
vm_page_queue_enter_clump(vm_page_queue_t head,vm_page_t elt)305 vm_page_queue_enter_clump(
306 vm_page_queue_t head,
307 vm_page_t elt)
308 {
309 vm_page_queue_entry_t first = NULL; /* first page in the clump */
310 vm_page_queue_entry_t last = NULL; /* last page in the clump */
311 vm_page_queue_entry_t prev = NULL;
312 vm_page_queue_entry_t next;
313 uint_t n_free = 1;
314 extern unsigned int vm_clump_size, vm_clump_promote_threshold;
315 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
316
317 /*
318 * If elt is part of the vm_pages[] array, find its neighboring buddies in the array.
319 */
320 if (vm_page_in_array(elt)) {
321 vm_page_t p;
322 uint_t i;
323 uint_t n;
324 ppnum_t clump_num;
325
326 first = last = (vm_page_queue_entry_t)elt;
327 clump_num = VM_PAGE_GET_CLUMP(elt);
328 n = VM_PAGE_GET_PHYS_PAGE(elt) & vm_clump_mask;
329
330 /*
331 * Check for preceeding vm_pages[] entries in the same chunk
332 */
333 for (i = 0, p = elt - 1; i < n && vm_page_get(0) <= p; i++, p--) {
334 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
335 if (prev == NULL) {
336 prev = (vm_page_queue_entry_t)p;
337 }
338 first = (vm_page_queue_entry_t)p;
339 n_free++;
340 }
341 }
342
343 /*
344 * Check the following vm_pages[] entries in the same chunk
345 */
346 for (i = n + 1, p = elt + 1; i < vm_clump_size && p < vm_page_get(vm_pages_count); i++, p++) {
347 if (p->vmp_q_state == VM_PAGE_ON_FREE_Q && clump_num == VM_PAGE_GET_CLUMP(p)) {
348 if (last == (vm_page_queue_entry_t)elt) { /* first one only */
349 __DEBUG_CHECK_BUDDIES(prev, p, vmp_pageq);
350 }
351
352 if (prev == NULL) {
353 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev);
354 }
355 last = (vm_page_queue_entry_t)p;
356 n_free++;
357 }
358 }
359 __DEBUG_STAT_INCREMENT_INRANGE;
360 }
361
362 /* if elt is not part of vm_pages or if 1st page in clump, insert at tail */
363 if (prev == NULL) {
364 prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->prev);
365 }
366
367 /* insert the element */
368 next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(prev->next);
369 elt->vmp_pageq.next = prev->next;
370 elt->vmp_pageq.prev = next->prev;
371 prev->next = next->prev = VM_PAGE_PACK_PTR(elt);
372 __DEBUG_STAT_INCREMENT_INSERTS;
373
374 /*
375 * Check if clump needs to be promoted to head.
376 */
377 if (n_free >= vm_clump_promote_threshold && n_free > 1) {
378 vm_page_queue_entry_t first_prev;
379
380 first_prev = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(first->prev);
381
382 /* If not at head already */
383 if (first_prev != head) {
384 vm_page_queue_entry_t last_next;
385 vm_page_queue_entry_t head_next;
386
387 last_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(last->next);
388
389 /* verify that the links within the clump are consistent */
390 __DEBUG_VERIFY_LINKS(first, n_free, last_next);
391
392 /* promote clump to head */
393 first_prev->next = last->next;
394 last_next->prev = first->prev;
395 first->prev = VM_PAGE_PACK_PTR(head);
396 last->next = head->next;
397
398 head_next = (vm_page_queue_entry_t)VM_PAGE_UNPACK_PTR(head->next);
399 head_next->prev = VM_PAGE_PACK_PTR(last);
400 head->next = VM_PAGE_PACK_PTR(first);
401 __DEBUG_STAT_INCREMENT_PROMOTES(n_free);
402 }
403 }
404 }
405 #endif /* __x86_64__ */
406 #endif /* __LP64__ */
407
408
409 /*!
410 * @abstract
411 * The number of pages to try to free/process at once while under
412 * the free page queue lock.
413 *
414 * @discussion
415 * The value is chosen to be a trade off between:
416 * - creating a lot of contention on the free page queue lock
417 * taking and dropping it all the time,
418 * - avoiding to hold the free page queue lock for too long periods of time.
419 */
420 #define VMP_FREE_BATCH_SIZE 64
421
422 /*!
423 * @function vm_page_free_queue_init()
424 *
425 * @abstract
426 * Initialize a free queue.
427 *
428 * @param free_queue The free queue to initialize.
429 */
430 extern void vm_page_free_queue_init(
431 vm_page_free_queue_t free_queue);
432
433 /*!
434 * @function vm_page_free_queue_enter()
435 *
436 * @abstract
437 * Add a page to a free queue.
438 *
439 * @discussion
440 * Internally, the free queue is not synchronized, so any locking must be done
441 * outside of this function.
442 *
443 * The page queue state will be set to the appropriate free queue state for the
444 * memory class (typically VM_PAGE_ON_FREE_Q).
445 *
446 * Note that the callers are responsible for making sure that this operation is
447 * a valid transition. This is a helper to abstract handling of the several
448 * free page queues on the system which sits above vm_page_queue_enter() and
449 * maintains counters as well, but is otherwise oblivious to the page state
450 * machine.
451 *
452 * Most clients should use a wrapper around this function (typically
453 * vm_page_release() or vm_page_free_list()) and not call it directly.
454 *
455 * @param mem_class The memory class to free pages to.
456 * @param page The page to free.
457 * @param pnum the physical address of @c page
458 */
459 extern void vm_page_free_queue_enter(
460 vm_memory_class_t mem_class,
461 vm_page_t page,
462 ppnum_t pnum);
463
464 /*!
465 * @function vm_page_free_queue_remove()
466 *
467 * @abstract
468 * Removes an arbitrary free page from the given free queue.
469 *
470 * @discussion
471 * The given page must be in the given free queue, or state may be corrupted.
472 *
473 * Internally, the free queue is not synchronized, so any locking must be done
474 * outside of this function.
475 *
476 * Note that the callers are responsible for making sure that the requested
477 * queue state corresponds to a valid transition. This is a helper to abstract
478 * handling of the several free page queues on the system which sits above
479 * vm_page_queue_remove() and maintains counters as well, but is otherwise
480 * oblivious to the page state machine.
481 *
482 * Most clients should use a wrapper around this function (typically
483 * vm_page_free_queue_steal()) and not call it directly.
484 *
485 * @param class The memory class corresponding to the free queue
486 * @c page is enqueued on.
487 * @param mem The page to remove.
488 * @param pnum The physical address of @c page
489 * @param q_state The desired queue state for the page.
490 */
491 __attribute__((always_inline))
492 extern void vm_page_free_queue_remove(
493 vm_memory_class_t class,
494 vm_page_t mem,
495 ppnum_t pnum,
496 vm_page_q_state_t q_state);
497
498 /*!
499 * @function vm_page_free_queue_grab()
500 *
501 * @abstract
502 * Gets pages from the free queue.
503 *
504 * @discussion
505 * Clients cannot get more pages than the free queue has; attempting to do so
506 * will cause a panic.
507 *
508 * Internally, the free queue is not synchronized, so any locking must be done
509 * outside of this function.
510 *
511 * Note that the callers are responsible for making sure that the requested
512 * queue state corresponds to a valid transition. This is a helper to abstract
513 * handling of the several free page queues on the system which sits above
514 * vm_page_queue_remove() and maintains counters as well, but is otherwise
515 * oblivious to the page state machine.
516 *
517 * Most clients should use a wrapper (typically vm_page_grab_options())
518 * around this function and not call it directly.
519 *
520 * @param options The grab options.
521 * @param mem_class The memory class to allocate from.
522 * @param num_pages The number of pages to grab.
523 * @param q_state The vmp_q_state to set on the page.
524 *
525 * @returns
526 * A list of pages; the list will be num_pages long.
527 */
528 extern vm_page_list_t vm_page_free_queue_grab(
529 vm_grab_options_t options,
530 vm_memory_class_t mem_class,
531 unsigned int num_pages,
532 vm_page_q_state_t q_state);
533
534 /*!
535 * @abstract
536 * Perform a wakeup for a free page queue wait event.
537 *
538 * @param event the free page queue event to wake up
539 * @param n the number of threads to try to wake up
540 * (UINT32_MAX means all).
541 */
542 extern void vm_page_free_wakeup(event_t event, uint32_t n);
543
544
545 extern void vm_page_assign_special_state(vm_page_t mem, vm_page_specialq_t mode);
546 extern void vm_page_update_special_state(vm_page_t mem);
547 extern void vm_page_add_to_specialq(vm_page_t mem, boolean_t first);
548 extern void vm_page_remove_from_specialq(vm_page_t mem);
549
550
551 /*
552 * Prototypes for functions exported by this module.
553 */
554 extern void vm_page_bootstrap(
555 vm_offset_t *startp,
556 vm_offset_t *endp);
557
558 extern vm_page_t kdp_vm_page_lookup(
559 vm_object_t object,
560 vm_object_offset_t offset);
561
562 extern vm_page_t vm_page_lookup(
563 vm_object_t object,
564 vm_object_offset_t offset);
565
566 /*!
567 * @abstract
568 * Creates a fictitious page.
569 *
570 * @discussion
571 * This function never returns VM_PAGE_NULL;
572 *
573 * Pages made by this function have the @c vm_page_fictitious_addr
574 * fake physical address.
575 */
576 extern vm_page_t vm_page_create_fictitious(void);
577
578 /*!
579 * @abstract
580 * Returns a kernel guard page (used by @c kmem_alloc_guard()).
581 *
582 * @discussion
583 * Pages returned by this function have the @c vm_page_guard_addr
584 * fake physical address.
585 *
586 * @param canwait Whether the caller can wait, if true,
587 * this function never returns VM_PAGE_NULL.
588 */
589 extern vm_page_t vm_page_create_guard(bool canwait);
590
591 /*!
592 * @abstract
593 * Create a private VM page.
594 *
595 * @discussion
596 * These pages allow for non canonical references to the same physical page.
597 * Its @c VM_PAGE_GET_PHYS_PAGE() will be @c base_page.
598 *
599 * Such pages must not be released back to the free queues directly,
600 * @c vm_page_reset_private() must be called first.
601 *
602 * This function never returns VM_PAGE_NULL
603 *
604 * @param base_page The physical page this private page represents.
605 */
606 extern vm_page_t vm_page_create_private(ppnum_t base_page);
607
608 /*!
609 * @abstract
610 * Returns whether this is the canonical page for a regular managed kernel page.
611 *
612 * @discussion
613 * A kernel page is the canonical @c vm_page_t for a given pmap managed physical
614 * page. These pages are made at startup, or when @c ml_static_mfree() is
615 * called, and are never freed.
616 *
617 * Its @c VM_PAGE_GET_PHYS_PAGE() will be a valid @c ppnum_t value.
618 *
619 * A page can either be:
620 * - a canonical page (@c vm_page_is_canonical())
621 * - a fictitious page (@c vm_page_is_fictitious()),
622 * of which guard pages are a special case (@c vm_page_is_guard())
623 * - a private page (@c vm_page_is_private())
624 */
625 extern bool vm_page_is_canonical(const struct vm_page *m) __pure2;
626
627 /*!
628 * @abstract
629 * Returns whether this page is fictitious (made by @c vm_page_create_guard()
630 * or by @c vm_page_create_fictitious()).
631 *
632 * @discussion
633 * A page can either be:
634 * - a canonical page (@c vm_page_is_canonical())
635 * - a fictitious page (@c vm_page_is_fictitious()),
636 * of which guard pages are a special case (@c vm_page_is_guard())
637 * - a private page (@c vm_page_is_private())
638 */
639 extern bool vm_page_is_fictitious(const struct vm_page *m);
640
641 /*!
642 * @abstract
643 * Returns whether this is a kernel guard page that was made by
644 * @c vm_page_create_guard().
645 */
646 extern bool vm_page_is_guard(const struct vm_page *m) __pure2;
647
648 /*!
649 * @abstract
650 * Returns whether a page is private (made by @c vm_page_create_private(),
651 * or converted from a fictitious page by @c vm_page_make_private()).
652 *
653 * @discussion
654 * A page can either be:
655 * - a canonical page (@c vm_page_is_canonical())
656 * - a fictitious page (@c vm_page_is_fictitious()),
657 * of which guard pages are a special case (@c vm_page_is_guard())
658 * - a private page (@c vm_page_is_private())
659 */
660 extern bool vm_page_is_private(const struct vm_page *m);
661
662 /*!
663 * @abstract
664 * Converts a fictitious page made by @c vm_page_create_fictitious()
665 * into a private page.
666 *
667 * @param m The fictitious page to convert into a private one.
668 * @param base_page The physical page that this page will represent
669 * (@c vm_page_create_private()).
670 */
671 extern void vm_page_make_private(vm_page_t m, ppnum_t base_page);
672
673 /*!
674 * @abstract
675 * Converts a private page into a fictitious page (as if made by
676 * @c vm_page_create_fictitious()).
677 *
678 * @discussion
679 * Private pages can't be released with @c vm_page_release()
680 * without being turned into a fictitious page first using this function.
681 */
682 extern void vm_page_reset_private(vm_page_t m);
683
684
685 extern bool vm_pool_low(void);
686
687 /*!
688 * @abstract
689 * Grabs a page.
690 *
691 * @discussion
692 * Allocate a page by looking at:
693 * - per-cpu queues,
694 * - global free queues,
695 * - magical queues (delayed, secluded, ...)
696 *
697 * This function always succeeds for VM privileged threads,
698 * unless VM_PAGE_GRAB_NOPAGEWAIT is passed.
699 *
700 * This function might return VM_PAGE_NULL if there are no pages left.
701 */
702 extern vm_page_t vm_page_grab_options(vm_grab_options_t options);
703
704 static inline vm_page_t
vm_page_grab(void)705 vm_page_grab(void)
706 {
707 return vm_page_grab_options(VM_PAGE_GRAB_OPTIONS_NONE);
708 }
709
710 /*!
711 * @abstract
712 * Returns the proper grab options for the specified object.
713 */
714 extern vm_grab_options_t vm_page_grab_options_for_object(vm_object_t object);
715
716 #if XNU_VM_HAS_LOPAGE
717 extern vm_page_t vm_page_grablo(vm_grab_options_t options);
718 #else
719 static inline vm_page_t
vm_page_grablo(vm_grab_options_t options)720 vm_page_grablo(vm_grab_options_t options)
721 {
722 return vm_page_grab_options(options);
723 }
724 #endif
725
726
727 __options_closed_decl(vmp_release_options_t, uint32_t, {
728 VMP_RELEASE_NONE = 0x00,
729 VMP_RELEASE_Q_LOCKED = 0x01,
730 VMP_RELEASE_SKIP_FREE_CHECK = 0x02,
731 VMP_RELEASE_HIBERNATE = 0x04,
732 VMP_RELEASE_STARTUP = 0x08,
733 });
734
735 extern void vm_page_release(
736 vm_page_t page,
737 vmp_release_options_t options);
738
739 extern boolean_t vm_page_wait(
740 int interruptible);
741
742 extern void vm_page_init(
743 vm_page_t page,
744 ppnum_t phys_page);
745
746 extern void vm_page_free(
747 vm_page_t page);
748
749 extern void vm_page_free_unlocked(
750 vm_page_t page,
751 boolean_t remove_from_hash);
752
753
754 extern void vm_page_balance_inactive(
755 int max_to_move);
756
757 extern void vm_page_activate(
758 vm_page_t page);
759
760 extern void vm_page_deactivate(
761 vm_page_t page);
762
763 extern void vm_page_deactivate_internal(
764 vm_page_t page,
765 boolean_t clear_hw_reference);
766
767 extern void vm_page_enqueue_cleaned(vm_page_t page);
768
769 extern void vm_page_lru(
770 vm_page_t page);
771
772 extern void vm_page_speculate(
773 vm_page_t page,
774 boolean_t new);
775
776 extern void vm_page_speculate_ageit(
777 struct vm_speculative_age_q *aq);
778
779 extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
780
781 extern void vm_page_rename(
782 vm_page_t page,
783 vm_object_t new_object,
784 vm_object_offset_t new_offset);
785
786 extern void vm_page_insert(
787 vm_page_t page,
788 vm_object_t object,
789 vm_object_offset_t offset);
790
791 extern void vm_page_insert_wired(
792 vm_page_t page,
793 vm_object_t object,
794 vm_object_offset_t offset,
795 vm_tag_t tag);
796
797
798 extern void vm_page_insert_internal(
799 vm_page_t page,
800 vm_object_t object,
801 vm_object_offset_t offset,
802 vm_tag_t tag,
803 boolean_t queues_lock_held,
804 boolean_t insert_in_hash,
805 boolean_t batch_pmap_op,
806 boolean_t delayed_accounting,
807 uint64_t *delayed_ledger_update);
808
809 extern void vm_page_replace(
810 vm_page_t mem,
811 vm_object_t object,
812 vm_object_offset_t offset);
813
814 extern void vm_page_remove(
815 vm_page_t page,
816 boolean_t remove_from_hash);
817
818 extern void vm_page_zero_fill(
819 vm_page_t page);
820
821 extern void vm_page_part_zero_fill(
822 vm_page_t m,
823 vm_offset_t m_pa,
824 vm_size_t len);
825
826 extern void vm_page_copy(
827 vm_page_t src_page,
828 vm_page_t dest_page);
829
830 extern void vm_page_part_copy(
831 vm_page_t src_m,
832 vm_offset_t src_pa,
833 vm_page_t dst_m,
834 vm_offset_t dst_pa,
835 vm_size_t len);
836
837 extern void vm_page_wire(
838 vm_page_t page,
839 vm_tag_t tag,
840 boolean_t check_memorystatus);
841
842 extern void vm_page_unwire(
843 vm_page_t page,
844 boolean_t queueit);
845
846 extern void vm_set_page_size(void);
847
848 extern void vm_page_validate_cs(
849 vm_page_t page,
850 vm_map_size_t fault_page_size,
851 vm_map_offset_t fault_phys_offset);
852
853 extern void vm_page_validate_cs_mapped(
854 vm_page_t page,
855 vm_map_size_t fault_page_size,
856 vm_map_offset_t fault_phys_offset,
857 const void *kaddr);
858 extern void vm_page_validate_cs_mapped_slow(
859 vm_page_t page,
860 const void *kaddr);
861 extern void vm_page_validate_cs_mapped_chunk(
862 vm_page_t page,
863 const void *kaddr,
864 vm_offset_t chunk_offset,
865 vm_size_t chunk_size,
866 boolean_t *validated,
867 unsigned *tainted);
868
869 extern void vm_page_free_prepare_queues(
870 vm_page_t page);
871
872 extern void vm_page_free_prepare_object(
873 vm_page_t page,
874 boolean_t remove_from_hash);
875
876 extern wait_result_t vm_page_sleep(
877 vm_object_t object,
878 vm_page_t m,
879 wait_interrupt_t interruptible,
880 lck_sleep_action_t action);
881
882 extern void vm_page_wakeup(
883 vm_object_t object,
884 vm_page_t m);
885
886 extern void vm_page_wakeup_done(
887 vm_object_t object,
888 vm_page_t m);
889
890 typedef struct page_worker_token {
891 thread_pri_floor_t pwt_floor_token;
892 bool pwt_did_register_inheritor;
893 } page_worker_token_t;
894
895 extern void vm_page_wakeup_done_with_inheritor(
896 vm_object_t object,
897 vm_page_t m,
898 page_worker_token_t *token);
899
900 extern void page_worker_register_worker(
901 event_t event,
902 page_worker_token_t *out_token);
903
904 extern boolean_t vm_page_is_relocatable(
905 vm_page_t m,
906 vm_relocate_reason_t reloc_reason);
907
908 extern kern_return_t vm_page_relocate(
909 vm_page_t m1,
910 int * compressed_pages,
911 vm_relocate_reason_t reason,
912 vm_page_t* new_page);
913
914 extern bool vm_page_is_restricted(
915 vm_page_t mem);
916
917 /*
918 * Functions implemented as macros. m->vmp_wanted and m->vmp_busy are
919 * protected by the object lock.
920 */
921
922 #if !XNU_TARGET_OS_OSX
923 #define SET_PAGE_DIRTY(m, set_pmap_modified) \
924 MACRO_BEGIN \
925 vm_page_t __page__ = (m); \
926 if (__page__->vmp_pmapped == TRUE && \
927 __page__->vmp_wpmapped == TRUE && \
928 __page__->vmp_dirty == FALSE && \
929 (set_pmap_modified)) { \
930 pmap_set_modify(VM_PAGE_GET_PHYS_PAGE(__page__)); \
931 } \
932 __page__->vmp_dirty = TRUE; \
933 MACRO_END
934 #else /* !XNU_TARGET_OS_OSX */
935 #define SET_PAGE_DIRTY(m, set_pmap_modified) \
936 MACRO_BEGIN \
937 vm_page_t __page__ = (m); \
938 __page__->vmp_dirty = TRUE; \
939 MACRO_END
940 #endif /* !XNU_TARGET_OS_OSX */
941
942 #define VM_PAGE_FREE(p) \
943 MACRO_BEGIN \
944 vm_page_free_unlocked(p, TRUE); \
945 MACRO_END
946
947
948 #define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
949
950 static inline void
vm_free_page_lock(void)951 vm_free_page_lock(void)
952 {
953 lck_mtx_lock(&vm_page_queue_free_lock);
954 }
955
956 static inline void
vm_free_page_lock_spin(void)957 vm_free_page_lock_spin(void)
958 {
959 lck_mtx_lock_spin(&vm_page_queue_free_lock);
960 }
961
962 static inline void
vm_free_page_lock_convert(void)963 vm_free_page_lock_convert(void)
964 {
965 lck_mtx_convert_spin(&vm_page_queue_free_lock);
966 }
967
968 static inline void
vm_free_page_unlock(void)969 vm_free_page_unlock(void)
970 {
971 lck_mtx_unlock(&vm_page_queue_free_lock);
972 }
973
974
975 #define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
976
977
978 #ifdef VPL_LOCK_SPIN
979 extern lck_grp_t vm_page_lck_grp_local;
980
981 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
982 #define VPL_LOCK(vpl) lck_spin_lock_grp(vpl, &vm_page_lck_grp_local)
983 #define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
984 #else
985 #define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
986 #define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
987 #define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
988 #endif
989
990 #if DEVELOPMENT || DEBUG
991 #define VM_PAGE_SPECULATIVE_USED_ADD() \
992 MACRO_BEGIN \
993 OSAddAtomic(1, &vm_page_speculative_used); \
994 MACRO_END
995 #else
996 #define VM_PAGE_SPECULATIVE_USED_ADD()
997 #endif
998
999 #define VM_PAGE_CONSUME_CLUSTERED(mem) \
1000 MACRO_BEGIN \
1001 ppnum_t __phys_page; \
1002 __phys_page = VM_PAGE_GET_PHYS_PAGE(mem); \
1003 pmap_lock_phys_page(__phys_page); \
1004 if (mem->vmp_clustered) { \
1005 vm_object_t o; \
1006 o = VM_PAGE_OBJECT(mem); \
1007 assert(o); \
1008 o->pages_used++; \
1009 mem->vmp_clustered = FALSE; \
1010 VM_PAGE_SPECULATIVE_USED_ADD(); \
1011 } \
1012 pmap_unlock_phys_page(__phys_page); \
1013 MACRO_END
1014
1015
1016 #define VM_PAGE_COUNT_AS_PAGEIN(mem) \
1017 MACRO_BEGIN \
1018 { \
1019 vm_object_t o; \
1020 o = VM_PAGE_OBJECT(mem); \
1021 DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
1022 counter_inc(¤t_task()->pageins); \
1023 if (o->internal) { \
1024 DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
1025 } else { \
1026 DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
1027 } \
1028 } \
1029 MACRO_END
1030
1031
1032 /* adjust for stolen pages accounted elsewhere */
1033 #define VM_PAGE_MOVE_STOLEN(page_count) \
1034 MACRO_BEGIN \
1035 vm_page_stolen_count -= (page_count); \
1036 vm_page_wire_count_initial -= (page_count); \
1037 MACRO_END
1038
1039 kern_return_t
1040 pmap_enter_object_options_check(
1041 pmap_t pmap,
1042 vm_map_address_t virtual_address,
1043 vm_map_offset_t fault_phys_offset,
1044 vm_object_t object,
1045 ppnum_t pn,
1046 vm_prot_t protection,
1047 vm_prot_t fault_type,
1048 boolean_t wired,
1049 unsigned int options);
1050
1051 extern kern_return_t pmap_enter_options_check(
1052 pmap_t pmap,
1053 vm_map_address_t virtual_address,
1054 vm_map_offset_t fault_phys_offset,
1055 vm_page_t page,
1056 vm_prot_t protection,
1057 vm_prot_t fault_type,
1058 boolean_t wired,
1059 unsigned int options);
1060
1061 extern kern_return_t pmap_enter_check(
1062 pmap_t pmap,
1063 vm_map_address_t virtual_address,
1064 vm_page_t page,
1065 vm_prot_t protection,
1066 vm_prot_t fault_type,
1067 boolean_t wired);
1068
1069 #define DW_vm_page_unwire 0x01
1070 #define DW_vm_page_wire 0x02
1071 #define DW_vm_page_free 0x04
1072 #define DW_vm_page_activate 0x08
1073 #define DW_vm_page_deactivate_internal 0x10
1074 #define DW_vm_page_speculate 0x20
1075 #define DW_vm_page_lru 0x40
1076 #define DW_vm_pageout_throttle_up 0x80
1077 #define DW_PAGE_WAKEUP 0x100
1078 #define DW_clear_busy 0x200
1079 #define DW_clear_reference 0x400
1080 #define DW_set_reference 0x800
1081 #define DW_move_page 0x1000
1082 #define DW_VM_PAGE_QUEUES_REMOVE 0x2000
1083 #define DW_enqueue_cleaned 0x4000
1084 #define DW_vm_phantom_cache_update 0x8000
1085 #define DW_vm_page_iopl_wire 0x20000
1086 #define DW_vm_page_iopl_wire_write 0x40000
1087
1088 struct vm_page_delayed_work {
1089 vm_page_t dw_m;
1090 int dw_mask;
1091 };
1092
1093 #define DEFAULT_DELAYED_WORK_LIMIT 32
1094
1095 struct vm_page_delayed_work_ctx {
1096 struct vm_page_delayed_work dwp[DEFAULT_DELAYED_WORK_LIMIT];
1097 thread_t delayed_owner;
1098 };
1099
1100 kern_return_t vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count);
1101
1102 #define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
1103
1104 /*
1105 * vm_page_do_delayed_work may need to drop the object lock...
1106 * if it does, we need the pages it's looking at to
1107 * be held stable via the busy bit, so if busy isn't already
1108 * set, we need to set it and ask vm_page_do_delayed_work
1109 * to clear it and wakeup anyone that might have blocked on
1110 * it once we're done processing the page.
1111 */
1112
1113 #define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
1114 MACRO_BEGIN \
1115 if (mem->vmp_busy == FALSE) { \
1116 mem->vmp_busy = TRUE; \
1117 if ( !(dwp->dw_mask & DW_vm_page_free)) \
1118 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
1119 } \
1120 dwp->dw_m = mem; \
1121 dwp++; \
1122 dw_cnt++; \
1123 MACRO_END
1124
1125
1126 //todo int
1127 extern vm_page_t vm_object_page_grab(vm_object_t);
1128
1129 //todo int
1130 #if VM_PAGE_BUCKETS_CHECK
1131 extern void vm_page_buckets_check(void);
1132 #endif /* VM_PAGE_BUCKETS_CHECK */
1133
1134 //todo int
1135 extern void vm_page_queues_remove(vm_page_t mem, boolean_t remove_from_specialq);
1136 extern void vm_page_remove_internal(vm_page_t page);
1137 extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first);
1138 extern void vm_page_enqueue_active(vm_page_t mem, boolean_t first);
1139 extern void vm_page_check_pageable_safe(vm_page_t page);
1140 //end int
1141
1142 //todo int
1143 extern void vm_retire_boot_pages(void);
1144
1145 //todo all int
1146
1147 #define VMP_ERROR_GET(p) ((p)->vmp_error)
1148
1149
1150 #endif /* XNU_KERNEL_PRIVATE */
1151 __END_DECLS
1152
1153 #endif /* _VM_VM_PAGE_INTERNAL_H_ */
1154