1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_object.h
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 * Date: 1985
62 *
63 * Virtual memory object module definitions.
64 */
65
66 #ifndef _VM_VM_OBJECT_H_
67 #define _VM_VM_OBJECT_H_
68
69 #include <debug.h>
70 #include <mach_assert.h>
71
72 #include <mach/kern_return.h>
73 #include <mach/boolean.h>
74 #include <mach/memory_object_types.h>
75 #include <mach/port.h>
76 #include <mach/vm_prot.h>
77 #include <mach/vm_param.h>
78 #include <mach/machine/vm_types.h>
79 #include <kern/queue.h>
80 #include <kern/locks.h>
81 #include <kern/assert.h>
82 #include <kern/misc_protos.h>
83 #include <kern/macro_help.h>
84 #include <ipc/ipc_types.h>
85 #include <vm/pmap.h>
86
87 #include <vm/vm_external.h>
88
89 #include <vm/vm_options.h>
90 #include <vm/vm_page.h>
91
92 #if VM_OBJECT_TRACKING
93 #include <libkern/OSDebug.h>
94 #include <kern/btlog.h>
95 extern void vm_object_tracking_init(void);
96 extern btlog_t vm_object_tracking_btlog;
97 #define VM_OBJECT_TRACKING_NUM_RECORDS 50000
98 #define VM_OBJECT_TRACKING_OP_CREATED 1
99 #define VM_OBJECT_TRACKING_OP_MODIFIED 2
100 #define VM_OBJECT_TRACKING_OP_TRUESHARE 3
101 #endif /* VM_OBJECT_TRACKING */
102
103 struct vm_page;
104
105 /*
106 * Types defined:
107 *
108 * vm_object_t Virtual memory object.
109 * vm_object_fault_info_t Used to determine cluster size.
110 */
111
112 struct vm_object_fault_info {
113 int interruptible;
114 uint32_t user_tag;
115 vm_size_t cluster_size;
116 vm_behavior_t behavior;
117 vm_object_offset_t lo_offset;
118 vm_object_offset_t hi_offset;
119 unsigned int
120 /* boolean_t */ no_cache:1,
121 /* boolean_t */ stealth:1,
122 /* boolean_t */ io_sync:1,
123 /* boolean_t */ cs_bypass:1,
124 /* boolean_t */ csm_associated:1,
125 /* boolean_t */ mark_zf_absent:1,
126 /* boolean_t */ batch_pmap_op:1,
127 /* boolean_t */ resilient_media:1,
128 /* boolean_t */ no_copy_on_read:1,
129 /* boolean_t */ fi_xnu_user_debug:1,
130 /* boolean_t */ fi_used_for_tpro:1,
131 __vm_object_fault_info_unused_bits:21;
132 int pmap_options;
133 };
134
135
136 #define vo_size vo_un1.vou_size
137 #define vo_cache_pages_to_scan vo_un1.vou_cache_pages_to_scan
138 #define vo_shadow_offset vo_un2.vou_shadow_offset
139 #define vo_cache_ts vo_un2.vou_cache_ts
140 #define vo_owner vo_un2.vou_owner
141
142 struct vm_object {
143 /*
144 * on 64 bit systems we pack the pointers hung off the memq.
145 * those pointers have to be able to point back to the memq.
146 * the packed pointers are required to be on a 64 byte boundary
147 * which means 2 things for the vm_object... (1) the memq
148 * struct has to be the first element of the structure so that
149 * we can control it's alignment... (2) the vm_object must be
150 * aligned on a 64 byte boundary... for static vm_object's
151 * this is accomplished via the 'aligned' attribute... for
152 * vm_object's in the zone pool, this is accomplished by
153 * rounding the size of the vm_object element to the nearest
154 * 64 byte size before creating the zone.
155 */
156 vm_page_queue_head_t memq; /* Resident memory - must be first */
157 lck_rw_t Lock; /* Synchronization */
158
159 union {
160 vm_object_size_t vou_size; /* Object size (only valid if internal) */
161 int vou_cache_pages_to_scan; /* pages yet to be visited in an
162 * external object in cache
163 */
164 } vo_un1;
165
166 struct vm_page *memq_hint;
167 int ref_count; /* Number of references */
168 unsigned int resident_page_count;
169 /* number of resident pages */
170 unsigned int wired_page_count; /* number of wired pages
171 * use VM_OBJECT_WIRED_PAGE_UPDATE macros to update */
172 unsigned int reusable_page_count;
173
174 struct vm_object *vo_copy; /* Object that should receive
175 * a copy of my changed pages,
176 * for copy_delay, or just the
177 * temporary object that
178 * shadows this object, for
179 * copy_call.
180 */
181 uint32_t vo_copy_version;
182 uint32_t __vo_unused_padding;
183 struct vm_object *shadow; /* My shadow */
184 memory_object_t pager; /* Where to get data */
185
186 union {
187 vm_object_offset_t vou_shadow_offset; /* Offset into shadow */
188 clock_sec_t vou_cache_ts; /* age of an external object
189 * present in cache
190 */
191 task_t vou_owner; /* If the object is purgeable
192 * or has a "ledger_tag", this
193 * is the task that owns it.
194 */
195 } vo_un2;
196
197 vm_object_offset_t paging_offset; /* Offset into memory object */
198 memory_object_control_t pager_control; /* Where data comes back */
199
200 memory_object_copy_strategy_t
201 copy_strategy; /* How to handle data copy */
202
203 /*
204 * Some user processes (mostly VirtualMachine software) take a large
205 * number of UPLs (via IOMemoryDescriptors) to wire pages in large
206 * VM objects and overflow the 16-bit "activity_in_progress" counter.
207 * Since we never enforced any limit there, let's give them 32 bits
208 * for backwards compatibility's sake.
209 */
210 unsigned short paging_in_progress:16;
211 unsigned short vo_size_delta;
212 unsigned int activity_in_progress;
213
214 /* The memory object ports are
215 * being used (e.g., for pagein
216 * or pageout) -- don't change
217 * any of these fields (i.e.,
218 * don't collapse, destroy or
219 * terminate)
220 */
221
222 unsigned int
223 /* boolean_t array */ all_wanted:6, /* Bit array of "want to be
224 * awakened" notations. See
225 * VM_OBJECT_EVENT_* items
226 * below */
227 /* boolean_t */ pager_created:1, /* Has pager been created? */
228 /* boolean_t */ pager_initialized:1, /* Are fields ready to use? */
229 /* boolean_t */ pager_ready:1, /* Will pager take requests? */
230
231 /* boolean_t */ pager_trusted:1, /* The pager for this object
232 * is trusted. This is true for
233 * all internal objects (backed
234 * by the default pager)
235 */
236 /* boolean_t */ can_persist:1, /* The kernel may keep the data
237 * for this object (and rights
238 * to the memory object) after
239 * all address map references
240 * are deallocated?
241 */
242 /* boolean_t */ internal:1, /* Created by the kernel (and
243 * therefore, managed by the
244 * default memory manger)
245 */
246 /* boolean_t */ private:1, /* magic device_pager object,
247 * holds private pages only */
248 /* boolean_t */ pageout:1, /* pageout object. contains
249 * private pages that refer to
250 * a real memory object. */
251 /* boolean_t */ alive:1, /* Not yet terminated */
252
253 /* boolean_t */ purgable:2, /* Purgable state. See
254 * VM_PURGABLE_*
255 */
256 /* boolean_t */ purgeable_only_by_kernel:1,
257 /* boolean_t */ purgeable_when_ripe:1, /* Purgeable when a token
258 * becomes ripe.
259 */
260 /* boolean_t */ shadowed:1, /* Shadow may exist */
261 /* boolean_t */ true_share:1,
262 /* This object is mapped
263 * in more than one place
264 * and hence cannot be
265 * coalesced */
266 /* boolean_t */ terminating:1,
267 /* Allows vm_object_lookup
268 * and vm_object_deallocate
269 * to special case their
270 * behavior when they are
271 * called as a result of
272 * page cleaning during
273 * object termination
274 */
275 /* boolean_t */ named:1, /* An enforces an internal
276 * naming convention, by
277 * calling the right routines
278 * for allocation and
279 * destruction, UBC references
280 * against the vm_object are
281 * checked.
282 */
283 /* boolean_t */ shadow_severed:1,
284 /* When a permanent object
285 * backing a COW goes away
286 * unexpectedly. This bit
287 * allows vm_fault to return
288 * an error rather than a
289 * zero filled page.
290 */
291 /* boolean_t */ phys_contiguous:1,
292 /* Memory is wired and
293 * guaranteed physically
294 * contiguous. However
295 * it is not device memory
296 * and obeys normal virtual
297 * memory rules w.r.t pmap
298 * access bits.
299 */
300 /* boolean_t */ nophyscache:1,
301 /* When mapped at the
302 * pmap level, don't allow
303 * primary caching. (for
304 * I/O)
305 */
306 /* boolean_t */ for_realtime:1,
307 /* Might be needed for realtime code path */
308 /* vm_object_destroy_reason_t */ no_pager_reason:2,
309 /* differentiate known and unknown causes */
310 #if FBDP_DEBUG_OBJECT_NO_PAGER
311 /* boolean_t */ fbdp_tracked:1,
312 __object1_unused_bits:2;
313 #else /* FBDP_DEBUG_OBJECT_NO_PAGER */
314 __object1_unused_bits:3;
315 #endif /* FBDP_DEBUG_OBJECT_NO_PAGER */
316
317 queue_chain_t cached_list; /* Attachment point for the
318 * list of objects cached as a
319 * result of their can_persist
320 * value
321 */
322 /*
323 * the following fields are not protected by any locks
324 * they are updated via atomic compare and swap
325 */
326 vm_object_offset_t last_alloc; /* last allocation offset */
327 vm_offset_t cow_hint; /* last page present in */
328 /* shadow but not in object */
329 int sequential; /* sequential access size */
330
331 uint32_t pages_created;
332 uint32_t pages_used;
333 /* hold object lock when altering */
334 unsigned int
335 wimg_bits:8, /* cache WIMG bits */
336 code_signed:1, /* pages are signed and should be
337 * validated; the signatures are stored
338 * with the pager */
339 transposed:1, /* object was transposed with another */
340 mapping_in_progress:1, /* pager being mapped/unmapped */
341 phantom_isssd:1,
342 volatile_empty:1,
343 volatile_fault:1,
344 all_reusable:1,
345 blocked_access:1,
346 set_cache_attr:1,
347 object_is_shared_cache:1,
348 purgeable_queue_type:2,
349 purgeable_queue_group:3,
350 io_tracking:1,
351 no_tag_update:1, /* */
352 #if CONFIG_SECLUDED_MEMORY
353 eligible_for_secluded:1,
354 can_grab_secluded:1,
355 #else /* CONFIG_SECLUDED_MEMORY */
356 __object3_unused_bits:2,
357 #endif /* CONFIG_SECLUDED_MEMORY */
358 #if VM_OBJECT_ACCESS_TRACKING
359 access_tracking:1,
360 #else /* VM_OBJECT_ACCESS_TRACKING */
361 __unused_access_tracking:1,
362 #endif /* VM_OBJECT_ACCESS_TRACKING */
363 vo_ledger_tag:3,
364 vo_no_footprint:1;
365
366 #if VM_OBJECT_ACCESS_TRACKING
367 uint32_t access_tracking_reads;
368 uint32_t access_tracking_writes;
369 #endif /* VM_OBJECT_ACCESS_TRACKING */
370
371 uint8_t scan_collisions;
372 uint8_t __object4_unused_bits[1];
373 vm_tag_t wire_tag;
374
375 #if CONFIG_PHANTOM_CACHE
376 uint32_t phantom_object_id;
377 #endif
378 #if CONFIG_IOSCHED || UPL_DEBUG
379 queue_head_t uplq; /* List of outstanding upls */
380 #endif
381
382 #ifdef VM_PIP_DEBUG
383 /*
384 * Keep track of the stack traces for the first holders
385 * of a "paging_in_progress" reference for this VM object.
386 */
387 #define VM_PIP_DEBUG_STACK_FRAMES 25 /* depth of each stack trace */
388 #define VM_PIP_DEBUG_MAX_REFS 10 /* track that many references */
389 struct __pip_backtrace {
390 void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
391 } pip_holders[VM_PIP_DEBUG_MAX_REFS];
392 #endif /* VM_PIP_DEBUG */
393
394 queue_chain_t objq; /* object queue - currently used for purgable queues */
395 queue_chain_t task_objq; /* objects owned by task - protected by task lock */
396
397 #if !VM_TAG_ACTIVE_UPDATE
398 queue_chain_t wired_objq;
399 #endif /* !VM_TAG_ACTIVE_UPDATE */
400
401 #if DEBUG
402 void *purgeable_owner_bt[16];
403 task_t vo_purgeable_volatilizer; /* who made it volatile? */
404 void *purgeable_volatilizer_bt[16];
405 #endif /* DEBUG */
406 };
407
408 #define VM_OBJECT_PURGEABLE_FAULT_ERROR(object) \
409 ((object)->volatile_fault && \
410 ((object)->purgable == VM_PURGABLE_VOLATILE || \
411 (object)->purgable == VM_PURGABLE_EMPTY))
412
413 #if VM_OBJECT_ACCESS_TRACKING
414 extern uint64_t vm_object_access_tracking_reads;
415 extern uint64_t vm_object_access_tracking_writes;
416 extern void vm_object_access_tracking(vm_object_t object,
417 int *access_tracking,
418 uint32_t *access_tracking_reads,
419 uint32_t *acess_tracking_writes);
420 #endif /* VM_OBJECT_ACCESS_TRACKING */
421
422 extern const vm_object_t kernel_object_default; /* the default kernel object */
423
424 extern const vm_object_t compressor_object; /* the single compressor object */
425
426 extern const vm_object_t retired_pages_object; /* pages retired due to ECC, should never be used */
427
428 #define is_kernel_object(object) ((object) == kernel_object_default)
429
430 # define VM_MSYNC_INITIALIZED 0
431 # define VM_MSYNC_SYNCHRONIZING 1
432 # define VM_MSYNC_DONE 2
433
434
435 extern lck_grp_t vm_map_lck_grp;
436 extern lck_attr_t vm_map_lck_attr;
437
438 #ifndef VM_TAG_ACTIVE_UPDATE
439 #error VM_TAG_ACTIVE_UPDATE
440 #endif
441
442 #if VM_TAG_ACTIVE_UPDATE
443 #define VM_OBJECT_WIRED_ENQUEUE(object) panic("VM_OBJECT_WIRED_ENQUEUE")
444 #define VM_OBJECT_WIRED_DEQUEUE(object) panic("VM_OBJECT_WIRED_DEQUEUE")
445 #else /* VM_TAG_ACTIVE_UPDATE */
446 #define VM_OBJECT_WIRED_ENQUEUE(object) \
447 MACRO_BEGIN \
448 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket); \
449 assert(!(object)->wired_objq.next); \
450 assert(!(object)->wired_objq.prev); \
451 queue_enter(&vm_objects_wired, (object), \
452 vm_object_t, wired_objq); \
453 lck_spin_unlock(&vm_objects_wired_lock); \
454 MACRO_END
455 #define VM_OBJECT_WIRED_DEQUEUE(object) \
456 MACRO_BEGIN \
457 if ((object)->wired_objq.next) { \
458 lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket); \
459 queue_remove(&vm_objects_wired, (object), \
460 vm_object_t, wired_objq); \
461 lck_spin_unlock(&vm_objects_wired_lock); \
462 } \
463 MACRO_END
464 #endif /* VM_TAG_ACTIVE_UPDATE */
465
466 #define VM_OBJECT_WIRED(object, tag) \
467 MACRO_BEGIN \
468 assert(VM_KERN_MEMORY_NONE != (tag)); \
469 assert(VM_KERN_MEMORY_NONE == (object)->wire_tag); \
470 (object)->wire_tag = (tag); \
471 if (!VM_TAG_ACTIVE_UPDATE) { \
472 VM_OBJECT_WIRED_ENQUEUE((object)); \
473 } \
474 MACRO_END
475
476 #define VM_OBJECT_UNWIRED(object) \
477 MACRO_BEGIN \
478 if (!VM_TAG_ACTIVE_UPDATE) { \
479 VM_OBJECT_WIRED_DEQUEUE((object)); \
480 } \
481 if (VM_KERN_MEMORY_NONE != (object)->wire_tag) { \
482 vm_tag_update_size((object)->wire_tag, -ptoa_64((object)->wired_page_count), (object)); \
483 (object)->wire_tag = VM_KERN_MEMORY_NONE; \
484 } \
485 MACRO_END
486
487 // These two macros start & end a C block
488 #define VM_OBJECT_WIRED_PAGE_UPDATE_START(object) \
489 MACRO_BEGIN \
490 { \
491 int64_t __wireddelta = 0; vm_tag_t __waswired = (object)->wire_tag;
492
493 #define VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag) \
494 if (__wireddelta) { \
495 boolean_t __overflow __assert_only = \
496 os_add_overflow((object)->wired_page_count, __wireddelta, \
497 &(object)->wired_page_count); \
498 assert(!__overflow); \
499 if (!(object)->pageout && !(object)->no_tag_update) { \
500 if (__wireddelta > 0) { \
501 assert (VM_KERN_MEMORY_NONE != (tag)); \
502 if (VM_KERN_MEMORY_NONE == __waswired) { \
503 VM_OBJECT_WIRED((object), (tag)); \
504 } \
505 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta), (object)); \
506 } else if (VM_KERN_MEMORY_NONE != __waswired) { \
507 assert (VM_KERN_MEMORY_NONE != (object)->wire_tag); \
508 vm_tag_update_size((object)->wire_tag, ptoa_64(__wireddelta), (object)); \
509 if (!(object)->wired_page_count) { \
510 VM_OBJECT_UNWIRED((object)); \
511 } \
512 } \
513 } \
514 } \
515 } \
516 MACRO_END
517
518 #define VM_OBJECT_WIRED_PAGE_COUNT(object, delta) \
519 __wireddelta += delta; \
520
521 #define VM_OBJECT_WIRED_PAGE_ADD(object, m) \
522 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta++;
523
524 #define VM_OBJECT_WIRED_PAGE_REMOVE(object, m) \
525 if (!(m)->vmp_private && !(m)->vmp_fictitious) __wireddelta--;
526
527
528
529 #define OBJECT_LOCK_SHARED 0
530 #define OBJECT_LOCK_EXCLUSIVE 1
531
532 extern lck_grp_t vm_object_lck_grp;
533 extern lck_attr_t vm_object_lck_attr;
534 extern lck_attr_t kernel_object_lck_attr;
535 extern lck_attr_t compressor_object_lck_attr;
536
537 extern vm_object_t vm_pageout_scan_wants_object;
538
539 extern void vm_object_lock(vm_object_t);
540 extern bool vm_object_lock_check_contended(vm_object_t);
541 extern boolean_t vm_object_lock_try(vm_object_t);
542 extern boolean_t _vm_object_lock_try(vm_object_t);
543 extern boolean_t vm_object_lock_avoid(vm_object_t);
544 extern void vm_object_lock_shared(vm_object_t);
545 extern boolean_t vm_object_lock_yield_shared(vm_object_t);
546 extern boolean_t vm_object_lock_try_shared(vm_object_t);
547 extern void vm_object_unlock(vm_object_t);
548 extern boolean_t vm_object_lock_upgrade(vm_object_t);
549
550 /*
551 * Object locking macros
552 */
553
554 #define vm_object_lock_init(object) \
555 lck_rw_init(&(object)->Lock, &vm_object_lck_grp, \
556 (is_kernel_object(object) ? \
557 &kernel_object_lck_attr : \
558 (((object) == compressor_object) ? \
559 &compressor_object_lck_attr : \
560 &vm_object_lck_attr)))
561 #define vm_object_lock_destroy(object) lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
562
563 #define vm_object_lock_try_scan(object) _vm_object_lock_try(object)
564
565 /*
566 * CAUTION: the following vm_object_lock_assert_held*() macros merely
567 * check if anyone is holding the lock, but the holder may not necessarily
568 * be the caller...
569 */
570 #define vm_object_lock_assert_held(object) \
571 LCK_RW_ASSERT(&(object)->Lock, LCK_RW_ASSERT_HELD)
572 #define vm_object_lock_assert_shared(object) \
573 LCK_RW_ASSERT(&(object)->Lock, LCK_RW_ASSERT_SHARED)
574 #define vm_object_lock_assert_exclusive(object) \
575 LCK_RW_ASSERT(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
576 #define vm_object_lock_assert_notheld(object) \
577 LCK_RW_ASSERT(&(object)->Lock, LCK_RW_ASSERT_NOTHELD)
578
579
580 /*
581 * Declare procedures that operate on VM objects.
582 */
583
584 __private_extern__ void vm_object_bootstrap(void);
585
586 __private_extern__ void vm_object_reaper_init(void);
587
588 __private_extern__ vm_object_t vm_object_allocate(vm_object_size_t size);
589
590 __private_extern__ void _vm_object_allocate(vm_object_size_t size,
591 vm_object_t object);
592
593 __private_extern__ void vm_object_set_size(
594 vm_object_t object,
595 vm_object_size_t outer_size,
596 vm_object_size_t inner_size);
597
598 #define vm_object_reference_locked(object) \
599 MACRO_BEGIN \
600 vm_object_t RLObject = (object); \
601 vm_object_lock_assert_exclusive(object); \
602 assert((RLObject)->ref_count > 0); \
603 (RLObject)->ref_count++; \
604 assert((RLObject)->ref_count > 1); \
605 MACRO_END
606
607
608 #define vm_object_reference_shared(object) \
609 MACRO_BEGIN \
610 vm_object_t RLObject = (object); \
611 vm_object_lock_assert_shared(object); \
612 assert((RLObject)->ref_count > 0); \
613 OSAddAtomic(1, &(RLObject)->ref_count); \
614 assert((RLObject)->ref_count > 0); \
615 MACRO_END
616
617
618 __private_extern__ void vm_object_reference(
619 vm_object_t object);
620
621 #if !MACH_ASSERT
622
623 #define vm_object_reference(object) \
624 MACRO_BEGIN \
625 vm_object_t RObject = (object); \
626 if (RObject) { \
627 vm_object_lock_shared(RObject); \
628 vm_object_reference_shared(RObject); \
629 vm_object_unlock(RObject); \
630 } \
631 MACRO_END
632
633 #endif /* MACH_ASSERT */
634
635 __private_extern__ void vm_object_deallocate(
636 vm_object_t object);
637
638 __private_extern__ void vm_object_pmap_protect(
639 vm_object_t object,
640 vm_object_offset_t offset,
641 vm_object_size_t size,
642 pmap_t pmap,
643 vm_map_size_t pmap_page_size,
644 vm_map_offset_t pmap_start,
645 vm_prot_t prot);
646
647 __private_extern__ void vm_object_pmap_protect_options(
648 vm_object_t object,
649 vm_object_offset_t offset,
650 vm_object_size_t size,
651 pmap_t pmap,
652 vm_map_size_t pmap_page_size,
653 vm_map_offset_t pmap_start,
654 vm_prot_t prot,
655 int options);
656
657 __private_extern__ void vm_object_page_remove(
658 vm_object_t object,
659 vm_object_offset_t start,
660 vm_object_offset_t end);
661
662 __private_extern__ void vm_object_deactivate_pages(
663 vm_object_t object,
664 vm_object_offset_t offset,
665 vm_object_size_t size,
666 boolean_t kill_page,
667 boolean_t reusable_page,
668 boolean_t reusable_no_write,
669 struct pmap *pmap,
670 /* XXX TODO4K: need pmap_page_size here too? */
671 vm_map_offset_t pmap_offset);
672
673 __private_extern__ void vm_object_reuse_pages(
674 vm_object_t object,
675 vm_object_offset_t start_offset,
676 vm_object_offset_t end_offset,
677 boolean_t allow_partial_reuse);
678
679 __private_extern__ uint64_t vm_object_purge(
680 vm_object_t object,
681 int flags);
682
683 __private_extern__ kern_return_t vm_object_purgable_control(
684 vm_object_t object,
685 vm_purgable_t control,
686 int *state);
687
688 __private_extern__ kern_return_t vm_object_get_page_counts(
689 vm_object_t object,
690 vm_object_offset_t offset,
691 vm_object_size_t size,
692 unsigned int *resident_page_count,
693 unsigned int *dirty_page_count);
694
695 __private_extern__ boolean_t vm_object_coalesce(
696 vm_object_t prev_object,
697 vm_object_t next_object,
698 vm_object_offset_t prev_offset,
699 vm_object_offset_t next_offset,
700 vm_object_size_t prev_size,
701 vm_object_size_t next_size);
702
703 __private_extern__ boolean_t vm_object_shadow(
704 vm_object_t *object,
705 vm_object_offset_t *offset,
706 vm_object_size_t length,
707 boolean_t always_shadow);
708
709 __private_extern__ void vm_object_collapse(
710 vm_object_t object,
711 vm_object_offset_t offset,
712 boolean_t can_bypass);
713
714 __private_extern__ boolean_t vm_object_copy_quickly(
715 vm_object_t object,
716 vm_object_offset_t src_offset,
717 vm_object_size_t size,
718 boolean_t *_src_needs_copy,
719 boolean_t *_dst_needs_copy);
720
721 __private_extern__ kern_return_t vm_object_copy_strategically(
722 vm_object_t src_object,
723 vm_object_offset_t src_offset,
724 vm_object_size_t size,
725 bool forking,
726 vm_object_t *dst_object,
727 vm_object_offset_t *dst_offset,
728 boolean_t *dst_needs_copy);
729
730 __private_extern__ kern_return_t vm_object_copy_slowly(
731 vm_object_t src_object,
732 vm_object_offset_t src_offset,
733 vm_object_size_t size,
734 boolean_t interruptible,
735 vm_object_t *_result_object);
736
737 __private_extern__ vm_object_t vm_object_copy_delayed(
738 vm_object_t src_object,
739 vm_object_offset_t src_offset,
740 vm_object_size_t size,
741 boolean_t src_object_shared);
742
743
744
745 __private_extern__ kern_return_t vm_object_destroy(
746 vm_object_t object,
747 vm_object_destroy_reason_t reason);
748
749 __private_extern__ void vm_object_pager_create(
750 vm_object_t object);
751
752 __private_extern__ void vm_object_compressor_pager_create(
753 vm_object_t object);
754
755 __private_extern__ void vm_object_page_map(
756 vm_object_t object,
757 vm_object_offset_t offset,
758 vm_object_size_t size,
759 vm_object_offset_t (*map_fn)
760 (void *, vm_object_offset_t),
761 void *map_fn_data);
762
763 __private_extern__ kern_return_t vm_object_upl_request(
764 vm_object_t object,
765 vm_object_offset_t offset,
766 upl_size_t size,
767 upl_t *upl,
768 upl_page_info_t *page_info,
769 unsigned int *count,
770 upl_control_flags_t flags,
771 vm_tag_t tag);
772
773 __private_extern__ kern_return_t vm_object_transpose(
774 vm_object_t object1,
775 vm_object_t object2,
776 vm_object_size_t transpose_size);
777
778 __private_extern__ boolean_t vm_object_sync(
779 vm_object_t object,
780 vm_object_offset_t offset,
781 vm_object_size_t size,
782 boolean_t should_flush,
783 boolean_t should_return,
784 boolean_t should_iosync);
785
786 __private_extern__ kern_return_t vm_object_update(
787 vm_object_t object,
788 vm_object_offset_t offset,
789 vm_object_size_t size,
790 vm_object_offset_t *error_offset,
791 int *io_errno,
792 memory_object_return_t should_return,
793 int flags,
794 vm_prot_t prot);
795
796 __private_extern__ kern_return_t vm_object_lock_request(
797 vm_object_t object,
798 vm_object_offset_t offset,
799 vm_object_size_t size,
800 memory_object_return_t should_return,
801 int flags,
802 vm_prot_t prot);
803
804
805
806 __private_extern__ vm_object_t vm_object_memory_object_associate(
807 memory_object_t pager,
808 vm_object_t object,
809 vm_object_size_t size,
810 boolean_t check_named);
811
812
813 __private_extern__ void vm_object_cluster_size(
814 vm_object_t object,
815 vm_object_offset_t *start,
816 vm_size_t *length,
817 vm_object_fault_info_t fault_info,
818 uint32_t *io_streaming);
819
820 __private_extern__ kern_return_t vm_object_populate_with_private(
821 vm_object_t object,
822 vm_object_offset_t offset,
823 ppnum_t phys_page,
824 vm_size_t size);
825
826 __private_extern__ void vm_object_change_wimg_mode(
827 vm_object_t object,
828 unsigned int wimg_mode);
829
830 extern kern_return_t adjust_vm_object_cache(
831 vm_size_t oval,
832 vm_size_t nval);
833
834 extern kern_return_t vm_object_page_op(
835 vm_object_t object,
836 vm_object_offset_t offset,
837 int ops,
838 ppnum_t *phys_entry,
839 int *flags);
840
841 extern kern_return_t vm_object_range_op(
842 vm_object_t object,
843 vm_object_offset_t offset_beg,
844 vm_object_offset_t offset_end,
845 int ops,
846 uint32_t *range);
847
848
849 __private_extern__ void vm_object_reap_pages(
850 vm_object_t object,
851 int reap_type);
852 #define REAP_REAP 0
853 #define REAP_TERMINATE 1
854 #define REAP_PURGEABLE 2
855 #define REAP_DATA_FLUSH 3
856
857 #if CONFIG_FREEZE
858
859 __private_extern__ uint32_t
860 vm_object_compressed_freezer_pageout(
861 vm_object_t object, uint32_t dirty_budget);
862
863 __private_extern__ void
864 vm_object_compressed_freezer_done(
865 void);
866
867 #endif /* CONFIG_FREEZE */
868
869 __private_extern__ void
870 vm_object_pageout(
871 vm_object_t object);
872
873 #if CONFIG_IOSCHED
874 struct io_reprioritize_req {
875 uint64_t blkno;
876 uint32_t len;
877 int priority;
878 struct vnode *devvp;
879 queue_chain_t io_reprioritize_list;
880 };
881 typedef struct io_reprioritize_req *io_reprioritize_req_t;
882
883 extern void vm_io_reprioritize_init(void);
884 #endif
885
886 /*
887 * Event waiting handling
888 */
889
890 #define VM_OBJECT_EVENT_INITIALIZED 0
891 #define VM_OBJECT_EVENT_PAGER_READY 1
892 #define VM_OBJECT_EVENT_PAGING_IN_PROGRESS 2
893 #define VM_OBJECT_EVENT_MAPPING_IN_PROGRESS 3
894 #define VM_OBJECT_EVENT_UNBLOCKED 4
895 #define VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS 5
896
897 #define VM_OBJECT_EVENT_MAX 5 /* 6 bits in "all_wanted", so 0->5 */
898
899 static __inline__ wait_result_t
vm_object_assert_wait(vm_object_t object,int event,wait_interrupt_t interruptible)900 vm_object_assert_wait(
901 vm_object_t object,
902 int event,
903 wait_interrupt_t interruptible)
904 {
905 wait_result_t wr;
906
907 vm_object_lock_assert_exclusive(object);
908 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
909
910 object->all_wanted |= 1 << event;
911 wr = assert_wait((event_t)((vm_offset_t)object + (vm_offset_t)event),
912 interruptible);
913 return wr;
914 }
915
916 static __inline__ wait_result_t
vm_object_wait(vm_object_t object,int event,wait_interrupt_t interruptible)917 vm_object_wait(
918 vm_object_t object,
919 int event,
920 wait_interrupt_t interruptible)
921 {
922 wait_result_t wr;
923
924 vm_object_assert_wait(object, event, interruptible);
925 vm_object_unlock(object);
926 wr = thread_block(THREAD_CONTINUE_NULL);
927 return wr;
928 }
929
930 static __inline__ wait_result_t
thread_sleep_vm_object(vm_object_t object,event_t event,wait_interrupt_t interruptible)931 thread_sleep_vm_object(
932 vm_object_t object,
933 event_t event,
934 wait_interrupt_t interruptible)
935 {
936 wait_result_t wr;
937
938 wr = lck_rw_sleep(&object->Lock,
939 LCK_SLEEP_PROMOTED_PRI,
940 event,
941 interruptible);
942 return wr;
943 }
944
945 static __inline__ wait_result_t
vm_object_sleep(vm_object_t object,int event,wait_interrupt_t interruptible)946 vm_object_sleep(
947 vm_object_t object,
948 int event,
949 wait_interrupt_t interruptible)
950 {
951 wait_result_t wr;
952
953 vm_object_lock_assert_exclusive(object);
954 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
955
956 object->all_wanted |= 1 << event;
957 wr = thread_sleep_vm_object(object,
958 (event_t)((vm_offset_t)object + (vm_offset_t)event),
959 interruptible);
960 return wr;
961 }
962
963 static __inline__ void
vm_object_wakeup(vm_object_t object,int event)964 vm_object_wakeup(
965 vm_object_t object,
966 int event)
967 {
968 vm_object_lock_assert_exclusive(object);
969 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
970
971 if (object->all_wanted & (1 << event)) {
972 thread_wakeup((event_t)((vm_offset_t)object + (vm_offset_t)event));
973 }
974 object->all_wanted &= ~(1 << event);
975 }
976
977 static __inline__ void
vm_object_set_wanted(vm_object_t object,int event)978 vm_object_set_wanted(
979 vm_object_t object,
980 int event)
981 {
982 vm_object_lock_assert_exclusive(object);
983 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
984
985 object->all_wanted |= (1 << event);
986 }
987
988 static __inline__ int
vm_object_wanted(vm_object_t object,int event)989 vm_object_wanted(
990 vm_object_t object,
991 int event)
992 {
993 vm_object_lock_assert_held(object);
994 assert(event >= 0 && event <= VM_OBJECT_EVENT_MAX);
995
996 return object->all_wanted & (1 << event);
997 }
998
999 /*
1000 * Routines implemented as macros
1001 */
1002 #ifdef VM_PIP_DEBUG
1003 #include <libkern/OSDebug.h>
1004 #define VM_PIP_DEBUG_BEGIN(object) \
1005 MACRO_BEGIN \
1006 int pip = ((object)->paging_in_progress + \
1007 (object)->activity_in_progress); \
1008 if (pip < VM_PIP_DEBUG_MAX_REFS) { \
1009 (void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
1010 VM_PIP_DEBUG_STACK_FRAMES); \
1011 } \
1012 MACRO_END
1013 #else /* VM_PIP_DEBUG */
1014 #define VM_PIP_DEBUG_BEGIN(object)
1015 #endif /* VM_PIP_DEBUG */
1016
1017 #define vm_object_activity_begin(object) \
1018 MACRO_BEGIN \
1019 vm_object_lock_assert_exclusive((object)); \
1020 VM_PIP_DEBUG_BEGIN((object)); \
1021 (object)->activity_in_progress++; \
1022 if ((object)->activity_in_progress == 0) { \
1023 panic("vm_object_activity_begin(%p): overflow\n", (object));\
1024 } \
1025 MACRO_END
1026
1027 #define vm_object_activity_end(object) \
1028 MACRO_BEGIN \
1029 vm_object_lock_assert_exclusive((object)); \
1030 if ((object)->activity_in_progress == 0) { \
1031 panic("vm_object_activity_end(%p): underflow\n", (object));\
1032 } \
1033 (object)->activity_in_progress--; \
1034 if ((object)->paging_in_progress == 0 && \
1035 (object)->activity_in_progress == 0) \
1036 vm_object_wakeup((object), \
1037 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1038 MACRO_END
1039
1040 #define vm_object_paging_begin(object) \
1041 MACRO_BEGIN \
1042 vm_object_lock_assert_exclusive((object)); \
1043 VM_PIP_DEBUG_BEGIN((object)); \
1044 (object)->paging_in_progress++; \
1045 if ((object)->paging_in_progress == 0) { \
1046 panic("vm_object_paging_begin(%p): overflow\n", (object));\
1047 } \
1048 MACRO_END
1049
1050 #define vm_object_paging_end(object) \
1051 MACRO_BEGIN \
1052 vm_object_lock_assert_exclusive((object)); \
1053 if ((object)->paging_in_progress == 0) { \
1054 panic("vm_object_paging_end(%p): underflow\n", (object));\
1055 } \
1056 (object)->paging_in_progress--; \
1057 if ((object)->paging_in_progress == 0) { \
1058 vm_object_wakeup((object), \
1059 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS); \
1060 if ((object)->activity_in_progress == 0) \
1061 vm_object_wakeup((object), \
1062 VM_OBJECT_EVENT_PAGING_IN_PROGRESS); \
1063 } \
1064 MACRO_END
1065
1066 #define vm_object_paging_wait(object, interruptible) \
1067 MACRO_BEGIN \
1068 vm_object_lock_assert_exclusive((object)); \
1069 while ((object)->paging_in_progress != 0 || \
1070 (object)->activity_in_progress != 0) { \
1071 wait_result_t _wr; \
1072 \
1073 _wr = vm_object_sleep((object), \
1074 VM_OBJECT_EVENT_PAGING_IN_PROGRESS, \
1075 (interruptible)); \
1076 \
1077 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1078 /*XXX break; */ \
1079 } \
1080 MACRO_END
1081
1082 #define vm_object_paging_only_wait(object, interruptible) \
1083 MACRO_BEGIN \
1084 vm_object_lock_assert_exclusive((object)); \
1085 while ((object)->paging_in_progress != 0) { \
1086 wait_result_t _wr; \
1087 \
1088 _wr = vm_object_sleep((object), \
1089 VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS,\
1090 (interruptible)); \
1091 \
1092 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1093 /*XXX break; */ \
1094 } \
1095 MACRO_END
1096
1097
1098 #define vm_object_mapping_begin(object) \
1099 MACRO_BEGIN \
1100 vm_object_lock_assert_exclusive((object)); \
1101 assert(! (object)->mapping_in_progress); \
1102 (object)->mapping_in_progress = TRUE; \
1103 MACRO_END
1104
1105 #define vm_object_mapping_end(object) \
1106 MACRO_BEGIN \
1107 vm_object_lock_assert_exclusive((object)); \
1108 assert((object)->mapping_in_progress); \
1109 (object)->mapping_in_progress = FALSE; \
1110 vm_object_wakeup((object), \
1111 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS); \
1112 MACRO_END
1113
1114 #define vm_object_mapping_wait(object, interruptible) \
1115 MACRO_BEGIN \
1116 vm_object_lock_assert_exclusive((object)); \
1117 while ((object)->mapping_in_progress) { \
1118 wait_result_t _wr; \
1119 \
1120 _wr = vm_object_sleep((object), \
1121 VM_OBJECT_EVENT_MAPPING_IN_PROGRESS, \
1122 (interruptible)); \
1123 /*XXX if ((interruptible) && (_wr != THREAD_AWAKENED))*/ \
1124 /*XXX break; */ \
1125 } \
1126 assert(!(object)->mapping_in_progress); \
1127 MACRO_END
1128
1129
1130
1131 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
1132 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
1133
1134 extern void vm_object_cache_add(vm_object_t);
1135 extern void vm_object_cache_remove(vm_object_t);
1136 extern int vm_object_cache_evict(int, int);
1137
1138 #define VM_OBJECT_OWNER_DISOWNED ((task_t) -1)
1139 #define VM_OBJECT_OWNER_UNCHANGED ((task_t) -2)
1140 #define VM_OBJECT_OWNER(object) \
1141 ((object == VM_OBJECT_NULL || \
1142 ((object)->purgable == VM_PURGABLE_DENY && \
1143 (object)->vo_ledger_tag == 0) || \
1144 (object)->vo_owner == TASK_NULL) \
1145 ? TASK_NULL /* not owned */ \
1146 : (((object)->vo_owner == VM_OBJECT_OWNER_DISOWNED) \
1147 ? kernel_task /* disowned -> kernel */ \
1148 : (object)->vo_owner)) /* explicit owner */ \
1149
1150 extern void vm_object_ledger_tag_ledgers(
1151 vm_object_t object,
1152 int *ledger_idx_volatile,
1153 int *ledger_idx_nonvolatile,
1154 int *ledger_idx_volatile_compressed,
1155 int *ledger_idx_nonvolatile_compressed,
1156 boolean_t *do_footprint);
1157 extern kern_return_t vm_object_ownership_change(
1158 vm_object_t object,
1159 int new_ledger_tag,
1160 task_t new_owner,
1161 int new_ledger_flags,
1162 boolean_t task_objq_locked);
1163
1164 // LP64todo: all the current tools are 32bit, obviously never worked for 64b
1165 // so probably should be a real 32b ID vs. ptr.
1166 // Current users just check for equality
1167 #define VM_OBJECT_ID(o) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM((o)))
1168
1169 static inline void
VM_OBJECT_COPY_SET(vm_object_t object,vm_object_t copy)1170 VM_OBJECT_COPY_SET(
1171 vm_object_t object,
1172 vm_object_t copy)
1173 {
1174 vm_object_lock_assert_exclusive(object);
1175 object->vo_copy = copy;
1176 if (copy != VM_OBJECT_NULL) {
1177 object->vo_copy_version++;
1178 }
1179 }
1180
1181 #endif /* _VM_VM_OBJECT_H_ */
1182