1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * File: vm/vm_map.h
61 * Author: Avadis Tevanian, Jr., Michael Wayne Young
62 * Date: 1985
63 *
64 * Virtual memory map module definitions.
65 *
66 * Contributors:
67 * avie, dlb, mwyoung
68 */
69
70 #ifndef _VM_VM_MAP_H_
71 #define _VM_VM_MAP_H_
72
73 #include <sys/cdefs.h>
74
75 #include <mach/mach_types.h>
76 #include <mach/kern_return.h>
77 #include <mach/boolean.h>
78 #include <mach/vm_types.h>
79 #include <mach/vm_prot.h>
80 #include <mach/vm_inherit.h>
81 #include <mach/vm_behavior.h>
82 #include <mach/vm_param.h>
83 #include <mach/sdt.h>
84 #include <vm/pmap.h>
85 #include <os/overflow.h>
86 #ifdef XNU_KERNEL_PRIVATE
87 #include <vm/vm_protos.h>
88 #endif /* XNU_KERNEL_PRIVATE */
89 #ifdef MACH_KERNEL_PRIVATE
90 #include <mach_assert.h>
91 #include <vm/vm_map_store.h>
92 #include <vm/vm_object.h>
93 #include <vm/vm_page.h>
94 #include <kern/locks.h>
95 #include <kern/zalloc.h>
96 #include <kern/macro_help.h>
97
98 #include <kern/thread.h>
99 #include <os/refcnt.h>
100 #endif /* MACH_KERNEL_PRIVATE */
101
102 __BEGIN_DECLS
103
104 #ifdef KERNEL_PRIVATE
105
106 extern void vm_map_reference(vm_map_t map);
107 extern vm_map_t current_map(void);
108
109 /* Setup reserved areas in a new VM map */
110 extern kern_return_t vm_map_exec(
111 vm_map_t new_map,
112 task_t task,
113 boolean_t is64bit,
114 void *fsroot,
115 cpu_type_t cpu,
116 cpu_subtype_t cpu_subtype,
117 boolean_t reslide,
118 boolean_t is_driverkit,
119 uint32_t rsr_version);
120
121 #ifdef MACH_KERNEL_PRIVATE
122
123 #define current_map_fast() (current_thread()->map)
124 #define current_map() (current_map_fast())
125
126 /*
127 * Types defined:
128 *
129 * vm_map_t the high-level address map data structure.
130 * vm_map_entry_t an entry in an address map.
131 * vm_map_version_t a timestamp of a map, for use with vm_map_lookup
132 * vm_map_copy_t represents memory copied from an address map,
133 * used for inter-map copy operations
134 */
135 typedef struct vm_map_entry *vm_map_entry_t;
136 #define VM_MAP_ENTRY_NULL ((vm_map_entry_t) NULL)
137
138
139 #define named_entry_lock_init(object) lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr)
140 #define named_entry_lock_destroy(object) lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp)
141 #define named_entry_lock(object) lck_mtx_lock(&(object)->Lock)
142 #define named_entry_unlock(object) lck_mtx_unlock(&(object)->Lock)
143
144 /*
145 * Type: vm_named_entry_t [internal use only]
146 *
147 * Description:
148 * Description of a mapping to a memory cache object.
149 *
150 * Implementation:
151 * While the handle to this object is used as a means to map
152 * and pass around the right to map regions backed by pagers
153 * of all sorts, the named_entry itself is only manipulated
154 * by the kernel. Named entries hold information on the
155 * right to map a region of a cached object. Namely,
156 * the target cache object, the beginning and ending of the
157 * region to be mapped, and the permissions, (read, write)
158 * with which it can be mapped.
159 *
160 */
161
162 struct vm_named_entry {
163 decl_lck_mtx_data(, Lock); /* Synchronization */
164 union {
165 vm_map_t map; /* map backing submap */
166 vm_map_copy_t copy; /* a VM map copy */
167 } backing;
168 vm_object_offset_t offset; /* offset into object */
169 vm_object_size_t size; /* size of region */
170 vm_object_offset_t data_offset; /* offset to first byte of data */
171 unsigned int /* Is backing.xxx : */
172 /* unsigned */ access:8, /* MAP_MEM_* */
173 /* vm_prot_t */ protection:4, /* access permissions */
174 /* boolean_t */ is_object:1, /* ... a VM object (wrapped in a VM map copy) */
175 /* boolean_t */ internal:1, /* ... an internal object */
176 /* boolean_t */ is_sub_map:1, /* ... a submap? */
177 /* boolean_t */ is_copy:1, /* ... a VM map copy */
178 /* boolean_t */ is_fully_owned:1; /* ... all objects are owned */
179 #if VM_NAMED_ENTRY_DEBUG
180 uint32_t named_entry_bt; /* btref_t */
181 #endif /* VM_NAMED_ENTRY_DEBUG */
182 };
183
184 /*
185 * Bit 3 of the protection and max_protection bitfields in a vm_map_entry
186 * does not correspond to bit 3 of a vm_prot_t, so these macros provide a means
187 * to convert between the "packed" representation in the vm_map_entry's fields
188 * and the equivalent bits defined in vm_prot_t.
189 */
190 #if defined(__x86_64__)
191 #define VM_VALID_VMPROTECT_FLAGS (VM_PROT_ALL | VM_PROT_COPY | VM_PROT_UEXEC)
192 #else
193 #define VM_VALID_VMPROTECT_FLAGS (VM_PROT_ALL | VM_PROT_COPY)
194 #endif
195
196 /*
197 * FOOTPRINT ACCOUNTING:
198 * The "memory footprint" is better described in the pmap layer.
199 *
200 * At the VM level, these 2 vm_map_entry_t fields are relevant:
201 * iokit_mapped:
202 * For an "iokit_mapped" entry, we add the size of the entry to the
203 * footprint when the entry is entered into the map and we subtract that
204 * size when the entry is removed. No other accounting should take place.
205 * "use_pmap" should be FALSE but is not taken into account.
206 * use_pmap: (only when is_sub_map is FALSE)
207 * This indicates if we should ask the pmap layer to account for pages
208 * in this mapping. If FALSE, we expect that another form of accounting
209 * is being used (e.g. "iokit_mapped" or the explicit accounting of
210 * non-volatile purgable memory).
211 *
212 * So the logic is mostly:
213 * if entry->is_sub_map == TRUE
214 * anything in a submap does not count for the footprint
215 * else if entry->iokit_mapped == TRUE
216 * footprint includes the entire virtual size of this entry
217 * else if entry->use_pmap == FALSE
218 * tell pmap NOT to account for pages being pmap_enter()'d from this
219 * mapping (i.e. use "alternate accounting")
220 * else
221 * pmap will account for pages being pmap_enter()'d from this mapping
222 * as it sees fit (only if anonymous, etc...)
223 */
224
225 #define VME_ALIAS_BITS 12
226 #define VME_ALIAS_MASK ((1u << VME_ALIAS_BITS) - 1)
227 #define VME_OFFSET_SHIFT VME_ALIAS_BITS
228 #define VME_OFFSET_BITS (64 - VME_ALIAS_BITS)
229 #define VME_SUBMAP_SHIFT 2
230 #define VME_SUBMAP_BITS (sizeof(vm_offset_t) * 8 - VME_SUBMAP_SHIFT)
231
232 struct vm_map_entry {
233 struct vm_map_links links; /* links to other entries */
234 #define vme_prev links.prev
235 #define vme_next links.next
236 #define vme_start links.start
237 #define vme_end links.end
238
239 struct vm_map_store store;
240
241 union {
242 vm_offset_t vme_object_value;
243 struct {
244 vm_offset_t vme_atomic:1; /* entry cannot be split/coalesced */
245 vm_offset_t is_sub_map:1; /* Is "object" a submap? */
246 vm_offset_t vme_submap:VME_SUBMAP_BITS;
247 };
248 struct {
249 uint32_t vme_ctx_atomic : 1;
250 uint32_t vme_ctx_is_sub_map : 1;
251 uint32_t vme_context : 30;
252
253 /**
254 * If vme_kernel_object==1 && KASAN,
255 * vme_object_or_delta holds the delta.
256 *
257 * If vme_kernel_object==1 && !KASAN,
258 * vme_tag_btref holds a btref when vme_alias is equal to the "vmtaglog"
259 * boot-arg.
260 *
261 * If vme_kernel_object==0,
262 * vme_object_or_delta holds the packed vm object.
263 */
264 union {
265 vm_page_object_t vme_object_or_delta;
266 btref_t vme_tag_btref;
267 };
268 };
269 };
270
271 unsigned long long
272 /* vm_tag_t */ vme_alias:VME_ALIAS_BITS, /* entry VM tag */
273 /* vm_object_offset_t*/ vme_offset:VME_OFFSET_BITS, /* offset into object */
274
275 /* boolean_t */ is_shared:1, /* region is shared */
276 /* boolean_t */ __unused1:1,
277 /* boolean_t */ in_transition:1, /* Entry being changed */
278 /* boolean_t */ needs_wakeup:1, /* Waiters on in_transition */
279 /* behavior is not defined for submap type */
280 /* vm_behavior_t */ behavior:2, /* user paging behavior hint */
281 /* boolean_t */ needs_copy:1, /* object need to be copied? */
282
283 /* Only in task maps: */
284 #if defined(__arm64e__)
285 /*
286 * On ARM, the fourth protection bit is unused (UEXEC is x86_64 only).
287 * We reuse it here to keep track of mappings that have hardware support
288 * for read-only/read-write trusted paths.
289 */
290 /* vm_prot_t-like */ protection:3, /* protection code */
291 /* boolean_t */ used_for_tpro:1,
292 #else /* __arm64e__ */
293 /* vm_prot_t-like */protection:4, /* protection code, bit3=UEXEC */
294 #endif /* __arm64e__ */
295
296 /* vm_prot_t-like */ max_protection:4, /* maximum protection, bit3=UEXEC */
297 /* vm_inherit_t */ inheritance:2, /* inheritance */
298
299 /*
300 * use_pmap is overloaded:
301 * if "is_sub_map":
302 * use a nested pmap?
303 * else (i.e. if object):
304 * use pmap accounting
305 * for footprint?
306 */
307 /* boolean_t */ use_pmap:1,
308 /* boolean_t */ no_cache:1, /* should new pages be cached? */
309 /* boolean_t */ vme_permanent:1, /* mapping can not be removed */
310 /* boolean_t */ superpage_size:1, /* use superpages of a certain size */
311 /* boolean_t */ map_aligned:1, /* align to map's page size */
312 /*
313 * zero out the wired pages of this entry
314 * if is being deleted without unwiring them
315 */
316 /* boolean_t */ zero_wired_pages:1,
317 /* boolean_t */ used_for_jit:1,
318 /* boolean_t */ csm_associated:1, /* code signing monitor will validate */
319
320 /* iokit accounting: use the virtual size rather than resident size: */
321 /* boolean_t */ iokit_acct:1,
322 /* boolean_t */ vme_resilient_codesign:1,
323 /* boolean_t */ vme_resilient_media:1,
324 /* boolean_t */ vme_xnu_user_debug:1,
325 /* boolean_t */ vme_no_copy_on_read:1,
326 /* boolean_t */ translated_allow_execute:1, /* execute in translated processes */
327 /* boolean_t */ vme_kernel_object:1; /* vme_object is kernel_object */
328
329 unsigned short wired_count; /* can be paged if = 0 */
330 unsigned short user_wired_count; /* for vm_wire */
331
332 #if DEBUG
333 #define MAP_ENTRY_CREATION_DEBUG (1)
334 #define MAP_ENTRY_INSERTION_DEBUG (1)
335 #endif /* DEBUG */
336 #if MAP_ENTRY_CREATION_DEBUG
337 struct vm_map_header *vme_creation_maphdr;
338 uint32_t vme_creation_bt; /* btref_t */
339 #endif /* MAP_ENTRY_CREATION_DEBUG */
340 #if MAP_ENTRY_INSERTION_DEBUG
341 uint32_t vme_insertion_bt; /* btref_t */
342 vm_map_offset_t vme_start_original;
343 vm_map_offset_t vme_end_original;
344 #endif /* MAP_ENTRY_INSERTION_DEBUG */
345 };
346
347 #define VME_ALIAS(entry) \
348 ((entry)->vme_alias)
349
350 static inline vm_map_t
_VME_SUBMAP(vm_map_entry_t entry)351 _VME_SUBMAP(
352 vm_map_entry_t entry)
353 {
354 __builtin_assume(entry->vme_submap);
355 return (vm_map_t)(entry->vme_submap << VME_SUBMAP_SHIFT);
356 }
357 #define VME_SUBMAP(entry) ({ assert((entry)->is_sub_map); _VME_SUBMAP(entry); })
358
359 static inline void
VME_SUBMAP_SET(vm_map_entry_t entry,vm_map_t submap)360 VME_SUBMAP_SET(
361 vm_map_entry_t entry,
362 vm_map_t submap)
363 {
364 __builtin_assume(((vm_offset_t)submap & 3) == 0);
365
366 entry->is_sub_map = true;
367 entry->vme_submap = (vm_offset_t)submap >> VME_SUBMAP_SHIFT;
368 }
369
370 static inline vm_object_t
_VME_OBJECT(vm_map_entry_t entry)371 _VME_OBJECT(
372 vm_map_entry_t entry)
373 {
374 vm_object_t object;
375
376 if (!entry->vme_kernel_object) {
377 object = VM_OBJECT_UNPACK(entry->vme_object_or_delta);
378 __builtin_assume(!is_kernel_object(object));
379 } else {
380 object = kernel_object_default;
381 }
382 return object;
383 }
384 #define VME_OBJECT(entry) ({ assert(!(entry)->is_sub_map); _VME_OBJECT(entry); })
385
386 static inline void
VME_OBJECT_SET(vm_map_entry_t entry,vm_object_t object,bool atomic,uint32_t context)387 VME_OBJECT_SET(
388 vm_map_entry_t entry,
389 vm_object_t object,
390 bool atomic,
391 uint32_t context)
392 {
393 __builtin_assume(((vm_offset_t)object & 3) == 0);
394
395 entry->vme_atomic = atomic;
396 entry->is_sub_map = false;
397 if (atomic) {
398 entry->vme_context = context;
399 } else {
400 entry->vme_context = 0;
401 }
402
403 if (!object) {
404 entry->vme_object_or_delta = 0;
405 } else if (is_kernel_object(object)) {
406 #if VM_BTLOG_TAGS
407 if (!(entry->vme_kernel_object && entry->vme_tag_btref))
408 #endif /* VM_BTLOG_TAGS */
409 {
410 entry->vme_object_or_delta = 0;
411 }
412 } else {
413 #if VM_BTLOG_TAGS
414 if (entry->vme_kernel_object && entry->vme_tag_btref) {
415 btref_put(entry->vme_tag_btref);
416 }
417 #endif /* VM_BTLOG_TAGS */
418 entry->vme_object_or_delta = VM_OBJECT_PACK(object);
419 }
420
421 entry->vme_kernel_object = is_kernel_object(object);
422 entry->vme_resilient_codesign = false;
423 entry->used_for_jit = false;
424 }
425
426 static inline vm_object_offset_t
VME_OFFSET(vm_map_entry_t entry)427 VME_OFFSET(
428 vm_map_entry_t entry)
429 {
430 return entry->vme_offset << VME_OFFSET_SHIFT;
431 }
432
433 static inline void
VME_OFFSET_SET(vm_map_entry_t entry,vm_object_offset_t offset)434 VME_OFFSET_SET(
435 vm_map_entry_t entry,
436 vm_object_offset_t offset)
437 {
438 entry->vme_offset = offset >> VME_OFFSET_SHIFT;
439 assert3u(VME_OFFSET(entry), ==, offset);
440 }
441
442 /*
443 * IMPORTANT:
444 * The "alias" field can be updated while holding the VM map lock
445 * "shared". It's OK as along as it's the only field that can be
446 * updated without the VM map "exclusive" lock.
447 */
448 static inline void
VME_ALIAS_SET(vm_map_entry_t entry,unsigned int alias)449 VME_ALIAS_SET(
450 vm_map_entry_t entry,
451 unsigned int alias)
452 {
453 assert3u(alias & VME_ALIAS_MASK, ==, alias);
454 entry->vme_alias = alias;
455 }
456
457 static inline void
VME_OBJECT_SHADOW(vm_map_entry_t entry,vm_object_size_t length,bool always)458 VME_OBJECT_SHADOW(
459 vm_map_entry_t entry,
460 vm_object_size_t length,
461 bool always)
462 {
463 vm_object_t object;
464 vm_object_offset_t offset;
465
466 object = VME_OBJECT(entry);
467 offset = VME_OFFSET(entry);
468 vm_object_shadow(&object, &offset, length, always);
469 if (object != VME_OBJECT(entry)) {
470 entry->vme_object_or_delta = VM_OBJECT_PACK(object);
471 entry->use_pmap = true;
472 }
473 if (offset != VME_OFFSET(entry)) {
474 VME_OFFSET_SET(entry, offset);
475 }
476 }
477
478 #if (DEBUG || DEVELOPMENT) && !KASAN
479 #define VM_BTLOG_TAGS 1
480 #else
481 #define VM_BTLOG_TAGS 0
482 #endif
483
484 extern vm_tag_t vmtaglog_tag; /* Collected from a tunable in vm_resident.c */
485 static inline void
vme_btref_consider_and_set(__unused vm_map_entry_t entry,__unused void * fp)486 vme_btref_consider_and_set(__unused vm_map_entry_t entry, __unused void *fp)
487 {
488 #if VM_BTLOG_TAGS
489 if (vmtaglog_tag && (VME_ALIAS(entry) == vmtaglog_tag) && entry->vme_kernel_object && entry->wired_count) {
490 assert(!entry->vme_tag_btref); /* We should have already zeroed and freed the btref if we're here. */
491 entry->vme_tag_btref = btref_get(fp, BTREF_GET_NOWAIT);
492 }
493 #endif /* VM_BTLOG_TAGS */
494 }
495
496 static inline void
vme_btref_consider_and_put(__unused vm_map_entry_t entry)497 vme_btref_consider_and_put(__unused vm_map_entry_t entry)
498 {
499 #if VM_BTLOG_TAGS
500 if (entry->vme_tag_btref && entry->vme_kernel_object && (entry->wired_count == 0) && (entry->user_wired_count == 0)) {
501 btref_put(entry->vme_tag_btref);
502 entry->vme_tag_btref = 0;
503 }
504 #endif /* VM_BTLOG_TAGS */
505 }
506
507
508 /*
509 * Convenience macros for dealing with superpages
510 * SUPERPAGE_NBASEPAGES is architecture dependent and defined in pmap.h
511 */
512 #define SUPERPAGE_SIZE (PAGE_SIZE*SUPERPAGE_NBASEPAGES)
513 #define SUPERPAGE_MASK (-SUPERPAGE_SIZE)
514 #define SUPERPAGE_ROUND_DOWN(a) (a & SUPERPAGE_MASK)
515 #define SUPERPAGE_ROUND_UP(a) ((a + SUPERPAGE_SIZE-1) & SUPERPAGE_MASK)
516
517 /*
518 * wired_counts are unsigned short. This value is used to safeguard
519 * against any mishaps due to runaway user programs.
520 */
521 #define MAX_WIRE_COUNT 65535
522
523 typedef struct vm_map_user_range {
524 vm_map_address_t vmur_min_address __kernel_data_semantics;
525
526 vm_map_address_t vmur_max_address : 56 __kernel_data_semantics;
527 vm_map_range_id_t vmur_range_id : 8;
528 } *vm_map_user_range_t;
529
530 /*
531 * Type: vm_map_t [exported; contents invisible]
532 *
533 * Description:
534 * An address map -- a directory relating valid
535 * regions of a task's address space to the corresponding
536 * virtual memory objects.
537 *
538 * Implementation:
539 * Maps are doubly-linked lists of map entries, sorted
540 * by address. One hint is used to start
541 * searches again from the last successful search,
542 * insertion, or removal. Another hint is used to
543 * quickly find free space.
544 *
545 * Note:
546 * vm_map_relocate_early_elem() knows about this layout,
547 * and needs to be kept in sync.
548 */
549 struct _vm_map {
550 lck_rw_t lock; /* map lock */
551 struct vm_map_header hdr; /* Map entry header */
552 #define min_offset hdr.links.start /* start of range */
553 #define max_offset hdr.links.end /* end of range */
554 pmap_t XNU_PTRAUTH_SIGNED_PTR("_vm_map.pmap") pmap; /* Physical map */
555 vm_map_size_t size; /* virtual size */
556 uint64_t size_limit; /* rlimit on address space size */
557 uint64_t data_limit; /* rlimit on data size */
558 vm_map_size_t user_wire_limit;/* rlimit on user locked memory */
559 vm_map_size_t user_wire_size; /* current size of user locked memory in this map */
560 #if __x86_64__
561 vm_map_offset_t vmmap_high_start;
562 #endif /* __x86_64__ */
563
564 os_ref_atomic_t map_refcnt; /* Reference count */
565
566 #if CONFIG_MAP_RANGES
567 #define VM_MAP_EXTRA_RANGES_MAX 1024
568 struct mach_vm_range default_range;
569 struct mach_vm_range data_range;
570
571 uint16_t extra_ranges_count;
572 vm_map_user_range_t extra_ranges;
573 #endif /* CONFIG_MAP_RANGES */
574
575 union {
576 /*
577 * If map->disable_vmentry_reuse == TRUE:
578 * the end address of the highest allocated vm_map_entry_t.
579 */
580 vm_map_offset_t vmu1_highest_entry_end;
581 /*
582 * For a nested VM map:
583 * the lowest address in this nested VM map that we would
584 * expect to be unnested under normal operation (i.e. for
585 * regular copy-on-write on DATA section).
586 */
587 vm_map_offset_t vmu1_lowest_unnestable_start;
588 } vmu1;
589 #define highest_entry_end vmu1.vmu1_highest_entry_end
590 #define lowest_unnestable_start vmu1.vmu1_lowest_unnestable_start
591 vm_map_entry_t hint; /* hint for quick lookups */
592 union {
593 struct vm_map_links* vmmap_hole_hint; /* hint for quick hole lookups */
594 struct vm_map_corpse_footprint_header *vmmap_corpse_footprint;
595 } vmmap_u_1;
596 #define hole_hint vmmap_u_1.vmmap_hole_hint
597 #define vmmap_corpse_footprint vmmap_u_1.vmmap_corpse_footprint
598 union {
599 vm_map_entry_t _first_free; /* First free space hint */
600 struct vm_map_links* _holes; /* links all holes between entries */
601 } f_s; /* Union for free space data structures being used */
602
603 #define first_free f_s._first_free
604 #define holes_list f_s._holes
605
606 unsigned int
607 /* boolean_t */ wait_for_space:1, /* Should callers wait for space? */
608 /* boolean_t */ wiring_required:1, /* All memory wired? */
609 /* boolean_t */ no_zero_fill:1, /* No zero fill absent pages */
610 /* boolean_t */ mapped_in_other_pmaps:1, /* has this submap been mapped in maps that use a different pmap */
611 /* boolean_t */ switch_protect:1, /* Protect map from write faults while switched */
612 /* boolean_t */ disable_vmentry_reuse:1, /* All vm entries should keep using newer and higher addresses in the map */
613 /* boolean_t */ map_disallow_data_exec:1, /* Disallow execution from data pages on exec-permissive architectures */
614 /* boolean_t */ holelistenabled:1,
615 /* boolean_t */ is_nested_map:1,
616 /* boolean_t */ map_disallow_new_exec:1, /* Disallow new executable code */
617 /* boolean_t */ jit_entry_exists:1,
618 /* boolean_t */ has_corpse_footprint:1,
619 /* boolean_t */ terminated:1,
620 /* boolean_t */ is_alien:1, /* for platform simulation, i.e. PLATFORM_IOS on OSX */
621 /* boolean_t */ cs_enforcement:1, /* code-signing enforcement */
622 /* boolean_t */ cs_debugged:1, /* code-signed but debugged */
623 /* boolean_t */ reserved_regions:1, /* has reserved regions. The map size that userspace sees should ignore these. */
624 /* boolean_t */ single_jit:1, /* only allow one JIT mapping */
625 /* boolean_t */ never_faults:1, /* this map should never cause faults */
626 /* boolean_t */ uses_user_ranges:1, /* has the map been configured to use user VM ranges */
627 /* boolean_t */ tpro_enforcement:1, /* enforce TPRO propagation */
628 /* boolean_t */ corpse_source:1, /* map is being used to create a corpse for diagnostics.*/
629 /* reserved */ res0:1,
630 /* reserved */pad:9;
631 unsigned int timestamp; /* Version number */
632 };
633
634 #define CAST_TO_VM_MAP_ENTRY(x) ((struct vm_map_entry *)(uintptr_t)(x))
635 #define vm_map_to_entry(map) CAST_TO_VM_MAP_ENTRY(&(map)->hdr.links)
636 #define vm_map_first_entry(map) ((map)->hdr.links.next)
637 #define vm_map_last_entry(map) ((map)->hdr.links.prev)
638
639 /*
640 * Type: vm_map_version_t [exported; contents invisible]
641 *
642 * Description:
643 * Map versions may be used to quickly validate a previous
644 * lookup operation.
645 *
646 * Usage note:
647 * Because they are bulky objects, map versions are usually
648 * passed by reference.
649 *
650 * Implementation:
651 * Just a timestamp for the main map.
652 */
653 typedef struct vm_map_version {
654 unsigned int main_timestamp;
655 } vm_map_version_t;
656
657 /*
658 * Type: vm_map_copy_t [exported; contents invisible]
659 *
660 * Description:
661 * A map copy object represents a region of virtual memory
662 * that has been copied from an address map but is still
663 * in transit.
664 *
665 * A map copy object may only be used by a single thread
666 * at a time.
667 *
668 * Implementation:
669 * There are two formats for map copy objects.
670 * The first is very similar to the main
671 * address map in structure, and as a result, some
672 * of the internal maintenance functions/macros can
673 * be used with either address maps or map copy objects.
674 *
675 * The map copy object contains a header links
676 * entry onto which the other entries that represent
677 * the region are chained.
678 *
679 * The second format is a kernel buffer copy object - for data
680 * small enough that physical copies were the most efficient
681 * method. This method uses a zero-sized array unioned with
682 * other format-specific data in the 'c_u' member. This unsized
683 * array overlaps the other elements and allows us to use this
684 * extra structure space for physical memory copies. On 64-bit
685 * systems this saves ~64 bytes per vm_map_copy.
686 */
687
688 struct vm_map_copy {
689 #define VM_MAP_COPY_ENTRY_LIST 1
690 #define VM_MAP_COPY_KERNEL_BUFFER 2
691 uint16_t type;
692 bool is_kernel_range;
693 bool is_user_range;
694 vm_map_range_id_t orig_range;
695 vm_object_offset_t offset;
696 vm_map_size_t size;
697 union {
698 struct vm_map_header hdr; /* ENTRY_LIST */
699 void *XNU_PTRAUTH_SIGNED_PTR("vm_map_copy.kdata") kdata; /* KERNEL_BUFFER */
700 } c_u;
701 };
702
703
704 ZONE_DECLARE_ID(ZONE_ID_VM_MAP_ENTRY, struct vm_map_entry);
705 #define vm_map_entry_zone (&zone_array[ZONE_ID_VM_MAP_ENTRY])
706
707 ZONE_DECLARE_ID(ZONE_ID_VM_MAP_HOLES, struct vm_map_links);
708 #define vm_map_holes_zone (&zone_array[ZONE_ID_VM_MAP_HOLES])
709
710 ZONE_DECLARE_ID(ZONE_ID_VM_MAP, struct _vm_map);
711 #define vm_map_zone (&zone_array[ZONE_ID_VM_MAP])
712
713
714 #define cpy_hdr c_u.hdr
715 #define cpy_kdata c_u.kdata
716
717 #define VM_MAP_COPY_PAGE_SHIFT(copy) ((copy)->cpy_hdr.page_shift)
718 #define VM_MAP_COPY_PAGE_SIZE(copy) (1 << VM_MAP_COPY_PAGE_SHIFT((copy)))
719 #define VM_MAP_COPY_PAGE_MASK(copy) (VM_MAP_COPY_PAGE_SIZE((copy)) - 1)
720
721 /*
722 * Useful macros for entry list copy objects
723 */
724
725 #define vm_map_copy_to_entry(copy) CAST_TO_VM_MAP_ENTRY(&(copy)->cpy_hdr.links)
726 #define vm_map_copy_first_entry(copy) \
727 ((copy)->cpy_hdr.links.next)
728 #define vm_map_copy_last_entry(copy) \
729 ((copy)->cpy_hdr.links.prev)
730
731 extern kern_return_t
732 vm_map_copy_adjust_to_target(
733 vm_map_copy_t copy_map,
734 vm_map_offset_t offset,
735 vm_map_size_t size,
736 vm_map_t target_map,
737 boolean_t copy,
738 vm_map_copy_t *target_copy_map_p,
739 vm_map_offset_t *overmap_start_p,
740 vm_map_offset_t *overmap_end_p,
741 vm_map_offset_t *trimmed_start_p);
742
743 /*
744 * Macros: vm_map_lock, etc. [internal use only]
745 * Description:
746 * Perform locking on the data portion of a map.
747 * When multiple maps are to be locked, order by map address.
748 * (See vm_map.c::vm_remap())
749 */
750
751 #define vm_map_lock_init(map) \
752 ((map)->timestamp = 0 , \
753 lck_rw_init(&(map)->lock, &vm_map_lck_grp, &vm_map_lck_rw_attr))
754
755 #define vm_map_lock(map) \
756 MACRO_BEGIN \
757 DTRACE_VM(vm_map_lock_w); \
758 lck_rw_lock_exclusive(&(map)->lock); \
759 MACRO_END
760
761 #define vm_map_unlock(map) \
762 MACRO_BEGIN \
763 DTRACE_VM(vm_map_unlock_w); \
764 (map)->timestamp++; \
765 lck_rw_done(&(map)->lock); \
766 MACRO_END
767
768 #define vm_map_lock_read(map) \
769 MACRO_BEGIN \
770 DTRACE_VM(vm_map_lock_r); \
771 lck_rw_lock_shared(&(map)->lock); \
772 MACRO_END
773
774 #define vm_map_unlock_read(map) \
775 MACRO_BEGIN \
776 DTRACE_VM(vm_map_unlock_r); \
777 lck_rw_done(&(map)->lock); \
778 MACRO_END
779
780 #define vm_map_lock_write_to_read(map) \
781 MACRO_BEGIN \
782 DTRACE_VM(vm_map_lock_downgrade); \
783 (map)->timestamp++; \
784 lck_rw_lock_exclusive_to_shared(&(map)->lock); \
785 MACRO_END
786
787 __attribute__((always_inline))
788 int vm_map_lock_read_to_write(vm_map_t map);
789
790 __attribute__((always_inline))
791 boolean_t vm_map_try_lock(vm_map_t map);
792
793 __attribute__((always_inline))
794 boolean_t vm_map_try_lock_read(vm_map_t map);
795
796 int vm_self_region_page_shift(vm_map_t target_map);
797 int vm_self_region_page_shift_safely(vm_map_t target_map);
798
799 #define vm_map_lock_assert_held(map) \
800 LCK_RW_ASSERT(&(map)->lock, LCK_RW_ASSERT_HELD)
801 #define vm_map_lock_assert_shared(map) \
802 LCK_RW_ASSERT(&(map)->lock, LCK_RW_ASSERT_SHARED)
803 #define vm_map_lock_assert_exclusive(map) \
804 LCK_RW_ASSERT(&(map)->lock, LCK_RW_ASSERT_EXCLUSIVE)
805 #define vm_map_lock_assert_notheld(map) \
806 LCK_RW_ASSERT(&(map)->lock, LCK_RW_ASSERT_NOTHELD)
807
808 /*
809 * Exported procedures that operate on vm_map_t.
810 */
811
812 /* Lookup map entry containing or the specified address in the given map */
813 extern boolean_t vm_map_lookup_entry(
814 vm_map_t map,
815 vm_map_address_t address,
816 vm_map_entry_t *entry); /* OUT */
817
818 /* Lookup map entry containing or the specified address in the given map */
819 extern boolean_t vm_map_lookup_entry_or_next(
820 vm_map_t map,
821 vm_map_address_t address,
822 vm_map_entry_t *entry); /* OUT */
823
824 /* like vm_map_lookup_entry without the PGZ bear trap */
825 #if CONFIG_PROB_GZALLOC
826 extern boolean_t vm_map_lookup_entry_allow_pgz(
827 vm_map_t map,
828 vm_map_address_t address,
829 vm_map_entry_t *entry); /* OUT */
830 #else /* !CONFIG_PROB_GZALLOC */
831 #define vm_map_lookup_entry_allow_pgz vm_map_lookup_entry
832 #endif /* !CONFIG_PROB_GZALLOC */
833
834 extern void vm_map_copy_remap(
835 vm_map_t map,
836 vm_map_entry_t where,
837 vm_map_copy_t copy,
838 vm_map_offset_t adjustment,
839 vm_prot_t cur_prot,
840 vm_prot_t max_prot,
841 vm_inherit_t inheritance);
842
843 /* Find the VM object, offset, and protection for a given virtual address
844 * in the specified map, assuming a page fault of the type specified. */
845 extern kern_return_t vm_map_lookup_and_lock_object(
846 vm_map_t *var_map, /* IN/OUT */
847 vm_map_address_t vaddr,
848 vm_prot_t fault_type,
849 int object_lock_type,
850 vm_map_version_t *out_version, /* OUT */
851 vm_object_t *object, /* OUT */
852 vm_object_offset_t *offset, /* OUT */
853 vm_prot_t *out_prot, /* OUT */
854 boolean_t *wired, /* OUT */
855 vm_object_fault_info_t fault_info, /* OUT */
856 vm_map_t *real_map, /* OUT */
857 bool *contended); /* OUT */
858
859 /* Verifies that the map has not changed since the given version. */
860 extern boolean_t vm_map_verify(
861 vm_map_t map,
862 vm_map_version_t *version); /* REF */
863
864
865 /*
866 * Functions implemented as macros
867 */
868 #define vm_map_min(map) ((map)->min_offset)
869 /* Lowest valid address in
870 * a map */
871
872 #define vm_map_max(map) ((map)->max_offset)
873 /* Highest valid address */
874
875 #define vm_map_pmap(map) ((map)->pmap)
876 /* Physical map associated
877 * with this address map */
878
879 /* Gain a reference to an existing map */
880 extern void vm_map_reference(
881 vm_map_t map);
882
883 /*
884 * Wait and wakeup macros for in_transition map entries.
885 */
886 #define vm_map_entry_wait(map, interruptible) \
887 ((map)->timestamp++ , \
888 lck_rw_sleep(&(map)->lock, LCK_SLEEP_EXCLUSIVE|LCK_SLEEP_PROMOTED_PRI, \
889 (event_t)&(map)->hdr, interruptible))
890
891
892 #define vm_map_entry_wakeup(map) \
893 thread_wakeup((event_t)(&(map)->hdr))
894
895
896 /* simplify map entries */
897 extern void vm_map_simplify_entry(
898 vm_map_t map,
899 vm_map_entry_t this_entry);
900 extern void vm_map_simplify(
901 vm_map_t map,
902 vm_map_offset_t start);
903
904 #if XNU_PLATFORM_MacOSX
905
906 /* Move the information in a map copy object to a new map copy object */
907 extern vm_map_copy_t vm_map_copy_copy(
908 vm_map_copy_t copy);
909
910 #endif /* XNU_PLATFORM_MacOSX */
911
912 /* Enter a mapping */
913 extern kern_return_t vm_map_enter(
914 vm_map_t map,
915 vm_map_offset_t *address,
916 vm_map_size_t size,
917 vm_map_offset_t mask,
918 vm_map_kernel_flags_t vmk_flags,
919 vm_object_t object,
920 vm_object_offset_t offset,
921 boolean_t needs_copy,
922 vm_prot_t cur_protection,
923 vm_prot_t max_protection,
924 vm_inherit_t inheritance);
925
926 #if __arm64__
927 extern kern_return_t vm_map_enter_fourk(
928 vm_map_t map,
929 vm_map_offset_t *address,
930 vm_map_size_t size,
931 vm_map_offset_t mask,
932 vm_map_kernel_flags_t vmk_flags,
933 vm_object_t object,
934 vm_object_offset_t offset,
935 boolean_t needs_copy,
936 vm_prot_t cur_protection,
937 vm_prot_t max_protection,
938 vm_inherit_t inheritance);
939 #endif /* __arm64__ */
940
941 /* XXX should go away - replaced with regular enter of contig object */
942 extern kern_return_t vm_map_enter_cpm(
943 vm_map_t map,
944 vm_map_address_t *addr,
945 vm_map_size_t size,
946 vm_map_kernel_flags_t vmk_flags);
947
948 extern kern_return_t vm_map_remap(
949 vm_map_t target_map,
950 vm_map_offset_t *address,
951 vm_map_size_t size,
952 vm_map_offset_t mask,
953 vm_map_kernel_flags_t vmk_flags,
954 vm_map_t src_map,
955 vm_map_offset_t memory_address,
956 boolean_t copy,
957 vm_prot_t *cur_protection,
958 vm_prot_t *max_protection,
959 vm_inherit_t inheritance);
960
961
962 /*
963 * Read and write from a kernel buffer to a specified map.
964 */
965 extern kern_return_t vm_map_write_user(
966 vm_map_t map,
967 void *src_p,
968 vm_map_offset_t dst_addr,
969 vm_size_t size);
970
971 extern kern_return_t vm_map_read_user(
972 vm_map_t map,
973 vm_map_offset_t src_addr,
974 void *dst_p,
975 vm_size_t size);
976
977 extern void vm_map_inherit_limits(
978 vm_map_t new_map,
979 const struct _vm_map *old_map);
980
981 /* Create a new task map using an existing task map as a template. */
982 extern vm_map_t vm_map_fork(
983 ledger_t ledger,
984 vm_map_t old_map,
985 int options);
986 #define VM_MAP_FORK_SHARE_IF_INHERIT_NONE 0x00000001
987 #define VM_MAP_FORK_PRESERVE_PURGEABLE 0x00000002
988 #define VM_MAP_FORK_CORPSE_FOOTPRINT 0x00000004
989
990 /* Change inheritance */
991 extern kern_return_t vm_map_inherit(
992 vm_map_t map,
993 vm_map_offset_t start,
994 vm_map_offset_t end,
995 vm_inherit_t new_inheritance);
996
997 /* Add or remove machine-dependent attributes from map regions */
998 extern kern_return_t vm_map_machine_attribute(
999 vm_map_t map,
1000 vm_map_offset_t start,
1001 vm_map_offset_t end,
1002 vm_machine_attribute_t attribute,
1003 vm_machine_attribute_val_t* value); /* IN/OUT */
1004
1005 extern kern_return_t vm_map_msync(
1006 vm_map_t map,
1007 vm_map_address_t address,
1008 vm_map_size_t size,
1009 vm_sync_t sync_flags);
1010
1011 /* Set paging behavior */
1012 extern kern_return_t vm_map_behavior_set(
1013 vm_map_t map,
1014 vm_map_offset_t start,
1015 vm_map_offset_t end,
1016 vm_behavior_t new_behavior);
1017
1018 extern kern_return_t vm_map_region(
1019 vm_map_t map,
1020 vm_map_offset_t *address,
1021 vm_map_size_t *size,
1022 vm_region_flavor_t flavor,
1023 vm_region_info_t info,
1024 mach_msg_type_number_t *count,
1025 mach_port_t *object_name);
1026
1027 extern kern_return_t vm_map_region_recurse_64(
1028 vm_map_t map,
1029 vm_map_offset_t *address,
1030 vm_map_size_t *size,
1031 natural_t *nesting_depth,
1032 vm_region_submap_info_64_t info,
1033 mach_msg_type_number_t *count);
1034
1035 extern kern_return_t vm_map_page_query_internal(
1036 vm_map_t map,
1037 vm_map_offset_t offset,
1038 int *disposition,
1039 int *ref_count);
1040
1041 extern kern_return_t vm_map_query_volatile(
1042 vm_map_t map,
1043 mach_vm_size_t *volatile_virtual_size_p,
1044 mach_vm_size_t *volatile_resident_size_p,
1045 mach_vm_size_t *volatile_compressed_size_p,
1046 mach_vm_size_t *volatile_pmap_size_p,
1047 mach_vm_size_t *volatile_compressed_pmap_size_p);
1048
1049 /* Convert from a map entry port to a map */
1050 extern vm_map_t convert_port_entry_to_map(
1051 ipc_port_t port);
1052
1053
1054 extern kern_return_t vm_map_set_cache_attr(
1055 vm_map_t map,
1056 vm_map_offset_t va);
1057
1058
1059 /* definitions related to overriding the NX behavior */
1060
1061 #define VM_ABI_32 0x1
1062 #define VM_ABI_64 0x2
1063
1064 extern int override_nx(vm_map_t map, uint32_t user_tag);
1065
1066 extern void vm_map_region_top_walk(
1067 vm_map_entry_t entry,
1068 vm_region_top_info_t top);
1069 extern void vm_map_region_walk(
1070 vm_map_t map,
1071 vm_map_offset_t va,
1072 vm_map_entry_t entry,
1073 vm_object_offset_t offset,
1074 vm_object_size_t range,
1075 vm_region_extended_info_t extended,
1076 boolean_t look_for_pages,
1077 mach_msg_type_number_t count);
1078
1079
1080
1081 extern void vm_map_copy_footprint_ledgers(
1082 task_t old_task,
1083 task_t new_task);
1084 extern void vm_map_copy_ledger(
1085 task_t old_task,
1086 task_t new_task,
1087 int ledger_entry);
1088
1089 /**
1090 * Represents a single region of virtual address space that should be reserved
1091 * (pre-mapped) in a user address space.
1092 */
1093 struct vm_reserved_region {
1094 const char *vmrr_name;
1095 vm_map_offset_t vmrr_addr;
1096 vm_map_size_t vmrr_size;
1097 };
1098
1099 /**
1100 * Return back a machine-dependent array of address space regions that should be
1101 * reserved by the VM. This function is defined in the machine-dependent
1102 * machine_routines.c files.
1103 */
1104 extern size_t ml_get_vm_reserved_regions(
1105 bool vm_is64bit,
1106 const struct vm_reserved_region **regions);
1107
1108 #endif /* MACH_KERNEL_PRIVATE */
1109
1110 /* Create an empty map */
1111 extern vm_map_t vm_map_create(
1112 pmap_t pmap,
1113 vm_map_offset_t min_off,
1114 vm_map_offset_t max_off,
1115 boolean_t pageable);
1116
1117 extern vm_map_size_t vm_map_adjusted_size(vm_map_t map);
1118
1119 extern void vm_map_disable_hole_optimization(vm_map_t map);
1120
1121 /* Get rid of a map */
1122 extern void vm_map_destroy(
1123 vm_map_t map);
1124
1125 /* Lose a reference */
1126 extern void vm_map_deallocate(
1127 vm_map_t map);
1128
1129 /* Lose a reference */
1130 extern void vm_map_inspect_deallocate(
1131 vm_map_inspect_t map);
1132
1133 /* Lose a reference */
1134 extern void vm_map_read_deallocate(
1135 vm_map_read_t map);
1136
1137 extern vm_map_t vm_map_switch(
1138 vm_map_t map);
1139
1140 /* Change protection */
1141 extern kern_return_t vm_map_protect(
1142 vm_map_t map,
1143 vm_map_offset_t start,
1144 vm_map_offset_t end,
1145 vm_prot_t new_prot,
1146 boolean_t set_max);
1147
1148 /* Check protection */
1149 extern boolean_t vm_map_check_protection(
1150 vm_map_t map,
1151 vm_map_offset_t start,
1152 vm_map_offset_t end,
1153 vm_prot_t protection);
1154
1155 extern boolean_t vm_map_cs_enforcement(
1156 vm_map_t map);
1157 extern void vm_map_cs_enforcement_set(
1158 vm_map_t map,
1159 boolean_t val);
1160
1161 extern void vm_map_cs_debugged_set(
1162 vm_map_t map,
1163 boolean_t val);
1164
1165 extern kern_return_t vm_map_cs_wx_enable(vm_map_t map);
1166 extern kern_return_t vm_map_csm_allow_jit(vm_map_t map);
1167
1168 /* wire down a region */
1169
1170 #ifdef XNU_KERNEL_PRIVATE
1171
1172 extern void vm_map_will_allocate_early_map(
1173 vm_map_t *map_owner);
1174
1175 extern void vm_map_relocate_early_maps(
1176 vm_offset_t delta);
1177
1178 extern void vm_map_relocate_early_elem(
1179 uint32_t zone_id,
1180 vm_offset_t new_addr,
1181 vm_offset_t delta);
1182
1183 /* never fails */
1184 extern vm_map_t vm_map_create_options(
1185 pmap_t pmap,
1186 vm_map_offset_t min_off,
1187 vm_map_offset_t max_off,
1188 vm_map_create_options_t options);
1189
1190 extern kern_return_t vm_map_wire_kernel(
1191 vm_map_t map,
1192 vm_map_offset_t start,
1193 vm_map_offset_t end,
1194 vm_prot_t access_type,
1195 vm_tag_t tag,
1196 boolean_t user_wire);
1197
1198 extern kern_return_t vm_map_wire_and_extract_kernel(
1199 vm_map_t map,
1200 vm_map_offset_t start,
1201 vm_prot_t access_type,
1202 vm_tag_t tag,
1203 boolean_t user_wire,
1204 ppnum_t *physpage_p);
1205
1206 /* kext exported versions */
1207
1208 extern kern_return_t vm_map_wire_external(
1209 vm_map_t map,
1210 vm_map_offset_t start,
1211 vm_map_offset_t end,
1212 vm_prot_t access_type,
1213 boolean_t user_wire);
1214
1215 extern kern_return_t vm_map_wire_and_extract_external(
1216 vm_map_t map,
1217 vm_map_offset_t start,
1218 vm_prot_t access_type,
1219 boolean_t user_wire,
1220 ppnum_t *physpage_p);
1221
1222 #else /* XNU_KERNEL_PRIVATE */
1223
1224 extern kern_return_t vm_map_wire(
1225 vm_map_t map,
1226 vm_map_offset_t start,
1227 vm_map_offset_t end,
1228 vm_prot_t access_type,
1229 boolean_t user_wire);
1230
1231 extern kern_return_t vm_map_wire_and_extract(
1232 vm_map_t map,
1233 vm_map_offset_t start,
1234 vm_prot_t access_type,
1235 boolean_t user_wire,
1236 ppnum_t *physpage_p);
1237
1238 #endif /* !XNU_KERNEL_PRIVATE */
1239
1240 /* unwire a region */
1241 extern kern_return_t vm_map_unwire(
1242 vm_map_t map,
1243 vm_map_offset_t start,
1244 vm_map_offset_t end,
1245 boolean_t user_wire);
1246
1247 #ifdef XNU_KERNEL_PRIVATE
1248
1249 /* Enter a mapping of a memory object */
1250 extern kern_return_t vm_map_enter_mem_object(
1251 vm_map_t map,
1252 vm_map_offset_t *address,
1253 vm_map_size_t size,
1254 vm_map_offset_t mask,
1255 vm_map_kernel_flags_t vmk_flags,
1256 ipc_port_t port,
1257 vm_object_offset_t offset,
1258 boolean_t needs_copy,
1259 vm_prot_t cur_protection,
1260 vm_prot_t max_protection,
1261 vm_inherit_t inheritance);
1262
1263 /* Enter a mapping of a memory object */
1264 extern kern_return_t vm_map_enter_mem_object_prefault(
1265 vm_map_t map,
1266 vm_map_offset_t *address,
1267 vm_map_size_t size,
1268 vm_map_offset_t mask,
1269 vm_map_kernel_flags_t vmk_flags,
1270 ipc_port_t port,
1271 vm_object_offset_t offset,
1272 vm_prot_t cur_protection,
1273 vm_prot_t max_protection,
1274 upl_page_list_ptr_t page_list,
1275 unsigned int page_list_count);
1276
1277 /* Enter a mapping of a memory object */
1278 extern kern_return_t vm_map_enter_mem_object_control(
1279 vm_map_t map,
1280 vm_map_offset_t *address,
1281 vm_map_size_t size,
1282 vm_map_offset_t mask,
1283 vm_map_kernel_flags_t vmk_flags,
1284 memory_object_control_t control,
1285 vm_object_offset_t offset,
1286 boolean_t needs_copy,
1287 vm_prot_t cur_protection,
1288 vm_prot_t max_protection,
1289 vm_inherit_t inheritance);
1290
1291 extern kern_return_t vm_map_terminate(
1292 vm_map_t map);
1293
1294 extern void vm_map_require(
1295 vm_map_t map);
1296
1297 extern void vm_map_copy_require(
1298 vm_map_copy_t copy);
1299
1300 extern kern_return_t vm_map_copy_extract(
1301 vm_map_t src_map,
1302 vm_map_address_t src_addr,
1303 vm_map_size_t len,
1304 boolean_t copy,
1305 vm_map_copy_t *copy_result, /* OUT */
1306 vm_prot_t *cur_prot, /* OUT */
1307 vm_prot_t *max_prot, /* OUT */
1308 vm_inherit_t inheritance,
1309 vm_map_kernel_flags_t vmk_flags);
1310
1311 #endif /* !XNU_KERNEL_PRIVATE */
1312
1313 /* Discard a copy without using it */
1314 extern void vm_map_copy_discard(
1315 vm_map_copy_t copy);
1316
1317 /* Overwrite existing memory with a copy */
1318 extern kern_return_t vm_map_copy_overwrite(
1319 vm_map_t dst_map,
1320 vm_map_address_t dst_addr,
1321 vm_map_copy_t copy,
1322 vm_map_size_t copy_size,
1323 boolean_t interruptible);
1324
1325 #define VM_MAP_COPY_OVERWRITE_OPTIMIZATION_THRESHOLD_PAGES (3)
1326
1327
1328 /* returns TRUE if size of vm_map_copy == size parameter FALSE otherwise */
1329 extern boolean_t vm_map_copy_validate_size(
1330 vm_map_t dst_map,
1331 vm_map_copy_t copy,
1332 vm_map_size_t *size);
1333
1334 /* Place a copy into a map */
1335 extern kern_return_t vm_map_copyout(
1336 vm_map_t dst_map,
1337 vm_map_address_t *dst_addr, /* OUT */
1338 vm_map_copy_t copy);
1339
1340 extern kern_return_t vm_map_copyout_size(
1341 vm_map_t dst_map,
1342 vm_map_address_t *dst_addr, /* OUT */
1343 vm_map_copy_t copy,
1344 vm_map_size_t copy_size);
1345
1346 extern kern_return_t vm_map_copyout_internal(
1347 vm_map_t dst_map,
1348 vm_map_address_t *dst_addr, /* OUT */
1349 vm_map_copy_t copy,
1350 vm_map_size_t copy_size,
1351 boolean_t consume_on_success,
1352 vm_prot_t cur_protection,
1353 vm_prot_t max_protection,
1354 vm_inherit_t inheritance);
1355
1356 extern kern_return_t vm_map_copyin(
1357 vm_map_t src_map,
1358 vm_map_address_t src_addr,
1359 vm_map_size_t len,
1360 boolean_t src_destroy,
1361 vm_map_copy_t *copy_result); /* OUT */
1362
1363 extern kern_return_t vm_map_copyin_common(
1364 vm_map_t src_map,
1365 vm_map_address_t src_addr,
1366 vm_map_size_t len,
1367 boolean_t src_destroy,
1368 boolean_t src_volatile,
1369 vm_map_copy_t *copy_result, /* OUT */
1370 boolean_t use_maxprot);
1371
1372 #define VM_MAP_COPYIN_SRC_DESTROY 0x00000001
1373 #define VM_MAP_COPYIN_USE_MAXPROT 0x00000002
1374 #define VM_MAP_COPYIN_ENTRY_LIST 0x00000004
1375 #define VM_MAP_COPYIN_PRESERVE_PURGEABLE 0x00000008
1376 #define VM_MAP_COPYIN_FORK 0x00000010
1377 #define VM_MAP_COPYIN_ALL_FLAGS 0x0000001F
1378 extern kern_return_t vm_map_copyin_internal(
1379 vm_map_t src_map,
1380 vm_map_address_t src_addr,
1381 vm_map_size_t len,
1382 int flags,
1383 vm_map_copy_t *copy_result); /* OUT */
1384
1385
1386 extern void vm_map_disable_NX(
1387 vm_map_t map);
1388
1389 extern void vm_map_disallow_data_exec(
1390 vm_map_t map);
1391
1392 extern void vm_map_set_64bit(
1393 vm_map_t map);
1394
1395 extern void vm_map_set_32bit(
1396 vm_map_t map);
1397
1398 extern void vm_map_set_jumbo(
1399 vm_map_t map);
1400
1401 extern void vm_map_set_jit_entitled(
1402 vm_map_t map);
1403
1404 extern void vm_map_set_max_addr(
1405 vm_map_t map, vm_map_offset_t new_max_offset);
1406
1407 extern boolean_t vm_map_has_hard_pagezero(
1408 vm_map_t map,
1409 vm_map_offset_t pagezero_size);
1410 extern void vm_commit_pagezero_status(vm_map_t tmap);
1411
1412 extern boolean_t vm_map_tpro(
1413 vm_map_t map);
1414
1415 extern void vm_map_set_tpro(
1416 vm_map_t map);
1417
1418 extern boolean_t vm_map_tpro_enforcement(
1419 vm_map_t map);
1420
1421 extern void vm_map_set_tpro_enforcement(
1422 vm_map_t map);
1423
1424 extern boolean_t vm_map_set_tpro_range(
1425 vm_map_t map,
1426 vm_map_address_t start,
1427 vm_map_address_t end);
1428
1429 extern boolean_t vm_map_is_64bit(
1430 vm_map_t map);
1431
1432 extern kern_return_t vm_map_raise_max_offset(
1433 vm_map_t map,
1434 vm_map_offset_t new_max_offset);
1435
1436 extern kern_return_t vm_map_raise_min_offset(
1437 vm_map_t map,
1438 vm_map_offset_t new_min_offset);
1439
1440 #if XNU_TARGET_OS_OSX
1441 extern void vm_map_set_high_start(
1442 vm_map_t map,
1443 vm_map_offset_t high_start);
1444 #endif /* XNU_TARGET_OS_OSX */
1445
1446 extern vm_map_offset_t vm_compute_max_offset(
1447 boolean_t is64);
1448
1449 extern void vm_map_get_max_aslr_slide_section(
1450 vm_map_t map,
1451 int64_t *max_sections,
1452 int64_t *section_size);
1453
1454 extern uint64_t vm_map_get_max_aslr_slide_pages(
1455 vm_map_t map);
1456
1457 extern uint64_t vm_map_get_max_loader_aslr_slide_pages(
1458 vm_map_t map);
1459
1460 extern kern_return_t vm_map_set_size_limit(
1461 vm_map_t map,
1462 uint64_t limit);
1463
1464 extern kern_return_t vm_map_set_data_limit(
1465 vm_map_t map,
1466 uint64_t limit);
1467
1468 extern void vm_map_set_user_wire_limit(
1469 vm_map_t map,
1470 vm_size_t limit);
1471
1472 extern void vm_map_switch_protect(
1473 vm_map_t map,
1474 boolean_t val);
1475
1476 extern void vm_map_iokit_mapped_region(
1477 vm_map_t map,
1478 vm_size_t bytes);
1479
1480 extern void vm_map_iokit_unmapped_region(
1481 vm_map_t map,
1482 vm_size_t bytes);
1483
1484
1485 extern boolean_t first_free_is_valid(vm_map_t);
1486
1487 extern int vm_map_page_shift(
1488 vm_map_t map);
1489
1490 extern vm_map_offset_t vm_map_page_mask(
1491 vm_map_t map);
1492
1493 extern int vm_map_page_size(
1494 vm_map_t map);
1495
1496 extern vm_map_offset_t vm_map_round_page_mask(
1497 vm_map_offset_t offset,
1498 vm_map_offset_t mask);
1499
1500 extern vm_map_offset_t vm_map_trunc_page_mask(
1501 vm_map_offset_t offset,
1502 vm_map_offset_t mask);
1503
1504 extern boolean_t vm_map_page_aligned(
1505 vm_map_offset_t offset,
1506 vm_map_offset_t mask);
1507
1508 extern bool vm_map_range_overflows(
1509 vm_map_t map,
1510 vm_map_offset_t addr,
1511 vm_map_size_t size);
1512 #ifdef XNU_KERNEL_PRIVATE
1513
1514 /* Support for vm_map ranges */
1515 extern kern_return_t vm_map_range_configure(
1516 vm_map_t map);
1517
1518 extern void vm_map_range_fork(
1519 vm_map_t new_map,
1520 vm_map_t old_map);
1521
1522 extern int vm_map_get_user_range(
1523 vm_map_t map,
1524 vm_map_range_id_t range_id,
1525 mach_vm_range_t range);
1526
1527 /*!
1528 * @function vm_map_kernel_flags_update_range_id()
1529 *
1530 * @brief
1531 * Updates the @c vmkf_range_id field with the adequate value
1532 * according to the policy for specified map and tag set in @c vmk_flags.
1533 *
1534 * @discussion
1535 * This function is meant to be called by Mach VM entry points,
1536 * which matters for the kernel: allocations with pointers _MUST_
1537 * be allocated with @c kmem_*() functions.
1538 *
1539 * If the range ID is already set, it is preserved.
1540 */
1541 extern void vm_map_kernel_flags_update_range_id(
1542 vm_map_kernel_flags_t *flags,
1543 vm_map_t map);
1544
1545 #if XNU_TARGET_OS_OSX
1546 extern void vm_map_mark_alien(vm_map_t map);
1547 extern void vm_map_single_jit(vm_map_t map);
1548 #endif /* XNU_TARGET_OS_OSX */
1549
1550 extern kern_return_t vm_map_page_info(
1551 vm_map_t map,
1552 vm_map_offset_t offset,
1553 vm_page_info_flavor_t flavor,
1554 vm_page_info_t info,
1555 mach_msg_type_number_t *count);
1556 extern kern_return_t vm_map_page_range_info_internal(
1557 vm_map_t map,
1558 vm_map_offset_t start_offset,
1559 vm_map_offset_t end_offset,
1560 int effective_page_shift,
1561 vm_page_info_flavor_t flavor,
1562 vm_page_info_t info,
1563 mach_msg_type_number_t *count);
1564
1565 #endif /* XNU_KERNEL_PRIVATE */
1566 #ifdef MACH_KERNEL_PRIVATE
1567
1568
1569 /*
1570 * Internal macros for rounding and truncation of vm_map offsets and sizes
1571 */
1572 #define VM_MAP_ROUND_PAGE(x, pgmask) (((vm_map_offset_t)(x) + (pgmask)) & ~((signed)(pgmask)))
1573 #define VM_MAP_TRUNC_PAGE(x, pgmask) ((vm_map_offset_t)(x) & ~((signed)(pgmask)))
1574
1575 /*
1576 * Macros for rounding and truncation of vm_map offsets and sizes
1577 */
1578 static inline int
VM_MAP_PAGE_SHIFT(vm_map_t map)1579 VM_MAP_PAGE_SHIFT(
1580 vm_map_t map)
1581 {
1582 int shift = map ? map->hdr.page_shift : PAGE_SHIFT;
1583 /*
1584 * help ubsan and codegen in general,
1585 * cannot use PAGE_{MIN,MAX}_SHIFT
1586 * because of testing code which
1587 * tests 16k aligned maps on 4k only systems.
1588 */
1589 __builtin_assume(shift >= 12 && shift <= 14);
1590 return shift;
1591 }
1592
1593 #define VM_MAP_PAGE_SIZE(map) (1 << VM_MAP_PAGE_SHIFT((map)))
1594 #define VM_MAP_PAGE_MASK(map) (VM_MAP_PAGE_SIZE((map)) - 1)
1595 #define VM_MAP_PAGE_ALIGNED(x, pgmask) (((x) & (pgmask)) == 0)
1596
1597 static inline bool
VM_MAP_IS_EXOTIC(vm_map_t map __unused)1598 VM_MAP_IS_EXOTIC(
1599 vm_map_t map __unused)
1600 {
1601 #if __arm64__
1602 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT ||
1603 pmap_is_exotic(map->pmap)) {
1604 return true;
1605 }
1606 #endif /* __arm64__ */
1607 return false;
1608 }
1609
1610 static inline bool
VM_MAP_IS_ALIEN(vm_map_t map __unused)1611 VM_MAP_IS_ALIEN(
1612 vm_map_t map __unused)
1613 {
1614 /*
1615 * An "alien" process/task/map/pmap should mostly behave
1616 * as it currently would on iOS.
1617 */
1618 #if XNU_TARGET_OS_OSX
1619 if (map->is_alien) {
1620 return true;
1621 }
1622 return false;
1623 #else /* XNU_TARGET_OS_OSX */
1624 return true;
1625 #endif /* XNU_TARGET_OS_OSX */
1626 }
1627
1628 static inline bool
VM_MAP_POLICY_WX_FAIL(vm_map_t map __unused)1629 VM_MAP_POLICY_WX_FAIL(
1630 vm_map_t map __unused)
1631 {
1632 if (VM_MAP_IS_ALIEN(map)) {
1633 return false;
1634 }
1635 return true;
1636 }
1637
1638 static inline bool
VM_MAP_POLICY_WX_STRIP_X(vm_map_t map __unused)1639 VM_MAP_POLICY_WX_STRIP_X(
1640 vm_map_t map __unused)
1641 {
1642 if (VM_MAP_IS_ALIEN(map)) {
1643 return true;
1644 }
1645 return false;
1646 }
1647
1648 static inline bool
VM_MAP_POLICY_ALLOW_MULTIPLE_JIT(vm_map_t map __unused)1649 VM_MAP_POLICY_ALLOW_MULTIPLE_JIT(
1650 vm_map_t map __unused)
1651 {
1652 if (VM_MAP_IS_ALIEN(map) || map->single_jit) {
1653 return false;
1654 }
1655 return true;
1656 }
1657
1658 static inline bool
VM_MAP_POLICY_ALLOW_JIT_RANDOM_ADDRESS(vm_map_t map)1659 VM_MAP_POLICY_ALLOW_JIT_RANDOM_ADDRESS(
1660 vm_map_t map)
1661 {
1662 return VM_MAP_IS_ALIEN(map);
1663 }
1664
1665 static inline bool
VM_MAP_POLICY_ALLOW_JIT_INHERIT(vm_map_t map __unused)1666 VM_MAP_POLICY_ALLOW_JIT_INHERIT(
1667 vm_map_t map __unused)
1668 {
1669 if (VM_MAP_IS_ALIEN(map)) {
1670 return false;
1671 }
1672 return true;
1673 }
1674
1675 static inline bool
VM_MAP_POLICY_ALLOW_JIT_SHARING(vm_map_t map __unused)1676 VM_MAP_POLICY_ALLOW_JIT_SHARING(
1677 vm_map_t map __unused)
1678 {
1679 if (VM_MAP_IS_ALIEN(map)) {
1680 return false;
1681 }
1682 return true;
1683 }
1684
1685 static inline bool
VM_MAP_POLICY_ALLOW_JIT_COPY(vm_map_t map __unused)1686 VM_MAP_POLICY_ALLOW_JIT_COPY(
1687 vm_map_t map __unused)
1688 {
1689 if (VM_MAP_IS_ALIEN(map)) {
1690 return false;
1691 }
1692 return true;
1693 }
1694
1695 static inline bool
VM_MAP_POLICY_WRITABLE_SHARED_REGION(vm_map_t map __unused)1696 VM_MAP_POLICY_WRITABLE_SHARED_REGION(
1697 vm_map_t map __unused)
1698 {
1699 #if __x86_64__
1700 return true;
1701 #else /* __x86_64__ */
1702 if (VM_MAP_IS_EXOTIC(map)) {
1703 return true;
1704 }
1705 return false;
1706 #endif /* __x86_64__ */
1707 }
1708
1709 static inline void
vm_prot_to_wimg(unsigned int prot,unsigned int * wimg)1710 vm_prot_to_wimg(unsigned int prot, unsigned int *wimg)
1711 {
1712 switch (prot) {
1713 case MAP_MEM_NOOP: break;
1714 case MAP_MEM_IO: *wimg = VM_WIMG_IO; break;
1715 case MAP_MEM_COPYBACK: *wimg = VM_WIMG_USE_DEFAULT; break;
1716 case MAP_MEM_INNERWBACK: *wimg = VM_WIMG_INNERWBACK; break;
1717 case MAP_MEM_POSTED: *wimg = VM_WIMG_POSTED; break;
1718 case MAP_MEM_POSTED_REORDERED: *wimg = VM_WIMG_POSTED_REORDERED; break;
1719 case MAP_MEM_POSTED_COMBINED_REORDERED: *wimg = VM_WIMG_POSTED_COMBINED_REORDERED; break;
1720 case MAP_MEM_WTHRU: *wimg = VM_WIMG_WTHRU; break;
1721 case MAP_MEM_WCOMB: *wimg = VM_WIMG_WCOMB; break;
1722 case MAP_MEM_RT: *wimg = VM_WIMG_RT; break;
1723 default: break;
1724 }
1725 }
1726
1727 static inline boolean_t
vm_map_always_shadow(vm_map_t map)1728 vm_map_always_shadow(vm_map_t map)
1729 {
1730 if (map->mapped_in_other_pmaps) {
1731 /*
1732 * This is a submap, mapped in other maps.
1733 * Even if a VM object is mapped only once in this submap,
1734 * the submap itself could be mapped multiple times,
1735 * so vm_object_shadow() should always create a shadow
1736 * object, even if the object has only 1 reference.
1737 */
1738 return TRUE;
1739 }
1740 return FALSE;
1741 }
1742
1743 #endif /* MACH_KERNEL_PRIVATE */
1744 #ifdef XNU_KERNEL_PRIVATE
1745
1746 extern kern_return_t vm_map_set_page_shift(vm_map_t map, int pageshift);
1747 extern bool vm_map_is_exotic(vm_map_t map);
1748 extern bool vm_map_is_alien(vm_map_t map);
1749 extern pmap_t vm_map_get_pmap(vm_map_t map);
1750
1751 extern bool vm_map_is_corpse_source(vm_map_t map);
1752 extern void vm_map_set_corpse_source(vm_map_t map);
1753 extern void vm_map_unset_corpse_source(vm_map_t map);
1754 #endif /* XNU_KERNEL_PRIVATE */
1755
1756 #define vm_map_round_page(x, pgmask) (((vm_map_offset_t)(x) + (pgmask)) & ~((signed)(pgmask)))
1757 #define vm_map_trunc_page(x, pgmask) ((vm_map_offset_t)(x) & ~((signed)(pgmask)))
1758
1759 /* Support for UPLs from vm_maps */
1760
1761 #ifdef XNU_KERNEL_PRIVATE
1762
1763 extern kern_return_t vm_map_get_upl(
1764 vm_map_t target_map,
1765 vm_map_offset_t map_offset,
1766 upl_size_t *size,
1767 upl_t *upl,
1768 upl_page_info_array_t page_info,
1769 unsigned int *page_infoCnt,
1770 upl_control_flags_t *flags,
1771 vm_tag_t tag,
1772 int force_data_sync);
1773
1774 #endif /* XNU_KERNEL_PRIVATE */
1775
1776 extern void
1777 vm_map_sizes(vm_map_t map,
1778 vm_map_size_t * psize,
1779 vm_map_size_t * pfree,
1780 vm_map_size_t * plargest_free);
1781
1782 #if CONFIG_DYNAMIC_CODE_SIGNING
1783
1784 extern kern_return_t vm_map_sign(vm_map_t map,
1785 vm_map_offset_t start,
1786 vm_map_offset_t end);
1787
1788 #endif /* CONFIG_DYNAMIC_CODE_SIGNING */
1789
1790 extern kern_return_t vm_map_partial_reap(
1791 vm_map_t map,
1792 unsigned int *reclaimed_resident,
1793 unsigned int *reclaimed_compressed);
1794
1795
1796 #if DEVELOPMENT || DEBUG
1797
1798 extern int vm_map_disconnect_page_mappings(
1799 vm_map_t map,
1800 boolean_t);
1801
1802 extern kern_return_t vm_map_inject_error(vm_map_t map, vm_map_offset_t vaddr);
1803
1804 #endif /* DEVELOPMENT || DEBUG */
1805
1806 #if CONFIG_FREEZE
1807
1808 extern kern_return_t vm_map_freeze(
1809 task_t task,
1810 unsigned int *purgeable_count,
1811 unsigned int *wired_count,
1812 unsigned int *clean_count,
1813 unsigned int *dirty_count,
1814 unsigned int dirty_budget,
1815 unsigned int *shared_count,
1816 int *freezer_error_code,
1817 boolean_t eval_only);
1818
1819 __enum_decl(freezer_error_code_t, int, {
1820 FREEZER_ERROR_GENERIC = -1,
1821 FREEZER_ERROR_EXCESS_SHARED_MEMORY = -2,
1822 FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO = -3,
1823 FREEZER_ERROR_NO_COMPRESSOR_SPACE = -4,
1824 FREEZER_ERROR_NO_SWAP_SPACE = -5,
1825 FREEZER_ERROR_NO_SLOTS = -6,
1826 });
1827
1828 #endif /* CONFIG_FREEZE */
1829 #if XNU_KERNEL_PRIVATE
1830
1831 boolean_t kdp_vm_map_is_acquired_exclusive(vm_map_t map);
1832
1833 boolean_t vm_map_entry_has_device_pager(vm_map_t, vm_map_offset_t vaddr);
1834
1835 #endif /* XNU_KERNEL_PRIVATE */
1836
1837 /*
1838 * In some cases, we don't have a real VM object but still want to return a
1839 * unique ID (to avoid a memory region looking like shared memory), so build
1840 * a fake pointer based on the map's ledger and the index of the ledger being
1841 * reported.
1842 */
1843 #define VM_OBJECT_ID_FAKE(map, ledger_id) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM((int*)((map)->pmap->ledger)+(ledger_id)))
1844
1845 #endif /* KERNEL_PRIVATE */
1846
1847 __END_DECLS
1848
1849 #endif /* _VM_VM_MAP_H_ */
1850