1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _VM_VM_KERN_XNU_H_
30 #define _VM_VM_KERN_XNU_H_
31
32 #include <sys/cdefs.h>
33 #include <vm/vm_kern.h>
34
35 __BEGIN_DECLS
36 #pragma GCC visibility push(hidden)
37 #ifdef XNU_KERNEL_PRIVATE
38
39
40 #pragma mark - the kmem subsystem
41
42 /*
43 * "kmem" is a set of methods that provide interfaces suitable
44 * to allocate memory from the VM in the kernel map or submaps.
45 *
46 * It provide leaner alternatives to some of the VM functions,
47 * closer to a typical allocator.
48 */
49
50 struct mach_memory_info;
51 struct vm_page;
52 struct vm_map_entry;
53
54 /*!
55 * @typedef
56 *
57 * @brief
58 * Pair of a return code and size/address/... used by kmem interfaces.
59 *
60 * @discussion
61 * Using a pair of integers allows the compiler to return everything
62 * through registers, and doesn't need to use stack values to get results,
63 * which yields significantly better codegen.
64 *
65 * If @c kmr_return is not @c KERN_SUCCESS, then the other field
66 * of the union is always supposed to be 0.
67 */
68 typedef struct {
69 kern_return_t kmr_return;
70 union {
71 vm_address_t kmr_address;
72 vm_size_t kmr_size;
73 void *kmr_ptr;
74 vm_map_t kmr_submap;
75 };
76 } kmem_return_t;
77
78 /*!
79 * @typedef kmem_guard_t
80 *
81 * @brief
82 * KMEM guards are used by the kmem_* subsystem to secure atomic allocations.
83 *
84 * @discussion
85 * This parameter is used to transmit the tag for the allocation.
86 *
87 * If @c kmg_atomic is set, then the other fields are also taken into account
88 * and will affect the allocation behavior for this allocation.
89 *
90 * @field kmg_tag The VM_KERN_MEMORY_* tag for this entry.
91 * @field kmg_type_hash Some hash related to the type of the allocation.
92 * @field kmg_atomic Whether the entry is atomic.
93 * @field kmg_submap Whether the entry is for a submap.
94 * @field kmg_context A use defined 30 bits that will be stored
95 * on the entry on allocation and checked
96 * on other operations.
97 */
98 typedef struct {
99 uint16_t kmg_tag;
100 uint16_t kmg_type_hash;
101 uint32_t kmg_atomic : 1;
102 uint32_t kmg_submap : 1;
103 uint32_t kmg_context : 30;
104 } kmem_guard_t;
105 #define KMEM_GUARD_NONE (kmem_guard_t){ }
106 #define KMEM_GUARD_SUBMAP (kmem_guard_t){ .kmg_atomic = 0, .kmg_submap = 1 }
107
108
109 /*!
110 * @typedef kmem_flags_t
111 *
112 * @brief
113 * Sets of flags taken by several of the @c kmem_* family of functions.
114 *
115 * @discussion
116 * This type is not used directly by any function, it is an underlying raw
117 * type that is re-vended under different namespaces for each @c kmem_*
118 * interface.
119 *
120 * - @c kmem_alloc uses @c kma_flags_t / @c KMA_* namespaced values.
121 * - @c kmem_suballoc uses @c kms_flags_t / @c KMS_* namespaced values.
122 * - @c kmem_realloc uses @c kmr_flags_t / @c KMR_* namespaced values.
123 * - @c kmem_free uses @c kmf_flags_t / @c KMF_* napespaced values.
124 *
125 *
126 * <h2>Call behavior</h2>
127 *
128 * @const KMEM_NONE (all)
129 * Pass this when no special options is to be used.
130 *
131 * @const KMEM_NOFAIL (alloc, suballoc)
132 * When this flag is passed, any allocation failure results into a panic().
133 * Using this flag should really be limited to cases when failure is not
134 * recoverable and possibly during early boot only.
135 *
136 * @const KMEM_NOPAGEWAIT (alloc, realloc)
137 * Pass this flag if the system should not wait in VM_PAGE_WAIT().
138 *
139 * @const KMEM_FREEOLD (realloc)
140 * Pass this flag if @c kmem_realloc should free the old mapping
141 * (when the address changed) as part of the call.
142 *
143 * @const KMEM_REALLOCF (realloc)
144 * Similar to @c Z_REALLOCF: if the call is failing,
145 * then free the old allocation too.
146 *
147 *
148 * <h2>How the entry is populated</h2>
149 *
150 * @const KMEM_VAONLY (alloc)
151 * By default memory allocated by the kmem subsystem is wired and mapped.
152 * Passing @c KMEM_VAONLY will cause the range to still be wired,
153 * but no page is actually mapped.
154 *
155 * @const KMEM_PAGEABLE (alloc)
156 * By default memory allocated by the kmem subsystem is wired and mapped.
157 * Passing @c KMEM_PAGEABLE makes the entry non wired, and pages will be
158 * added to the entry as it faults.
159 *
160 * @const KMEM_ZERO (alloc, realloc)
161 * Any new page added is zeroed.
162 *
163 *
164 * <h2>VM object to use for the entry</h2>
165 *
166 * @const KMEM_KOBJECT (alloc, realloc)
167 * The entry will be made for the @c kernel_object.
168 *
169 * Note that the @c kernel_object is just a "collection of pages".
170 * Pages in that object can't be remaped or present in several VM maps
171 * like traditional objects.
172 *
173 * If neither @c KMEM_KOBJECT nor @c KMEM_COMPRESSOR is passed,
174 * the a new fresh VM object will be made for this allocation.
175 * This is expensive and should be limited to allocations that
176 * need the features associated with a VM object.
177 *
178 * @const KMEM_COMPRESSOR (alloc)
179 * The entry is allocated for the @c compressor_object.
180 * Pages belonging to the compressor are not on the paging queues,
181 * nor are they counted as wired.
182 *
183 * Only the VM Compressor subsystem should use this.
184 *
185 *
186 * <h2>How to look for addresses</h2>
187 *
188 * @const KMEM_LOMEM (alloc, realloc)
189 * The physical memory allocated must be in the first 4G of memory,
190 * in order to support hardware controllers incapable of generating DMAs
191 * with more than 32bits of physical address.
192 *
193 * @const KMEM_LAST_FREE (alloc, suballoc, realloc)
194 * When looking for space in the specified map,
195 * start scanning for addresses from the end of the map
196 * rather than the start.
197 *
198 * @const KMEM_DATA (alloc, suballoc, realloc)
199 * The memory must be allocated from the "Data" range.
200 *
201 * @const KMEM_SPRAYQTN (alloc, realloc)
202 * The memory must be allocated from the "spray quarantine" range. For more
203 * details on what allocations qualify to use this flag see
204 * @c KMEM_RANGE_ID_SPRAYQTN.
205 *
206 * @const KMEM_GUESS_SIZE (free)
207 * When freeing an atomic entry (requires a valid kmem guard),
208 * then look up the entry size because the caller didn't
209 * preserve it.
210 *
211 * This flag is only here in order to support kfree_data_addr(),
212 * and shall not be used by any other clients.
213 *
214 * <h2>Entry properties</h2>
215 *
216 * @const KMEM_PERMANENT (alloc, suballoc)
217 * The entry is made permanent.
218 *
219 * In the kernel maps, permanent entries can never be deleted.
220 * Calling @c kmem_free() on such a range will panic.
221 *
222 * In user maps, permanent entries will only be deleted
223 * whenthe map is terminated.
224 *
225 * @const KMEM_GUARD_FIRST (alloc, realloc)
226 * @const KMEM_GUARD_LAST (alloc, realloc)
227 * Asks @c kmem_* to put a guard page at the beginning (resp. end)
228 * of the allocation.
229 *
230 * The allocation size will not be extended to accomodate for guards,
231 * and the client of this interface must take them into account.
232 * Typically if a usable range of 3 pages is needed with both guards,
233 * then 5 pages must be asked.
234 *
235 * Alignment constraints take guards into account (the aligment applies
236 * to the address right after the first guard page).
237 *
238 * The returned address for allocation will pointing at the entry start,
239 * which is the address of the left guard page if any.
240 *
241 * Note that if @c kmem_realloc* is called, the *exact* same
242 * guard flags must be passed for this entry. The KMEM subsystem
243 * is generally oblivious to guards, and passing inconsistent flags
244 * will cause pages to be moved incorrectly.
245 *
246 * @const KMEM_KSTACK (alloc)
247 * This flag must be passed when the allocation is for kernel stacks.
248 * This only has an effect on Intel.
249 *
250 * @const KMEM_NOENCRYPT (alloc)
251 * Obsolete, will be repurposed soon.
252 *
253 * @const KMEM_KASAN_GUARD (alloc, realloc, free)
254 * Under KASAN_CLASSIC add guards left and right to this allocation
255 * in order to detect out of bounds.
256 *
257 * This can't be passed if any of @c KMEM_GUARD_FIRST
258 * or @c KMEM_GUARD_LAST is used.
259 *
260 * @const KMEM_TAG (alloc, realloc, free)
261 * Under KASAN_TBI, this allocation is tagged non canonically.
262 */
263 __options_decl(kmem_flags_t, uint32_t, {
264 KMEM_NONE = 0x00000000,
265
266 /* Call behavior */
267 KMEM_NOFAIL = 0x00000001,
268 KMEM_NOPAGEWAIT = 0x00000002,
269 KMEM_FREEOLD = 0x00000004,
270 KMEM_REALLOCF = 0x00000008,
271
272 /* How the entry is populated */
273 KMEM_VAONLY = 0x00000010,
274 KMEM_PAGEABLE = 0x00000020,
275 KMEM_ZERO = 0x00000040,
276
277 /* VM object to use for the entry */
278 KMEM_KOBJECT = 0x00000100,
279 KMEM_COMPRESSOR = 0x00000200,
280
281 /* How to look for addresses */
282 KMEM_LOMEM = 0x00001000,
283 KMEM_LAST_FREE = 0x00002000,
284 KMEM_GUESS_SIZE = 0x00004000,
285 KMEM_DATA = 0x00008000,
286 KMEM_SPRAYQTN = 0x00010000,
287
288 /* Entry properties */
289 KMEM_PERMANENT = 0x00100000,
290 KMEM_GUARD_FIRST = 0x00200000,
291 KMEM_GUARD_LAST = 0x00400000,
292 KMEM_KSTACK = 0x00800000,
293 KMEM_NOENCRYPT = 0x01000000,
294 KMEM_KASAN_GUARD = 0x02000000,
295 KMEM_TAG = 0x04000000,
296 });
297
298
299 /*
300 * @function kmem_range_id_size
301 *
302 * @abstract Return the addressable size of the memory range.
303 */
304 __pure2
305 extern vm_map_size_t kmem_range_id_size(
306 kmem_range_id_t range_id);
307
308 /**
309 * @enum kmem_claims_flags_t
310 *
311 * @abstract
312 * Set of flags used in the processing of kmem_range claims
313 *
314 * @discussion
315 * These flags are used by the kmem subsytem while processing kmem_range
316 * claims and are not explicitly passed by the caller registering the claim.
317 *
318 * @const KC_NO_ENTRY
319 * A vm map entry should not be created for the respective claim.
320 *
321 * @const KC_NO_MOVE
322 * The range shouldn't be moved once it has been placed as it has constraints.
323 */
324 __options_decl(kmem_claims_flags_t, uint32_t, {
325 KC_NONE = 0x00000000,
326 KC_NO_ENTRY = 0x00000001,
327 KC_NO_MOVE = 0x00000002,
328 });
329
330 /*
331 * Security config that creates the additional splits in non data part of
332 * kernel_map
333 */
334 #if KASAN || (__arm64__ && !defined(KERNEL_INTEGRITY_KTRR) && !defined(KERNEL_INTEGRITY_CTRR))
335 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT OFF
336 #else
337 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT ON
338 #endif
339
340 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__OFF() 0
341 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__ON() 1
342 #define ZSECURITY_CONFIG2(v) ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__##v()
343 #define ZSECURITY_CONFIG1(v) ZSECURITY_CONFIG2(v)
344 #define ZSECURITY_CONFIG(opt) ZSECURITY_CONFIG1(ZSECURITY_CONFIG_##opt)
345
346 struct kmem_range_startup_spec {
347 const char *kc_name;
348 struct mach_vm_range *kc_range;
349 vm_map_size_t kc_size;
350 vm_map_size_t (^kc_calculate_sz)(void);
351 kmem_claims_flags_t kc_flags;
352 };
353
354 extern void kmem_range_startup_init(
355 struct kmem_range_startup_spec *sp);
356
357 /*!
358 * @macro KMEM_RANGE_REGISTER_*
359 *
360 * @abstract
361 * Register a claim for kmem range or submap.
362 *
363 * @discussion
364 * Claims are shuffled during startup to randomize the layout of the kernel map.
365 * Temporary entries are created in place of the claims, therefore the caller
366 * must provide the start of the assigned range as a hint and
367 * @c{VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE} to kmem_suballoc to replace the mapping.
368 *
369 * Min/max constraints can be provided in the range when the claim is
370 * registered.
371 *
372 * This macro comes in 2 flavors:
373 * - STATIC : When the size of the range/submap is known at compile time
374 * - DYNAMIC: When the size of the range/submap needs to be computed
375 * Temporary entries are create
376 * The start of the
377 *
378 * @param name the name of the claim
379 * @param range the assigned range for the claim
380 * @param size the size of submap/range (if known at compile time)
381 * @param calculate_sz a block that returns the computed size of submap/range
382 */
383 #define KMEM_RANGE_REGISTER_STATIC(name, range, size) \
384 static __startup_data struct kmem_range_startup_spec \
385 __startup_kmem_range_spec_ ## name = { #name, range, size, NULL, KC_NONE}; \
386 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
387 &__startup_kmem_range_spec_ ## name)
388
389 #define KMEM_RANGE_REGISTER_DYNAMIC(name, range, calculate_sz) \
390 static __startup_data struct kmem_range_startup_spec \
391 __startup_kmem_range_spec_ ## name = { #name, range, 0, calculate_sz, \
392 KC_NONE}; \
393 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
394 &__startup_kmem_range_spec_ ## name)
395
396
397 #pragma mark kmem entry parameters
398
399 /*!
400 * @function kmem_entry_validate_guard()
401 *
402 * @brief
403 * Validates that the entry matches the input parameters, panic otherwise.
404 *
405 * @discussion
406 * If the guard has a zero @c kmg_guard value,
407 * then the entry must be non atomic.
408 *
409 * The guard tag is not used for validation as the VM subsystems
410 * (particularly in IOKit) might decide to substitute it in ways
411 * that are difficult to predict for the programmer.
412 *
413 * @param entry the entry to validate
414 * @param addr the supposed start address
415 * @param size the supposed size of the entry
416 * @param guard the guard to use to "authenticate" the allocation.
417 */
418 extern void kmem_entry_validate_guard(
419 vm_map_t map,
420 struct vm_map_entry *entry,
421 vm_offset_t addr,
422 vm_size_t size,
423 kmem_guard_t guard);
424
425 /*!
426 * @function kmem_size_guard()
427 *
428 * @brief
429 * Returns the size of an atomic kalloc allocation made in the specified map,
430 * according to the guard.
431 *
432 * @param map a kernel map to lookup the entry into.
433 * @param addr the kernel address to lookup.
434 * @param guard the guard to use to "authenticate" the allocation.
435 */
436 extern vm_size_t kmem_size_guard(
437 vm_map_t map,
438 vm_offset_t addr,
439 kmem_guard_t guard);
440
441
442 #pragma mark kmem allocations
443
444 /*!
445 * @typedef kma_flags_t
446 *
447 * @brief
448 * Flags used by the @c kmem_alloc* family of flags.
449 */
450 __options_decl(kma_flags_t, uint32_t, {
451 KMA_NONE = KMEM_NONE,
452
453 /* Call behavior */
454 KMA_NOFAIL = KMEM_NOFAIL,
455 KMA_NOPAGEWAIT = KMEM_NOPAGEWAIT,
456
457 /* How the entry is populated */
458 KMA_VAONLY = KMEM_VAONLY,
459 KMA_PAGEABLE = KMEM_PAGEABLE,
460 KMA_ZERO = KMEM_ZERO,
461
462 /* VM object to use for the entry */
463 KMA_KOBJECT = KMEM_KOBJECT,
464 KMA_COMPRESSOR = KMEM_COMPRESSOR,
465
466 /* How to look for addresses */
467 KMA_LOMEM = KMEM_LOMEM,
468 KMA_LAST_FREE = KMEM_LAST_FREE,
469 KMA_DATA = KMEM_DATA,
470 KMA_SPRAYQTN = KMEM_SPRAYQTN,
471
472 /* Entry properties */
473 KMA_PERMANENT = KMEM_PERMANENT,
474 KMA_GUARD_FIRST = KMEM_GUARD_FIRST,
475 KMA_GUARD_LAST = KMEM_GUARD_LAST,
476 KMA_KSTACK = KMEM_KSTACK,
477 KMA_NOENCRYPT = KMEM_NOENCRYPT,
478 KMA_KASAN_GUARD = KMEM_KASAN_GUARD,
479 KMA_TAG = KMEM_TAG,
480 });
481
482
483 /*!
484 * @function kmem_alloc_guard()
485 *
486 * @brief
487 * Master entry point for allocating kernel memory.
488 *
489 * @param map map to allocate into, must be a kernel map.
490 * @param size the size of the entry to allocate, must not be 0.
491 * @param mask an alignment mask that the returned allocation
492 * will be aligned to (ignoring guards, see @const
493 * KMEM_GUARD_FIRST).
494 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
495 * @param guard how to guard the allocation.
496 *
497 * @returns
498 * - the non zero address of the allocaation on success in @c kmr_address.
499 * - @c KERN_NO_SPACE if the target map is out of address space.
500 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
501 */
502 extern kmem_return_t kmem_alloc_guard(
503 vm_map_t map,
504 vm_size_t size,
505 vm_offset_t mask,
506 kma_flags_t flags,
507 kmem_guard_t guard) __result_use_check;
508
509 static inline kern_return_t
kernel_memory_allocate(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,kma_flags_t flags,vm_tag_t tag)510 kernel_memory_allocate(
511 vm_map_t map,
512 vm_offset_t *addrp,
513 vm_size_t size,
514 vm_offset_t mask,
515 kma_flags_t flags,
516 vm_tag_t tag)
517 {
518 kmem_guard_t guard = {
519 .kmg_tag = tag,
520 };
521 kmem_return_t kmr;
522
523 kmr = kmem_alloc_guard(map, size, mask, flags, guard);
524 if (kmr.kmr_return == KERN_SUCCESS) {
525 __builtin_assume(kmr.kmr_address != 0);
526 } else {
527 __builtin_assume(kmr.kmr_address == 0);
528 }
529 *addrp = kmr.kmr_address;
530 return kmr.kmr_return;
531 }
532
533 static inline kern_return_t
kmem_alloc(vm_map_t map,vm_offset_t * addrp,vm_size_t size,kma_flags_t flags,vm_tag_t tag)534 kmem_alloc(
535 vm_map_t map,
536 vm_offset_t *addrp,
537 vm_size_t size,
538 kma_flags_t flags,
539 vm_tag_t tag)
540 {
541 return kernel_memory_allocate(map, addrp, size, 0, flags, tag);
542 }
543
544 /*!
545 * @function kmem_alloc_contig_guard()
546 *
547 * @brief
548 * Variant of kmem_alloc_guard() that allocates a contiguous range
549 * of physical memory.
550 *
551 * @param map map to allocate into, must be a kernel map.
552 * @param size the size of the entry to allocate, must not be 0.
553 * @param mask an alignment mask that the returned allocation
554 * will be aligned to (ignoring guards, see @const
555 * KMEM_GUARD_FIRST).
556 * @param max_pnum The maximum page number to allocate, or 0.
557 * @param pnum_mask A page number alignment mask for the first allocated
558 * page, or 0.
559 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
560 * @param guard how to guard the allocation.
561 *
562 * @returns
563 * - the non zero address of the allocaation on success in @c kmr_address.
564 * - @c KERN_NO_SPACE if the target map is out of address space.
565 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
566 */
567 extern kmem_return_t kmem_alloc_contig_guard(
568 vm_map_t map,
569 vm_size_t size,
570 vm_offset_t mask,
571 ppnum_t max_pnum,
572 ppnum_t pnum_mask,
573 kma_flags_t flags,
574 kmem_guard_t guard);
575
576 static inline kern_return_t
kmem_alloc_contig(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,ppnum_t max_pnum,ppnum_t pnum_mask,kma_flags_t flags,vm_tag_t tag)577 kmem_alloc_contig(
578 vm_map_t map,
579 vm_offset_t *addrp,
580 vm_size_t size,
581 vm_offset_t mask,
582 ppnum_t max_pnum,
583 ppnum_t pnum_mask,
584 kma_flags_t flags,
585 vm_tag_t tag)
586 {
587 kmem_guard_t guard = {
588 .kmg_tag = tag,
589 };
590 kmem_return_t kmr;
591
592 kmr = kmem_alloc_contig_guard(map, size, mask,
593 max_pnum, pnum_mask, flags, guard);
594 if (kmr.kmr_return == KERN_SUCCESS) {
595 __builtin_assume(kmr.kmr_address != 0);
596 } else {
597 __builtin_assume(kmr.kmr_address == 0);
598 }
599 *addrp = kmr.kmr_address;
600 return kmr.kmr_return;
601 }
602
603
604 /*!
605 * @typedef kms_flags_t
606 *
607 * @brief
608 * Flags used by @c kmem_suballoc.
609 */
610 __options_decl(kms_flags_t, uint32_t, {
611 KMS_NONE = KMEM_NONE,
612
613 /* Call behavior */
614 KMS_NOFAIL = KMEM_NOFAIL,
615
616 /* How to look for addresses */
617 KMS_LAST_FREE = KMEM_LAST_FREE,
618 KMS_DATA = KMEM_DATA,
619
620 /* Entry properties */
621 KMS_PERMANENT = KMEM_PERMANENT,
622 });
623
624 /*!
625 * @function kmem_suballoc()
626 *
627 * @brief
628 * Create a kernel submap, in an atomic entry guarded with KMEM_GUARD_SUBMAP.
629 *
630 * @param parent map to allocate into, must be a kernel map.
631 * @param addr (in/out) the address for the map (see vm_map_enter)
632 * @param size the size of the entry to allocate, must not be 0.
633 * @param vmc_options the map creation options
634 * @param vm_flags a set of @c VM_FLAGS_* flags
635 * @param flags a set of @c KMS_* flags, (@see @c kmem_flags_t)
636 * @param tag the tag for this submap's entry.
637 */
638 extern kmem_return_t kmem_suballoc(
639 vm_map_t parent,
640 mach_vm_offset_t *addr,
641 vm_size_t size,
642 vm_map_create_options_t vmc_options,
643 int vm_flags,
644 kms_flags_t flags,
645 vm_tag_t tag);
646
647
648 #pragma mark kmem reallocation
649
650 /*!
651 * @typedef kmr_flags_t
652 *
653 * @brief
654 * Flags used by the @c kmem_realloc* family of flags.
655 */
656 __options_decl(kmr_flags_t, uint32_t, {
657 KMR_NONE = KMEM_NONE,
658
659 /* Call behavior */
660 KMR_NOPAGEWAIT = KMEM_NOPAGEWAIT,
661 KMR_FREEOLD = KMEM_FREEOLD,
662 KMR_REALLOCF = KMEM_REALLOCF,
663
664 /* How the entry is populated */
665 KMR_ZERO = KMEM_ZERO,
666
667 /* VM object to use for the entry */
668 KMR_KOBJECT = KMEM_KOBJECT,
669
670 /* How to look for addresses */
671 KMR_LOMEM = KMEM_LOMEM,
672 KMR_LAST_FREE = KMEM_LAST_FREE,
673 KMR_DATA = KMEM_DATA,
674 KMR_SPRAYQTN = KMEM_SPRAYQTN,
675
676 /* Entry properties */
677 KMR_GUARD_FIRST = KMEM_GUARD_FIRST,
678 KMR_GUARD_LAST = KMEM_GUARD_LAST,
679 KMR_KASAN_GUARD = KMEM_KASAN_GUARD,
680 KMR_TAG = KMEM_TAG,
681 });
682
683 #define KMEM_REALLOC_FLAGS_VALID(flags) \
684 (((flags) & (KMR_KOBJECT | KMEM_GUARD_LAST | KMEM_KASAN_GUARD | KMR_DATA)) == KMR_DATA || ((flags) & KMR_FREEOLD))
685
686 /*!
687 * @function kmem_realloc_guard()
688 *
689 * @brief
690 * Reallocates memory allocated with kmem_alloc_guard()
691 *
692 * @discussion
693 * @c kmem_realloc_guard() either mandates a guard with atomicity set,
694 * or must use KMR_DATA (this is not an implementation limitation but
695 * but a security policy).
696 *
697 * If kmem_realloc_guard() is called for the kernel object
698 * (with @c KMR_KOBJECT) or with any trailing guard page,
699 * then the use of @c KMR_FREEOLD is mandatory.
700 *
701 * When @c KMR_FREEOLD isn't used, if the allocation was relocated
702 * as opposed to be extended or truncated in place, the caller
703 * must free its old mapping manually by calling @c kmem_free_guard().
704 *
705 * Note that if the entry is truncated, it will always be done in place.
706 *
707 *
708 * @param map map to allocate into, must be a kernel map.
709 * @param oldaddr the address to reallocate,
710 * passing 0 means @c kmem_alloc_guard() will be called.
711 * @param oldsize the current size of the entry
712 * @param newsize the new size of the entry,
713 * 0 means kmem_free_guard() will be called.
714 * @param flags a set of @c KMR_* flags, (@see @c kmem_flags_t)
715 * the exact same set of @c KMR_GUARD_* flags must
716 * be passed for all calls (@see kmem_flags_t).
717 * @param guard the allocation guard.
718 *
719 * @returns
720 * - the newly allocated address on success in @c kmr_address
721 * (note that if newsize is 0, then address will be 0 too).
722 * - @c KERN_NO_SPACE if the target map is out of address space.
723 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
724 */
725 extern kmem_return_t kmem_realloc_guard(
726 vm_map_t map,
727 vm_offset_t oldaddr,
728 vm_size_t oldsize,
729 vm_size_t newsize,
730 kmr_flags_t flags,
731 kmem_guard_t guard) __result_use_check
732 __attribute__((diagnose_if(!KMEM_REALLOC_FLAGS_VALID(flags),
733 "invalid realloc flags passed", "error")));
734
735 /*!
736 * @function kmem_realloc_should_free()
737 *
738 * @brief
739 * Returns whether the old address passed to a @c kmem_realloc_guard()
740 * call without @c KMR_FREEOLD must be freed.
741 *
742 * @param oldaddr the "oldaddr" passed to @c kmem_realloc_guard().
743 * @param kmr the result of that @c kmem_realloc_should_free() call.
744 */
745 static inline bool
kmem_realloc_should_free(vm_offset_t oldaddr,kmem_return_t kmr)746 kmem_realloc_should_free(
747 vm_offset_t oldaddr,
748 kmem_return_t kmr)
749 {
750 return oldaddr && oldaddr != kmr.kmr_address;
751 }
752
753
754 #pragma mark kmem free
755
756 /*!
757 * @typedef kmf_flags_t
758 *
759 * @brief
760 * Flags used by the @c kmem_free* family of flags.
761 */
762 __options_decl(kmf_flags_t, uint32_t, {
763 KMF_NONE = KMEM_NONE,
764
765 /* Call behavior */
766
767 /* How the entry is populated */
768
769 /* How to look for addresses */
770 KMF_GUESS_SIZE = KMEM_GUESS_SIZE,
771 KMF_KASAN_GUARD = KMEM_KASAN_GUARD,
772 KMF_TAG = KMEM_TAG,
773 });
774
775
776 /*!
777 * @function kmem_free_guard()
778 *
779 * @brief
780 * Frees memory allocated with @c kmem_alloc or @c kmem_realloc.
781 *
782 * @param map map to free from, must be a kernel map.
783 * @param addr the address to free
784 * @param size the size of the memory to free
785 * @param flags a set of @c KMF_* flags, (@see @c kmem_flags_t)
786 * @param guard the allocation guard.
787 *
788 * @returns the size of the entry that was deleted.
789 * (useful when @c KMF_GUESS_SIZE was used)
790 */
791 extern vm_size_t kmem_free_guard(
792 vm_map_t map,
793 vm_offset_t addr,
794 vm_size_t size,
795 kmf_flags_t flags,
796 kmem_guard_t guard);
797
798 static inline void
kmem_free(vm_map_t map,vm_offset_t addr,vm_size_t size)799 kmem_free(
800 vm_map_t map,
801 vm_offset_t addr,
802 vm_size_t size)
803 {
804 kmem_free_guard(map, addr, size, KMF_NONE, KMEM_GUARD_NONE);
805 }
806
807
808 #pragma mark kmem population
809
810 /*!
811 * @function kernel_memory_populate()
812 *
813 * @brief
814 * Populate pages for a given kernel map allocation.
815 *
816 * @discussion
817 * Allocations made against the kernel object (@c KMEM_KOBJECT)
818 * or the compressor object (@c KMEM_COMPRESSOR) must have their
819 * backing store explicitly managed by clients.
820 *
821 * This function will cause pages in the specified range to be allocated
822 * explicitly. No page must have been allocated for that range at this time,
823 * either because the allocation was done with @c KMEM_VAONLY or because pages
824 * were explicitly depopulated.
825 *
826 * @param addr the aligned starting address to populate.
827 * @param size the aligned size of the region to populate.
828 * @param flags a set flags that must match the flags passed at
829 * @c kmem_alloc*() time. In particular, one of
830 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
831 * @param tag the kernel memory tag to use for accounting purposes.
832 * @returns
833 * - KERN_SUCCESS the operation succeeded.
834 * - KERN_RESOURCE_SHORTAGE
835 * the kernel was out of physical pages.
836 */
837 extern kern_return_t kernel_memory_populate(
838 vm_offset_t addr,
839 vm_size_t size,
840 kma_flags_t flags,
841 vm_tag_t tag);
842
843
844 /*!
845 * @function kernel_memory_depopulate()
846 *
847 * @brief
848 * Depopulate pages for a given kernel map allocation.
849 *
850 * @discussion
851 * Allocations made against the kernel object (@c KMEM_KOBJECT)
852 * or the compressor object (@c KMEM_COMPRESSOR) must have their
853 * backing store explicitly managed by clients.
854 *
855 * This function will cause pages in the specified range to be deallocated
856 * explicitly. This range must be populated at the time of the call, either
857 * because the @c kmem_alloc*() call asked for pages, or because
858 * @c kernel_memory_populate() has been called explicitly.
859 *
860 * It is not necessary to explicitly depopulate ranges prior to calling
861 * @c kmem_free*(), even if populating the range was made explicitly with
862 * @c kernel_memory_populate() rather than implicitly at allocation time.
863 *
864 *
865 * @param addr the aligned starting address to depopulate.
866 * @param size the aligned size of the region to depopulate.
867 * @param flags a set flags that must match the flags passed at
868 * @c kmem_alloc*() time. In particular, one of
869 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
870 * @param tag the kernel memory tag to use for accounting purposes,
871 * which must match the tag used for population.
872 */
873 extern void kernel_memory_depopulate(
874 vm_offset_t addr,
875 vm_size_t size,
876 kma_flags_t flags,
877 vm_tag_t tag);
878
879
880 #pragma mark - VM_FLAGS_* / vm_map_kernel_flags_t conversions
881
882 /*!
883 * @function vm_map_kernel_flags_vmflags()
884 *
885 * @return The vmflags set in the specified @c vmk_flags.
886 */
887 extern int vm_map_kernel_flags_vmflags(
888 vm_map_kernel_flags_t vmk_flags);
889
890 /*!
891 * @function vm_map_kernel_flags_set_vmflags()
892 *
893 * @brief
894 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
895 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
896 */
897 __attribute__((overloadable))
898 extern void vm_map_kernel_flags_set_vmflags(
899 vm_map_kernel_flags_t *vmk_flags,
900 int vm_flags,
901 vm_tag_t vm_tag);
902
903 /*!
904 * @function vm_map_kernel_flags_set_vmflags()
905 *
906 * @brief
907 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
908 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
909 *
910 * @discussion
911 * This variant takes the tag from the top byte of the flags.
912 */
913 __attribute__((overloadable))
914 extern void vm_map_kernel_flags_set_vmflags(
915 vm_map_kernel_flags_t *vmk_flags,
916 int vm_flags_and_tag);
917
918 /*!
919 * @function vm_map_kernel_flags_and_vmflags()
920 *
921 * @brief
922 * Apply a mask to the vmflags.
923 */
924 extern void vm_map_kernel_flags_and_vmflags(
925 vm_map_kernel_flags_t *vmk_flags,
926 int vm_flags_mask);
927
928 /*!
929 * @function vm_map_kernel_flags_check_vmflags()
930 *
931 * @return
932 * Whether the @c vmk_flags @c vmf_* fields
933 * are limited to the specified mask.
934 */
935 extern bool vm_map_kernel_flags_check_vmflags(
936 vm_map_kernel_flags_t vmk_flags,
937 int vm_flags_mask);
938
939 /*!
940 * @function vm_map_kernel_flags_check_vm_and_kflags()
941 *
942 * @return
943 * Whether the @c vmk_flags @c vmf_* fields
944 * are limited to the specified mask.
945 */
946 extern bool vm_map_kernel_flags_check_vm_and_kflags(
947 vm_map_kernel_flags_t vmk_flags,
948 int vm_flags_mask);
949
950
951 #pragma mark - kernel variants of the Mach VM interfaces
952
953 /*!
954 * @function mach_vm_allocate_kernel()
955 *
956 * @brief
957 * Allocate memory in the specified map.
958 *
959 * @discussion
960 * This is the in-kernel equivalent to the @c mach_vm_allocate() MIG call,
961 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
962 * @c VM_FLAGS_.
963 *
964 * Memory will not be pre-faulted and touching it will cause page faults.
965 *
966 * Memory will be zero-filled when faulted, unless the map's @c no_zero_fill
967 * property is set (only the @c ipc_kernel_map is marked this way).
968 *
969 * The allocation being made will have:
970 * - @c VM_PROT_DEFAULT (rw-) current protections,
971 * - @c VM_PROT_ALL (rwx) max protections,
972 * - @c VM_INHERIT_COPY inheritance.
973 *
974 *
975 * @param map the map to allocate memory into.
976 *
977 * @param addr_u [in] when @c vmk_flags.vmf_fixed is set, @c *addr
978 * is used as the address at which the allocation
979 * must be made. If it is misaligned for the map's
980 * page size, its low bits are truncated and ignored.
981 *
982 * when @c vmk_flags.vmf_fixed is not set,
983 * the value of @c *addrp is ignored.
984 *
985 * [out] filled with the address at which the allocation
986 * was made on success, unmodified otherwise.
987 *
988 * @c vmk_flags.vmf_return_data_addr has no effect
989 * on the returned address.
990 *
991 * @param size_u the size of the allocation to make. zero sizes are
992 * allowed and result in @c *addr being zero.
993 *
994 * misaligned sizes for the page size of the target map
995 * will be rounded up to the nearest page size.
996 *
997 * @param vmk_flags a set of flags that influence the properties of the
998 * allocation being made.
999 *
1000 * @returns KERN_SUCCESS when the operation succeeds,
1001 * or an error denoting the reason for failure.
1002 */
1003 extern kern_return_t mach_vm_allocate_kernel(
1004 vm_map_t map,
1005 mach_vm_offset_ut *addr_u,
1006 mach_vm_size_ut size_u,
1007 vm_map_kernel_flags_t vmk_flags);
1008
1009
1010 /*!
1011 * @function mach_vm_map_kernel()
1012 *
1013 * @brief
1014 * Map some range of an object into an address space.
1015 *
1016 * @discussion
1017 * This is the in-kernel equivalent to the @c mach_vm_map() MIG call,
1018 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1019 * @c VM_FLAGS_.
1020 *
1021 *
1022 * @param target_map the target address space.
1023 *
1024 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1025 * is used as the address at which the allocation
1026 * must be made. If it is misaligned for the map's
1027 * page size, its low bits are truncated and ignored.
1028 *
1029 * when @c vmk_flags.vmf_fixed is not set,
1030 * the value of @c *address is used as a starting
1031 * point from which to scan for memory in the direction
1032 * specified by @c vmk_flags.vmkf_last_free.
1033 *
1034 * [out] filled with the address at which the allocation
1035 * was made on success, unmodified otherwise.
1036 *
1037 * if @c vmk_flags.vmf_return_data_addr is
1038 * specified, @c *address will maintain
1039 * "misalignment" from the requested @c offset
1040 * inside the object, otherwise it will be page
1041 * aligned for the target map.
1042 *
1043 * @param size the size of the allocation to make.
1044 * zero sizes are disallowed.
1045 *
1046 * @param mask a mask to specify the allocation alignment.
1047 * this value should be of the form (2^n-1).
1048 *
1049 * allocations will always at least be page aligned
1050 * for the specified target map, but this can be used to
1051 * require larger alignments.
1052 *
1053 * @param vmk_flags a set of flags that influence the properties of the
1054 * allocation being made.
1055 *
1056 * @param port the object to map into the target address space:
1057 * - @c IP_NULL means anonymous memory, and this call
1058 * behaves like a more versatile
1059 * mach_vm_allocate_kernel(),
1060 * - a kobject port of type @c IKOT_NAMED_ENTRY,
1061 * pointing to a @c vm_named_entry_t,
1062 * - a naked @c memory_object_t (a VM pager).
1063 *
1064 * @param offset an offset within the specified object to map.
1065 *
1066 * @param copy whether to make a copy-on-write (true) mapping
1067 * or a shared (false) mapping.
1068 *
1069 * @param cur_prot the effective protections for the mapping.
1070 *
1071 * @param max_prot the maximum protections for the mapping,
1072 * which must at least cover @c cur_prot.
1073 *
1074 * @param inheritance the inheritance policy for the mapping.
1075 *
1076 * @returns KERN_SUCCESS when the operation succeeds,
1077 * or an error denoting the reason for failure.
1078 */
1079 extern kern_return_t mach_vm_map_kernel(
1080 vm_map_t target_map,
1081 mach_vm_offset_ut *address,
1082 mach_vm_size_ut size,
1083 mach_vm_offset_ut mask,
1084 vm_map_kernel_flags_t vmk_flags,
1085 ipc_port_t port,
1086 memory_object_offset_ut offset,
1087 boolean_t copy,
1088 vm_prot_ut cur_prot,
1089 vm_prot_ut max_prot,
1090 vm_inherit_ut inheritance);
1091
1092
1093 /*!
1094 * @function mach_vm_remap_new_kernel()
1095 *
1096 * @brief
1097 * Remap a range of memory from one address space to another.
1098 *
1099 * @discussion
1100 * This is the in-kernel equivalent to the @c mach_vm_remap() MIG call,
1101 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1102 * @c VM_FLAGS_.
1103 *
1104 * This call forces vmk_flags.vmf_return_data_addr to true regardless
1105 * of what the caller specified.
1106 *
1107 *
1108 * @param target_map the target address space.
1109 *
1110 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1111 * is used as the address at which the allocation
1112 * must be made. If it is misaligned for the map's
1113 * page size, its low bits are truncated and ignored.
1114 *
1115 * when @c vmk_flags.vmf_fixed is not set,
1116 * the value of @c *address is used as a starting
1117 * point from which to scan for memory in the direction
1118 * specified by @c vmk_flags.vmkf_last_free.
1119 *
1120 * [out] filled with the address at which the allocation
1121 * was made on success, unmodified otherwise.
1122 * @c *address has the same "misalignment"
1123 * as @c src_address.
1124 *
1125 * @param size the size of the region to remap.
1126 *
1127 * @param mask a mask to specify the allocation alignment.
1128 * this value should be of the form (2^n-1).
1129 *
1130 * allocations will always at least be page aligned
1131 * for the specified target map, but this can be used to
1132 * require larger alignments.
1133 *
1134 * @param vmk_flags a set of flags that influence the properties of the
1135 * allocation being made.
1136 *
1137 * @param src_map the address space to remap the memory from.
1138 *
1139 * @param src_address the start address within @c src_map to remap.
1140 *
1141 * @param copy whether to make a copy-on-write (true) mapping
1142 * or a shared (false) mapping.
1143 *
1144 * @param cur_prot [in] for shared mappings, the minimum set of effective
1145 * permissions the source mapping must have
1146 *
1147 * [out] the resulting effective permissions for the mapping
1148 *
1149 * @param max_prot [in] for shared mappings, the minimum set of maximum
1150 * permissions the source mappings must have.
1151 *
1152 * [out] the resulting maximum permissions for the mapping
1153 *
1154 * @param inheritance the inheritance policy for the mapping.
1155 *
1156 * @returns KERN_SUCCESS when the operation succeeds,
1157 * or an error denoting the reason for failure.
1158 */
1159 extern kern_return_t mach_vm_remap_new_kernel(
1160 vm_map_t target_map,
1161 mach_vm_offset_ut *address,
1162 mach_vm_size_ut size,
1163 mach_vm_offset_ut mask,
1164 vm_map_kernel_flags_t vmk_flags,
1165 vm_map_t src_map,
1166 mach_vm_offset_ut src_address,
1167 boolean_t copy,
1168 vm_prot_ut *cur_prot,
1169 vm_prot_ut *max_prot,
1170 vm_inherit_ut inheritance);
1171
1172 /*!
1173 * @function vm_map_wire_kernel()
1174 *
1175 * @brief
1176 * Sets the pageability of the specified address range in the
1177 * target map as wired.
1178 *
1179 * @discussion
1180 * This call is the kernel version of @c vm_map_wire(). The main difference
1181 * is that the caller must specify a valid @c VM_KERN_MEMORY_* tag for kernel
1182 * wirings, or a valid @c VM_MEMORY_* tag for user wirings.
1183 *
1184 * Consult the documentation of @c vm_map_wire() in @c <vm/vm_map.h> for details.
1185 */
1186 extern kern_return_t vm_map_wire_kernel(
1187 vm_map_t map,
1188 vm_map_offset_ut start_u,
1189 vm_map_offset_ut end_u,
1190 vm_prot_ut prot_u,
1191 vm_tag_t tag,
1192 boolean_t user_wire);
1193
1194
1195 /*!
1196 * @function vm_purgable_control()
1197 *
1198 * @brief
1199 * Perform a purgeability operation on a VM object at a given address
1200 * in an address space.
1201 *
1202 * @discussion
1203 * This is the in-kernel equivalent to the @c mach_vm_map_purgable_control()
1204 * MIG call, except that it allows all operations, including
1205 * @c VM_PURGABLE_*_FROM_KERNEL ones).
1206 *
1207 * Valid @c control operations and the meaning of @c state is documented
1208 * in @c <mach/vm_purgable.h>.
1209 *
1210 *
1211 * @param map the target address space
1212 * @param address the address to use to find the VM object to target
1213 * @param control a purgeability operation to perform.
1214 * @param state an in/out parameter that is operation dependent.
1215 */
1216 extern kern_return_t vm_purgable_control(
1217 vm_map_t map,
1218 vm_offset_t address,
1219 vm_purgable_t control,
1220 int *state);
1221
1222 extern kern_return_t mach_vm_purgable_control(
1223 vm_map_t map,
1224 mach_vm_offset_t address,
1225 vm_purgable_t control,
1226 int *state);
1227
1228
1229 #ifdef MACH_KERNEL_PRIVATE
1230 #pragma mark - map copyio
1231
1232
1233 /*!
1234 * @function copyinmap()
1235 *
1236 * @brief
1237 * Like copyin, except that @c fromaddr is an address in the specified VM map.
1238 *
1239 * @param map the map @c fromaddr is relative to.
1240 * @param fromaddr the address to copy from within @c map.
1241 * @param todata the kernel buffer to write into.
1242 * @param length the number of bytes to copy.
1243 * @returns
1244 * - KERN_SUCCESS the copy was successful
1245 * - KERN_INVALID_ADDRESS
1246 * a fault occurred during copyio and couldn't be resolved
1247 * (similar to copyin returning EFAULT).
1248 */
1249 extern kern_return_t copyinmap(
1250 vm_map_t map,
1251 vm_map_offset_t fromaddr,
1252 void *todata __sized_by(length),
1253 vm_size_t length);
1254
1255
1256 /*!
1257 * @function copyoutmap()
1258 *
1259 * @brief
1260 * Like copyout, except that @c toaddr is an address in the specified VM map.
1261 *
1262 * @param map the map @c toaddr is relative to
1263 * @param fromdata the kernel buffer to copy from.
1264 * @param toaddr the address within @c map to copy into
1265 * @param length the number of bytes to copy.
1266 * @returns
1267 * - KERN_SUCCESS the copy was successful
1268 * - KERN_INVALID_ADDRESS
1269 * a fault occurred during copyio and couldn't be resolved
1270 * (similar to copyin returning EFAULT).
1271 */
1272 extern kern_return_t copyoutmap(
1273 vm_map_t map,
1274 void *fromdata __sized_by(length),
1275 vm_map_offset_t toaddr,
1276 vm_size_t length);
1277
1278
1279 /*!
1280 * @function copyoutmap_atomic32()
1281 *
1282 * @brief
1283 * Copies out a 32bit value atomically at a given address in a specified VM map.
1284 *
1285 * @param map the specified map.
1286 * @param value the 32 bit value to write at @c toaddr.
1287 * @param toaddr the address within @c map to copy into
1288 * @returns
1289 * - KERN_SUCCESS the copy was successful
1290 * - KERN_INVALID_ADDRESS
1291 * a fault occurred during copyio and couldn't be resolved
1292 * (similar to copyin returning EFAULT).
1293 */
1294 extern kern_return_t copyoutmap_atomic32(
1295 vm_map_t map,
1296 uint32_t value,
1297 vm_map_offset_t toaddr);
1298
1299
1300 /*!
1301 * @function copyoutmap_atomic64()
1302 *
1303 * @brief
1304 * Copies out a 64bit value atomically at a given address in a specified VM map.
1305 *
1306 * @param map the specified map.
1307 * @param value the 64 bit value to write at @c toaddr.
1308 * @param toaddr the address within @c map to copy into
1309 * @returns
1310 * - KERN_SUCCESS the copy was successful
1311 * - KERN_INVALID_ADDRESS
1312 * a fault occurred during copyio and couldn't be resolved
1313 * (similar to copyin returning EFAULT).
1314 */
1315 extern kern_return_t copyoutmap_atomic64(
1316 vm_map_t map,
1317 uint64_t value,
1318 vm_map_offset_t toaddr);
1319
1320
1321 #endif /* MACH_KERNEL_PRIVATE */
1322 #pragma mark - accounting
1323
1324 #pragma mark accounting: kern allocation name
1325
1326 /*!
1327 * @function kern_allocation_update_size()
1328 *
1329 * @brief
1330 * Update accounting for a specified kern allocation name.
1331 *
1332 * @discussion
1333 * This is to be called when memory gets wired/unwired in order
1334 * to update accounting information.
1335 *
1336 * [development kernels] If the @c vmtaglog boot-arg is used,
1337 * and its value matches the VM tag of this allocation name, then VM tag
1338 * log entries will be added on the specified object.
1339 *
1340 *
1341 * @param allocation a @c kern_allocation_name_t made with
1342 * @c kern_allocation_name_allocate().
1343 * @param delta the amount to update the accounting with,
1344 * positive values increment,
1345 * negative values decrement.
1346 * @param object an optional object this wiring/unwiring applies to.
1347 */
1348 extern void kern_allocation_update_size(
1349 kern_allocation_name_t allocation,
1350 int64_t delta,
1351 vm_object_t object);
1352
1353
1354 /*!
1355 * @function kern_allocation_update_subtotal()
1356 *
1357 * @brief
1358 * Update subtotal accounting for a specified kern allocation name.
1359 *
1360 * @discussion
1361 * IOKit uses global kern allocation names that cover the VM tags
1362 * (@see @c vm_tag_bt()) of several kexts and uses this to perform
1363 * accounting per kext within these meta accounting data structures.
1364 *
1365 * @param allocation a @c kern_allocation_name_t made with
1366 * @c kern_allocation_name_allocate()
1367 * and a non 0 "subtotalscount".
1368 * @param subtag a @c vm_tag_t subtag to update accounting for.
1369 * @param delta the amount to update the accounting with,
1370 * positive values increment,
1371 * negative values decrement.
1372 */
1373 extern void kern_allocation_update_subtotal(
1374 kern_allocation_name_t allocation,
1375 vm_tag_t subtag,
1376 int64_t delta);
1377
1378
1379 #pragma mark accounting: vm tags
1380
1381 /*
1382 * VM Tags come in 3 flavors:
1383 *
1384 * - static user VM tags, defined by the @c VM_MEMORY_* constants
1385 * (@see <mach/vm_statistics.h>),
1386 *
1387 * - static kernel VM tags, defined by the @c VM_KERN_MEMORY_* constants
1388 * (@see <mach/vm_statistics.h>),
1389 *
1390 * - dynamically allocated kernel VM tags typically associated with kexts
1391 * lazily (as kexts wire down memory and accounting needs to be made).
1392 *
1393 * By default, kernel VM tags track wired memory at the VM layer,
1394 * but no insight is given to allocations done by @c kalloc*()
1395 * and its wrappers like @c IOMalloc*(), when backed by zone memory
1396 * (for sizes below @c KHEAP_MAX_SIZE).
1397 *
1398 *
1399 * Implementation details
1400 * ~~~~~~~~~~~~~~~~~~~~~~
1401 *
1402 * Static tags are limited to values from 0 to 255, as they are passed
1403 * in the bits reserved by the @c VM_FLAGS_ALIAS_MASK of VM flags.
1404 *
1405 * However, dynamic flags are an internal kernel concept which is limited
1406 * by the size of the storage in VM map entries, which reserves
1407 * @c VME_ALIAS_BITS (12) for this, effectively limiting dynamic
1408 * tags to about 4000.
1409 *
1410 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE) defines the range of
1411 * possible values for dynamic tags.
1412 *
1413 *
1414 * Zone accounting [development kernels]
1415 * ~~~~~~~~~~~~~~~
1416 *
1417 * On development kernels the "-zt" boot-arg can be used, in which case
1418 * precise accounting of @c kalloc*() allocations is enabled per bucket,
1419 * which can be observed by the @c zprint(1) command.
1420 *
1421 * This effectively makes accounting a vector composed of @c VM_TAG_SIZECLASSES
1422 * possible zone size classes in addition to the regular VM wired memory
1423 * accounting that is always performed.
1424 *
1425 * The tag for which an allocation is recorded will be (in first hit order):
1426 *
1427 * - the tag specified as part of the @c zalloc_flags_t explicitly
1428 * (@see @c kalloc_*_tag() or @c Z_VM_TAG()). In core XNU, non @c _tag
1429 * variants of @c kalloc*() will generate a per-call site dynamic VM tag,
1430 * using the @c VM_ALLOC_SITE_TAG() macro,
1431 *
1432 * - if the allocation was made by a kernel extension, the dynamic VM tag
1433 * for this extension (@see @c vm_tag_bt()),
1434 *
1435 * - as a fallback:
1436 * o @c VM_KERN_MEMORY_KALLOC_DATA for @c kalloc_data() calls,
1437 * o @c VM_KERN_MEMORY_KALLOC_TYPE for @c kalloc_type() calls,
1438 * o @c VM_KERN_MEMORY_KALLOC for legacy @c kalloc() calls.
1439 */
1440
1441 /*!
1442 * @brief
1443 * Lock used to serialize the @c vm_tag_alloc() operation, used by IOKit.
1444 */
1445 extern lck_ticket_t vm_allocation_sites_lock;
1446
1447
1448 /*!
1449 * @function vm_tag_bt()
1450 *
1451 * @brief
1452 * Returns the dynamic kernel extension tag based on backtracing from this call.
1453 *
1454 * @discussion
1455 * This function will lazily allocate a dynamic tag for the current kernel
1456 * extension based on the backtrace, and then return a stable identifier.
1457 *
1458 * This might fail for cases where the kernel is out of dynamic tags.
1459 *
1460 *
1461 * @returns A dynamic kernel extension tag within
1462 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1463 * or @c VM_KERN_MEMORY_NONE if the call was not made
1464 * from a kernel extension, or allocating a dynamic tag
1465 * for it failed.
1466 */
1467 extern vm_tag_t vm_tag_bt(void);
1468
1469
1470 /*!
1471 * @function vm_tag_alloc()
1472 *
1473 * @brief
1474 * Lazily allocates a dynamic VM tag for a given allocation site.
1475 *
1476 * @description
1477 * This is used by IOKit and the zalloc subsystem to generate dynamic VM tags
1478 * for kernel extensions (@see vm_tag_bt()) or core kernel @c kalloc*()
1479 * call sites (@see VM_ALLOC_SITE_TAG()).
1480 *
1481 *
1482 * @param site the allocation site to generate a tag for.
1483 * @returns A dynamic kernel extension tag within
1484 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1485 * or @c VM_KERN_MEMORY_NONE if the kernel
1486 * is out of dynamic tags.
1487 */
1488 extern vm_tag_t vm_tag_alloc(
1489 vm_allocation_site_t *site);
1490
1491 /*!
1492 * @function vm_tag_alloc_locked()
1493 *
1494 * @brief
1495 * Lazily allocates a dynamic VM tag for a given allocation site,
1496 * while the caller holds the @c vm_allocation_sites_lock lock.
1497 *
1498 * @description
1499 * This is used by IOKit to generate dynamic VM tags for kernel extensions
1500 * (@see vm_tag_bt()).
1501 *
1502 *
1503 * @param site the allocation site to generate a tag for.
1504 * @param releasesiteP an optional allocation site data structure
1505 * that the caller is responsible for releasing
1506 * with @c kern_allocation_name_release().
1507 */
1508 extern void vm_tag_alloc_locked(
1509 vm_allocation_site_t *site,
1510 vm_allocation_site_t **releasesiteP);
1511
1512
1513 /*!
1514 * @function vm_tag_update_size()
1515 *
1516 * @brief
1517 * Update accounting for a specified VM kernel tag (static or dynamic).
1518 *
1519 * @discussion
1520 * This is to be called when memory gets wired/unwired in order
1521 * to update accounting information for a given tag.
1522 *
1523 * [development kernels] If the @c vmtaglog boot-arg is used,
1524 * and its value matches the VM tag of this allocation name, then VM tag
1525 * log entries will be added on the specified object.
1526 *
1527 *
1528 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1529 * @param delta the amount to update the accounting with,
1530 * positive values increment,
1531 * negative values decrement.
1532 * @param object an optional object this wiring/unwiring applies to.
1533 */
1534 extern void vm_tag_update_size(
1535 vm_tag_t tag,
1536 int64_t delta,
1537 vm_object_t object);
1538
1539
1540 /*!
1541 * @function vm_tag_get_size()
1542 *
1543 * @brief
1544 * Returns how much wired memory is accounted for the specified VM kernel tag.
1545 *
1546 *
1547 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1548 * @returns the amount of wired memory for the specified tag.
1549 */
1550 extern uint64_t vm_tag_get_size(
1551 vm_tag_t tag);
1552
1553 #if VM_TAG_SIZECLASSES
1554
1555 /*!
1556 * @function vm_tag_will_update_zone()
1557 *
1558 * @brief
1559 * Lazily allocate the vector of zone-level accounting for a given VM tag.
1560 *
1561 * @discussion
1562 * This is to be called when memory gets wired/unwired in order
1563 * to update accounting information for a given tag.
1564 *
1565 * This function might fail when the first time it is called is
1566 * for a @c Z_NOWAIT allocation. However this is transient and
1567 * will always eventually resolve. Once the data structure is allocated,
1568 * this function always succeeds. The consequence is a slight misaccounting
1569 * of a few allocations.
1570 *
1571 * This call will never fail for the following tags that are always
1572 * pre-allocated:
1573 * - @c VM_KERN_MEMORY_DIAG,
1574 * - @c VM_KERN_MEMORY_KALLOC,
1575 * - @c VM_KERN_MEMORY_KALLOC_DATA,
1576 * - @c VM_KERN_MEMORY_KALLOC_TYPE,
1577 * - @c VM_KERN_MEMORY_LIBKERN,
1578 * - @c VM_KERN_MEMORY_OSFMK,
1579 * - @c VM_KERN_MEMORY_RECOUNT.
1580 *
1581 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1582 *
1583 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1584 * @param zflags the @c zalloc_flags_t passed to the current
1585 * @c kalloc*() call.
1586 *
1587 * @returns @c tag if the allocation was successful,
1588 * @c VM_KERN_MEMORY_NONE if the accounting
1589 * data structure couldn't be allocated.
1590 */
1591 extern vm_tag_t vm_tag_will_update_zone(
1592 vm_tag_t tag,
1593 uint32_t zflags);
1594
1595
1596 /*!
1597 * @function vm_tag_update_zone_size()
1598 *
1599 * @brief
1600 * Update the per size class zone level accounting for a given kernel VM tag.
1601 *
1602 * @discussion
1603 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1604 *
1605 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1606 * @param size_class the zone size class index to account against
1607 * (@see zone_t::z_tags_sizeclass).
1608 * @param delta the amount to update the accounting with,
1609 * positive values increment,
1610 * negative values decrement.
1611 */
1612 extern void vm_tag_update_zone_size(
1613 vm_tag_t tag,
1614 uint32_t size_class,
1615 long delta);
1616
1617 #endif /* VM_TAG_SIZECLASSES */
1618
1619 #pragma mark accounting: diagnostics and query interfaces
1620
1621 /*!
1622 * @function vm_page_diagnose_estimate()
1623 *
1624 * @brief
1625 * Estimate how many @c mach_memory_info_t structures
1626 * are needed in order to return information about VM kernel tags,
1627 * and per size class zone level accounting when enabled.
1628 */
1629 extern uint32_t vm_page_diagnose_estimate(void);
1630
1631
1632 /*!
1633 * @function vm_page_diagnose()
1634 *
1635 * @brief
1636 * Fills out a @c mach_memory_info_t array of information about VM tags
1637 * and per size class zone level information.
1638 *
1639 * @param info a pointer to an array of @c num_info Mach memory
1640 * info data structures to fill.
1641 * @param num_info the number of entries in the @c info array.
1642 * @param zones_collectable_bytes
1643 * how much memory is collectable in the zone subsystem
1644 * if a @c zone_gc() was running right now.
1645 * @param redact_info whether information that could leak the type to bucket
1646 * mapping in @c kalloc_type() can be returned or not.
1647 *
1648 * @returns
1649 * - KERN_SUCCESS the call was successful.
1650 * - KERN_ABORTED the accounting subsytem isn't inititalized yet.
1651 */
1652 extern kern_return_t vm_page_diagnose(
1653 struct mach_memory_info *info __counted_by(num_info),
1654 unsigned int num_info,
1655 uint64_t zones_collectable_bytes,
1656 bool redact_info);
1657
1658 #if DEBUG || DEVELOPMENT
1659
1660 /*!
1661 * @typedef kmem_gobj_stats
1662 *
1663 * @brief
1664 * Statistics about the "guard objects" allocator for the pointer ranges
1665 * of kmem.
1666 */
1667 typedef struct {
1668 vm_map_size_t meta_sz; /**< total faulted size of metadata */
1669 vm_map_size_t pte_sz; /**< total faulted leaf PTE size */
1670 vm_map_size_t total_va; /**< total amount of VA ever used */
1671 vm_map_size_t total_used; /**< current amount of VA allocated */
1672 } kmem_gobj_stats;
1673
1674
1675 /*!
1676 * @function kmem_get_gobj_stats()
1677 *
1678 * @brief
1679 * Returns statistics about the guard objects allocator.
1680 *
1681 * @description
1682 * This is the backend of the @c vm.kmem_gobj_stats sysctl.
1683 */
1684 extern kmem_gobj_stats kmem_get_gobj_stats(void);
1685
1686
1687 /*!
1688 * @function vm_kern_allocation_info()
1689 *
1690 * @brief
1691 * Returns information about a given kernel heap allocation.
1692 *
1693 *
1694 * @param [in] addr the heap allocation pointer.
1695 * @param [out] size a guess at the size of this allocation.
1696 * @param [out] tag the kernel VM tag for this allocation
1697 * (only filled if the "-zt" boot-arg is set
1698 * for zone allocations)
1699 * @param [out] zone_size
1700 * the zone size class if the allocation
1701 * is from a zone, 0 for VM.
1702 *
1703 * @returns
1704 * - KERN_SUCCESS if a guess could be made about this pointer.
1705 * - KERN_INVALID_ADDRESS
1706 * if the address couldn't be resolved in the kernel heap.
1707 */
1708 extern kern_return_t vm_kern_allocation_info(
1709 uintptr_t addr,
1710 vm_size_t *size,
1711 vm_tag_t *tag,
1712 vm_size_t *zone_size);
1713
1714 #endif /* DEBUG || DEVELOPMENT */
1715
1716 #pragma mark - init methods
1717
1718 /*!
1719 * @function vm_init_before_launchd()
1720 *
1721 * @brief
1722 * Memorize how many wired pages were used at boot before launchd starts.
1723 *
1724 * @discussion
1725 * The captured number can be seen as the @c VM_KERN_COUNT_WIRED_BOOT value
1726 * in the output of @c zprint(1).
1727 */
1728 extern void vm_init_before_launchd(void);
1729
1730 #if VM_TAG_SIZECLASSES
1731
1732 /*!
1733 * @function vm_allocation_zones_init()
1734 *
1735 * @brief
1736 * Initialize the per-zone accounting tags subsystem
1737 * if the "-zt" boot-arg is present.
1738 */
1739 extern void vm_allocation_zones_init(void);
1740
1741 #endif /* VM_TAG_SIZECLASSES */
1742
1743 #endif /* XNU_KERNEL_PRIVATE */
1744 #pragma GCC visibility pop
1745 __END_DECLS
1746
1747 #endif /* _VM_VM_KERN_XNU_H_ */
1748