1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _VM_VM_KERN_XNU_H_
30 #define _VM_VM_KERN_XNU_H_
31
32 #include <sys/cdefs.h>
33 #include <vm/vm_kern.h>
34
35 __BEGIN_DECLS
36 #pragma GCC visibility push(hidden)
37 #ifdef XNU_KERNEL_PRIVATE
38
39
40 #pragma mark - the kmem subsystem
41
42 /*
43 * "kmem" is a set of methods that provide interfaces suitable
44 * to allocate memory from the VM in the kernel map or submaps.
45 *
46 * It provide leaner alternatives to some of the VM functions,
47 * closer to a typical allocator.
48 */
49
50 struct mach_memory_info;
51 struct vm_page;
52 struct vm_map_entry;
53
54 /*!
55 * @typedef
56 *
57 * @brief
58 * Pair of a return code and size/address/... used by kmem interfaces.
59 *
60 * @discussion
61 * Using a pair of integers allows the compiler to return everything
62 * through registers, and doesn't need to use stack values to get results,
63 * which yields significantly better codegen.
64 *
65 * If @c kmr_return is not @c KERN_SUCCESS, then the other field
66 * of the union is always supposed to be 0.
67 */
68 typedef struct {
69 kern_return_t kmr_return;
70 union {
71 vm_address_t kmr_address;
72 vm_size_t kmr_size;
73 void *kmr_ptr;
74 vm_map_t kmr_submap;
75 };
76 } kmem_return_t;
77
78 /*!
79 * @typedef kmem_guard_t
80 *
81 * @brief
82 * KMEM guards are used by the kmem_* subsystem to secure atomic allocations.
83 *
84 * @discussion
85 * This parameter is used to transmit the tag for the allocation.
86 *
87 * If @c kmg_atomic is set, then the other fields are also taken into account
88 * and will affect the allocation behavior for this allocation.
89 *
90 * @field kmg_tag The VM_KERN_MEMORY_* tag for this entry.
91 * @field kmg_type_hash Some hash related to the type of the allocation.
92 * @field kmg_atomic Whether the entry is atomic.
93 * @field kmg_submap Whether the entry is for a submap.
94 * @field kmg_context A use defined 30 bits that will be stored
95 * on the entry on allocation and checked
96 * on other operations.
97 */
98 typedef struct {
99 uint16_t kmg_tag;
100 uint16_t kmg_type_hash;
101 uint32_t kmg_atomic : 1;
102 uint32_t kmg_submap : 1;
103 uint32_t kmg_context : 30;
104 } kmem_guard_t;
105 #define KMEM_GUARD_NONE (kmem_guard_t){ }
106 #define KMEM_GUARD_SUBMAP (kmem_guard_t){ .kmg_atomic = 0, .kmg_submap = 1 }
107
108
109 /*!
110 * @typedef kmem_flags_t
111 *
112 * @brief
113 * Sets of flags taken by several of the @c kmem_* family of functions.
114 *
115 * @discussion
116 * This type is not used directly by any function, it is an underlying raw
117 * type that is re-vended under different namespaces for each @c kmem_*
118 * interface.
119 *
120 * - @c kmem_alloc uses @c kma_flags_t / @c KMA_* namespaced values.
121 * - @c kmem_suballoc uses @c kms_flags_t / @c KMS_* namespaced values.
122 * - @c kmem_realloc uses @c kmr_flags_t / @c KMR_* namespaced values.
123 * - @c kmem_free uses @c kmf_flags_t / @c KMF_* napespaced values.
124 *
125 *
126 * <h2>Call behavior</h2>
127 *
128 * @const KMEM_NONE (all)
129 * Pass this when no special options is to be used.
130 *
131 * @const KMEM_NOFAIL (alloc, suballoc)
132 * When this flag is passed, any allocation failure results into a panic().
133 * Using this flag should really be limited to cases when failure is not
134 * recoverable and possibly during early boot only.
135 *
136 * @const KMEM_NOPAGEWAIT (alloc, realloc)
137 * Pass this flag if the system should not wait in VM_PAGE_WAIT().
138 *
139 * @const KMEM_FREEOLD (realloc)
140 * Pass this flag if @c kmem_realloc should free the old mapping
141 * (when the address changed) as part of the call.
142 *
143 * @const KMEM_REALLOCF (realloc)
144 * Similar to @c Z_REALLOCF: if the call is failing,
145 * then free the old allocation too.
146 *
147 * @const KMEM_NOSOFTLIMIT (alloc, realloc)
148 * Kernel private.
149 * Override soft allocation size limits and attempt to make the allocation
150 * anyways.
151 *
152 * <h2>How the entry is populated</h2>
153 *
154 * @const KMEM_VAONLY (alloc)
155 * By default memory allocated by the kmem subsystem is wired and mapped.
156 * Passing @c KMEM_VAONLY will cause the range to still be wired,
157 * but no page is actually mapped.
158 *
159 * @const KMEM_PAGEABLE (alloc)
160 * By default memory allocated by the kmem subsystem is wired and mapped.
161 * Passing @c KMEM_PAGEABLE makes the entry non wired, and pages will be
162 * added to the entry as it faults.
163 *
164 * @const KMEM_ZERO (alloc, realloc)
165 * Any new page added is zeroed.
166 *
167 *
168 * <h2>VM object to use for the entry</h2>
169 *
170 * @const KMEM_KOBJECT (alloc, realloc)
171 * The entry will be made for the @c kernel_object.
172 *
173 * Note that the @c kernel_object is just a "collection of pages".
174 * Pages in that object can't be remaped or present in several VM maps
175 * like traditional objects.
176 *
177 * If neither @c KMEM_KOBJECT nor @c KMEM_COMPRESSOR is passed,
178 * the a new fresh VM object will be made for this allocation.
179 * This is expensive and should be limited to allocations that
180 * need the features associated with a VM object.
181 *
182 * @const KMEM_COMPRESSOR (alloc)
183 * The entry is allocated for the @c compressor_object.
184 * Pages belonging to the compressor are not on the paging queues,
185 * nor are they counted as wired.
186 *
187 * Only the VM Compressor subsystem should use this.
188 *
189 *
190 * <h2>How to look for addresses</h2>
191 *
192 * @const KMEM_LOMEM (alloc, realloc)
193 * The physical memory allocated must be in the first 4G of memory,
194 * in order to support hardware controllers incapable of generating DMAs
195 * with more than 32bits of physical address.
196 *
197 * @const KMEM_LAST_FREE (alloc, suballoc, realloc)
198 * When looking for space in the specified map,
199 * start scanning for addresses from the end of the map
200 * rather than the start.
201 *
202 * @const KMEM_DATA (alloc, suballoc, realloc)
203 * The memory must be allocated from the "Data" range.
204 *
205 * @const KMEM_SPRAYQTN (alloc, realloc)
206 * The memory must be allocated from the "spray quarantine" range. For more
207 * details on what allocations qualify to use this flag see
208 * @c KMEM_RANGE_ID_SPRAYQTN.
209 *
210 * @const KMEM_GUESS_SIZE (free)
211 * When freeing an atomic entry (requires a valid kmem guard),
212 * then look up the entry size because the caller didn't
213 * preserve it.
214 *
215 * This flag is only here in order to support kfree_data_addr(),
216 * and shall not be used by any other clients.
217 *
218 * <h2>Entry properties</h2>
219 *
220 * @const KMEM_PERMANENT (alloc, suballoc)
221 * The entry is made permanent.
222 *
223 * In the kernel maps, permanent entries can never be deleted.
224 * Calling @c kmem_free() on such a range will panic.
225 *
226 * In user maps, permanent entries will only be deleted
227 * whenthe map is terminated.
228 *
229 * @const KMEM_GUARD_FIRST (alloc, realloc, free)
230 * @const KMEM_GUARD_LAST (alloc, realloc, free)
231 * Asks @c kmem_* to put a guard page at the beginning (resp. end)
232 * of the allocation.
233 *
234 * The allocation size will not be extended to accomodate for guards,
235 * and the client of this interface must take them into account.
236 * Typically if a usable range of 3 pages is needed with both guards,
237 * then 5 pages must be asked.
238 *
239 * Alignment constraints take guards into account (the aligment applies
240 * to the address right after the first guard page).
241 *
242 * The returned address for allocation will pointing at the entry start,
243 * which is the address of the left guard page if any.
244 *
245 * Note that if @c kmem_realloc* or @c kmem_free* is called, the *exact*
246 * same guard flags must be passed for this entry. The KMEM subsystem
247 * is generally oblivious to guards, and passing inconsistent flags
248 * will cause pages to be moved incorrectly.
249 *
250 * @const KMEM_KSTACK (alloc)
251 * This flag must be passed when the allocation is for kernel stacks.
252 * This only has an effect on Intel.
253 *
254 * @const KMEM_NOENCRYPT (alloc)
255 * Obsolete, will be repurposed soon.
256 *
257 * @const KMEM_KASAN_GUARD (alloc, realloc, free)
258 * Under KASAN_CLASSIC add guards left and right to this allocation
259 * in order to detect out of bounds.
260 *
261 * This can't be passed if any of @c KMEM_GUARD_FIRST
262 * or @c KMEM_GUARD_LAST is used.
263 *
264 * @const KMEM_TAG (alloc, realloc, free)
265 * Under KASAN_TBI, this allocation is tagged non canonically.
266 */
267 __options_decl(kmem_flags_t, uint32_t, {
268 KMEM_NONE = 0x00000000,
269
270 /* Call behavior */
271 KMEM_NOFAIL = 0x00000001,
272 KMEM_NOPAGEWAIT = 0x00000002,
273 KMEM_FREEOLD = 0x00000004,
274 KMEM_REALLOCF = 0x00000008,
275 KMEM_NOSOFTLIMIT = 0x00000010,
276
277 /* How the entry is populated */
278 KMEM_VAONLY = 0x00000020,
279 KMEM_PAGEABLE = 0x00000040,
280 KMEM_ZERO = 0x00000080,
281
282 /* VM object to use for the entry */
283 KMEM_KOBJECT = 0x00000100,
284 KMEM_COMPRESSOR = 0x00000200,
285
286 /* How to look for addresses */
287 KMEM_LOMEM = 0x00001000,
288 KMEM_LAST_FREE = 0x00002000,
289 KMEM_GUESS_SIZE = 0x00004000,
290 KMEM_DATA = 0x00008000,
291 KMEM_DATA_SHARED = 0x00010000,
292 KMEM_SPRAYQTN = 0x00020000,
293
294 /* Entry properties */
295 KMEM_PERMANENT = 0x00200000,
296 KMEM_GUARD_FIRST = 0x00400000,
297 KMEM_GUARD_LAST = 0x00800000,
298 KMEM_KSTACK = 0x01000000,
299 KMEM_NOENCRYPT = 0x02000000,
300 KMEM_KASAN_GUARD = 0x04000000,
301 KMEM_TAG = 0x08000000,
302 });
303
304
305 /*
306 * @function kmem_range_id_size
307 *
308 * @abstract Return the addressable size of the memory range.
309 */
310 __pure2
311 extern vm_map_size_t kmem_range_id_size(
312 kmem_range_id_t range_id);
313
314 /**
315 * @enum kmem_claims_flags_t
316 *
317 * @abstract
318 * Set of flags used in the processing of kmem_range claims
319 *
320 * @discussion
321 * These flags are used by the kmem subsytem while processing kmem_range
322 * claims and are not explicitly passed by the caller registering the claim.
323 *
324 * @const KC_NO_ENTRY
325 * A vm map entry should not be created for the respective claim.
326 *
327 * @const KC_NO_MOVE
328 * The range shouldn't be moved once it has been placed as it has constraints.
329 */
330 __options_decl(kmem_claims_flags_t, uint32_t, {
331 KC_NONE = 0x00000000,
332 KC_NO_ENTRY = 0x00000001,
333 KC_NO_MOVE = 0x00000002,
334 });
335
336 /*
337 * Security config that creates the additional splits in non data part of
338 * kernel_map
339 */
340 #if KASAN || (__arm64__ && !defined(KERNEL_INTEGRITY_KTRR) && !defined(KERNEL_INTEGRITY_CTRR))
341 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT OFF
342 #else
343 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT ON
344 #endif
345
346 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__OFF() 0
347 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__ON() 1
348 #define ZSECURITY_CONFIG2(v) ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__##v()
349 #define ZSECURITY_CONFIG1(v) ZSECURITY_CONFIG2(v)
350 #define ZSECURITY_CONFIG(opt) ZSECURITY_CONFIG1(ZSECURITY_CONFIG_##opt)
351
352
353 struct kmem_range_startup_spec {
354 const char *kc_name;
355 struct mach_vm_range *kc_range;
356 vm_map_size_t kc_size;
357 vm_map_size_t (^kc_calculate_sz)(void);
358 kmem_claims_flags_t kc_flags;
359 };
360
361 extern void kmem_range_startup_init(
362 struct kmem_range_startup_spec *sp);
363
364 /*!
365 * @macro KMEM_RANGE_REGISTER_*
366 *
367 * @abstract
368 * Register a claim for kmem range or submap.
369 *
370 * @discussion
371 * Claims are shuffled during startup to randomize the layout of the kernel map.
372 * Temporary entries are created in place of the claims, therefore the caller
373 * must provide the start of the assigned range as a hint and
374 * @c{VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE} to kmem_suballoc to replace the mapping.
375 *
376 * Min/max constraints can be provided in the range when the claim is
377 * registered.
378 *
379 * This macro comes in 2 flavors:
380 * - STATIC : When the size of the range/submap is known at compile time
381 * - DYNAMIC: When the size of the range/submap needs to be computed
382 * Temporary entries are create
383 * The start of the
384 *
385 * @param name the name of the claim
386 * @param range the assigned range for the claim
387 * @param size the size of submap/range (if known at compile time)
388 * @param calculate_sz a block that returns the computed size of submap/range
389 */
390 #define KMEM_RANGE_REGISTER_STATIC(name, range, size) \
391 static __startup_data struct kmem_range_startup_spec \
392 __startup_kmem_range_spec_ ## name = { #name, range, size, NULL, KC_NONE}; \
393 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
394 &__startup_kmem_range_spec_ ## name)
395
396 #define KMEM_RANGE_REGISTER_DYNAMIC(name, range, calculate_sz) \
397 static __startup_data struct kmem_range_startup_spec \
398 __startup_kmem_range_spec_ ## name = { #name, range, 0, calculate_sz, \
399 KC_NONE}; \
400 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
401 &__startup_kmem_range_spec_ ## name)
402
403
404 #pragma mark kmem entry parameters
405
406 /*!
407 * @function kmem_entry_validate_guard()
408 *
409 * @brief
410 * Validates that the entry matches the input parameters, panic otherwise.
411 *
412 * @discussion
413 * If the guard has a zero @c kmg_guard value,
414 * then the entry must be non atomic.
415 *
416 * The guard tag is not used for validation as the VM subsystems
417 * (particularly in IOKit) might decide to substitute it in ways
418 * that are difficult to predict for the programmer.
419 *
420 * @param entry the entry to validate
421 * @param addr the supposed start address
422 * @param size the supposed size of the entry
423 * @param guard the guard to use to "authenticate" the allocation.
424 */
425 extern void kmem_entry_validate_guard(
426 vm_map_t map,
427 struct vm_map_entry *entry,
428 vm_offset_t addr,
429 vm_size_t size,
430 kmem_guard_t guard);
431
432 /*!
433 * @function kmem_size_guard()
434 *
435 * @brief
436 * Returns the size of an atomic kalloc allocation made in the specified map,
437 * according to the guard.
438 *
439 * @param map a kernel map to lookup the entry into.
440 * @param addr the kernel address to lookup.
441 * @param guard the guard to use to "authenticate" the allocation.
442 */
443 extern vm_size_t kmem_size_guard(
444 vm_map_t map,
445 vm_offset_t addr,
446 kmem_guard_t guard);
447
448
449 #pragma mark kmem allocations
450
451 /*!
452 * @typedef kma_flags_t
453 *
454 * @brief
455 * Flags used by the @c kmem_alloc* family of flags.
456 */
457 __options_decl(kma_flags_t, uint32_t, {
458 KMA_NONE = KMEM_NONE,
459
460 /* Call behavior */
461 KMA_NOFAIL = KMEM_NOFAIL,
462 KMA_NOPAGEWAIT = KMEM_NOPAGEWAIT,
463
464 /* How the entry is populated */
465 KMA_VAONLY = KMEM_VAONLY,
466 KMA_PAGEABLE = KMEM_PAGEABLE,
467 KMA_ZERO = KMEM_ZERO,
468 KMA_NOSOFTLIMIT = KMEM_NOSOFTLIMIT,
469
470 /* VM object to use for the entry */
471 KMA_KOBJECT = KMEM_KOBJECT,
472 KMA_COMPRESSOR = KMEM_COMPRESSOR,
473
474 /* How to look for addresses */
475 KMA_LOMEM = KMEM_LOMEM,
476 KMA_LAST_FREE = KMEM_LAST_FREE,
477 KMA_DATA = KMEM_DATA,
478 KMA_DATA_SHARED = KMEM_DATA_SHARED,
479 KMA_SPRAYQTN = KMEM_SPRAYQTN,
480
481 /* Entry properties */
482 KMA_PERMANENT = KMEM_PERMANENT,
483 KMA_GUARD_FIRST = KMEM_GUARD_FIRST,
484 KMA_GUARD_LAST = KMEM_GUARD_LAST,
485 KMA_KSTACK = KMEM_KSTACK,
486 KMA_NOENCRYPT = KMEM_NOENCRYPT,
487 KMA_KASAN_GUARD = KMEM_KASAN_GUARD,
488 KMA_TAG = KMEM_TAG,
489 });
490
491
492 /*!
493 * @function kmem_alloc_guard()
494 *
495 * @brief
496 * Master entry point for allocating kernel memory.
497 *
498 * @param map map to allocate into, must be a kernel map.
499 * @param size the size of the entry to allocate, must not be 0.
500 * @param mask an alignment mask that the returned allocation
501 * will be aligned to (ignoring guards, see @const
502 * KMEM_GUARD_FIRST).
503 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
504 * @param guard how to guard the allocation.
505 *
506 * @returns
507 * - the non zero address of the allocaation on success in @c kmr_address.
508 * - @c KERN_NO_SPACE if the target map is out of address space.
509 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
510 */
511 extern kmem_return_t kmem_alloc_guard(
512 vm_map_t map,
513 vm_size_t size,
514 vm_offset_t mask,
515 kma_flags_t flags,
516 kmem_guard_t guard) __result_use_check;
517
518 static inline kern_return_t
kernel_memory_allocate(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,kma_flags_t flags,vm_tag_t tag)519 kernel_memory_allocate(
520 vm_map_t map,
521 vm_offset_t *addrp,
522 vm_size_t size,
523 vm_offset_t mask,
524 kma_flags_t flags,
525 vm_tag_t tag)
526 {
527 kmem_guard_t guard = {
528 .kmg_tag = tag,
529 };
530 kmem_return_t kmr;
531
532 kmr = kmem_alloc_guard(map, size, mask, flags, guard);
533 if (kmr.kmr_return == KERN_SUCCESS) {
534 __builtin_assume(kmr.kmr_address != 0);
535 } else {
536 __builtin_assume(kmr.kmr_address == 0);
537 }
538 *addrp = kmr.kmr_address;
539 return kmr.kmr_return;
540 }
541
542 static inline kern_return_t
kmem_alloc(vm_map_t map,vm_offset_t * addrp,vm_size_t size,kma_flags_t flags,vm_tag_t tag)543 kmem_alloc(
544 vm_map_t map,
545 vm_offset_t *addrp,
546 vm_size_t size,
547 kma_flags_t flags,
548 vm_tag_t tag)
549 {
550 return kernel_memory_allocate(map, addrp, size, 0, flags, tag);
551 }
552
553 /*!
554 * @function kmem_alloc_contig_guard()
555 *
556 * @brief
557 * Variant of kmem_alloc_guard() that allocates a contiguous range
558 * of physical memory.
559 *
560 * @param map map to allocate into, must be a kernel map.
561 * @param size the size of the entry to allocate, must not be 0.
562 * @param mask an alignment mask that the returned allocation
563 * will be aligned to (ignoring guards, see @const
564 * KMEM_GUARD_FIRST).
565 * @param max_pnum The maximum page number to allocate, or 0.
566 * @param pnum_mask A page number alignment mask for the first allocated
567 * page, or 0.
568 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
569 * @param guard how to guard the allocation.
570 *
571 * @returns
572 * - the non zero address of the allocaation on success in @c kmr_address.
573 * - @c KERN_NO_SPACE if the target map is out of address space.
574 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
575 */
576 extern kmem_return_t kmem_alloc_contig_guard(
577 vm_map_t map,
578 vm_size_t size,
579 vm_offset_t mask,
580 ppnum_t max_pnum,
581 ppnum_t pnum_mask,
582 kma_flags_t flags,
583 kmem_guard_t guard);
584
585 static inline kern_return_t
kmem_alloc_contig(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,ppnum_t max_pnum,ppnum_t pnum_mask,kma_flags_t flags,vm_tag_t tag)586 kmem_alloc_contig(
587 vm_map_t map,
588 vm_offset_t *addrp,
589 vm_size_t size,
590 vm_offset_t mask,
591 ppnum_t max_pnum,
592 ppnum_t pnum_mask,
593 kma_flags_t flags,
594 vm_tag_t tag)
595 {
596 kmem_guard_t guard = {
597 .kmg_tag = tag,
598 };
599 kmem_return_t kmr;
600
601 kmr = kmem_alloc_contig_guard(map, size, mask,
602 max_pnum, pnum_mask, flags, guard);
603 if (kmr.kmr_return == KERN_SUCCESS) {
604 __builtin_assume(kmr.kmr_address != 0);
605 } else {
606 __builtin_assume(kmr.kmr_address == 0);
607 }
608 *addrp = kmr.kmr_address;
609 return kmr.kmr_return;
610 }
611
612
613 /*!
614 * @typedef kms_flags_t
615 *
616 * @brief
617 * Flags used by @c kmem_suballoc.
618 */
619 __options_decl(kms_flags_t, uint32_t, {
620 KMS_NONE = KMEM_NONE,
621
622 /* Call behavior */
623 KMS_NOFAIL = KMEM_NOFAIL,
624 KMS_NOSOFTLIMIT = KMEM_NOSOFTLIMIT,
625
626 /* How to look for addresses */
627 KMS_LAST_FREE = KMEM_LAST_FREE,
628 KMS_DATA = KMEM_DATA,
629
630 /* Entry properties */
631 KMS_PERMANENT = KMEM_PERMANENT,
632 });
633
634 /*!
635 * @function kmem_suballoc()
636 *
637 * @brief
638 * Create a kernel submap, in an atomic entry guarded with KMEM_GUARD_SUBMAP.
639 *
640 * @param parent map to allocate into, must be a kernel map.
641 * @param addr (in/out) the address for the map (see vm_map_enter)
642 * @param size the size of the entry to allocate, must not be 0.
643 * @param vmc_options the map creation options
644 * @param vm_flags a set of @c VM_FLAGS_* flags
645 * @param flags a set of @c KMS_* flags, (@see @c kmem_flags_t)
646 * @param tag the tag for this submap's entry.
647 */
648 extern kmem_return_t kmem_suballoc(
649 vm_map_t parent,
650 mach_vm_offset_t *addr,
651 vm_size_t size,
652 vm_map_create_options_t vmc_options,
653 int vm_flags,
654 kms_flags_t flags,
655 vm_tag_t tag);
656
657
658 #pragma mark kmem reallocation
659
660 /*!
661 * @typedef kmr_flags_t
662 *
663 * @brief
664 * Flags used by the @c kmem_realloc* family of flags.
665 */
666 __options_decl(kmr_flags_t, uint32_t, {
667 KMR_NONE = KMEM_NONE,
668
669 /* Call behavior */
670 KMR_NOPAGEWAIT = KMEM_NOPAGEWAIT,
671 KMR_FREEOLD = KMEM_FREEOLD,
672 KMR_REALLOCF = KMEM_REALLOCF,
673
674 /* How the entry is populated */
675 KMR_ZERO = KMEM_ZERO,
676
677 /* VM object to use for the entry */
678 KMR_KOBJECT = KMEM_KOBJECT,
679
680 /* How to look for addresses */
681 KMR_LOMEM = KMEM_LOMEM,
682 KMR_LAST_FREE = KMEM_LAST_FREE,
683 KMR_DATA = KMEM_DATA,
684 KMR_DATA_SHARED = KMEM_DATA_SHARED,
685 KMR_SPRAYQTN = KMEM_SPRAYQTN,
686
687 /* Entry properties */
688 KMR_GUARD_FIRST = KMEM_GUARD_FIRST,
689 KMR_GUARD_LAST = KMEM_GUARD_LAST,
690 KMR_KASAN_GUARD = KMEM_KASAN_GUARD,
691 KMR_TAG = KMEM_TAG,
692 });
693
694 #define KMEM_REALLOC_FLAGS_VALID(flags) \
695 (((flags) & (KMR_KOBJECT | KMEM_GUARD_LAST | KMEM_KASAN_GUARD | KMR_DATA)) == KMR_DATA \
696 || ((flags) & (KMR_KOBJECT | KMEM_GUARD_LAST | KMEM_KASAN_GUARD | KMR_DATA_SHARED)) == KMR_DATA_SHARED \
697 || ((flags) & KMR_FREEOLD))
698
699 /*!
700 * @function kmem_realloc_guard()
701 *
702 * @brief
703 * Reallocates memory allocated with kmem_alloc_guard()
704 *
705 * @discussion
706 * @c kmem_realloc_guard() either mandates a guard with atomicity set,
707 * or must use KMR_DATA (this is not an implementation limitation but
708 * but a security policy).
709 *
710 * If kmem_realloc_guard() is called for the kernel object
711 * (with @c KMR_KOBJECT) or with any trailing guard page,
712 * then the use of @c KMR_FREEOLD is mandatory.
713 *
714 * When @c KMR_FREEOLD isn't used, if the allocation was relocated
715 * as opposed to be extended or truncated in place, the caller
716 * must free its old mapping manually by calling @c kmem_free_guard().
717 *
718 * Note that if the entry is truncated, it will always be done in place.
719 *
720 *
721 * @param map map to allocate into, must be a kernel map.
722 * @param oldaddr the address to reallocate,
723 * passing 0 means @c kmem_alloc_guard() will be called.
724 * @param oldsize the current size of the entry
725 * @param newsize the new size of the entry,
726 * 0 means kmem_free_guard() will be called.
727 * @param flags a set of @c KMR_* flags, (@see @c kmem_flags_t)
728 * the exact same set of @c KMR_GUARD_* flags must
729 * be passed for all calls (@see kmem_flags_t).
730 * @param guard the allocation guard.
731 *
732 * @returns
733 * - the newly allocated address on success in @c kmr_address
734 * (note that if newsize is 0, then address will be 0 too).
735 * - @c KERN_NO_SPACE if the target map is out of address space.
736 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
737 */
738 extern kmem_return_t kmem_realloc_guard(
739 vm_map_t map,
740 vm_offset_t oldaddr,
741 vm_size_t oldsize,
742 vm_size_t newsize,
743 kmr_flags_t flags,
744 kmem_guard_t guard) __result_use_check
745 __attribute__((diagnose_if(!KMEM_REALLOC_FLAGS_VALID(flags),
746 "invalid realloc flags passed", "error")));
747
748 /*!
749 * @function kmem_realloc_should_free()
750 *
751 * @brief
752 * Returns whether the old address passed to a @c kmem_realloc_guard()
753 * call without @c KMR_FREEOLD must be freed.
754 *
755 * @param oldaddr the "oldaddr" passed to @c kmem_realloc_guard().
756 * @param kmr the result of that @c kmem_realloc_should_free() call.
757 */
758 static inline bool
kmem_realloc_should_free(vm_offset_t oldaddr,kmem_return_t kmr)759 kmem_realloc_should_free(
760 vm_offset_t oldaddr,
761 kmem_return_t kmr)
762 {
763 return oldaddr && oldaddr != kmr.kmr_address;
764 }
765
766
767 #pragma mark kmem free
768
769 /*!
770 * @typedef kmf_flags_t
771 *
772 * @brief
773 * Flags used by the @c kmem_free* family of flags.
774 */
775 __options_decl(kmf_flags_t, uint32_t, {
776 KMF_NONE = KMEM_NONE,
777
778 /* Call behavior */
779
780 /* How the entry is populated */
781
782 /* How to look for addresses */
783 KMF_GUESS_SIZE = KMEM_GUESS_SIZE,
784
785 /* Entry properties */
786 KMF_GUARD_FIRST = KMEM_GUARD_FIRST,
787 KMF_GUARD_LAST = KMEM_GUARD_LAST,
788 KMF_KASAN_GUARD = KMEM_KASAN_GUARD,
789 KMF_TAG = KMEM_TAG,
790 });
791
792
793 /*!
794 * @function kmem_free_guard()
795 *
796 * @brief
797 * Frees memory allocated with @c kmem_alloc or @c kmem_realloc.
798 *
799 * @param map map to free from, must be a kernel map.
800 * @param addr the address to free
801 * @param size the size of the memory to free
802 * @param flags a set of @c KMF_* flags, (@see @c kmem_flags_t)
803 * @param guard the allocation guard.
804 *
805 * @returns the size of the entry that was deleted.
806 * (useful when @c KMF_GUESS_SIZE was used)
807 */
808 extern vm_size_t kmem_free_guard(
809 vm_map_t map,
810 vm_offset_t addr,
811 vm_size_t size,
812 kmf_flags_t flags,
813 kmem_guard_t guard);
814
815 __attribute__((overloadable))
816 static inline void
kmem_free(vm_map_t map,vm_offset_t addr,vm_size_t size,kmf_flags_t flags)817 kmem_free(
818 vm_map_t map,
819 vm_offset_t addr,
820 vm_size_t size,
821 kmf_flags_t flags)
822 {
823 kmem_free_guard(map, addr, size, flags, KMEM_GUARD_NONE);
824 }
825
826 __attribute__((overloadable))
827 static inline void
kmem_free(vm_map_t map,vm_offset_t addr,vm_size_t size)828 kmem_free(
829 vm_map_t map,
830 vm_offset_t addr,
831 vm_size_t size)
832 {
833 kmem_free(map, addr, size, KMF_NONE);
834 }
835
836
837 #pragma mark kmem population
838
839 /*!
840 * @function kernel_memory_populate()
841 *
842 * @brief
843 * Populate pages for a given kernel map allocation.
844 *
845 * @discussion
846 * Allocations made against the kernel object (@c KMEM_KOBJECT)
847 * or the compressor object (@c KMEM_COMPRESSOR) must have their
848 * backing store explicitly managed by clients.
849 *
850 * This function will cause pages in the specified range to be allocated
851 * explicitly. No page must have been allocated for that range at this time,
852 * either because the allocation was done with @c KMEM_VAONLY or because pages
853 * were explicitly depopulated.
854 *
855 * @param addr the aligned starting address to populate.
856 * @param size the aligned size of the region to populate.
857 * @param flags a set flags that must match the flags passed at
858 * @c kmem_alloc*() time. In particular, one of
859 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
860 * @param tag the kernel memory tag to use for accounting purposes.
861 * @returns
862 * - KERN_SUCCESS the operation succeeded.
863 * - KERN_RESOURCE_SHORTAGE
864 * the kernel was out of physical pages.
865 */
866 extern kern_return_t kernel_memory_populate(
867 vm_offset_t addr,
868 vm_size_t size,
869 kma_flags_t flags,
870 vm_tag_t tag);
871
872
873 /*!
874 * @function kernel_memory_depopulate()
875 *
876 * @brief
877 * Depopulate pages for a given kernel map allocation.
878 *
879 * @discussion
880 * Allocations made against the kernel object (@c KMEM_KOBJECT)
881 * or the compressor object (@c KMEM_COMPRESSOR) must have their
882 * backing store explicitly managed by clients.
883 *
884 * This function will cause pages in the specified range to be deallocated
885 * explicitly. This range must be populated at the time of the call, either
886 * because the @c kmem_alloc*() call asked for pages, or because
887 * @c kernel_memory_populate() has been called explicitly.
888 *
889 * It is not necessary to explicitly depopulate ranges prior to calling
890 * @c kmem_free*(), even if populating the range was made explicitly with
891 * @c kernel_memory_populate() rather than implicitly at allocation time.
892 *
893 *
894 * @param addr the aligned starting address to depopulate.
895 * @param size the aligned size of the region to depopulate.
896 * @param flags a set flags that must match the flags passed at
897 * @c kmem_alloc*() time. In particular, one of
898 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
899 * @param tag the kernel memory tag to use for accounting purposes,
900 * which must match the tag used for population.
901 */
902 extern void kernel_memory_depopulate(
903 vm_offset_t addr,
904 vm_size_t size,
905 kma_flags_t flags,
906 vm_tag_t tag);
907
908
909 #pragma mark - VM_FLAGS_* / vm_map_kernel_flags_t conversions
910
911 /*!
912 * @function vm_map_kernel_flags_vmflags()
913 *
914 * @return The vmflags set in the specified @c vmk_flags.
915 */
916 extern int vm_map_kernel_flags_vmflags(
917 vm_map_kernel_flags_t vmk_flags);
918
919 /*!
920 * @function vm_map_kernel_flags_set_vmflags()
921 *
922 * @brief
923 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
924 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
925 */
926 __attribute__((overloadable))
927 extern void vm_map_kernel_flags_set_vmflags(
928 vm_map_kernel_flags_t *vmk_flags,
929 int vm_flags,
930 vm_tag_t vm_tag);
931
932 /*!
933 * @function vm_map_kernel_flags_set_vmflags()
934 *
935 * @brief
936 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
937 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
938 *
939 * @discussion
940 * This variant takes the tag from the top byte of the flags.
941 */
942 __attribute__((overloadable))
943 extern void vm_map_kernel_flags_set_vmflags(
944 vm_map_kernel_flags_t *vmk_flags,
945 int vm_flags_and_tag);
946
947 /*!
948 * @function vm_map_kernel_flags_and_vmflags()
949 *
950 * @brief
951 * Apply a mask to the vmflags.
952 */
953 extern void vm_map_kernel_flags_and_vmflags(
954 vm_map_kernel_flags_t *vmk_flags,
955 int vm_flags_mask);
956
957 /*!
958 * @function vm_map_kernel_flags_check_vmflags()
959 *
960 * @return
961 * Whether the @c vmk_flags @c vmf_* fields
962 * are limited to the specified mask.
963 */
964 extern bool vm_map_kernel_flags_check_vmflags(
965 vm_map_kernel_flags_t vmk_flags,
966 int vm_flags_mask);
967
968 /*!
969 * @function vm_map_kernel_flags_check_vm_and_kflags()
970 *
971 * @return
972 * Whether the @c vmk_flags @c vmf_* fields
973 * are limited to the specified mask.
974 */
975 extern bool vm_map_kernel_flags_check_vm_and_kflags(
976 vm_map_kernel_flags_t vmk_flags,
977 int vm_flags_mask);
978
979
980 #pragma mark - kernel variants of the Mach VM interfaces
981
982 /*!
983 * @function mach_vm_allocate_kernel()
984 *
985 * @brief
986 * Allocate memory in the specified map.
987 *
988 * @discussion
989 * This is the in-kernel equivalent to the @c mach_vm_allocate() MIG call,
990 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
991 * @c VM_FLAGS_.
992 *
993 * Memory will not be pre-faulted and touching it will cause page faults.
994 *
995 * Memory will be zero-filled when faulted, unless the map's @c no_zero_fill
996 * property is set (only the @c ipc_kernel_map is marked this way).
997 *
998 * The allocation being made will have:
999 * - @c VM_PROT_DEFAULT (rw-) current protections,
1000 * - @c VM_PROT_ALL (rwx) max protections,
1001 * - @c VM_INHERIT_COPY inheritance.
1002 *
1003 *
1004 * @param map the map to allocate memory into.
1005 *
1006 * @param addr_u [in] when @c vmk_flags.vmf_fixed is set, @c *addr
1007 * is used as the address at which the allocation
1008 * must be made. If it is misaligned for the map's
1009 * page size, its low bits are truncated and ignored.
1010 *
1011 * when @c vmk_flags.vmf_fixed is not set,
1012 * the value of @c *addrp is ignored.
1013 *
1014 * [out] filled with the address at which the allocation
1015 * was made on success, unmodified otherwise.
1016 *
1017 * @c vmk_flags.vmf_return_data_addr has no effect
1018 * on the returned address.
1019 *
1020 * @param size_u the size of the allocation to make. zero sizes are
1021 * allowed and result in @c *addr being zero.
1022 *
1023 * misaligned sizes for the page size of the target map
1024 * will be rounded up to the nearest page size.
1025 *
1026 * @param vmk_flags a set of flags that influence the properties of the
1027 * allocation being made.
1028 *
1029 * @returns KERN_SUCCESS when the operation succeeds,
1030 * or an error denoting the reason for failure.
1031 */
1032 extern kern_return_t mach_vm_allocate_kernel(
1033 vm_map_t map,
1034 mach_vm_offset_ut *addr_u,
1035 mach_vm_size_ut size_u,
1036 vm_map_kernel_flags_t vmk_flags);
1037
1038
1039 /*!
1040 * @function mach_vm_map_kernel()
1041 *
1042 * @brief
1043 * Map some range of an object into an address space.
1044 *
1045 * @discussion
1046 * This is the in-kernel equivalent to the @c mach_vm_map() MIG call,
1047 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1048 * @c VM_FLAGS_.
1049 *
1050 *
1051 * @param target_map the target address space.
1052 *
1053 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1054 * is used as the address at which the allocation
1055 * must be made. If it is misaligned for the map's
1056 * page size, its low bits are truncated and ignored.
1057 *
1058 * when @c vmk_flags.vmf_fixed is not set,
1059 * the value of @c *address is used as a starting
1060 * point from which to scan for memory in the direction
1061 * specified by @c vmk_flags.vmkf_last_free.
1062 *
1063 * [out] filled with the address at which the allocation
1064 * was made on success, unmodified otherwise.
1065 *
1066 * if @c vmk_flags.vmf_return_data_addr is
1067 * specified, @c *address will maintain
1068 * "misalignment" from the requested @c offset
1069 * inside the object, otherwise it will be page
1070 * aligned for the target map.
1071 *
1072 * @param size the size of the allocation to make.
1073 * zero sizes are disallowed.
1074 *
1075 * @param mask a mask to specify the allocation alignment.
1076 * this value should be of the form (2^n-1).
1077 *
1078 * allocations will always at least be page aligned
1079 * for the specified target map, but this can be used to
1080 * require larger alignments.
1081 *
1082 * @param vmk_flags a set of flags that influence the properties of the
1083 * allocation being made.
1084 *
1085 * @param port the object to map into the target address space:
1086 * - @c IP_NULL means anonymous memory, and this call
1087 * behaves like a more versatile
1088 * mach_vm_allocate_kernel(),
1089 * - a kobject port of type @c IKOT_NAMED_ENTRY,
1090 * pointing to a @c vm_named_entry_t,
1091 * - a naked @c memory_object_t (a VM pager).
1092 *
1093 * @param offset an offset within the specified object to map.
1094 *
1095 * @param copy whether to make a copy-on-write (true) mapping
1096 * or a shared (false) mapping.
1097 *
1098 * @param cur_prot the effective protections for the mapping.
1099 *
1100 * @param max_prot the maximum protections for the mapping,
1101 * which must at least cover @c cur_prot.
1102 *
1103 * @param inheritance the inheritance policy for the mapping.
1104 *
1105 * @returns KERN_SUCCESS when the operation succeeds,
1106 * or an error denoting the reason for failure.
1107 */
1108 extern kern_return_t mach_vm_map_kernel(
1109 vm_map_t target_map,
1110 mach_vm_offset_ut *address,
1111 mach_vm_size_ut size,
1112 mach_vm_offset_ut mask,
1113 vm_map_kernel_flags_t vmk_flags,
1114 ipc_port_t port,
1115 memory_object_offset_ut offset,
1116 boolean_t copy,
1117 vm_prot_ut cur_prot,
1118 vm_prot_ut max_prot,
1119 vm_inherit_ut inheritance);
1120
1121
1122 /*!
1123 * @function mach_vm_remap_new_kernel()
1124 *
1125 * @brief
1126 * Remap a range of memory from one address space to another.
1127 *
1128 * @discussion
1129 * This is the in-kernel equivalent to the @c mach_vm_remap() MIG call,
1130 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1131 * @c VM_FLAGS_.
1132 *
1133 * This call forces vmk_flags.vmf_return_data_addr to true regardless
1134 * of what the caller specified.
1135 *
1136 *
1137 * @param target_map the target address space.
1138 *
1139 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1140 * is used as the address at which the allocation
1141 * must be made. If it is misaligned for the map's
1142 * page size, its low bits are truncated and ignored.
1143 *
1144 * when @c vmk_flags.vmf_fixed is not set,
1145 * the value of @c *address is used as a starting
1146 * point from which to scan for memory in the direction
1147 * specified by @c vmk_flags.vmkf_last_free.
1148 *
1149 * [out] filled with the address at which the allocation
1150 * was made on success, unmodified otherwise.
1151 * @c *address has the same "misalignment"
1152 * as @c src_address.
1153 *
1154 * @param size the size of the region to remap.
1155 *
1156 * @param mask a mask to specify the allocation alignment.
1157 * this value should be of the form (2^n-1).
1158 *
1159 * allocations will always at least be page aligned
1160 * for the specified target map, but this can be used to
1161 * require larger alignments.
1162 *
1163 * @param vmk_flags a set of flags that influence the properties of the
1164 * allocation being made.
1165 *
1166 * @param src_map the address space to remap the memory from.
1167 *
1168 * @param src_address the start address within @c src_map to remap.
1169 *
1170 * @param copy whether to make a copy-on-write (true) mapping
1171 * or a shared (false) mapping.
1172 *
1173 * @param cur_prot [in] for shared mappings, the minimum set of effective
1174 * permissions the source mapping must have
1175 *
1176 * [out] the resulting effective permissions for the mapping
1177 *
1178 * @param max_prot [in] for shared mappings, the minimum set of maximum
1179 * permissions the source mappings must have.
1180 *
1181 * [out] the resulting maximum permissions for the mapping
1182 *
1183 * @param inheritance the inheritance policy for the mapping.
1184 *
1185 * @returns KERN_SUCCESS when the operation succeeds,
1186 * or an error denoting the reason for failure.
1187 */
1188 extern kern_return_t mach_vm_remap_new_kernel(
1189 vm_map_t target_map,
1190 mach_vm_offset_ut *address,
1191 mach_vm_size_ut size,
1192 mach_vm_offset_ut mask,
1193 vm_map_kernel_flags_t vmk_flags,
1194 vm_map_t src_map,
1195 mach_vm_offset_ut src_address,
1196 boolean_t copy,
1197 vm_prot_ut *cur_prot,
1198 vm_prot_ut *max_prot,
1199 vm_inherit_ut inheritance);
1200
1201 /*!
1202 * @function vm_map_wire_kernel()
1203 *
1204 * @brief
1205 * Sets the pageability of the specified address range in the
1206 * target map as wired.
1207 *
1208 * @discussion
1209 * This call is the kernel version of @c vm_map_wire(). The main difference
1210 * is that the caller must specify a valid @c VM_KERN_MEMORY_* tag for kernel
1211 * wirings, or a valid @c VM_MEMORY_* tag for user wirings.
1212 *
1213 * Consult the documentation of @c vm_map_wire() in @c <vm/vm_map.h> for details.
1214 */
1215 extern kern_return_t vm_map_wire_kernel(
1216 vm_map_t map,
1217 vm_map_offset_ut start_u,
1218 vm_map_offset_ut end_u,
1219 vm_prot_ut prot_u,
1220 vm_tag_t tag,
1221 boolean_t user_wire);
1222
1223 /*!
1224 * @function vm_map_purgable_control()
1225 *
1226 * @brief
1227 * Perform a purgeability operation on a VM object at a given address
1228 * in an address space.
1229 *
1230 * @discussion
1231 * This is the in-kernel equivalent to the @c mach_vm_map_purgable_control()
1232 * MIG call, except that it allows all operations, including
1233 * @c VM_PURGABLE_*_FROM_KERNEL ones).
1234 *
1235 * Valid @c control operations and the meaning of @c state is documented
1236 * in @c <mach/vm_purgable.h>.
1237 *
1238 *
1239 * @param map the target address space
1240 * @param address the address to use to find the VM object to target
1241 * @param control a purgeability operation to perform.
1242 * @param state an in/out parameter that is operation dependent.
1243 */
1244 extern kern_return_t vm_map_purgable_control(
1245 vm_map_t map,
1246 vm_map_offset_ut address,
1247 vm_purgable_t control,
1248 int *state);
1249
1250 extern kern_return_t mach_vm_purgable_control(
1251 vm_map_t map,
1252 mach_vm_offset_ut address_u,
1253 vm_purgable_t control,
1254 int *state);
1255
1256
1257 #ifdef MACH_KERNEL_PRIVATE
1258 #pragma mark - map copyio
1259
1260
1261 /*!
1262 * @function copyinmap()
1263 *
1264 * @brief
1265 * Like copyin, except that @c fromaddr is an address in the specified VM map.
1266 *
1267 * @param map the map @c fromaddr is relative to.
1268 * @param fromaddr the address to copy from within @c map.
1269 * @param todata the kernel buffer to write into.
1270 * @param length the number of bytes to copy.
1271 * @returns
1272 * - KERN_SUCCESS the copy was successful
1273 * - KERN_INVALID_ADDRESS
1274 * a fault occurred during copyio and couldn't be resolved
1275 * (similar to copyin returning EFAULT).
1276 */
1277 extern kern_return_t copyinmap(
1278 vm_map_t map,
1279 vm_map_offset_t fromaddr,
1280 void *todata __sized_by(length),
1281 vm_size_t length);
1282
1283
1284 /*!
1285 * @function copyoutmap()
1286 *
1287 * @brief
1288 * Like copyout, except that @c toaddr is an address in the specified VM map.
1289 *
1290 * @param map the map @c toaddr is relative to
1291 * @param fromdata the kernel buffer to copy from.
1292 * @param toaddr the address within @c map to copy into
1293 * @param length the number of bytes to copy.
1294 * @returns
1295 * - KERN_SUCCESS the copy was successful
1296 * - KERN_INVALID_ADDRESS
1297 * a fault occurred during copyio and couldn't be resolved
1298 * (similar to copyin returning EFAULT).
1299 */
1300 extern kern_return_t copyoutmap(
1301 vm_map_t map,
1302 void *fromdata __sized_by(length),
1303 vm_map_offset_t toaddr,
1304 vm_size_t length);
1305
1306
1307 /*!
1308 * @function copyoutmap_atomic32()
1309 *
1310 * @brief
1311 * Copies out a 32bit value atomically at a given address in a specified VM map.
1312 *
1313 * @param map the specified map.
1314 * @param value the 32 bit value to write at @c toaddr.
1315 * @param toaddr the address within @c map to copy into
1316 * @returns
1317 * - KERN_SUCCESS the copy was successful
1318 * - KERN_INVALID_ADDRESS
1319 * a fault occurred during copyio and couldn't be resolved
1320 * (similar to copyin returning EFAULT).
1321 */
1322 extern kern_return_t copyoutmap_atomic32(
1323 vm_map_t map,
1324 uint32_t value,
1325 vm_map_offset_t toaddr);
1326
1327
1328 /*!
1329 * @function copyoutmap_atomic64()
1330 *
1331 * @brief
1332 * Copies out a 64bit value atomically at a given address in a specified VM map.
1333 *
1334 * @param map the specified map.
1335 * @param value the 64 bit value to write at @c toaddr.
1336 * @param toaddr the address within @c map to copy into
1337 * @returns
1338 * - KERN_SUCCESS the copy was successful
1339 * - KERN_INVALID_ADDRESS
1340 * a fault occurred during copyio and couldn't be resolved
1341 * (similar to copyin returning EFAULT).
1342 */
1343 extern kern_return_t copyoutmap_atomic64(
1344 vm_map_t map,
1345 uint64_t value,
1346 vm_map_offset_t toaddr);
1347
1348
1349 #endif /* MACH_KERNEL_PRIVATE */
1350 #pragma mark - accounting
1351
1352 #pragma mark accounting: kern allocation name
1353
1354 /*!
1355 * @function kern_allocation_update_size()
1356 *
1357 * @brief
1358 * Update accounting for a specified kern allocation name.
1359 *
1360 * @discussion
1361 * This is to be called when memory gets wired/unwired in order
1362 * to update accounting information.
1363 *
1364 * [development kernels] If the @c vmtaglog boot-arg is used,
1365 * and its value matches the VM tag of this allocation name, then VM tag
1366 * log entries will be added on the specified object.
1367 *
1368 *
1369 * @param allocation a @c kern_allocation_name_t made with
1370 * @c kern_allocation_name_allocate().
1371 * @param delta the amount to update the accounting with,
1372 * positive values increment,
1373 * negative values decrement.
1374 * @param object an optional object this wiring/unwiring applies to.
1375 */
1376 extern void kern_allocation_update_size(
1377 kern_allocation_name_t allocation,
1378 int64_t delta,
1379 vm_object_t object);
1380
1381
1382 /*!
1383 * @function kern_allocation_update_subtotal()
1384 *
1385 * @brief
1386 * Update subtotal accounting for a specified kern allocation name.
1387 *
1388 * @discussion
1389 * IOKit uses global kern allocation names that cover the VM tags
1390 * (@see @c vm_tag_bt()) of several kexts and uses this to perform
1391 * accounting per kext within these meta accounting data structures.
1392 *
1393 * @param allocation a @c kern_allocation_name_t made with
1394 * @c kern_allocation_name_allocate()
1395 * and a non 0 "subtotalscount".
1396 * @param subtag a @c vm_tag_t subtag to update accounting for.
1397 * @param delta the amount to update the accounting with,
1398 * positive values increment,
1399 * negative values decrement.
1400 */
1401 extern void kern_allocation_update_subtotal(
1402 kern_allocation_name_t allocation,
1403 vm_tag_t subtag,
1404 int64_t delta);
1405
1406
1407 #pragma mark accounting: vm tags
1408
1409 /*
1410 * VM Tags come in 3 flavors:
1411 *
1412 * - static user VM tags, defined by the @c VM_MEMORY_* constants
1413 * (@see <mach/vm_statistics.h>),
1414 *
1415 * - static kernel VM tags, defined by the @c VM_KERN_MEMORY_* constants
1416 * (@see <mach/vm_statistics.h>),
1417 *
1418 * - dynamically allocated kernel VM tags typically associated with kexts
1419 * lazily (as kexts wire down memory and accounting needs to be made).
1420 *
1421 * By default, kernel VM tags track wired memory at the VM layer,
1422 * but no insight is given to allocations done by @c kalloc*()
1423 * and its wrappers like @c IOMalloc*(), when backed by zone memory
1424 * (for sizes below @c KHEAP_MAX_SIZE).
1425 *
1426 *
1427 * Implementation details
1428 * ~~~~~~~~~~~~~~~~~~~~~~
1429 *
1430 * Static tags are limited to values from 0 to 255, as they are passed
1431 * in the bits reserved by the @c VM_FLAGS_ALIAS_MASK of VM flags.
1432 *
1433 * However, dynamic flags are an internal kernel concept which is limited
1434 * by the size of the storage in VM map entries, which reserves
1435 * @c VME_ALIAS_BITS (12) for this, effectively limiting dynamic
1436 * tags to about 4000.
1437 *
1438 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE) defines the range of
1439 * possible values for dynamic tags.
1440 *
1441 *
1442 * Zone accounting [development kernels]
1443 * ~~~~~~~~~~~~~~~
1444 *
1445 * On development kernels the "-zt" boot-arg can be used, in which case
1446 * precise accounting of @c kalloc*() allocations is enabled per bucket,
1447 * which can be observed by the @c zprint(1) command.
1448 *
1449 * This effectively makes accounting a vector composed of @c VM_TAG_SIZECLASSES
1450 * possible zone size classes in addition to the regular VM wired memory
1451 * accounting that is always performed.
1452 *
1453 * The tag for which an allocation is recorded will be (in first hit order):
1454 *
1455 * - the tag specified as part of the @c zalloc_flags_t explicitly
1456 * (@see @c kalloc_*_tag() or @c Z_VM_TAG()). In core XNU, non @c _tag
1457 * variants of @c kalloc*() will generate a per-call site dynamic VM tag,
1458 * using the @c VM_ALLOC_SITE_TAG() macro,
1459 *
1460 * - if the allocation was made by a kernel extension, the dynamic VM tag
1461 * for this extension (@see @c vm_tag_bt()),
1462 *
1463 * - as a fallback:
1464 * o @c VM_KERN_MEMORY_KALLOC_DATA for @c kalloc_data() calls,
1465 * o @c VM_KERN_MEMORY_KALLOC_TYPE for @c kalloc_type() calls,
1466 * o @c VM_KERN_MEMORY_KALLOC for legacy @c kalloc() calls.
1467 */
1468
1469 /*!
1470 * @brief
1471 * Lock used to serialize the @c vm_tag_alloc() operation, used by IOKit.
1472 */
1473 extern lck_ticket_t vm_allocation_sites_lock;
1474
1475
1476 /*!
1477 * @function vm_tag_bt()
1478 *
1479 * @brief
1480 * Returns the dynamic kernel extension tag based on backtracing from this call.
1481 *
1482 * @discussion
1483 * This function will lazily allocate a dynamic tag for the current kernel
1484 * extension based on the backtrace, and then return a stable identifier.
1485 *
1486 * This might fail for cases where the kernel is out of dynamic tags.
1487 *
1488 *
1489 * @returns A dynamic kernel extension tag within
1490 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1491 * or @c VM_KERN_MEMORY_NONE if the call was not made
1492 * from a kernel extension, or allocating a dynamic tag
1493 * for it failed.
1494 */
1495 extern vm_tag_t vm_tag_bt(void);
1496
1497
1498 /*!
1499 * @function vm_tag_alloc()
1500 *
1501 * @brief
1502 * Lazily allocates a dynamic VM tag for a given allocation site.
1503 *
1504 * @description
1505 * This is used by IOKit and the zalloc subsystem to generate dynamic VM tags
1506 * for kernel extensions (@see vm_tag_bt()) or core kernel @c kalloc*()
1507 * call sites (@see VM_ALLOC_SITE_TAG()).
1508 *
1509 *
1510 * @param site the allocation site to generate a tag for.
1511 * @returns A dynamic kernel extension tag within
1512 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1513 * or @c VM_KERN_MEMORY_NONE if the kernel
1514 * is out of dynamic tags.
1515 */
1516 extern vm_tag_t vm_tag_alloc(
1517 vm_allocation_site_t *site);
1518
1519 /*!
1520 * @function vm_tag_alloc_locked()
1521 *
1522 * @brief
1523 * Lazily allocates a dynamic VM tag for a given allocation site,
1524 * while the caller holds the @c vm_allocation_sites_lock lock.
1525 *
1526 * @description
1527 * This is used by IOKit to generate dynamic VM tags for kernel extensions
1528 * (@see vm_tag_bt()).
1529 *
1530 *
1531 * @param site the allocation site to generate a tag for.
1532 * @param releasesiteP an optional allocation site data structure
1533 * that the caller is responsible for releasing
1534 * with @c kern_allocation_name_release().
1535 */
1536 extern void vm_tag_alloc_locked(
1537 vm_allocation_site_t *site,
1538 vm_allocation_site_t **releasesiteP);
1539
1540
1541 /*!
1542 * @function vm_tag_update_size()
1543 *
1544 * @brief
1545 * Update accounting for a specified VM kernel tag (static or dynamic).
1546 *
1547 * @discussion
1548 * This is to be called when memory gets wired/unwired in order
1549 * to update accounting information for a given tag.
1550 *
1551 * [development kernels] If the @c vmtaglog boot-arg is used,
1552 * and its value matches the VM tag of this allocation name, then VM tag
1553 * log entries will be added on the specified object.
1554 *
1555 *
1556 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1557 * @param delta the amount to update the accounting with,
1558 * positive values increment,
1559 * negative values decrement.
1560 * @param object an optional object this wiring/unwiring applies to.
1561 */
1562 extern void vm_tag_update_size(
1563 vm_tag_t tag,
1564 int64_t delta,
1565 vm_object_t object);
1566
1567
1568 /*!
1569 * @function vm_tag_get_size()
1570 *
1571 * @brief
1572 * Returns how much wired memory is accounted for the specified VM kernel tag.
1573 *
1574 *
1575 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1576 * @returns the amount of wired memory for the specified tag.
1577 */
1578 extern uint64_t vm_tag_get_size(
1579 vm_tag_t tag);
1580
1581 #if VM_TAG_SIZECLASSES
1582
1583 /*!
1584 * @function vm_tag_will_update_zone()
1585 *
1586 * @brief
1587 * Lazily allocate the vector of zone-level accounting for a given VM tag.
1588 *
1589 * @discussion
1590 * This is to be called when memory gets wired/unwired in order
1591 * to update accounting information for a given tag.
1592 *
1593 * This function might fail when the first time it is called is
1594 * for a @c Z_NOWAIT allocation. However this is transient and
1595 * will always eventually resolve. Once the data structure is allocated,
1596 * this function always succeeds. The consequence is a slight misaccounting
1597 * of a few allocations.
1598 *
1599 * This call will never fail for the following tags that are always
1600 * pre-allocated:
1601 * - @c VM_KERN_MEMORY_DIAG,
1602 * - @c VM_KERN_MEMORY_KALLOC,
1603 * - @c VM_KERN_MEMORY_KALLOC_DATA,
1604 * - @c VM_KERN_MEMORY_KALLOC_TYPE,
1605 * - @c VM_KERN_MEMORY_LIBKERN,
1606 * - @c VM_KERN_MEMORY_OSFMK,
1607 * - @c VM_KERN_MEMORY_RECOUNT.
1608 *
1609 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1610 *
1611 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1612 * @param zflags the @c zalloc_flags_t passed to the current
1613 * @c kalloc*() call.
1614 *
1615 * @returns @c tag if the allocation was successful,
1616 * @c VM_KERN_MEMORY_NONE if the accounting
1617 * data structure couldn't be allocated.
1618 */
1619 extern vm_tag_t vm_tag_will_update_zone(
1620 vm_tag_t tag,
1621 uint32_t zflags);
1622
1623
1624 /*!
1625 * @function vm_tag_update_zone_size()
1626 *
1627 * @brief
1628 * Update the per size class zone level accounting for a given kernel VM tag.
1629 *
1630 * @discussion
1631 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1632 *
1633 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1634 * @param size_class the zone size class index to account against
1635 * (@see zone_t::z_tags_sizeclass).
1636 * @param delta the amount to update the accounting with,
1637 * positive values increment,
1638 * negative values decrement.
1639 */
1640 extern void vm_tag_update_zone_size(
1641 vm_tag_t tag,
1642 uint32_t size_class,
1643 long delta);
1644
1645 #endif /* VM_TAG_SIZECLASSES */
1646
1647 #pragma mark accounting: diagnostics and query interfaces
1648
1649 /*!
1650 * @function vm_page_diagnose_estimate()
1651 *
1652 * @brief
1653 * Estimate how many @c mach_memory_info_t structures
1654 * are needed in order to return information about VM kernel tags,
1655 * and per size class zone level accounting when enabled.
1656 */
1657 extern uint32_t vm_page_diagnose_estimate(void);
1658
1659
1660 /*!
1661 * @function vm_page_diagnose()
1662 *
1663 * @brief
1664 * Fills out a @c mach_memory_info_t array of information about VM tags
1665 * and per size class zone level information.
1666 *
1667 * @param info a pointer to an array of @c num_info Mach memory
1668 * info data structures to fill.
1669 * @param num_info the number of entries in the @c info array.
1670 * @param zones_collectable_bytes
1671 * how much memory is collectable in the zone subsystem
1672 * if a @c zone_gc() was running right now.
1673 * @param redact_info whether information that could leak the type to bucket
1674 * mapping in @c kalloc_type() can be returned or not.
1675 *
1676 * @returns
1677 * - KERN_SUCCESS the call was successful.
1678 * - KERN_ABORTED the accounting subsytem isn't inititalized yet.
1679 */
1680 extern kern_return_t vm_page_diagnose(
1681 struct mach_memory_info *info __counted_by(num_info),
1682 unsigned int num_info,
1683 uint64_t zones_collectable_bytes,
1684 bool redact_info);
1685
1686 #if DEBUG || DEVELOPMENT
1687
1688 /*!
1689 * @typedef kmem_gobj_stats
1690 *
1691 * @brief
1692 * Statistics about the "guard objects" allocator for the pointer ranges
1693 * of kmem.
1694 */
1695 typedef struct {
1696 vm_map_size_t meta_sz; /**< total faulted size of metadata */
1697 vm_map_size_t pte_sz; /**< total faulted leaf PTE size */
1698 vm_map_size_t total_va; /**< total amount of VA ever used */
1699 vm_map_size_t total_used; /**< current amount of VA allocated */
1700 } kmem_gobj_stats;
1701
1702
1703 /*!
1704 * @function kmem_get_gobj_stats()
1705 *
1706 * @brief
1707 * Returns statistics about the guard objects allocator.
1708 *
1709 * @description
1710 * This is the backend of the @c vm.kmem_gobj_stats sysctl.
1711 */
1712 extern kmem_gobj_stats kmem_get_gobj_stats(void);
1713
1714
1715 /*!
1716 * @function vm_kern_allocation_info()
1717 *
1718 * @brief
1719 * Returns information about a given kernel heap allocation.
1720 *
1721 *
1722 * @param [in] addr the heap allocation pointer.
1723 * @param [out] size a guess at the size of this allocation.
1724 * @param [out] tag the kernel VM tag for this allocation
1725 * (only filled if the "-zt" boot-arg is set
1726 * for zone allocations)
1727 * @param [out] zone_size
1728 * the zone size class if the allocation
1729 * is from a zone, 0 for VM.
1730 *
1731 * @returns
1732 * - KERN_SUCCESS if a guess could be made about this pointer.
1733 * - KERN_INVALID_ADDRESS
1734 * if the address couldn't be resolved in the kernel heap.
1735 */
1736 extern kern_return_t vm_kern_allocation_info(
1737 uintptr_t addr,
1738 vm_size_t *size,
1739 vm_tag_t *tag,
1740 vm_size_t *zone_size);
1741
1742 #endif /* DEBUG || DEVELOPMENT */
1743
1744 #pragma mark - init methods
1745
1746 /*!
1747 * @function vm_init_before_launchd()
1748 *
1749 * @brief
1750 * Memorize how many wired pages were used at boot before launchd starts.
1751 *
1752 * @discussion
1753 * The captured number can be seen as the @c VM_KERN_COUNT_WIRED_BOOT value
1754 * in the output of @c zprint(1).
1755 */
1756 extern void vm_init_before_launchd(void);
1757
1758 #if VM_TAG_SIZECLASSES
1759
1760 /*!
1761 * @function vm_allocation_zones_init()
1762 *
1763 * @brief
1764 * Initialize the per-zone accounting tags subsystem
1765 * if the "-zt" boot-arg is present.
1766 */
1767 extern void vm_allocation_zones_init(void);
1768
1769 #endif /* VM_TAG_SIZECLASSES */
1770
1771 extern memory_object_t device_pager_setup(memory_object_t, uintptr_t, vm_size_t, int);
1772
1773 extern kern_return_t device_pager_populate_object( memory_object_t device,
1774 memory_object_offset_t offset, ppnum_t page_num, vm_size_t size);
1775
1776 #endif /* XNU_KERNEL_PRIVATE */
1777 #pragma GCC visibility pop
1778 __END_DECLS
1779
1780 #endif /* _VM_VM_KERN_XNU_H_ */
1781