1 /*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _VM_VM_KERN_XNU_H_
30 #define _VM_VM_KERN_XNU_H_
31
32 #include <sys/cdefs.h>
33 #include <vm/vm_kern.h>
34
35 __BEGIN_DECLS
36 __exported_push_hidden
37 #ifdef XNU_KERNEL_PRIVATE
38
39
40 #pragma mark - the kmem subsystem
41
42 /*
43 * "kmem" is a set of methods that provide interfaces suitable
44 * to allocate memory from the VM in the kernel map or submaps.
45 *
46 * It provide leaner alternatives to some of the VM functions,
47 * closer to a typical allocator.
48 */
49
50 struct mach_memory_info;
51 struct vm_page;
52 struct vm_map_entry;
53
54 /*!
55 * @typedef
56 *
57 * @brief
58 * Pair of a return code and size/address/... used by kmem interfaces.
59 *
60 * @discussion
61 * Using a pair of integers allows the compiler to return everything
62 * through registers, and doesn't need to use stack values to get results,
63 * which yields significantly better codegen.
64 *
65 * If @c kmr_return is not @c KERN_SUCCESS, then the other field
66 * of the union is always supposed to be 0.
67 */
68 typedef struct {
69 kern_return_t kmr_return;
70 union {
71 vm_address_t kmr_address;
72 vm_size_t kmr_size;
73 void *kmr_ptr;
74 vm_map_t kmr_submap;
75 };
76 } kmem_return_t;
77
78 /*!
79 * @typedef kmem_guard_t
80 *
81 * @brief
82 * KMEM guards are used by the kmem_* subsystem to secure atomic allocations.
83 *
84 * @discussion
85 * This parameter is used to transmit the tag for the allocation.
86 *
87 * If @c kmg_atomic is set, then the other fields are also taken into account
88 * and will affect the allocation behavior for this allocation.
89 *
90 * @field kmg_tag The VM_KERN_MEMORY_* tag for this entry.
91 * @field kmg_type_hash Some hash related to the type of the allocation.
92 * @field kmg_atomic Whether the entry is atomic.
93 * @field kmg_submap Whether the entry is for a submap.
94 * @field kmg_context A use defined 30 bits that will be stored
95 * on the entry on allocation and checked
96 * on other operations.
97 */
98 typedef struct {
99 uint16_t kmg_tag;
100 uint16_t kmg_type_hash;
101 uint32_t kmg_atomic : 1;
102 uint32_t kmg_submap : 1;
103 uint32_t kmg_context : 30;
104 } kmem_guard_t;
105 #define KMEM_GUARD_NONE (kmem_guard_t){ }
106 #define KMEM_GUARD_SUBMAP (kmem_guard_t){ .kmg_atomic = 0, .kmg_submap = 1 }
107
108
109 /*!
110 * @typedef kmem_flags_t
111 *
112 * @brief
113 * Sets of flags taken by several of the @c kmem_* family of functions.
114 *
115 * @discussion
116 * This type is not used directly by any function, it is an underlying raw
117 * type that is re-vended under different namespaces for each @c kmem_*
118 * interface.
119 *
120 * - @c kmem_alloc uses @c kma_flags_t / @c KMA_* namespaced values.
121 * - @c kmem_suballoc uses @c kms_flags_t / @c KMS_* namespaced values.
122 * - @c kmem_realloc uses @c kmr_flags_t / @c KMR_* namespaced values.
123 * - @c kmem_free uses @c kmf_flags_t / @c KMF_* napespaced values.
124 *
125 *
126 * <h2>Call behavior</h2>
127 *
128 * @const KMEM_NONE (all)
129 * Pass this when no special options is to be used.
130 *
131 * @const KMEM_NOFAIL (alloc, suballoc)
132 * When this flag is passed, any allocation failure results into a panic().
133 * Using this flag should really be limited to cases when failure is not
134 * recoverable and possibly during early boot only.
135 *
136 * @const KMEM_NOPAGEWAIT (alloc, realloc)
137 * Pass this flag if the system should not wait in VM_PAGE_WAIT().
138 *
139 * @const KMEM_FREEOLD (realloc)
140 * Pass this flag if @c kmem_realloc should free the old mapping
141 * (when the address changed) as part of the call.
142 *
143 * @const KMEM_REALLOCF (realloc)
144 * Similar to @c Z_REALLOCF: if the call is failing,
145 * then free the old allocation too.
146 *
147 * @const KMEM_NOSOFTLIMIT (alloc, realloc)
148 * Kernel private.
149 * Override soft allocation size limits and attempt to make the allocation
150 * anyways.
151 *
152 * <h2>How the entry is populated</h2>
153 *
154 * @const KMEM_VAONLY (alloc)
155 * By default memory allocated by the kmem subsystem is wired and mapped.
156 * Passing @c KMEM_VAONLY will cause the range to still be wired,
157 * but no page is actually mapped.
158 *
159 * @const KMEM_PAGEABLE (alloc)
160 * By default memory allocated by the kmem subsystem is wired and mapped.
161 * Passing @c KMEM_PAGEABLE makes the entry non wired, and pages will be
162 * added to the entry as it faults.
163 *
164 * @const KMEM_ZERO (alloc, realloc)
165 * Any new page added is zeroed.
166 *
167 *
168 * <h2>VM object to use for the entry</h2>
169 *
170 * @const KMEM_KOBJECT (alloc, realloc)
171 * The entry will be made for the @c kernel_object.
172 *
173 * Note that the @c kernel_object is just a "collection of pages".
174 * Pages in that object can't be remaped or present in several VM maps
175 * like traditional objects.
176 *
177 * If neither @c KMEM_KOBJECT nor @c KMEM_COMPRESSOR is passed,
178 * the a new fresh VM object will be made for this allocation.
179 * This is expensive and should be limited to allocations that
180 * need the features associated with a VM object.
181 *
182 * @const KMEM_COMPRESSOR (alloc)
183 * The entry is allocated for the @c compressor_object.
184 * Pages belonging to the compressor are not on the paging queues,
185 * nor are they counted as wired.
186 *
187 * Only the VM Compressor subsystem should use this.
188 *
189 *
190 * <h2>How to look for addresses</h2>
191 *
192 * @const KMEM_LOMEM (alloc, realloc)
193 * The physical memory allocated must be in the first 4G of memory,
194 * in order to support hardware controllers incapable of generating DMAs
195 * with more than 32bits of physical address.
196 *
197 * @const KMEM_LAST_FREE (alloc, suballoc, realloc)
198 * When looking for space in the specified map,
199 * start scanning for addresses from the end of the map
200 * rather than the start.
201 *
202 * @const KMEM_DATA (alloc, suballoc, realloc)
203 * The memory must be allocated from the "Data" range.
204 *
205 * @const KMEM_SPRAYQTN (alloc, realloc)
206 * The memory must be allocated from the "spray quarantine" range. For more
207 * details on what allocations qualify to use this flag see
208 * @c KMEM_RANGE_ID_SPRAYQTN.
209 *
210 * @const KMEM_GUESS_SIZE (free)
211 * When freeing an atomic entry (requires a valid kmem guard),
212 * then look up the entry size because the caller didn't
213 * preserve it.
214 *
215 * This flag is only here in order to support kfree_data_addr(),
216 * and shall not be used by any other clients.
217 *
218 * <h2>Entry properties</h2>
219 *
220 * @const KMEM_PERMANENT (alloc, suballoc)
221 * The entry is made permanent.
222 *
223 * In the kernel maps, permanent entries can never be deleted.
224 * Calling @c kmem_free() on such a range will panic.
225 *
226 * In user maps, permanent entries will only be deleted
227 * whenthe map is terminated.
228 *
229 * @const KMEM_GUARD_FIRST (alloc, realloc, free)
230 * @const KMEM_GUARD_LAST (alloc, realloc, free)
231 * Asks @c kmem_* to put a guard page at the beginning (resp. end)
232 * of the allocation.
233 *
234 * The allocation size will not be extended to accomodate for guards,
235 * and the client of this interface must take them into account.
236 * Typically if a usable range of 3 pages is needed with both guards,
237 * then 5 pages must be asked.
238 *
239 * Alignment constraints take guards into account (the aligment applies
240 * to the address right after the first guard page).
241 *
242 * The returned address for allocation will pointing at the entry start,
243 * which is the address of the left guard page if any.
244 *
245 * Note that if @c kmem_realloc* or @c kmem_free* is called, the *exact*
246 * same guard flags must be passed for this entry. The KMEM subsystem
247 * is generally oblivious to guards, and passing inconsistent flags
248 * will cause pages to be moved incorrectly.
249 *
250 * @const KMEM_KSTACK (alloc)
251 * This flag must be passed when the allocation is for kernel stacks.
252 * This only has an effect on Intel.
253 *
254 * @const KMEM_NOENCRYPT (alloc)
255 * Obsolete, will be repurposed soon.
256 *
257 * @const KMEM_KASAN_GUARD (alloc, realloc, free)
258 * Under KASAN_CLASSIC add guards left and right to this allocation
259 * in order to detect out of bounds.
260 *
261 * This can't be passed if any of @c KMEM_GUARD_FIRST
262 * or @c KMEM_GUARD_LAST is used.
263 *
264 * @const KMEM_TAG (alloc, realloc, free)
265 * Under KASAN_TBI, this allocation is tagged non canonically.
266 */
267 __options_decl(kmem_flags_t, uint32_t, {
268 KMEM_NONE = 0x00000000,
269
270 /* Call behavior */
271 KMEM_NOFAIL = 0x00000001,
272 KMEM_NOPAGEWAIT = 0x00000002,
273 KMEM_FREEOLD = 0x00000004,
274 KMEM_REALLOCF = 0x00000008,
275 KMEM_NOSOFTLIMIT = 0x00000010,
276
277 /* How the entry is populated */
278 KMEM_VAONLY = 0x00000020,
279 KMEM_PAGEABLE = 0x00000040,
280 KMEM_ZERO = 0x00000080,
281
282 /* VM object to use for the entry */
283 KMEM_KOBJECT = 0x00000100,
284 KMEM_COMPRESSOR = 0x00000200,
285
286 /* How to look for addresses */
287 KMEM_LOMEM = 0x00001000,
288 KMEM_LAST_FREE = 0x00002000,
289 KMEM_GUESS_SIZE = 0x00004000,
290 KMEM_DATA = 0x00008000,
291 KMEM_DATA_SHARED = 0x00010000,
292 KMEM_SPRAYQTN = 0x00020000,
293
294 /* Entry properties */
295 KMEM_PERMANENT = 0x00200000,
296 KMEM_GUARD_FIRST = 0x00400000,
297 KMEM_GUARD_LAST = 0x00800000,
298 KMEM_KSTACK = 0x01000000,
299 KMEM_NOENCRYPT = 0x02000000,
300 KMEM_KASAN_GUARD = 0x04000000,
301 KMEM_TAG = 0x08000000,
302 });
303
304
305 /*
306 * @function kmem_range_id_size
307 *
308 * @abstract Return the addressable size of the memory range.
309 */
310 __pure2
311 extern vm_map_size_t kmem_range_id_size(
312 kmem_range_id_t range_id);
313
314 /**
315 * @enum kmem_claims_flags_t
316 *
317 * @abstract
318 * Set of flags used in the processing of kmem_range claims
319 *
320 * @discussion
321 * These flags are used by the kmem subsytem while processing kmem_range
322 * claims and are not explicitly passed by the caller registering the claim.
323 *
324 * @const KC_NO_ENTRY
325 * A vm map entry should not be created for the respective claim.
326 *
327 * @const KC_NO_MOVE
328 * The range shouldn't be moved once it has been placed as it has constraints.
329 */
330 __options_decl(kmem_claims_flags_t, uint32_t, {
331 KC_NONE = 0x00000000,
332 KC_NO_ENTRY = 0x00000001,
333 KC_NO_MOVE = 0x00000002,
334 });
335
336 /*
337 * Security config that creates the additional splits in non data part of
338 * kernel_map
339 */
340 #if KASAN || (__arm64__ && !defined(KERNEL_INTEGRITY_KTRR) && !defined(KERNEL_INTEGRITY_CTRR) && !defined(KERNEL_INTEGRITY_PV_CTRR))
341 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT OFF
342 #else
343 # define ZSECURITY_CONFIG_KERNEL_PTR_SPLIT ON
344 #endif
345
346 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__OFF() 0
347 #define ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__ON() 1
348 #define ZSECURITY_CONFIG2(v) ZSECURITY_NOT_A_COMPILE_TIME_CONFIG__##v()
349 #define ZSECURITY_CONFIG1(v) ZSECURITY_CONFIG2(v)
350 #define ZSECURITY_CONFIG(opt) ZSECURITY_CONFIG1(ZSECURITY_CONFIG_##opt)
351
352
353 struct kmem_range_startup_spec {
354 const char *kc_name;
355 struct mach_vm_range *kc_range;
356 vm_map_size_t kc_size;
357 vm_map_size_t (^kc_calculate_sz)(void);
358 kmem_claims_flags_t kc_flags;
359 };
360
361 extern void kmem_range_startup_init(
362 struct kmem_range_startup_spec *sp);
363
364 /*!
365 * @macro KMEM_RANGE_REGISTER_*
366 *
367 * @abstract
368 * Register a claim for kmem range or submap.
369 *
370 * @discussion
371 * Claims are shuffled during startup to randomize the layout of the kernel map.
372 * Temporary entries are created in place of the claims, therefore the caller
373 * must provide the start of the assigned range as a hint and
374 * @c{VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE} to kmem_suballoc to replace the mapping.
375 *
376 * Min/max constraints can be provided in the range when the claim is
377 * registered.
378 *
379 * This macro comes in 2 flavors:
380 * - STATIC : When the size of the range/submap is known at compile time
381 * - DYNAMIC: When the size of the range/submap needs to be computed
382 * Temporary entries are create
383 * The start of the
384 *
385 * @param name the name of the claim
386 * @param range the assigned range for the claim
387 * @param size the size of submap/range (if known at compile time)
388 * @param calculate_sz a block that returns the computed size of submap/range
389 */
390 #define KMEM_RANGE_REGISTER_STATIC(name, range, size) \
391 static __startup_data struct kmem_range_startup_spec \
392 __startup_kmem_range_spec_ ## name = { #name, range, size, NULL, KC_NONE}; \
393 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
394 &__startup_kmem_range_spec_ ## name)
395
396 #define KMEM_RANGE_REGISTER_DYNAMIC(name, range, calculate_sz) \
397 static __startup_data struct kmem_range_startup_spec \
398 __startup_kmem_range_spec_ ## name = { #name, range, 0, calculate_sz, \
399 KC_NONE}; \
400 STARTUP_ARG(KMEM, STARTUP_RANK_SECOND, kmem_range_startup_init, \
401 &__startup_kmem_range_spec_ ## name)
402
403
404 #pragma mark kmem entry parameters
405
406 /*!
407 * @function kmem_entry_validate_guard()
408 *
409 * @brief
410 * Validates that the entry matches the input parameters, panic otherwise.
411 *
412 * @discussion
413 * If the guard has a zero @c kmg_guard value,
414 * then the entry must be non atomic.
415 *
416 * The guard tag is not used for validation as the VM subsystems
417 * (particularly in IOKit) might decide to substitute it in ways
418 * that are difficult to predict for the programmer.
419 *
420 * @param entry the entry to validate
421 * @param addr the supposed start address
422 * @param size the supposed size of the entry
423 * @param guard the guard to use to "authenticate" the allocation.
424 */
425 extern void kmem_entry_validate_guard(
426 vm_map_t map,
427 struct vm_map_entry *entry,
428 vm_offset_t addr,
429 vm_size_t size,
430 kmem_guard_t guard);
431
432 /*!
433 * @function kmem_size_guard()
434 *
435 * @brief
436 * Returns the size of an atomic kalloc allocation made in the specified map,
437 * according to the guard.
438 *
439 * @param map a kernel map to lookup the entry into.
440 * @param addr the kernel address to lookup.
441 * @param guard the guard to use to "authenticate" the allocation.
442 */
443 extern vm_size_t kmem_size_guard(
444 vm_map_t map,
445 vm_offset_t addr,
446 kmem_guard_t guard);
447
448
449 #pragma mark kmem allocations
450
451 /*!
452 * @typedef kma_flags_t
453 *
454 * @brief
455 * Flags used by the @c kmem_alloc* family of flags.
456 */
457 __options_decl(kma_flags_t, uint32_t, {
458 KMA_NONE = KMEM_NONE,
459
460 /* Call behavior */
461 KMA_NOFAIL = KMEM_NOFAIL,
462 KMA_NOPAGEWAIT = KMEM_NOPAGEWAIT,
463
464 /* How the entry is populated */
465 KMA_VAONLY = KMEM_VAONLY,
466 KMA_PAGEABLE = KMEM_PAGEABLE,
467 KMA_ZERO = KMEM_ZERO,
468 KMA_NOSOFTLIMIT = KMEM_NOSOFTLIMIT,
469
470 /* VM object to use for the entry */
471 KMA_KOBJECT = KMEM_KOBJECT,
472 KMA_COMPRESSOR = KMEM_COMPRESSOR,
473
474 /* How to look for addresses */
475 KMA_LOMEM = KMEM_LOMEM,
476 KMA_LAST_FREE = KMEM_LAST_FREE,
477 KMA_DATA = KMEM_DATA,
478 KMA_DATA_SHARED = KMEM_DATA_SHARED,
479 KMA_SPRAYQTN = KMEM_SPRAYQTN,
480
481 /* Entry properties */
482 KMA_PERMANENT = KMEM_PERMANENT,
483 KMA_GUARD_FIRST = KMEM_GUARD_FIRST,
484 KMA_GUARD_LAST = KMEM_GUARD_LAST,
485 KMA_KSTACK = KMEM_KSTACK,
486 KMA_NOENCRYPT = KMEM_NOENCRYPT,
487 KMA_KASAN_GUARD = KMEM_KASAN_GUARD,
488 KMA_TAG = KMEM_TAG,
489 });
490
491
492 /*!
493 * @function kmem_alloc_guard()
494 *
495 * @brief
496 * Master entry point for allocating kernel memory.
497 *
498 * @param map map to allocate into, must be a kernel map.
499 * @param size the size of the entry to allocate, must not be 0.
500 * @param mask an alignment mask that the returned allocation
501 * will be aligned to (ignoring guards, see @const
502 * KMEM_GUARD_FIRST).
503 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
504 * @param guard how to guard the allocation.
505 *
506 * @returns
507 * - the non zero address of the allocaation on success in @c kmr_address.
508 * - @c KERN_NO_SPACE if the target map is out of address space.
509 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
510 */
511 extern kmem_return_t kmem_alloc_guard(
512 vm_map_t map,
513 vm_size_t size,
514 vm_offset_t mask,
515 kma_flags_t flags,
516 kmem_guard_t guard) __result_use_check;
517
518 static inline kern_return_t
kernel_memory_allocate(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,kma_flags_t flags,vm_tag_t tag)519 kernel_memory_allocate(
520 vm_map_t map,
521 vm_offset_t *addrp,
522 vm_size_t size,
523 vm_offset_t mask,
524 kma_flags_t flags,
525 vm_tag_t tag)
526 {
527 kmem_guard_t guard = {
528 .kmg_tag = tag,
529 };
530 kmem_return_t kmr;
531
532 kmr = kmem_alloc_guard(map, size, mask, flags, guard);
533 if (kmr.kmr_return == KERN_SUCCESS) {
534 __builtin_assume(kmr.kmr_address != 0);
535 } else {
536 __builtin_assume(kmr.kmr_address == 0);
537 }
538 *addrp = kmr.kmr_address;
539 return kmr.kmr_return;
540 }
541
542 static inline kern_return_t
kmem_alloc(vm_map_t map,vm_offset_t * addrp,vm_size_t size,kma_flags_t flags,vm_tag_t tag)543 kmem_alloc(
544 vm_map_t map,
545 vm_offset_t *addrp,
546 vm_size_t size,
547 kma_flags_t flags,
548 vm_tag_t tag)
549 {
550 return kernel_memory_allocate(map, addrp, size, 0, flags, tag);
551 }
552
553 /*!
554 * @function kmem_alloc_contig_guard()
555 *
556 * @brief
557 * Variant of kmem_alloc_guard() that allocates a contiguous range
558 * of physical memory.
559 *
560 * @param map map to allocate into, must be a kernel map.
561 * @param size the size of the entry to allocate, must not be 0.
562 * @param mask an alignment mask that the returned allocation
563 * will be aligned to (ignoring guards, see @const
564 * KMEM_GUARD_FIRST).
565 * @param max_pnum The maximum page number to allocate, or 0.
566 * @param pnum_mask A page number alignment mask for the first allocated
567 * page, or 0.
568 * @param flags a set of @c KMA_* flags, (@see @c kmem_flags_t)
569 * @param guard how to guard the allocation.
570 *
571 * @returns
572 * - the non zero address of the allocaation on success in @c kmr_address.
573 * - @c KERN_NO_SPACE if the target map is out of address space.
574 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
575 */
576 extern kmem_return_t kmem_alloc_contig_guard(
577 vm_map_t map,
578 vm_size_t size,
579 vm_offset_t mask,
580 ppnum_t max_pnum,
581 ppnum_t pnum_mask,
582 kma_flags_t flags,
583 kmem_guard_t guard);
584
585 static inline kern_return_t
kmem_alloc_contig(vm_map_t map,vm_offset_t * addrp,vm_size_t size,vm_offset_t mask,ppnum_t max_pnum,ppnum_t pnum_mask,kma_flags_t flags,vm_tag_t tag)586 kmem_alloc_contig(
587 vm_map_t map,
588 vm_offset_t *addrp,
589 vm_size_t size,
590 vm_offset_t mask,
591 ppnum_t max_pnum,
592 ppnum_t pnum_mask,
593 kma_flags_t flags,
594 vm_tag_t tag)
595 {
596 kmem_guard_t guard = {
597 .kmg_tag = tag,
598 };
599 kmem_return_t kmr;
600
601 kmr = kmem_alloc_contig_guard(map, size, mask,
602 max_pnum, pnum_mask, flags, guard);
603 if (kmr.kmr_return == KERN_SUCCESS) {
604 __builtin_assume(kmr.kmr_address != 0);
605 } else {
606 __builtin_assume(kmr.kmr_address == 0);
607 }
608 *addrp = kmr.kmr_address;
609 return kmr.kmr_return;
610 }
611
612
613 /*!
614 * @typedef kms_flags_t
615 *
616 * @brief
617 * Flags used by @c kmem_suballoc.
618 */
619 __options_decl(kms_flags_t, uint32_t, {
620 KMS_NONE = KMEM_NONE,
621
622 /* Call behavior */
623 KMS_NOFAIL = KMEM_NOFAIL,
624 KMS_NOSOFTLIMIT = KMEM_NOSOFTLIMIT,
625
626 /* How to look for addresses */
627 KMS_LAST_FREE = KMEM_LAST_FREE,
628 KMS_DATA = KMEM_DATA,
629 KMS_DATA_SHARED = KMEM_DATA_SHARED,
630
631 /* Entry properties */
632 KMS_PERMANENT = KMEM_PERMANENT,
633 });
634
635 /*!
636 * @function kmem_suballoc()
637 *
638 * @brief
639 * Create a kernel submap, in an atomic entry guarded with KMEM_GUARD_SUBMAP.
640 *
641 * @param parent map to allocate into, must be a kernel map.
642 * @param addr (in/out) the address for the map (see vm_map_enter)
643 * @param size the size of the entry to allocate, must not be 0.
644 * @param vmc_options the map creation options
645 * @param vm_flags a set of @c VM_FLAGS_* flags
646 * @param flags a set of @c KMS_* flags, (@see @c kmem_flags_t)
647 * @param tag the tag for this submap's entry.
648 */
649 extern kmem_return_t kmem_suballoc(
650 vm_map_t parent,
651 mach_vm_offset_t *addr,
652 vm_size_t size,
653 vm_map_create_options_t vmc_options,
654 int vm_flags,
655 kms_flags_t flags,
656 vm_tag_t tag);
657
658
659 #pragma mark kmem reallocation
660
661 /*!
662 * @typedef kmr_flags_t
663 *
664 * @brief
665 * Flags used by the @c kmem_realloc* family of flags.
666 */
667 __options_decl(kmr_flags_t, uint32_t, {
668 KMR_NONE = KMEM_NONE,
669
670 /* Call behavior */
671 KMR_NOPAGEWAIT = KMEM_NOPAGEWAIT,
672 KMR_FREEOLD = KMEM_FREEOLD,
673 KMR_REALLOCF = KMEM_REALLOCF,
674
675 /* How the entry is populated */
676 KMR_ZERO = KMEM_ZERO,
677
678 /* VM object to use for the entry */
679 KMR_KOBJECT = KMEM_KOBJECT,
680
681 /* How to look for addresses */
682 KMR_LOMEM = KMEM_LOMEM,
683 KMR_LAST_FREE = KMEM_LAST_FREE,
684 KMR_DATA = KMEM_DATA,
685 KMR_DATA_SHARED = KMEM_DATA_SHARED,
686 KMR_SPRAYQTN = KMEM_SPRAYQTN,
687
688 /* Entry properties */
689 KMR_GUARD_FIRST = KMEM_GUARD_FIRST,
690 KMR_GUARD_LAST = KMEM_GUARD_LAST,
691 KMR_KASAN_GUARD = KMEM_KASAN_GUARD,
692 KMR_TAG = KMEM_TAG,
693 });
694
695 #define KMEM_REALLOC_FLAGS_VALID(flags) \
696 (((flags) & (KMR_KOBJECT | KMEM_GUARD_LAST | KMEM_KASAN_GUARD | KMR_DATA)) == KMR_DATA \
697 || ((flags) & (KMR_KOBJECT | KMEM_GUARD_LAST | KMEM_KASAN_GUARD | KMR_DATA_SHARED)) == KMR_DATA_SHARED \
698 || ((flags) & KMR_FREEOLD) \
699 && (((flags) & (KMR_DATA | KMR_DATA_SHARED)) != (KMR_DATA | KMR_DATA_SHARED)) \
700 && (((flags) & (KMA_PAGEABLE | KMA_DATA)) != (KMA_PAGEABLE | KMA_DATA)))
701
702
703 /*!
704 * @function kmem_realloc_guard()
705 *
706 * @brief
707 * Reallocates memory allocated with kmem_alloc_guard()
708 *
709 * @discussion
710 * @c kmem_realloc_guard() either mandates a guard with atomicity set,
711 * or must use KMR_DATA (this is not an implementation limitation but
712 * but a security policy).
713 *
714 * If kmem_realloc_guard() is called for the kernel object
715 * (with @c KMR_KOBJECT) or with any trailing guard page,
716 * then the use of @c KMR_FREEOLD is mandatory.
717 *
718 * When @c KMR_FREEOLD isn't used, if the allocation was relocated
719 * as opposed to be extended or truncated in place, the caller
720 * must free its old mapping manually by calling @c kmem_free_guard().
721 *
722 * Note that if the entry is truncated, it will always be done in place.
723 *
724 *
725 * @param map map to allocate into, must be a kernel map.
726 * @param oldaddr the address to reallocate,
727 * passing 0 means @c kmem_alloc_guard() will be called.
728 * @param oldsize the current size of the entry
729 * @param newsize the new size of the entry,
730 * 0 means kmem_free_guard() will be called.
731 * @param flags a set of @c KMR_* flags, (@see @c kmem_flags_t)
732 * the exact same set of @c KMR_GUARD_* flags must
733 * be passed for all calls (@see kmem_flags_t).
734 * @param guard the allocation guard.
735 *
736 * @returns
737 * - the newly allocated address on success in @c kmr_address
738 * (note that if newsize is 0, then address will be 0 too).
739 * - @c KERN_NO_SPACE if the target map is out of address space.
740 * - @c KERN_RESOURCE_SHORTAGE if the kernel is out of pages.
741 */
742 extern kmem_return_t kmem_realloc_guard(
743 vm_map_t map,
744 vm_offset_t oldaddr,
745 vm_size_t oldsize,
746 vm_size_t newsize,
747 kmr_flags_t flags,
748 kmem_guard_t guard) __result_use_check
749 __attribute__((diagnose_if(!KMEM_REALLOC_FLAGS_VALID(flags),
750 "invalid realloc flags passed", "error")));
751
752 /*!
753 * @function kmem_realloc_should_free()
754 *
755 * @brief
756 * Returns whether the old address passed to a @c kmem_realloc_guard()
757 * call without @c KMR_FREEOLD must be freed.
758 *
759 * @param oldaddr the "oldaddr" passed to @c kmem_realloc_guard().
760 * @param kmr the result of that @c kmem_realloc_should_free() call.
761 */
762 static inline bool
kmem_realloc_should_free(vm_offset_t oldaddr,kmem_return_t kmr)763 kmem_realloc_should_free(
764 vm_offset_t oldaddr,
765 kmem_return_t kmr)
766 {
767 return oldaddr && oldaddr != kmr.kmr_address;
768 }
769
770
771 #pragma mark kmem free
772
773 /*!
774 * @typedef kmf_flags_t
775 *
776 * @brief
777 * Flags used by the @c kmem_free* family of flags.
778 */
779 __options_decl(kmf_flags_t, uint32_t, {
780 KMF_NONE = KMEM_NONE,
781
782 /* Call behavior */
783
784 /* How the entry is populated */
785
786 /* How to look for addresses */
787 KMF_GUESS_SIZE = KMEM_GUESS_SIZE,
788
789 /* Entry properties */
790 KMF_GUARD_FIRST = KMEM_GUARD_FIRST,
791 KMF_GUARD_LAST = KMEM_GUARD_LAST,
792 KMF_KASAN_GUARD = KMEM_KASAN_GUARD,
793 KMF_TAG = KMEM_TAG,
794 });
795
796
797 /*!
798 * @function kmem_free_guard()
799 *
800 * @brief
801 * Frees memory allocated with @c kmem_alloc or @c kmem_realloc.
802 *
803 * @param map map to free from, must be a kernel map.
804 * @param addr the address to free
805 * @param size the size of the memory to free
806 * @param flags a set of @c KMF_* flags, (@see @c kmem_flags_t)
807 * @param guard the allocation guard.
808 *
809 * @returns the size of the entry that was deleted.
810 * (useful when @c KMF_GUESS_SIZE was used)
811 */
812 extern vm_size_t kmem_free_guard(
813 vm_map_t map,
814 vm_offset_t addr,
815 vm_size_t size,
816 kmf_flags_t flags,
817 kmem_guard_t guard);
818
819 __attribute__((overloadable))
820 static inline void
kmem_free(vm_map_t map,vm_offset_t addr,vm_size_t size,kmf_flags_t flags)821 kmem_free(
822 vm_map_t map,
823 vm_offset_t addr,
824 vm_size_t size,
825 kmf_flags_t flags)
826 {
827 kmem_free_guard(map, addr, size, flags, KMEM_GUARD_NONE);
828 }
829
830 __attribute__((overloadable))
831 static inline void
kmem_free(vm_map_t map,vm_offset_t addr,vm_size_t size)832 kmem_free(
833 vm_map_t map,
834 vm_offset_t addr,
835 vm_size_t size)
836 {
837 kmem_free(map, addr, size, KMF_NONE);
838 }
839
840
841 #pragma mark kmem population
842
843 /*!
844 * @function kernel_memory_populate()
845 *
846 * @brief
847 * Populate pages for a given kernel map allocation.
848 *
849 * @discussion
850 * Allocations made against the kernel object (@c KMEM_KOBJECT)
851 * or the compressor object (@c KMEM_COMPRESSOR) must have their
852 * backing store explicitly managed by clients.
853 *
854 * This function will cause pages in the specified range to be allocated
855 * explicitly. No page must have been allocated for that range at this time,
856 * either because the allocation was done with @c KMEM_VAONLY or because pages
857 * were explicitly depopulated.
858 *
859 * @param addr the aligned starting address to populate.
860 * @param size the aligned size of the region to populate.
861 * @param flags a set flags that must match the flags passed at
862 * @c kmem_alloc*() time. In particular, one of
863 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
864 * @param tag the kernel memory tag to use for accounting purposes.
865 * @returns
866 * - KERN_SUCCESS the operation succeeded.
867 * - KERN_RESOURCE_SHORTAGE
868 * the kernel was out of physical pages.
869 */
870 extern kern_return_t kernel_memory_populate(
871 vm_offset_t addr,
872 vm_size_t size,
873 kma_flags_t flags,
874 vm_tag_t tag);
875
876
877 /*!
878 * @function kernel_memory_depopulate()
879 *
880 * @brief
881 * Depopulate pages for a given kernel map allocation.
882 *
883 * @discussion
884 * Allocations made against the kernel object (@c KMEM_KOBJECT)
885 * or the compressor object (@c KMEM_COMPRESSOR) must have their
886 * backing store explicitly managed by clients.
887 *
888 * This function will cause pages in the specified range to be deallocated
889 * explicitly. This range must be populated at the time of the call, either
890 * because the @c kmem_alloc*() call asked for pages, or because
891 * @c kernel_memory_populate() has been called explicitly.
892 *
893 * It is not necessary to explicitly depopulate ranges prior to calling
894 * @c kmem_free*(), even if populating the range was made explicitly with
895 * @c kernel_memory_populate() rather than implicitly at allocation time.
896 *
897 *
898 * @param addr the aligned starting address to depopulate.
899 * @param size the aligned size of the region to depopulate.
900 * @param flags a set flags that must match the flags passed at
901 * @c kmem_alloc*() time. In particular, one of
902 * @c KMA_KOBJECT or @c KMA_COMPRESSOR must be passed.
903 * @param tag the kernel memory tag to use for accounting purposes,
904 * which must match the tag used for population.
905 */
906 extern void kernel_memory_depopulate(
907 vm_offset_t addr,
908 vm_size_t size,
909 kma_flags_t flags,
910 vm_tag_t tag);
911
912
913 #pragma mark - VM_FLAGS_* / vm_map_kernel_flags_t conversions
914
915 /*!
916 * @function vm_map_kernel_flags_vmflags()
917 *
918 * @return The vmflags set in the specified @c vmk_flags.
919 */
920 extern int vm_map_kernel_flags_vmflags(
921 vm_map_kernel_flags_t vmk_flags);
922
923 /*!
924 * @function vm_map_kernel_flags_set_vmflags()
925 *
926 * @brief
927 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
928 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
929 */
930 __attribute__((overloadable))
931 extern void vm_map_kernel_flags_set_vmflags(
932 vm_map_kernel_flags_t *vmk_flags,
933 int vm_flags,
934 vm_tag_t vm_tag);
935
936 /*!
937 * @function vm_map_kernel_flags_set_vmflags()
938 *
939 * @brief
940 * Populates the @c vmf_* and @c vm_tag fields of the vmk flags,
941 * with the specified vm flags (@c VM_FLAG_* from <mach/vm_statistics.h>).
942 *
943 * @discussion
944 * This variant takes the tag from the top byte of the flags.
945 */
946 __attribute__((overloadable))
947 extern void vm_map_kernel_flags_set_vmflags(
948 vm_map_kernel_flags_t *vmk_flags,
949 int vm_flags_and_tag);
950
951 /*!
952 * @function vm_map_kernel_flags_and_vmflags()
953 *
954 * @brief
955 * Apply a mask to the vmflags.
956 */
957 extern void vm_map_kernel_flags_and_vmflags(
958 vm_map_kernel_flags_t *vmk_flags,
959 int vm_flags_mask);
960
961 /*!
962 * @function vm_map_kernel_flags_check_vmflags()
963 *
964 * @return
965 * Whether the @c vmk_flags @c vmf_* fields
966 * are limited to the specified mask.
967 */
968 extern bool vm_map_kernel_flags_check_vmflags(
969 vm_map_kernel_flags_t vmk_flags,
970 int vm_flags_mask);
971
972 /*!
973 * @function vm_map_kernel_flags_check_vm_and_kflags()
974 *
975 * @return
976 * Whether the @c vmk_flags @c vmf_* fields
977 * are limited to the specified mask.
978 */
979 extern bool vm_map_kernel_flags_check_vm_and_kflags(
980 vm_map_kernel_flags_t vmk_flags,
981 int vm_flags_mask);
982
983
984 #pragma mark - kernel variants of the Mach VM interfaces
985
986 /*!
987 * @function mach_vm_allocate_kernel()
988 *
989 * @brief
990 * Allocate memory in the specified map.
991 *
992 * @discussion
993 * This is the in-kernel equivalent to the @c mach_vm_allocate() MIG call,
994 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
995 * @c VM_FLAGS_.
996 *
997 * Memory will not be pre-faulted and touching it will cause page faults.
998 *
999 * Memory will be zero-filled when faulted, unless the map's @c no_zero_fill
1000 * property is set (only the @c ipc_kernel_map is marked this way).
1001 *
1002 * The allocation being made will have:
1003 * - @c VM_PROT_DEFAULT (rw-) current protections,
1004 * - @c VM_PROT_ALL (rwx) max protections,
1005 * - @c VM_INHERIT_COPY inheritance.
1006 *
1007 *
1008 * @param map the map to allocate memory into.
1009 *
1010 * @param addr_u [in] when @c vmk_flags.vmf_fixed is set, @c *addr
1011 * is used as the address at which the allocation
1012 * must be made. If it is misaligned for the map's
1013 * page size, its low bits are truncated and ignored.
1014 *
1015 * when @c vmk_flags.vmf_fixed is not set,
1016 * the value of @c *addrp is ignored.
1017 *
1018 * [out] filled with the address at which the allocation
1019 * was made on success, unmodified otherwise.
1020 *
1021 * @c vmk_flags.vmf_return_data_addr has no effect
1022 * on the returned address.
1023 *
1024 * @param size_u the size of the allocation to make. zero sizes are
1025 * allowed and result in @c *addr being zero.
1026 *
1027 * misaligned sizes for the page size of the target map
1028 * will be rounded up to the nearest page size.
1029 *
1030 * @param vmk_flags a set of flags that influence the properties of the
1031 * allocation being made.
1032 *
1033 * @returns KERN_SUCCESS when the operation succeeds,
1034 * or an error denoting the reason for failure.
1035 */
1036 extern kern_return_t mach_vm_allocate_kernel(
1037 vm_map_t map,
1038 mach_vm_offset_ut *addr_u,
1039 mach_vm_size_ut size_u,
1040 vm_map_kernel_flags_t vmk_flags);
1041
1042
1043 /*!
1044 * @function mach_vm_map_kernel()
1045 *
1046 * @brief
1047 * Map some range of an object into an address space.
1048 *
1049 * @discussion
1050 * This is the in-kernel equivalent to the @c mach_vm_map() MIG call,
1051 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1052 * @c VM_FLAGS_.
1053 *
1054 *
1055 * @param target_map the target address space.
1056 *
1057 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1058 * is used as the address at which the allocation
1059 * must be made. If it is misaligned for the map's
1060 * page size, its low bits are truncated and ignored.
1061 *
1062 * when @c vmk_flags.vmf_fixed is not set,
1063 * the value of @c *address is used as a starting
1064 * point from which to scan for memory in the direction
1065 * specified by @c vmk_flags.vmkf_last_free.
1066 *
1067 * [out] filled with the address at which the allocation
1068 * was made on success, unmodified otherwise.
1069 *
1070 * if @c vmk_flags.vmf_return_data_addr is
1071 * specified, @c *address will maintain
1072 * "misalignment" from the requested @c offset
1073 * inside the object, otherwise it will be page
1074 * aligned for the target map.
1075 *
1076 * @param size the size of the allocation to make.
1077 * zero sizes are disallowed.
1078 *
1079 * @param mask a mask to specify the allocation alignment.
1080 * this value should be of the form (2^n-1).
1081 *
1082 * allocations will always at least be page aligned
1083 * for the specified target map, but this can be used to
1084 * require larger alignments.
1085 *
1086 * @param vmk_flags a set of flags that influence the properties of the
1087 * allocation being made.
1088 *
1089 * @param port the object to map into the target address space:
1090 * - @c IP_NULL means anonymous memory, and this call
1091 * behaves like a more versatile
1092 * mach_vm_allocate_kernel(),
1093 * - a kobject port of type @c IKOT_NAMED_ENTRY,
1094 * pointing to a @c vm_named_entry_t,
1095 * - a naked @c memory_object_t (a VM pager).
1096 *
1097 * @param offset an offset within the specified object to map.
1098 *
1099 * @param copy whether to make a copy-on-write (true) mapping
1100 * or a shared (false) mapping.
1101 *
1102 * @param cur_prot the effective protections for the mapping.
1103 *
1104 * @param max_prot the maximum protections for the mapping,
1105 * which must at least cover @c cur_prot.
1106 *
1107 * @param inheritance the inheritance policy for the mapping.
1108 *
1109 * @returns KERN_SUCCESS when the operation succeeds,
1110 * or an error denoting the reason for failure.
1111 */
1112 extern kern_return_t mach_vm_map_kernel(
1113 vm_map_t target_map,
1114 mach_vm_offset_ut *address,
1115 mach_vm_size_ut size,
1116 mach_vm_offset_ut mask,
1117 vm_map_kernel_flags_t vmk_flags,
1118 ipc_port_t port,
1119 memory_object_offset_ut offset,
1120 boolean_t copy,
1121 vm_prot_ut cur_prot,
1122 vm_prot_ut max_prot,
1123 vm_inherit_ut inheritance);
1124
1125
1126 /*!
1127 * @function mach_vm_remap_new_kernel()
1128 *
1129 * @brief
1130 * Remap a range of memory from one address space to another.
1131 *
1132 * @discussion
1133 * This is the in-kernel equivalent to the @c mach_vm_remap() MIG call,
1134 * except that it takes a full set of @c vm_map_kernel_flags_t rather than
1135 * @c VM_FLAGS_.
1136 *
1137 * This call forces vmk_flags.vmf_return_data_addr to true regardless
1138 * of what the caller specified.
1139 *
1140 *
1141 * @param target_map the target address space.
1142 *
1143 * @param address [in] when @c vmk_flags.vmf_fixed is set, @c *address
1144 * is used as the address at which the allocation
1145 * must be made. If it is misaligned for the map's
1146 * page size, its low bits are truncated and ignored.
1147 *
1148 * when @c vmk_flags.vmf_fixed is not set,
1149 * the value of @c *address is used as a starting
1150 * point from which to scan for memory in the direction
1151 * specified by @c vmk_flags.vmkf_last_free.
1152 *
1153 * [out] filled with the address at which the allocation
1154 * was made on success, unmodified otherwise.
1155 * @c *address has the same "misalignment"
1156 * as @c src_address.
1157 *
1158 * @param size the size of the region to remap.
1159 *
1160 * @param mask a mask to specify the allocation alignment.
1161 * this value should be of the form (2^n-1).
1162 *
1163 * allocations will always at least be page aligned
1164 * for the specified target map, but this can be used to
1165 * require larger alignments.
1166 *
1167 * @param vmk_flags a set of flags that influence the properties of the
1168 * allocation being made.
1169 *
1170 * @param src_map the address space to remap the memory from.
1171 *
1172 * @param src_address the start address within @c src_map to remap.
1173 *
1174 * @param copy whether to make a copy-on-write (true) mapping
1175 * or a shared (false) mapping.
1176 *
1177 * @param cur_prot [in] for shared mappings, the minimum set of effective
1178 * permissions the source mapping must have
1179 *
1180 * [out] the resulting effective permissions for the mapping
1181 *
1182 * @param max_prot [in] for shared mappings, the minimum set of maximum
1183 * permissions the source mappings must have.
1184 *
1185 * [out] the resulting maximum permissions for the mapping
1186 *
1187 * @param inheritance the inheritance policy for the mapping.
1188 *
1189 * @returns KERN_SUCCESS when the operation succeeds,
1190 * or an error denoting the reason for failure.
1191 */
1192 extern kern_return_t mach_vm_remap_new_kernel(
1193 vm_map_t target_map,
1194 mach_vm_offset_ut *address,
1195 mach_vm_size_ut size,
1196 mach_vm_offset_ut mask,
1197 vm_map_kernel_flags_t vmk_flags,
1198 vm_map_t src_map,
1199 mach_vm_offset_ut src_address,
1200 boolean_t copy,
1201 vm_prot_ut *cur_prot,
1202 vm_prot_ut *max_prot,
1203 vm_inherit_ut inheritance);
1204
1205 /*!
1206 * @function vm_map_wire_kernel()
1207 *
1208 * @brief
1209 * Sets the pageability of the specified address range in the
1210 * target map as wired.
1211 *
1212 * @discussion
1213 * This call is the kernel version of @c vm_map_wire(). The main difference
1214 * is that the caller must specify a valid @c VM_KERN_MEMORY_* tag for kernel
1215 * wirings, or a valid @c VM_MEMORY_* tag for user wirings.
1216 *
1217 * Consult the documentation of @c vm_map_wire() in @c <vm/vm_map.h> for details.
1218 */
1219 extern kern_return_t vm_map_wire_kernel(
1220 vm_map_t map,
1221 vm_map_offset_ut start_u,
1222 vm_map_offset_ut end_u,
1223 vm_prot_ut prot_u,
1224 vm_tag_t tag,
1225 boolean_t user_wire);
1226
1227 /*!
1228 * @function vm_map_purgable_control()
1229 *
1230 * @brief
1231 * Perform a purgeability operation on a VM object at a given address
1232 * in an address space.
1233 *
1234 * @discussion
1235 * This is the in-kernel equivalent to the @c mach_vm_map_purgable_control()
1236 * MIG call, except that it allows all operations, including
1237 * @c VM_PURGABLE_*_FROM_KERNEL ones).
1238 *
1239 * Valid @c control operations and the meaning of @c state is documented
1240 * in @c <mach/vm_purgable.h>.
1241 *
1242 *
1243 * @param map the target address space
1244 * @param address the address to use to find the VM object to target
1245 * @param control a purgeability operation to perform.
1246 * @param state an in/out parameter that is operation dependent.
1247 */
1248 extern kern_return_t vm_map_purgable_control(
1249 vm_map_t map,
1250 vm_map_offset_ut address,
1251 vm_purgable_t control,
1252 int *state);
1253
1254 extern kern_return_t mach_vm_purgable_control(
1255 vm_map_t map,
1256 mach_vm_offset_ut address_u,
1257 vm_purgable_t control,
1258 int *state);
1259
1260
1261 #ifdef MACH_KERNEL_PRIVATE
1262 #pragma mark - map copyio
1263
1264
1265 /*!
1266 * @function copyinmap()
1267 *
1268 * @brief
1269 * Like copyin, except that @c fromaddr is an address in the specified VM map.
1270 *
1271 * @param map the map @c fromaddr is relative to.
1272 * @param fromaddr the address to copy from within @c map.
1273 * @param todata the kernel buffer to write into.
1274 * @param length the number of bytes to copy.
1275 * @returns
1276 * - KERN_SUCCESS the copy was successful
1277 * - KERN_INVALID_ADDRESS
1278 * a fault occurred during copyio and couldn't be resolved
1279 * (similar to copyin returning EFAULT).
1280 */
1281 extern kern_return_t copyinmap(
1282 vm_map_t map,
1283 vm_map_offset_t fromaddr,
1284 void *todata __sized_by(length),
1285 vm_size_t length);
1286
1287
1288 /*!
1289 * @function copyoutmap()
1290 *
1291 * @brief
1292 * Like copyout, except that @c toaddr is an address in the specified VM map.
1293 *
1294 * @param map the map @c toaddr is relative to
1295 * @param fromdata the kernel buffer to copy from.
1296 * @param toaddr the address within @c map to copy into
1297 * @param length the number of bytes to copy.
1298 * @returns
1299 * - KERN_SUCCESS the copy was successful
1300 * - KERN_INVALID_ADDRESS
1301 * a fault occurred during copyio and couldn't be resolved
1302 * (similar to copyin returning EFAULT).
1303 */
1304 extern kern_return_t copyoutmap(
1305 vm_map_t map,
1306 void *fromdata __sized_by(length),
1307 vm_map_offset_t toaddr,
1308 vm_size_t length);
1309
1310
1311 /*!
1312 * @function copyoutmap_atomic32()
1313 *
1314 * @brief
1315 * Copies out a 32bit value atomically at a given address in a specified VM map.
1316 *
1317 * @param map the specified map.
1318 * @param value the 32 bit value to write at @c toaddr.
1319 * @param toaddr the address within @c map to copy into
1320 * @returns
1321 * - KERN_SUCCESS the copy was successful
1322 * - KERN_INVALID_ADDRESS
1323 * a fault occurred during copyio and couldn't be resolved
1324 * (similar to copyin returning EFAULT).
1325 */
1326 extern kern_return_t copyoutmap_atomic32(
1327 vm_map_t map,
1328 uint32_t value,
1329 vm_map_offset_t toaddr);
1330
1331
1332 /*!
1333 * @function copyoutmap_atomic64()
1334 *
1335 * @brief
1336 * Copies out a 64bit value atomically at a given address in a specified VM map.
1337 *
1338 * @param map the specified map.
1339 * @param value the 64 bit value to write at @c toaddr.
1340 * @param toaddr the address within @c map to copy into
1341 * @returns
1342 * - KERN_SUCCESS the copy was successful
1343 * - KERN_INVALID_ADDRESS
1344 * a fault occurred during copyio and couldn't be resolved
1345 * (similar to copyin returning EFAULT).
1346 */
1347 extern kern_return_t copyoutmap_atomic64(
1348 vm_map_t map,
1349 uint64_t value,
1350 vm_map_offset_t toaddr);
1351
1352
1353 #endif /* MACH_KERNEL_PRIVATE */
1354 #pragma mark - accounting
1355
1356 #pragma mark accounting: kern allocation name
1357
1358 /*!
1359 * @function kern_allocation_update_size()
1360 *
1361 * @brief
1362 * Update accounting for a specified kern allocation name.
1363 *
1364 * @discussion
1365 * This is to be called when memory gets wired/unwired in order
1366 * to update accounting information.
1367 *
1368 * [development kernels] If the @c vmtaglog boot-arg is used,
1369 * and its value matches the VM tag of this allocation name, then VM tag
1370 * log entries will be added on the specified object.
1371 *
1372 *
1373 * @param allocation a @c kern_allocation_name_t made with
1374 * @c kern_allocation_name_allocate().
1375 * @param delta the amount to update the accounting with,
1376 * positive values increment,
1377 * negative values decrement.
1378 * @param object an optional object this wiring/unwiring applies to.
1379 */
1380 extern void kern_allocation_update_size(
1381 kern_allocation_name_t allocation,
1382 int64_t delta,
1383 vm_object_t object);
1384
1385
1386 /*!
1387 * @function kern_allocation_update_subtotal()
1388 *
1389 * @brief
1390 * Update subtotal accounting for a specified kern allocation name.
1391 *
1392 * @discussion
1393 * IOKit uses global kern allocation names that cover the VM tags
1394 * (@see @c vm_tag_bt()) of several kexts and uses this to perform
1395 * accounting per kext within these meta accounting data structures.
1396 *
1397 * @param allocation a @c kern_allocation_name_t made with
1398 * @c kern_allocation_name_allocate()
1399 * and a non 0 "subtotalscount".
1400 * @param subtag a @c vm_tag_t subtag to update accounting for.
1401 * @param delta the amount to update the accounting with,
1402 * positive values increment,
1403 * negative values decrement.
1404 */
1405 extern void kern_allocation_update_subtotal(
1406 kern_allocation_name_t allocation,
1407 vm_tag_t subtag,
1408 int64_t delta);
1409
1410
1411 #pragma mark accounting: vm tags
1412
1413 /*
1414 * VM Tags come in 3 flavors:
1415 *
1416 * - static user VM tags, defined by the @c VM_MEMORY_* constants
1417 * (@see <mach/vm_statistics.h>),
1418 *
1419 * - static kernel VM tags, defined by the @c VM_KERN_MEMORY_* constants
1420 * (@see <mach/vm_statistics.h>),
1421 *
1422 * - dynamically allocated kernel VM tags typically associated with kexts
1423 * lazily (as kexts wire down memory and accounting needs to be made).
1424 *
1425 * By default, kernel VM tags track wired memory at the VM layer,
1426 * but no insight is given to allocations done by @c kalloc*()
1427 * and its wrappers like @c IOMalloc*(), when backed by zone memory
1428 * (for sizes below @c KHEAP_MAX_SIZE).
1429 *
1430 *
1431 * Implementation details
1432 * ~~~~~~~~~~~~~~~~~~~~~~
1433 *
1434 * Static tags are limited to values from 0 to 255, as they are passed
1435 * in the bits reserved by the @c VM_FLAGS_ALIAS_MASK of VM flags.
1436 *
1437 * However, dynamic flags are an internal kernel concept which is limited
1438 * by the size of the storage in VM map entries, which reserves
1439 * @c VME_ALIAS_BITS (12) for this, effectively limiting dynamic
1440 * tags to about 4000.
1441 *
1442 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE) defines the range of
1443 * possible values for dynamic tags.
1444 *
1445 *
1446 * Zone accounting [development kernels]
1447 * ~~~~~~~~~~~~~~~
1448 *
1449 * On development kernels the "-zt" boot-arg can be used, in which case
1450 * precise accounting of @c kalloc*() allocations is enabled per bucket,
1451 * which can be observed by the @c zprint(1) command.
1452 *
1453 * This effectively makes accounting a vector composed of @c VM_TAG_SIZECLASSES
1454 * possible zone size classes in addition to the regular VM wired memory
1455 * accounting that is always performed.
1456 *
1457 * The tag for which an allocation is recorded will be (in first hit order):
1458 *
1459 * - the tag specified as part of the @c zalloc_flags_t explicitly
1460 * (@see @c kalloc_*_tag() or @c Z_VM_TAG()). In core XNU, non @c _tag
1461 * variants of @c kalloc*() will generate a per-call site dynamic VM tag,
1462 * using the @c VM_ALLOC_SITE_TAG() macro,
1463 *
1464 * - if the allocation was made by a kernel extension, the dynamic VM tag
1465 * for this extension (@see @c vm_tag_bt()),
1466 *
1467 * - as a fallback:
1468 * o @c VM_KERN_MEMORY_KALLOC_DATA for @c kalloc_data() calls,
1469 * o @c VM_KERN_MEMORY_KALLOC_TYPE for @c kalloc_type() calls,
1470 * o @c VM_KERN_MEMORY_KALLOC for legacy @c kalloc() calls.
1471 */
1472
1473 /*!
1474 * @brief
1475 * Lock used to serialize the @c vm_tag_alloc() operation, used by IOKit.
1476 */
1477 extern lck_ticket_t vm_allocation_sites_lock;
1478
1479
1480 /*!
1481 * @function vm_tag_bt()
1482 *
1483 * @brief
1484 * Returns the dynamic kernel extension tag based on backtracing from this call.
1485 *
1486 * @discussion
1487 * This function will lazily allocate a dynamic tag for the current kernel
1488 * extension based on the backtrace, and then return a stable identifier.
1489 *
1490 * This might fail for cases where the kernel is out of dynamic tags.
1491 *
1492 *
1493 * @returns A dynamic kernel extension tag within
1494 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1495 * or @c VM_KERN_MEMORY_NONE if the call was not made
1496 * from a kernel extension, or allocating a dynamic tag
1497 * for it failed.
1498 */
1499 extern vm_tag_t vm_tag_bt(void);
1500
1501
1502 /*!
1503 * @function vm_tag_alloc()
1504 *
1505 * @brief
1506 * Lazily allocates a dynamic VM tag for a given allocation site.
1507 *
1508 * @description
1509 * This is used by IOKit and the zalloc subsystem to generate dynamic VM tags
1510 * for kernel extensions (@see vm_tag_bt()) or core kernel @c kalloc*()
1511 * call sites (@see VM_ALLOC_SITE_TAG()).
1512 *
1513 *
1514 * @param site the allocation site to generate a tag for.
1515 * @returns A dynamic kernel extension tag within
1516 * @c [VM_KERN_MEMORY_FIRST_DYNAMIC, VM_MAX_TAG_VALUE),
1517 * or @c VM_KERN_MEMORY_NONE if the kernel
1518 * is out of dynamic tags.
1519 */
1520 extern vm_tag_t vm_tag_alloc(
1521 vm_allocation_site_t *site);
1522
1523 /*!
1524 * @function vm_tag_alloc_locked()
1525 *
1526 * @brief
1527 * Lazily allocates a dynamic VM tag for a given allocation site,
1528 * while the caller holds the @c vm_allocation_sites_lock lock.
1529 *
1530 * @description
1531 * This is used by IOKit to generate dynamic VM tags for kernel extensions
1532 * (@see vm_tag_bt()).
1533 *
1534 *
1535 * @param site the allocation site to generate a tag for.
1536 * @param releasesiteP an optional allocation site data structure
1537 * that the caller is responsible for releasing
1538 * with @c kern_allocation_name_release().
1539 */
1540 extern void vm_tag_alloc_locked(
1541 vm_allocation_site_t *site,
1542 vm_allocation_site_t **releasesiteP);
1543
1544
1545 /*!
1546 * @function vm_tag_update_size()
1547 *
1548 * @brief
1549 * Update accounting for a specified VM kernel tag (static or dynamic).
1550 *
1551 * @discussion
1552 * This is to be called when memory gets wired/unwired in order
1553 * to update accounting information for a given tag.
1554 *
1555 * [development kernels] If the @c vmtaglog boot-arg is used,
1556 * and its value matches the VM tag of this allocation name, then VM tag
1557 * log entries will be added on the specified object.
1558 *
1559 *
1560 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1561 * @param delta the amount to update the accounting with,
1562 * positive values increment,
1563 * negative values decrement.
1564 * @param object an optional object this wiring/unwiring applies to.
1565 */
1566 extern void vm_tag_update_size(
1567 vm_tag_t tag,
1568 int64_t delta,
1569 vm_object_t object);
1570
1571
1572 /*!
1573 * @function vm_tag_get_size()
1574 *
1575 * @brief
1576 * Returns how much wired memory is accounted for the specified VM kernel tag.
1577 *
1578 *
1579 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1580 * @returns the amount of wired memory for the specified tag.
1581 */
1582 extern uint64_t vm_tag_get_size(
1583 vm_tag_t tag);
1584
1585 #if VM_TAG_SIZECLASSES
1586
1587 /*!
1588 * @function vm_tag_will_update_zone()
1589 *
1590 * @brief
1591 * Lazily allocate the vector of zone-level accounting for a given VM tag.
1592 *
1593 * @discussion
1594 * This is to be called when memory gets wired/unwired in order
1595 * to update accounting information for a given tag.
1596 *
1597 * This function might fail when the first time it is called is
1598 * for a @c Z_NOWAIT allocation. However this is transient and
1599 * will always eventually resolve. Once the data structure is allocated,
1600 * this function always succeeds. The consequence is a slight misaccounting
1601 * of a few allocations.
1602 *
1603 * This call will never fail for the following tags that are always
1604 * pre-allocated:
1605 * - @c VM_KERN_MEMORY_DIAG,
1606 * - @c VM_KERN_MEMORY_KALLOC,
1607 * - @c VM_KERN_MEMORY_KALLOC_DATA,
1608 * - @c VM_KERN_MEMORY_KALLOC_TYPE,
1609 * - @c VM_KERN_MEMORY_LIBKERN,
1610 * - @c VM_KERN_MEMORY_OSFMK,
1611 * - @c VM_KERN_MEMORY_RECOUNT.
1612 *
1613 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1614 *
1615 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1616 * @param zflags the @c zalloc_flags_t passed to the current
1617 * @c kalloc*() call.
1618 *
1619 * @returns @c tag if the allocation was successful,
1620 * @c VM_KERN_MEMORY_NONE if the accounting
1621 * data structure couldn't be allocated.
1622 */
1623 extern vm_tag_t vm_tag_will_update_zone(
1624 vm_tag_t tag,
1625 uint32_t zflags);
1626
1627
1628 /*!
1629 * @function vm_tag_update_zone_size()
1630 *
1631 * @brief
1632 * Update the per size class zone level accounting for a given kernel VM tag.
1633 *
1634 * @discussion
1635 * Note that this function isn't called if the "-zt" boot-arg isn't set.
1636 *
1637 * @param tag A non @c VM_KERN_MEMORY_NONE VM kernel tag.
1638 * @param size_class the zone size class index to account against
1639 * (@see zone_t::z_tags_sizeclass).
1640 * @param delta the amount to update the accounting with,
1641 * positive values increment,
1642 * negative values decrement.
1643 */
1644 extern void vm_tag_update_zone_size(
1645 vm_tag_t tag,
1646 uint32_t size_class,
1647 long delta);
1648
1649 #endif /* VM_TAG_SIZECLASSES */
1650
1651 #pragma mark accounting: diagnostics and query interfaces
1652
1653 /*!
1654 * @function vm_page_diagnose_estimate()
1655 *
1656 * @brief
1657 * Estimate how many @c mach_memory_info_t structures
1658 * are needed in order to return information about VM kernel tags,
1659 * and per size class zone level accounting when enabled.
1660 */
1661 extern uint32_t vm_page_diagnose_estimate(void);
1662
1663
1664 /*!
1665 * @function vm_page_diagnose()
1666 *
1667 * @brief
1668 * Fills out a @c mach_memory_info_t array of information about VM tags
1669 * and per size class zone level information.
1670 *
1671 * @param info a pointer to an array of @c num_info Mach memory
1672 * info data structures to fill.
1673 * @param num_info the number of entries in the @c info array.
1674 * @param zones_collectable_bytes
1675 * how much memory is collectable in the zone subsystem
1676 * if a @c zone_gc() was running right now.
1677 * @param redact_info whether information that could leak the type to bucket
1678 * mapping in @c kalloc_type() can be returned or not.
1679 *
1680 * @returns
1681 * - KERN_SUCCESS the call was successful.
1682 * - KERN_ABORTED the accounting subsytem isn't inititalized yet.
1683 */
1684 extern kern_return_t vm_page_diagnose(
1685 struct mach_memory_info *info __counted_by(num_info),
1686 unsigned int num_info,
1687 uint64_t zones_collectable_bytes,
1688 bool redact_info);
1689
1690 #if DEBUG || DEVELOPMENT
1691
1692 /*!
1693 * @typedef kmem_gobj_stats
1694 *
1695 * @brief
1696 * Statistics about the "guard objects" allocator for the pointer ranges
1697 * of kmem.
1698 */
1699 typedef struct {
1700 vm_map_size_t meta_sz; /**< total faulted size of metadata */
1701 vm_map_size_t pte_sz; /**< total faulted leaf PTE size */
1702 vm_map_size_t total_va; /**< total amount of VA ever used */
1703 vm_map_size_t total_used; /**< current amount of VA allocated */
1704 } kmem_gobj_stats;
1705
1706
1707 /*!
1708 * @function kmem_get_gobj_stats()
1709 *
1710 * @brief
1711 * Returns statistics about the guard objects allocator.
1712 *
1713 * @description
1714 * This is the backend of the @c vm.kmem_gobj_stats sysctl.
1715 */
1716 extern kmem_gobj_stats kmem_get_gobj_stats(void);
1717
1718
1719 /*!
1720 * @function vm_kern_allocation_info()
1721 *
1722 * @brief
1723 * Returns information about a given kernel heap allocation.
1724 *
1725 *
1726 * @param [in] addr the heap allocation pointer.
1727 * @param [out] size a guess at the size of this allocation.
1728 * @param [out] tag the kernel VM tag for this allocation
1729 * (only filled if the "-zt" boot-arg is set
1730 * for zone allocations)
1731 * @param [out] zone_size
1732 * the zone size class if the allocation
1733 * is from a zone, 0 for VM.
1734 *
1735 * @returns
1736 * - KERN_SUCCESS if a guess could be made about this pointer.
1737 * - KERN_INVALID_ADDRESS
1738 * if the address couldn't be resolved in the kernel heap.
1739 */
1740 extern kern_return_t vm_kern_allocation_info(
1741 uintptr_t addr,
1742 vm_size_t *size,
1743 vm_tag_t *tag,
1744 vm_size_t *zone_size);
1745
1746 #endif /* DEBUG || DEVELOPMENT */
1747
1748 #pragma mark - init methods
1749
1750 /*!
1751 * @function vm_init_before_launchd()
1752 *
1753 * @brief
1754 * Memorize how many wired pages were used at boot before launchd starts.
1755 *
1756 * @discussion
1757 * The captured number can be seen as the @c VM_KERN_COUNT_WIRED_BOOT value
1758 * in the output of @c zprint(1).
1759 */
1760 extern void vm_init_before_launchd(void);
1761
1762 #if DEVELOPMENT || DEBUG
1763
1764 extern kern_return_t vm_tag_reset_peak(vm_tag_t tag);
1765
1766 extern void vm_tag_reset_all_peaks(void);
1767
1768 #endif /* DEVELOPMENT || DEBUG */
1769
1770 #if VM_TAG_SIZECLASSES
1771
1772 /*!
1773 * @function vm_allocation_zones_init()
1774 *
1775 * @brief
1776 * Initialize the per-zone accounting tags subsystem
1777 * if the "-zt" boot-arg is present.
1778 */
1779 extern void vm_allocation_zones_init(void);
1780
1781 #endif /* VM_TAG_SIZECLASSES */
1782
1783 extern memory_object_t device_pager_setup(memory_object_t, uintptr_t, vm_size_t, int);
1784
1785 extern kern_return_t device_pager_populate_object( memory_object_t device,
1786 memory_object_offset_t offset, ppnum_t page_num, vm_size_t size);
1787
1788 #endif /* XNU_KERNEL_PRIVATE */
1789 __exported_pop
1790 __END_DECLS
1791
1792 #endif /* _VM_VM_KERN_XNU_H_ */
1793