1 /*
2 * Copyright (c) 2000-2021 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #ifdef KERNEL_PRIVATE
58
59 #ifndef _KERN_KALLOC_H_
60 #define _KERN_KALLOC_H_
61
62 #include <mach/machine/vm_types.h>
63 #include <mach/boolean.h>
64 #include <mach/vm_types.h>
65 #include <kern/zalloc.h>
66 #include <libkern/section_keywords.h>
67
68 __BEGIN_DECLS
69
70 #if XNU_KERNEL_PRIVATE
71
72 /*!
73 * @const KALLOC_SAFE_ALLOC_SIZE
74 *
75 * @brief
76 * The maximum allocation size that is safe to allocate with Z_NOFAIL in kalloc.
77 */
78 #if __LP64__
79 #define KALLOC_SAFE_ALLOC_SIZE (16u * 1024u)
80 #else
81 #define KALLOC_SAFE_ALLOC_SIZE (8u * 1024u)
82 #endif
83
84 /*!
85 * @typedef kalloc_heap_t
86 *
87 * @abstract
88 * A kalloc heap view represents a sub-accounting context
89 * for a given kalloc heap.
90 */
91 typedef struct kalloc_heap {
92 struct kheap_zones *kh_zones;
93 zone_stats_t kh_stats;
94 const char *kh_name;
95 struct kalloc_heap *kh_next;
96 zone_kheap_id_t kh_heap_id;
97 vm_map_t kh_large_map;
98 vm_map_t kh_fallback_map;
99 vm_tag_t kh_tag;
100 } *kalloc_heap_t;
101
102 /*!
103 * @macro KALLOC_HEAP_DECLARE
104 *
105 * @abstract
106 * (optionally) declare a kalloc heap view in a header.
107 *
108 * @discussion
109 * Unlike kernel zones, new full blown heaps cannot be instantiated.
110 * However new accounting views of the base heaps can be made.
111 */
112 #define KALLOC_HEAP_DECLARE(var) \
113 extern struct kalloc_heap var[1]
114
115 /**
116 * @const KHEAP_ANY
117 *
118 * @brief
119 * A value that represents either the default or kext heap for codepaths that
120 * need to allow @c kheap_free() to either one.
121 *
122 * @discussion
123 * When the memory provenance is not known, this value can be used to free
124 * memory indiscriminately.
125 *
126 * Note: code using this constant can likely be used as a gadget to free
127 * arbitrary memory and its use is strongly discouraged.
128 */
129 #define KHEAP_ANY ((struct kalloc_heap *)NULL)
130
131 /**
132 * @const KHEAP_DATA_BUFFERS
133 *
134 * @brief
135 * The builtin heap for bags of pure bytes.
136 *
137 * @discussion
138 * This set of kalloc zones should contain pure bags of bytes with no pointers
139 * or length/offset fields.
140 *
141 * The zones forming the heap aren't sequestered from each other, however the
142 * entire heap lives in a different submap from any other kernel allocation.
143 *
144 * The main motivation behind this separation is due to the fact that a lot of
145 * these objects have been used by attackers to spray the heap to make it more
146 * predictable while exploiting use-after-frees or overflows.
147 *
148 * Common attributes that make these objects useful for spraying includes
149 * control of:
150 * - Data in allocation
151 * - Time of alloc and free (lifetime)
152 * - Size of allocation
153 */
154 KALLOC_HEAP_DECLARE(KHEAP_DATA_BUFFERS);
155
156 /**
157 * @const KHEAP_KEXT
158 *
159 * @brief
160 * The builtin heap for allocations made by kexts.
161 *
162 * @discussion
163 * This set of kalloc zones should contain allocations from kexts and the
164 * individual zones in this heap are sequestered.
165 */
166 KALLOC_HEAP_DECLARE(KHEAP_KEXT);
167
168 /**
169 * @const KHEAP_DEFAULT
170 *
171 * @brief
172 * The builtin default core kernel kalloc heap.
173 *
174 * @discussion
175 * This set of kalloc zones should contain other objects that don't have their
176 * own security mitigations. The individual zones are themselves sequestered.
177 */
178 KALLOC_HEAP_DECLARE(KHEAP_DEFAULT);
179
180 /*!
181 * @macro KALLOC_HEAP_DEFINE
182 *
183 * @abstract
184 * Defines a given kalloc heap view and what it points to.
185 *
186 * @discussion
187 * Kalloc heaps are views over one of the pre-defined builtin heaps
188 * (such as @c KHEAP_DATA_BUFFERS or @c KHEAP_DEFAULT). Instantiating
189 * a new one allows for accounting of allocations through this view.
190 *
191 * Kalloc heap views are initialized during the @c STARTUP_SUB_ZALLOC phase,
192 * as the last rank. If views on zones are created, these must have been
193 * created before this stage.
194 *
195 * @param var the name for the zone view.
196 * @param name a string describing the zone view.
197 * @param heap_id a @c KHEAP_ID_* constant.
198 */
199 #define KALLOC_HEAP_DEFINE(var, name, heap_id) \
200 SECURITY_READ_ONLY_LATE(struct kalloc_heap) var[1] = { { \
201 .kh_name = name, \
202 .kh_heap_id = heap_id, \
203 } }; \
204 STARTUP_ARG(ZALLOC, STARTUP_RANK_LAST, kheap_startup_init, var)
205
206
207 /*
208 * Allocations of type SO_NAME are known to not have pointers for
209 * most platforms -- for macOS this is not guaranteed
210 */
211 #if XNU_TARGET_OS_OSX
212 #define KHEAP_SONAME KHEAP_DEFAULT
213 #else /* XNU_TARGET_OS_OSX */
214 #define KHEAP_SONAME KHEAP_DATA_BUFFERS
215 #endif /* XNU_TARGET_OS_OSX */
216
217 #endif/* XNU_KERNEL_PRIVATE */
218
219 /*!
220 * @enum kalloc_type_flags_t
221 *
222 * @brief
223 * Flags that can be passed to @c KALLOC_TYPE_DEFINE
224 *
225 * @discussion
226 * These flags can be used to request for a specific accounting
227 * behavior.
228 *
229 * @const KT_DEFAULT
230 * Passing this flag will provide default accounting behavior
231 * i.e shared accounting unless toggled with KT_OPTIONS_ACCT is
232 * set in kt boot-arg.
233 *
234 * @const KT_PRIV_ACCT
235 * Passing this flag will provide individual stats for your
236 * @c kalloc_type_view that is defined.
237 *
238 * @const KT_SHARED_ACCT
239 * Passing this flag will accumulate stats as a part of the
240 * zone that your @c kalloc_type_view points to.
241 *
242 #if XNU_KERNEL_PRIVATE
243 *
244 * @const KT_VM_TAG_MASK
245 * Represents bits in which a vm_tag_t for the allocation can be passed.
246 * (used for the zone tagging debugging feature).
247 #endif
248 */
249 __options_decl(kalloc_type_flags_t, uint32_t, {
250 KT_DEFAULT = 0x0001,
251 KT_PRIV_ACCT = 0x0002,
252 KT_SHARED_ACCT = 0x0004,
253 #if XNU_KERNEL_PRIVATE
254 /** used to propagate vm tags for -zt */
255 KT_VM_TAG_MASK = 0xffff0000,
256 #endif
257 });
258
259 /*!
260 * @typedef kalloc_type_view_t
261 *
262 * @abstract
263 * A kalloc type view is a structure used to redirect callers
264 * of @c kalloc_type to a particular zone based on the signature of
265 * their type.
266 *
267 * @discussion
268 * These structures are automatically created under the hood for every
269 * @c kalloc_type and @c kfree_type callsite. They are ingested during startup
270 * and are assigned zones based on the security policy for their signature.
271 *
272 * These structs are protected by the kernel lockdown and can't be initialized
273 * dynamically. They must be created using @c KALLOC_TYPE_DEFINE() or
274 * @c kalloc_type or @c kfree_type.
275 *
276 */
277 struct kalloc_type_view {
278 struct zone_view kt_zv;
279 const char *kt_signature;
280 kalloc_type_flags_t kt_flags;
281 uint32_t kt_size;
282 void *kt_site;
283 void *unused;
284 };
285
286 typedef struct kalloc_type_view *kalloc_type_view_t;
287
288 /*!
289 * @macro KALLOC_TYPE_DECLARE
290 *
291 * @abstract
292 * (optionally) declares a kalloc type view (in a header).
293 *
294 * @param var the name for the kalloc type view.
295 */
296 #define KALLOC_TYPE_DECLARE(var) \
297 extern struct kalloc_type_view var[1]
298
299 /*!
300 * @macro KALLOC_TYPE_DEFINE
301 *
302 * @abstract
303 * Defines a given kalloc type view with prefered accounting
304 *
305 * @discussion
306 * This macro allows you to define a kalloc type with private
307 * accounting. The defined kalloc_type_view can be used with
308 * kalloc_type_impl/kfree_type_impl to allocate/free memory.
309 * zalloc/zfree can also be used from inside xnu. However doing
310 * so doesn't handle freeing a NULL pointer or the use of tags.
311 *
312 * @param var the name for the kalloc type view.
313 * @param type the type of your allocation.
314 * @param flags a @c KT_* flag.
315 */
316 #define KALLOC_TYPE_DEFINE(var, type, flags) \
317 _KALLOC_TYPE_DEFINE(var, type, flags)
318
319
320 #ifdef XNU_KERNEL_PRIVATE
321
322 #define kalloc(size) \
323 kheap_alloc(KHEAP_DEFAULT, size, Z_WAITOK)
324
325 #define kalloc_flags(size, flags) \
326 kheap_alloc(KHEAP_DEFAULT, size, flags)
327
328 #define kalloc_tag(size, itag) \
329 kheap_alloc_tag(KHEAP_DEFAULT, size, Z_WAITOK, itag)
330
331 #define kalloc_tag_bt(size, itag) \
332 kheap_alloc_tag_bt(KHEAP_DEFAULT, size, Z_WAITOK, itag)
333
334 #define krealloc(elem, old_size, new_size, flags) \
335 kheap_realloc(KHEAP_DEFAULT, elem, old_size, new_size, flags)
336
337 #define krealloc_tag_bt(elem, old_size, new_size, flags, itag) \
338 kheap_realloc_tag_bt(KHEAP_DEFAULT, elem, old_size, new_size, flags, itag)
339
340 /*
341 * These versions allow specifying the kalloc heap to allocate memory
342 * from
343 */
344 #define kheap_alloc(kalloc_heap, size, flags) \
345 ({ VM_ALLOC_SITE_STATIC(0, 0); \
346 kalloc_ext(kalloc_heap, size, flags, &site).addr; })
347
348 #define kheap_alloc_tag(kalloc_heap, size, flags, itag) \
349 ({ VM_ALLOC_SITE_STATIC(0, (itag)); \
350 kalloc_ext(kalloc_heap, size, flags, &site).addr; })
351
352 #define kheap_alloc_tag_bt(kalloc_heap, size, flags, itag) \
353 ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \
354 kalloc_ext(kalloc_heap, size, flags, &site).addr; })
355
356 #define kheap_realloc(kalloc_heap, elem, old_size, new_size, flags) \
357 ({ VM_ALLOC_SITE_STATIC(0, 0); \
358 krealloc_ext(kalloc_heap, elem, old_size, new_size, flags, &site).addr; })
359
360 #define kheap_realloc_tag_bt(kalloc_heap, elem, old_size, new_size, flags, itag) \
361 ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \
362 krealloc_ext(kalloc_heap, elem, old_size, new_size, flags, &site).addr; })
363
364 /*
365 * These versions should be used for allocating pure data bytes that
366 * do not contain any pointers
367 */
368 #define kalloc_data(size, flags) \
369 kheap_alloc(KHEAP_DATA_BUFFERS, size, flags)
370
371 #define kalloc_data_tag(size, flags, itag) \
372 kheap_alloc_tag(KHEAP_DATA_BUFFERS, size, flags, itag)
373
374 #define kalloc_data_tag_bt(size, flags, itag) \
375 kheap_alloc_tag_bt(KHEAP_DATA_BUFFERS, size, flags, itag)
376
377 #define krealloc_data(elem, old_size, new_size, flags) \
378 kheap_realloc(KHEAP_DATA_BUFFERS, elem, old_size, new_size, flags)
379
380 #define krealloc_data_tag_bt(elem, old_size, new_size, flags, itag) \
381 kheap_realloc_tag_bt(KHEAP_DATA_BUFFERS, elem, old_size, new_size, flags, itag)
382
383 #define krealloc_data_addr(elem, new_size, flags) \
384 ({ VM_ALLOC_SITE_STATIC(0, 0); \
385 kheap_realloc_addr(KHEAP_DATA_BUFFERS, elem, new_size, flags, \
386 &site).addr; })
387
388 #define kfree_data(elem, size) \
389 kheap_free(KHEAP_DATA_BUFFERS, elem, size);
390
391 #define kfree_data_addr(elem) \
392 kheap_free_addr(KHEAP_DATA_BUFFERS, elem);
393
394 extern void
395 kfree(
396 void *data,
397 vm_size_t size);
398
399 extern void
400 kheap_free(
401 kalloc_heap_t heap,
402 void *data,
403 vm_size_t size);
404
405 extern void
406 kheap_free_addr(
407 kalloc_heap_t heap,
408 void *addr);
409
410 extern void
411 kheap_free_bounded(
412 kalloc_heap_t heap,
413 void *addr,
414 vm_size_t min_sz,
415 vm_size_t max_sz);
416
417 extern void
418 kalloc_data_require(
419 void *data,
420 vm_size_t size);
421
422 extern void
423 kalloc_non_data_require(
424 void *data,
425 vm_size_t size);
426
427 #else /* XNU_KERNEL_PRIVATE */
428
429 extern void *
430 kalloc(
431 vm_size_t size) __attribute__((alloc_size(1)));
432
433 extern void *
434 kalloc_data(
435 vm_size_t size,
436 zalloc_flags_t flags) __attribute__((alloc_size(1)));
437
438 extern void *
439 krealloc_data(
440 void *ptr,
441 vm_size_t old_size,
442 vm_size_t new_size,
443 zalloc_flags_t flags) __attribute__((alloc_size(3)));
444
445 extern void *
446 krealloc_data_addr(
447 void *ptr,
448 vm_size_t new_size,
449 zalloc_flags_t flags) __attribute__((alloc_size(2)));
450
451 extern void
452 kfree(
453 void *data,
454 vm_size_t size);
455
456 extern void
457 kfree_data(
458 void *ptr,
459 vm_size_t size);
460
461 extern void
462 kfree_data_addr(
463 void *ptr);
464
465 #endif /* !XNU_KERNEL_PRIVATE */
466
467 /*!
468 * @macro kalloc_type
469 *
470 * @abstract
471 * Allocates element of a particular type
472 *
473 * @discussion
474 * This family of allocators segregate kalloc allocations based on their type.
475 *
476 * This macro comes in 3 variants:
477 *
478 * 1. @c kalloc_type(type, flags)
479 * Use this macro for fixed sized allocation of a particular type.
480 *
481 * 2. @c kalloc_type(e_type, count, flags)
482 * Use this macro for variable sized allocations that form an array,
483 * do note that @c kalloc_type(e_type, 1, flags) is not equivalent to
484 * @c kalloc_type(e_type, flags).
485 *
486 * 3. @c kalloc_type(hdr_type, e_type, count, flags)
487 * Use this macro for variable sized allocations formed with
488 * a header of type @c hdr_type followed by a variable sized array
489 * with elements of type @c e_type, equivalent to this:
490 *
491 * <code>
492 * struct {
493 * hdr_type hdr;
494 * e_type arr[];
495 * }
496 * </code>
497 *
498 * @param flags @c zalloc_flags_t that get passed to zalloc_internal
499 */
500 #define kalloc_type(...) KALLOC_DISPATCH(kalloc_type, ##__VA_ARGS__)
501
502 /*!
503 * @macro kfree_type
504 *
505 * @abstract
506 * Allocates element of a particular type
507 *
508 * @discussion
509 * This pairs with the @c kalloc_type() that was made to allocate this element.
510 * Arguments passed to @c kfree_type() must match the one passed at allocation
511 * time precisely.
512 *
513 * This macro comes in the same 3 variants kalloc_type() does:
514 *
515 * 1. @c kfree_type(type, elem)
516 * 2. @c kfree_type(e_type, count, elem)
517 * 3. @c kfree_type(hdr_type, e_type, count, elem)
518 *
519 * @param elem The address of the element to free
520 */
521 #define kfree_type(...) KALLOC_DISPATCH(kfree_type, ##__VA_ARGS__)
522
523 #ifdef XNU_KERNEL_PRIVATE
524 #define kalloc_type_tag(...) KALLOC_DISPATCH(kalloc_type_tag, ##__VA_ARGS__)
525 #define kalloc_type_tag_bt(...) KALLOC_DISPATCH(kalloc_type_tag_bt, ##__VA_ARGS__)
526 #define kallocp_type_tag_bt(ty, countp, flags, tag) ({ \
527 VM_ALLOC_SITE_STATIC(VM_TAG_BT, (tag)); \
528 (ty *)kallocp_ext(KHEAP_DEFAULT, sizeof(ty), countp, flags, &site); \
529 })
530
531 /*
532 * kalloc_type_require can't be made available to kexts as the
533 * kalloc_type_view's zone could be NULL in the following cases:
534 * - Size greater than KALLOC_SAFE_ALLOC_SIZE
535 * - On macOS, if call is not in BootKC
536 * - All allocations in kext for armv7
537 */
538 #define kalloc_type_require(type, value) ({ \
539 static KALLOC_TYPE_DEFINE(kt_view_var, type, \
540 KT_SHARED_ACCT); \
541 zone_require(kt_view_var->kt_zv, value); \
542 })
543
544 #endif
545
546 #pragma mark implementation details
547
548 static inline vm_size_t
kt_size(vm_size_t s1,vm_size_t s2,vm_size_t c2)549 kt_size(vm_size_t s1, vm_size_t s2, vm_size_t c2)
550 {
551 /* kalloc_large() will reject this size before even asking the VM */
552 const vm_size_t limit = 1ull << (8 * sizeof(vm_size_t) - 1);
553
554 if (os_mul_and_add_overflow(s2, c2, s1, &s1) || (s1 & limit)) {
555 return limit;
556 }
557 return s1;
558 }
559
560 #define kalloc_type_2(type, flags) ({ \
561 static KALLOC_TYPE_DEFINE(kt_view_var, type, KT_SHARED_ACCT); \
562 (type *)kalloc_type_impl(kt_view_var, flags); \
563 })
564
565 #define kfree_type_2(type, elem) ({ \
566 static KALLOC_TYPE_DEFINE(kt_view_var, type, KT_SHARED_ACCT); \
567 kfree_type_impl(kt_view_var, __zalloc_ptr_load_and_erase(elem)); \
568 })
569
570 #define kfree_type_3(type, count, elem) \
571 kfree(elem, kt_size(0, sizeof(type), count))
572
573 #define kfree_type_4(hdr_ty, e_ty, count, elem) \
574 kfree(elem, kt_size(sizeof(hdr_ty), sizeof(e_ty), count))
575
576 #ifdef XNU_KERNEL_PRIVATE
577 #define kalloc_type_3(type, count, flags) \
578 ((type *)kalloc_flags(kt_size(0, sizeof(type), count), flags))
579
580 #define kalloc_type_4(hdr_ty, e_ty, count, flags) \
581 ((hdr_ty *)kalloc_flags(kt_size(sizeof(hdr_ty), sizeof(e_ty), count), flags))
582
583 #define kalloc_type_tag_3(type, flags, tag) ({ \
584 static KALLOC_TYPE_DEFINE(kt_view_var, type, \
585 (kalloc_type_flags_t) (KT_SHARED_ACCT | Z_VM_TAG(tag))); \
586 (type *) kalloc_type_impl(kt_view_var, flags); \
587 })
588
589 #define kalloc_type_tag_4(type, count, flags, tag) \
590 ((type *)kheap_alloc_tag(KHEAP_DEFAULT, kt_size(0, sizeof(type), count), flags, tag))
591
592 #define kalloc_type_tag_bt_4(type, count, flags, tag) \
593 ((type *)kheap_alloc_tag_bt(KHEAP_DEFAULT, kt_size(0, sizeof(type), count), flags, tag))
594
595 #define kalloc_type_tag_5(hdr_ty, e_ty, count, flags, tag) \
596 ((hdr_ty *)kheap_alloc_tag(KHEAP_DEFAULT,kt_size(sizeof(hdr_ty), sizeof(e_ty), count), flags, tag))
597
598 #define kalloc_type_tag_bt_5(hdr_ty, e_ty, count, flags, tag) \
599 ((hdr_ty *)kheap_alloc_tag_bt(KHEAP_DEFAULT,kt_size(sizeof(hdr_ty), sizeof(e_ty), count), flags, tag))
600
601 #else /* XNU_KERNEL_PRIVATE */
602 /* for now kexts do not have access to flags */
603 #define kalloc_type_3(type, count, flags) ({ \
604 _Static_assert((flags) == Z_WAITOK, "kexts can only pass Z_WAITOK"); \
605 ((type *)kalloc(kt_size(0, sizeof(type), count)); \
606 })
607
608 #define kalloc_type_4(hdr_ty, e_ty, count, flags) ({ \
609 _Static_assert((flags) == Z_WAITOK, "kexts can only pass Z_WAITOK"); \
610 ((hdr_ty *)kalloc(kt_size(sizeof(hdr_ty), sizeof(e_ty), count))); \
611 })
612 #endif /* !XNU_KERNEL_PRIVATE */
613
614 /*
615 * All k*free macros set "elem" to NULL on free.
616 *
617 * Note: all values passed to k*free() might be in the element to be freed,
618 * temporaries must be taken, and the resetting to be done prior to free.
619 */
620 #ifdef XNU_KERNEL_PRIVATE
621
622 #define kfree(elem, size) ({ \
623 __auto_type __kfree_size = (size); \
624 (kfree)((void *)__zalloc_ptr_load_and_erase(elem), \
625 __kfree_size); \
626 })
627
628 #define kheap_free(heap, elem, size) ({ \
629 __auto_type __kfree_heap = (heap); \
630 __auto_type __kfree_size = (size); \
631 (kheap_free)(__kfree_heap, \
632 (void *)__zalloc_ptr_load_and_erase(elem), \
633 __kfree_size); \
634 })
635
636 #define kheap_free_addr(heap, elem) ({ \
637 __auto_type __kfree_heap = (heap); \
638 (kheap_free_addr)(__kfree_heap, \
639 (void *)__zalloc_ptr_load_and_erase(elem)); \
640 })
641
642 #define kheap_free_bounded(heap, elem, min_sz, max_sz) ({ \
643 static_assert(max_sz <= KALLOC_SAFE_ALLOC_SIZE); \
644 __auto_type __kfree_heap = (heap); \
645 __auto_type __kfree_min_sz = (min_sz); \
646 __auto_type __kfree_max_sz = (max_sz); \
647 (kheap_free_bounded)(__kfree_heap, \
648 (void *)__zalloc_ptr_load_and_erase(elem), \
649 __kfree_min_sz, __kfree_max_sz); \
650 })
651
652 #else /* XNU_KERNEL_PRIVATE */
653
654 #define kfree_data(elem, size) ({ \
655 __auto_type __kfree_size = (size); \
656 (kfree_data)((void *)__zalloc_ptr_load_and_erase(elem), \
657 __kfree_size); \
658 })
659
660 #define kfree_data_addr(elem) \
661 (kfree_data_addr)((void *)__zalloc_ptr_load_and_erase(elem))
662
663 #endif /* !XNU_KERNEL_PRIVATE */
664
665 #if __has_feature(address_sanitizer)
666 # define __kalloc_no_kasan __attribute__((no_sanitize("address")))
667 #else
668 # define __kalloc_no_kasan
669 #endif
670
671 #define KALLOC_CONCAT(x, y) __CONCAT(x,y)
672
673 #define KALLOC_COUNT_ARGS1(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, N, ...) N
674 #define KALLOC_COUNT_ARGS(...) \
675 KALLOC_COUNT_ARGS1(, ##__VA_ARGS__, _9, _8, _7, _6, _5, _4, _3, _2, _1, _0)
676 #define KALLOC_DISPATCH1(base, N, ...) __CONCAT(base, N)(__VA_ARGS__)
677 #define KALLOC_DISPATCH(base, ...) \
678 KALLOC_DISPATCH1(base, KALLOC_COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__)
679
680 #define kt_view_var \
681 KALLOC_CONCAT(kalloc_type_view_, __LINE__)
682
683 #if __LP64__
684 #define KALLOC_TYPE_SEGMENT "__DATA_CONST"
685 #else
686 #define KALLOC_TYPE_SEGMENT "__DATA"
687 #endif
688
689 /*
690 * When kalloc_type_impl is called from xnu, it calls zalloc_flags
691 * directly and doesn't redirect zone-less sites to kheap_alloc.
692 * Passing a size larger than kalloc_max for these allocations will
693 * lead to a panic as the zone is null. Therefore assert that size
694 * is less than KALLOC_SAFE_ALLOC_SIZE.
695 */
696 #ifdef XNU_KERNEL_PRIVATE
697 #define KALLOC_TYPE_SIZE_CHECK(size) \
698 _Static_assert(size <= KALLOC_SAFE_ALLOC_SIZE, \
699 "type is too large");
700 #else
701 #define KALLOC_TYPE_SIZE_CHECK(size)
702 #endif
703
704 #define _KALLOC_TYPE_DEFINE(var, type, flags) \
705 __kalloc_no_kasan \
706 __PLACE_IN_SECTION(KALLOC_TYPE_SEGMENT ", __kalloc_type") \
707 struct kalloc_type_view var[1] = { { \
708 .kt_zv.zv_name = "site." #type, \
709 .kt_flags = flags, \
710 .kt_size = sizeof(type), \
711 .kt_signature = __builtin_xnu_type_signature(type), \
712 } }; \
713 KALLOC_TYPE_SIZE_CHECK(sizeof(type));
714
715 #ifndef XNU_KERNEL_PRIVATE
716 /*
717 * This macro is currently used by AppleImage4
718 */
719 #define KALLOC_TYPE_DEFINE_SITE(var, type, flags) \
720 static _KALLOC_TYPE_DEFINE(var, type, flags)
721
722 #endif /* !XNU_KERNEL_PRIVATE */
723
724 #ifdef XNU_KERNEL_PRIVATE
725
726 #define KT_VM_TAG(var) \
727 ((var)->kt_flags & KT_VM_TAG_MASK)
728
729 #define kalloc_type_impl(kt_view, flags) ({ \
730 zalloc_flags(kt_view, \
731 (zalloc_flags_t) (KT_VM_TAG(kt_view) | flags)); \
732 })
733
734 static inline void
kfree_type_impl(kalloc_type_view_t kt_view,void * ptr)735 kfree_type_impl(kalloc_type_view_t kt_view, void *ptr)
736 {
737 if (NULL == ptr) {
738 return;
739 }
740 zfree(kt_view, ptr);
741 }
742
743 #else /* XNU_KERNEL_PRIVATE */
744
745 extern void *
746 kalloc_type_impl(
747 kalloc_type_view_t kt_view,
748 zalloc_flags_t flags);
749
750 extern void
751 kfree_type_impl(
752 kalloc_type_view_t kt_view,
753 void *ptr);
754
755 #endif /* !XNU_KERNEL_PRIVATE */
756
757 void *
758 kalloc_type_impl_external(
759 kalloc_type_view_t kt_view,
760 zalloc_flags_t flags);
761
762 void
763 kfree_type_impl_external(
764 kalloc_type_view_t kt_view,
765 void *ptr);
766
767 extern void *
768 OSObject_typed_operator_new(
769 kalloc_type_view_t ktv,
770 vm_size_t size);
771
772 extern void
773 OSObject_typed_operator_delete(
774 kalloc_type_view_t ktv,
775 void *mem,
776 vm_size_t size);
777
778 #ifdef XNU_KERNEL_PRIVATE
779 #pragma GCC visibility push(hidden)
780
781 #define KALLOC_TYPE_SIZE_MASK 0xffffff
782 #define KALLOC_TYPE_IDX_SHIFT 24
783 #define KALLOC_TYPE_IDX_MASK 0xff
784
785 static inline uint16_t
kalloc_type_get_idx(uint32_t kt_size)786 kalloc_type_get_idx(uint32_t kt_size)
787 {
788 return (uint16_t) (kt_size >> KALLOC_TYPE_IDX_SHIFT);
789 }
790
791 static inline uint32_t
kalloc_type_set_idx(uint32_t kt_size,uint16_t idx)792 kalloc_type_set_idx(uint32_t kt_size, uint16_t idx)
793 {
794 return kt_size | ((uint32_t) idx << KALLOC_TYPE_IDX_SHIFT);
795 }
796
797 static inline uint32_t
kalloc_type_get_size(uint32_t kt_size)798 kalloc_type_get_size(uint32_t kt_size)
799 {
800 return kt_size & KALLOC_TYPE_SIZE_MASK;
801 }
802
803 bool
804 IOMallocType_from_vm(
805 uint32_t kt_idx,
806 uint32_t kt_size);
807
808 /* Used by kern_os_* and operator new */
809 KALLOC_HEAP_DECLARE(KERN_OS_MALLOC);
810
811 extern void
812 kheap_startup_init(
813 kalloc_heap_t heap);
814
815 /*
816 * This type is used so that kalloc_internal has good calling conventions
817 * for callers who want to cheaply both know the allocated address
818 * and the actual size of the allocation.
819 */
820 struct kalloc_result {
821 void *addr;
822 vm_size_t size;
823 };
824
825 extern struct kalloc_result
826 kalloc_ext(
827 kalloc_heap_t kheap,
828 vm_size_t size,
829 zalloc_flags_t flags,
830 vm_allocation_site_t *site);
831
832 extern struct kalloc_result
833 krealloc_ext(
834 kalloc_heap_t kheap,
835 void *addr,
836 vm_size_t old_size,
837 vm_size_t new_size,
838 zalloc_flags_t flags,
839 vm_allocation_site_t *site);
840
841 extern struct kalloc_result
842 kheap_realloc_addr(
843 kalloc_heap_t kheap,
844 void *addr,
845 vm_size_t new_size,
846 zalloc_flags_t flags,
847 vm_allocation_site_t *site);
848
849
850 /* these versions update the size reference with the actual size allocated */
851
852 static inline void *
kallocp_ext(kalloc_heap_t kheap,vm_size_t ty_size,vm_size_t * count,zalloc_flags_t flags,vm_allocation_site_t * site)853 kallocp_ext(
854 kalloc_heap_t kheap,
855 vm_size_t ty_size,
856 vm_size_t *count,
857 zalloc_flags_t flags,
858 vm_allocation_site_t *site)
859 {
860 vm_size_t size = kt_size(0, ty_size, *count);
861 struct kalloc_result kar = kalloc_ext(kheap, size, flags, site);
862 *count = kar.size / ty_size;
863 return kar.addr;
864 }
865
866
867 #define kallocp_tag_bt(sizep, itag) \
868 ({ VM_ALLOC_SITE_STATIC(VM_TAG_BT, (itag)); \
869 kallocp_ext(KHEAP_DEFAULT, sizep, Z_WAITOK, &site); })
870
871 extern vm_size_t
872 kheap_alloc_size(
873 kalloc_heap_t heap,
874 void *addr);
875
876 extern bool
877 kalloc_owned_map(
878 vm_map_t map);
879
880 extern vm_map_t
881 kalloc_large_map_get(void);
882
883 extern vm_map_t
884 kalloc_large_data_map_get(void);
885
886 extern vm_map_t
887 kernel_data_map_get(void);
888
889 extern zone_t
890 kalloc_heap_zone_for_size(
891 kalloc_heap_t heap,
892 vm_size_t size);
893
894 extern vm_size_t kalloc_max_prerounded;
895 extern vm_size_t kalloc_large_total;
896
897 extern void
898 kern_os_kfree(
899 void *addr,
900 vm_size_t size);
901
902 extern void
903 kern_os_typed_free(
904 kalloc_type_view_t ktv,
905 void *addr,
906 vm_size_t esize);
907
908 #pragma GCC visibility pop
909 #endif /* !XNU_KERNEL_PRIVATE */
910
911 extern void
912 kern_os_zfree(
913 zone_t zone,
914 void *addr,
915 vm_size_t size);
916
917 __END_DECLS
918
919 #endif /* _KERN_KALLOC_H_ */
920
921 #endif /* KERNEL_PRIVATE */
922